1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 80 #endif 81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 83 }; 84 85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 87 (const int []){ \ 88 __VA_ARGS__, 0, \ 89 } 90 91 /** Node object of input graph for mlx5_flow_expand_rss(). */ 92 struct mlx5_flow_expand_node { 93 const int *const next; 94 /**< 95 * List of next node indexes. Index 0 is interpreted as a terminator. 96 */ 97 const enum rte_flow_item_type type; 98 /**< Pattern item type of current node. */ 99 uint64_t rss_types; 100 /**< 101 * RSS types bit-field associated with this node 102 * (see RTE_ETH_RSS_* definitions). 103 */ 104 uint64_t node_flags; 105 /**< 106 * Bit-fields that define how the node is used in the expansion. 107 * (see MLX5_EXPANSION_NODE_* definitions). 108 */ 109 }; 110 111 /* Optional expand field. The expansion alg will not go deeper. */ 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 113 114 /* The node is not added implicitly as expansion to the flow pattern. 115 * If the node type does not match the flow pattern item type, the 116 * expansion alg will go deeper to its next items. 117 * In the current implementation, the list of next nodes indexes can 118 * have up to one node with this flag set and it has to be the last 119 * node index (before the list terminator). 120 */ 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 122 123 /** Object returned by mlx5_flow_expand_rss(). */ 124 struct mlx5_flow_expand_rss { 125 uint32_t entries; 126 /**< Number of entries @p patterns and @p priorities. */ 127 struct { 128 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 129 uint32_t priority; /**< Priority offset for each expansion. */ 130 } entry[]; 131 }; 132 133 static void 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 135 136 static const struct mlx5_flow_expand_node * 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 138 unsigned int item_idx, 139 const struct mlx5_flow_expand_node graph[], 140 const struct mlx5_flow_expand_node *node); 141 142 static bool 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 144 { 145 switch (item->type) { 146 case RTE_FLOW_ITEM_TYPE_ETH: 147 case RTE_FLOW_ITEM_TYPE_VLAN: 148 case RTE_FLOW_ITEM_TYPE_IPV4: 149 case RTE_FLOW_ITEM_TYPE_IPV6: 150 case RTE_FLOW_ITEM_TYPE_UDP: 151 case RTE_FLOW_ITEM_TYPE_TCP: 152 case RTE_FLOW_ITEM_TYPE_VXLAN: 153 case RTE_FLOW_ITEM_TYPE_NVGRE: 154 case RTE_FLOW_ITEM_TYPE_GRE: 155 case RTE_FLOW_ITEM_TYPE_GENEVE: 156 case RTE_FLOW_ITEM_TYPE_MPLS: 157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 158 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 160 case RTE_FLOW_ITEM_TYPE_GTP: 161 return true; 162 default: 163 break; 164 } 165 return false; 166 } 167 168 /** 169 * Network Service Header (NSH) and its next protocol values 170 * are described in RFC-8393. 171 */ 172 static enum rte_flow_item_type 173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 174 { 175 enum rte_flow_item_type type; 176 177 switch (proto_mask & proto_spec) { 178 case RTE_VXLAN_GPE_TYPE_IPV4: 179 type = RTE_FLOW_ITEM_TYPE_IPV4; 180 break; 181 case RTE_VXLAN_GPE_TYPE_IPV6: 182 type = RTE_VXLAN_GPE_TYPE_IPV6; 183 break; 184 case RTE_VXLAN_GPE_TYPE_ETH: 185 type = RTE_FLOW_ITEM_TYPE_ETH; 186 break; 187 default: 188 type = RTE_FLOW_ITEM_TYPE_END; 189 } 190 return type; 191 } 192 193 static enum rte_flow_item_type 194 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 195 { 196 enum rte_flow_item_type type; 197 198 switch (proto_mask & proto_spec) { 199 case IPPROTO_UDP: 200 type = RTE_FLOW_ITEM_TYPE_UDP; 201 break; 202 case IPPROTO_TCP: 203 type = RTE_FLOW_ITEM_TYPE_TCP; 204 break; 205 case IPPROTO_IP: 206 type = RTE_FLOW_ITEM_TYPE_IPV4; 207 break; 208 case IPPROTO_IPV6: 209 type = RTE_FLOW_ITEM_TYPE_IPV6; 210 break; 211 default: 212 type = RTE_FLOW_ITEM_TYPE_END; 213 } 214 return type; 215 } 216 217 static enum rte_flow_item_type 218 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 219 rte_be16_t type_mask, bool is_tunnel) 220 { 221 enum rte_flow_item_type type; 222 223 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 224 case RTE_ETHER_TYPE_TEB: 225 type = is_tunnel ? 226 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 227 break; 228 case RTE_ETHER_TYPE_VLAN: 229 type = !is_tunnel ? 230 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 231 break; 232 case RTE_ETHER_TYPE_IPV4: 233 type = RTE_FLOW_ITEM_TYPE_IPV4; 234 break; 235 case RTE_ETHER_TYPE_IPV6: 236 type = RTE_FLOW_ITEM_TYPE_IPV6; 237 break; 238 default: 239 type = RTE_FLOW_ITEM_TYPE_END; 240 } 241 return type; 242 } 243 244 static enum rte_flow_item_type 245 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 246 { 247 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 248 do { \ 249 const void *m = item->mask; \ 250 const void *s = item->spec; \ 251 mask = m ? \ 252 ((const struct rte_flow_item_##type *)m)->fld : \ 253 rte_flow_item_##type##_mask.fld; \ 254 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 255 } while (0) 256 257 enum rte_flow_item_type ret; 258 uint16_t spec, mask; 259 260 if (item == NULL || item->spec == NULL) 261 return RTE_FLOW_ITEM_TYPE_VOID; 262 switch (item->type) { 263 case RTE_FLOW_ITEM_TYPE_ETH: 264 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 265 if (!mask) 266 return RTE_FLOW_ITEM_TYPE_VOID; 267 ret = mlx5_ethertype_to_item_type(spec, mask, false); 268 break; 269 case RTE_FLOW_ITEM_TYPE_VLAN: 270 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 271 if (!mask) 272 return RTE_FLOW_ITEM_TYPE_VOID; 273 ret = mlx5_ethertype_to_item_type(spec, mask, false); 274 break; 275 case RTE_FLOW_ITEM_TYPE_IPV4: 276 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 277 if (!mask) 278 return RTE_FLOW_ITEM_TYPE_VOID; 279 ret = mlx5_inet_proto_to_item_type(spec, mask); 280 break; 281 case RTE_FLOW_ITEM_TYPE_IPV6: 282 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 283 if (!mask) 284 return RTE_FLOW_ITEM_TYPE_VOID; 285 ret = mlx5_inet_proto_to_item_type(spec, mask); 286 break; 287 case RTE_FLOW_ITEM_TYPE_GENEVE: 288 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 289 ret = mlx5_ethertype_to_item_type(spec, mask, true); 290 break; 291 case RTE_FLOW_ITEM_TYPE_GRE: 292 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 293 ret = mlx5_ethertype_to_item_type(spec, mask, true); 294 break; 295 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 296 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 297 ret = mlx5_nsh_proto_to_item_type(spec, mask); 298 break; 299 default: 300 ret = RTE_FLOW_ITEM_TYPE_VOID; 301 break; 302 } 303 return ret; 304 #undef MLX5_XSET_ITEM_MASK_SPEC 305 } 306 307 static const int * 308 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 309 const int *next_node) 310 { 311 const struct mlx5_flow_expand_node *node = NULL; 312 const int *next = next_node; 313 314 while (next && *next) { 315 /* 316 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 317 * flag set, because they were not found in the flow pattern. 318 */ 319 node = &graph[*next]; 320 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 321 break; 322 next = node->next; 323 } 324 return next; 325 } 326 327 #define MLX5_RSS_EXP_ELT_N 16 328 329 /** 330 * Expand RSS flows into several possible flows according to the RSS hash 331 * fields requested and the driver capabilities. 332 * 333 * @param[out] buf 334 * Buffer to store the result expansion. 335 * @param[in] size 336 * Buffer size in bytes. If 0, @p buf can be NULL. 337 * @param[in] pattern 338 * User flow pattern. 339 * @param[in] types 340 * RSS types to expand (see RTE_ETH_RSS_* definitions). 341 * @param[in] graph 342 * Input graph to expand @p pattern according to @p types. 343 * @param[in] graph_root_index 344 * Index of root node in @p graph, typically 0. 345 * 346 * @return 347 * A positive value representing the size of @p buf in bytes regardless of 348 * @p size on success, a negative errno value otherwise and rte_errno is 349 * set, the following errors are defined: 350 * 351 * -E2BIG: graph-depth @p graph is too deep. 352 * -EINVAL: @p size has not enough space for expanded pattern. 353 */ 354 static int 355 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 356 const struct rte_flow_item *pattern, uint64_t types, 357 const struct mlx5_flow_expand_node graph[], 358 int graph_root_index) 359 { 360 const struct rte_flow_item *item; 361 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 362 const int *next_node; 363 const int *stack[MLX5_RSS_EXP_ELT_N]; 364 int stack_pos = 0; 365 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 366 unsigned int i, item_idx, last_expand_item_idx = 0; 367 size_t lsize; 368 size_t user_pattern_size = 0; 369 void *addr = NULL; 370 const struct mlx5_flow_expand_node *next = NULL; 371 struct rte_flow_item missed_item; 372 int missed = 0; 373 int elt = 0; 374 const struct rte_flow_item *last_expand_item = NULL; 375 376 memset(&missed_item, 0, sizeof(missed_item)); 377 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 378 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 379 if (lsize > size) 380 return -EINVAL; 381 buf->entry[0].priority = 0; 382 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 383 buf->entries = 0; 384 addr = buf->entry[0].pattern; 385 for (item = pattern, item_idx = 0; 386 item->type != RTE_FLOW_ITEM_TYPE_END; 387 item++, item_idx++) { 388 if (!mlx5_flow_is_rss_expandable_item(item)) { 389 user_pattern_size += sizeof(*item); 390 continue; 391 } 392 last_expand_item = item; 393 last_expand_item_idx = item_idx; 394 i = 0; 395 while (node->next && node->next[i]) { 396 next = &graph[node->next[i]]; 397 if (next->type == item->type) 398 break; 399 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 400 node = next; 401 i = 0; 402 } else { 403 ++i; 404 } 405 } 406 if (next) 407 node = next; 408 user_pattern_size += sizeof(*item); 409 } 410 user_pattern_size += sizeof(*item); /* Handle END item. */ 411 lsize += user_pattern_size; 412 if (lsize > size) 413 return -EINVAL; 414 /* Copy the user pattern in the first entry of the buffer. */ 415 rte_memcpy(addr, pattern, user_pattern_size); 416 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 417 buf->entries = 1; 418 /* Start expanding. */ 419 memset(flow_items, 0, sizeof(flow_items)); 420 user_pattern_size -= sizeof(*item); 421 /* 422 * Check if the last valid item has spec set, need complete pattern, 423 * and the pattern can be used for expansion. 424 */ 425 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 426 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 427 /* Item type END indicates expansion is not required. */ 428 return lsize; 429 } 430 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 431 next = NULL; 432 missed = 1; 433 i = 0; 434 while (node->next && node->next[i]) { 435 next = &graph[node->next[i]]; 436 if (next->type == missed_item.type) { 437 flow_items[0].type = missed_item.type; 438 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 439 break; 440 } 441 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 442 node = next; 443 i = 0; 444 } else { 445 ++i; 446 } 447 next = NULL; 448 } 449 } 450 if (next && missed) { 451 elt = 2; /* missed item + item end. */ 452 node = next; 453 lsize += elt * sizeof(*item) + user_pattern_size; 454 if (lsize > size) 455 return -EINVAL; 456 if (node->rss_types & types) { 457 buf->entry[buf->entries].priority = 1; 458 buf->entry[buf->entries].pattern = addr; 459 buf->entries++; 460 rte_memcpy(addr, buf->entry[0].pattern, 461 user_pattern_size); 462 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 463 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 464 addr = (void *)(((uintptr_t)addr) + 465 elt * sizeof(*item)); 466 } 467 } else if (last_expand_item != NULL) { 468 node = mlx5_flow_expand_rss_adjust_node(pattern, 469 last_expand_item_idx, graph, node); 470 } 471 memset(flow_items, 0, sizeof(flow_items)); 472 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 473 node->next); 474 stack[stack_pos] = next_node; 475 node = next_node ? &graph[*next_node] : NULL; 476 while (node) { 477 flow_items[stack_pos].type = node->type; 478 if (node->rss_types & types) { 479 size_t n; 480 /* 481 * compute the number of items to copy from the 482 * expansion and copy it. 483 * When the stack_pos is 0, there are 1 element in it, 484 * plus the addition END item. 485 */ 486 elt = stack_pos + 2; 487 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 488 lsize += elt * sizeof(*item) + user_pattern_size; 489 if (lsize > size) 490 return -EINVAL; 491 n = elt * sizeof(*item); 492 buf->entry[buf->entries].priority = 493 stack_pos + 1 + missed; 494 buf->entry[buf->entries].pattern = addr; 495 buf->entries++; 496 rte_memcpy(addr, buf->entry[0].pattern, 497 user_pattern_size); 498 addr = (void *)(((uintptr_t)addr) + 499 user_pattern_size); 500 rte_memcpy(addr, &missed_item, 501 missed * sizeof(*item)); 502 addr = (void *)(((uintptr_t)addr) + 503 missed * sizeof(*item)); 504 rte_memcpy(addr, flow_items, n); 505 addr = (void *)(((uintptr_t)addr) + n); 506 } 507 /* Go deeper. */ 508 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 509 node->next) { 510 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 511 node->next); 512 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 513 rte_errno = E2BIG; 514 return -rte_errno; 515 } 516 stack[stack_pos] = next_node; 517 } else if (*(next_node + 1)) { 518 /* Follow up with the next possibility. */ 519 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 520 ++next_node); 521 } else if (!stack_pos) { 522 /* 523 * Completing the traverse over the different paths. 524 * The next_node is advanced to the terminator. 525 */ 526 ++next_node; 527 } else { 528 /* Move to the next path. */ 529 while (stack_pos) { 530 next_node = stack[--stack_pos]; 531 next_node++; 532 if (*next_node) 533 break; 534 } 535 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 536 next_node); 537 stack[stack_pos] = next_node; 538 } 539 node = next_node && *next_node ? &graph[*next_node] : NULL; 540 }; 541 return lsize; 542 } 543 544 enum mlx5_expansion { 545 MLX5_EXPANSION_ROOT, 546 MLX5_EXPANSION_ROOT_OUTER, 547 MLX5_EXPANSION_OUTER_ETH, 548 MLX5_EXPANSION_OUTER_VLAN, 549 MLX5_EXPANSION_OUTER_IPV4, 550 MLX5_EXPANSION_OUTER_IPV4_UDP, 551 MLX5_EXPANSION_OUTER_IPV4_TCP, 552 MLX5_EXPANSION_OUTER_IPV6, 553 MLX5_EXPANSION_OUTER_IPV6_UDP, 554 MLX5_EXPANSION_OUTER_IPV6_TCP, 555 MLX5_EXPANSION_VXLAN, 556 MLX5_EXPANSION_STD_VXLAN, 557 MLX5_EXPANSION_L3_VXLAN, 558 MLX5_EXPANSION_VXLAN_GPE, 559 MLX5_EXPANSION_GRE, 560 MLX5_EXPANSION_NVGRE, 561 MLX5_EXPANSION_GRE_KEY, 562 MLX5_EXPANSION_MPLS, 563 MLX5_EXPANSION_ETH, 564 MLX5_EXPANSION_VLAN, 565 MLX5_EXPANSION_IPV4, 566 MLX5_EXPANSION_IPV4_UDP, 567 MLX5_EXPANSION_IPV4_TCP, 568 MLX5_EXPANSION_IPV6, 569 MLX5_EXPANSION_IPV6_UDP, 570 MLX5_EXPANSION_IPV6_TCP, 571 MLX5_EXPANSION_IPV6_FRAG_EXT, 572 MLX5_EXPANSION_GTP, 573 MLX5_EXPANSION_GENEVE, 574 }; 575 576 /** Supported expansion of items. */ 577 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 578 [MLX5_EXPANSION_ROOT] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 580 MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6), 582 .type = RTE_FLOW_ITEM_TYPE_END, 583 }, 584 [MLX5_EXPANSION_ROOT_OUTER] = { 585 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 586 MLX5_EXPANSION_OUTER_IPV4, 587 MLX5_EXPANSION_OUTER_IPV6), 588 .type = RTE_FLOW_ITEM_TYPE_END, 589 }, 590 [MLX5_EXPANSION_OUTER_ETH] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 592 .type = RTE_FLOW_ITEM_TYPE_ETH, 593 .rss_types = 0, 594 }, 595 [MLX5_EXPANSION_OUTER_VLAN] = { 596 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 597 MLX5_EXPANSION_OUTER_IPV6), 598 .type = RTE_FLOW_ITEM_TYPE_VLAN, 599 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 600 }, 601 [MLX5_EXPANSION_OUTER_IPV4] = { 602 .next = MLX5_FLOW_EXPAND_RSS_NEXT 603 (MLX5_EXPANSION_OUTER_IPV4_UDP, 604 MLX5_EXPANSION_OUTER_IPV4_TCP, 605 MLX5_EXPANSION_GRE, 606 MLX5_EXPANSION_NVGRE, 607 MLX5_EXPANSION_IPV4, 608 MLX5_EXPANSION_IPV6), 609 .type = RTE_FLOW_ITEM_TYPE_IPV4, 610 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 611 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 612 }, 613 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 614 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 615 MLX5_EXPANSION_VXLAN_GPE, 616 MLX5_EXPANSION_MPLS, 617 MLX5_EXPANSION_GENEVE, 618 MLX5_EXPANSION_GTP), 619 .type = RTE_FLOW_ITEM_TYPE_UDP, 620 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 621 }, 622 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 623 .type = RTE_FLOW_ITEM_TYPE_TCP, 624 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 625 }, 626 [MLX5_EXPANSION_OUTER_IPV6] = { 627 .next = MLX5_FLOW_EXPAND_RSS_NEXT 628 (MLX5_EXPANSION_OUTER_IPV6_UDP, 629 MLX5_EXPANSION_OUTER_IPV6_TCP, 630 MLX5_EXPANSION_IPV4, 631 MLX5_EXPANSION_IPV6, 632 MLX5_EXPANSION_GRE, 633 MLX5_EXPANSION_NVGRE), 634 .type = RTE_FLOW_ITEM_TYPE_IPV6, 635 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 636 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 637 }, 638 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 639 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 640 MLX5_EXPANSION_VXLAN_GPE, 641 MLX5_EXPANSION_MPLS, 642 MLX5_EXPANSION_GENEVE, 643 MLX5_EXPANSION_GTP), 644 .type = RTE_FLOW_ITEM_TYPE_UDP, 645 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 646 }, 647 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 648 .type = RTE_FLOW_ITEM_TYPE_TCP, 649 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 650 }, 651 [MLX5_EXPANSION_VXLAN] = { 652 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 653 MLX5_EXPANSION_IPV4, 654 MLX5_EXPANSION_IPV6), 655 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 656 }, 657 [MLX5_EXPANSION_STD_VXLAN] = { 658 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 659 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 660 }, 661 [MLX5_EXPANSION_L3_VXLAN] = { 662 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 663 MLX5_EXPANSION_IPV6), 664 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 665 }, 666 [MLX5_EXPANSION_VXLAN_GPE] = { 667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 668 MLX5_EXPANSION_IPV4, 669 MLX5_EXPANSION_IPV6), 670 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 671 }, 672 [MLX5_EXPANSION_GRE] = { 673 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 674 MLX5_EXPANSION_IPV4, 675 MLX5_EXPANSION_IPV6, 676 MLX5_EXPANSION_GRE_KEY, 677 MLX5_EXPANSION_MPLS), 678 .type = RTE_FLOW_ITEM_TYPE_GRE, 679 }, 680 [MLX5_EXPANSION_GRE_KEY] = { 681 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 682 MLX5_EXPANSION_IPV6, 683 MLX5_EXPANSION_MPLS), 684 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 685 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 686 }, 687 [MLX5_EXPANSION_NVGRE] = { 688 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 689 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 690 }, 691 [MLX5_EXPANSION_MPLS] = { 692 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 693 MLX5_EXPANSION_IPV6, 694 MLX5_EXPANSION_ETH), 695 .type = RTE_FLOW_ITEM_TYPE_MPLS, 696 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 697 }, 698 [MLX5_EXPANSION_ETH] = { 699 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 700 .type = RTE_FLOW_ITEM_TYPE_ETH, 701 }, 702 [MLX5_EXPANSION_VLAN] = { 703 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 704 MLX5_EXPANSION_IPV6), 705 .type = RTE_FLOW_ITEM_TYPE_VLAN, 706 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 707 }, 708 [MLX5_EXPANSION_IPV4] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 710 MLX5_EXPANSION_IPV4_TCP), 711 .type = RTE_FLOW_ITEM_TYPE_IPV4, 712 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 713 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 714 }, 715 [MLX5_EXPANSION_IPV4_UDP] = { 716 .type = RTE_FLOW_ITEM_TYPE_UDP, 717 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 718 }, 719 [MLX5_EXPANSION_IPV4_TCP] = { 720 .type = RTE_FLOW_ITEM_TYPE_TCP, 721 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 722 }, 723 [MLX5_EXPANSION_IPV6] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 725 MLX5_EXPANSION_IPV6_TCP, 726 MLX5_EXPANSION_IPV6_FRAG_EXT), 727 .type = RTE_FLOW_ITEM_TYPE_IPV6, 728 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 729 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 730 }, 731 [MLX5_EXPANSION_IPV6_UDP] = { 732 .type = RTE_FLOW_ITEM_TYPE_UDP, 733 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 734 }, 735 [MLX5_EXPANSION_IPV6_TCP] = { 736 .type = RTE_FLOW_ITEM_TYPE_TCP, 737 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 738 }, 739 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 740 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 741 }, 742 [MLX5_EXPANSION_GTP] = { 743 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_GTP, 746 }, 747 [MLX5_EXPANSION_GENEVE] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 752 }, 753 }; 754 755 static struct rte_flow_action_handle * 756 mlx5_action_handle_create(struct rte_eth_dev *dev, 757 const struct rte_flow_indir_action_conf *conf, 758 const struct rte_flow_action *action, 759 struct rte_flow_error *error); 760 static int mlx5_action_handle_destroy 761 (struct rte_eth_dev *dev, 762 struct rte_flow_action_handle *handle, 763 struct rte_flow_error *error); 764 static int mlx5_action_handle_update 765 (struct rte_eth_dev *dev, 766 struct rte_flow_action_handle *handle, 767 const void *update, 768 struct rte_flow_error *error); 769 static int mlx5_action_handle_query 770 (struct rte_eth_dev *dev, 771 const struct rte_flow_action_handle *handle, 772 void *data, 773 struct rte_flow_error *error); 774 static int 775 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 776 struct rte_flow_tunnel *app_tunnel, 777 struct rte_flow_action **actions, 778 uint32_t *num_of_actions, 779 struct rte_flow_error *error); 780 static int 781 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 782 struct rte_flow_tunnel *app_tunnel, 783 struct rte_flow_item **items, 784 uint32_t *num_of_items, 785 struct rte_flow_error *error); 786 static int 787 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 788 struct rte_flow_item *pmd_items, 789 uint32_t num_items, struct rte_flow_error *err); 790 static int 791 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 792 struct rte_flow_action *pmd_actions, 793 uint32_t num_actions, 794 struct rte_flow_error *err); 795 static int 796 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 797 struct rte_mbuf *m, 798 struct rte_flow_restore_info *info, 799 struct rte_flow_error *err); 800 static struct rte_flow_item_flex_handle * 801 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 802 const struct rte_flow_item_flex_conf *conf, 803 struct rte_flow_error *error); 804 static int 805 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 806 const struct rte_flow_item_flex_handle *handle, 807 struct rte_flow_error *error); 808 static int 809 mlx5_flow_info_get(struct rte_eth_dev *dev, 810 struct rte_flow_port_info *port_info, 811 struct rte_flow_queue_info *queue_info, 812 struct rte_flow_error *error); 813 static int 814 mlx5_flow_port_configure(struct rte_eth_dev *dev, 815 const struct rte_flow_port_attr *port_attr, 816 uint16_t nb_queue, 817 const struct rte_flow_queue_attr *queue_attr[], 818 struct rte_flow_error *err); 819 820 static struct rte_flow_pattern_template * 821 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 822 const struct rte_flow_pattern_template_attr *attr, 823 const struct rte_flow_item items[], 824 struct rte_flow_error *error); 825 826 static int 827 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 828 struct rte_flow_pattern_template *template, 829 struct rte_flow_error *error); 830 static struct rte_flow_actions_template * 831 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 832 const struct rte_flow_actions_template_attr *attr, 833 const struct rte_flow_action actions[], 834 const struct rte_flow_action masks[], 835 struct rte_flow_error *error); 836 static int 837 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 838 struct rte_flow_actions_template *template, 839 struct rte_flow_error *error); 840 841 static struct rte_flow_template_table * 842 mlx5_flow_table_create(struct rte_eth_dev *dev, 843 const struct rte_flow_template_table_attr *attr, 844 struct rte_flow_pattern_template *item_templates[], 845 uint8_t nb_item_templates, 846 struct rte_flow_actions_template *action_templates[], 847 uint8_t nb_action_templates, 848 struct rte_flow_error *error); 849 static int 850 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 851 struct rte_flow_template_table *table, 852 struct rte_flow_error *error); 853 static struct rte_flow * 854 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 855 uint32_t queue, 856 const struct rte_flow_op_attr *attr, 857 struct rte_flow_template_table *table, 858 const struct rte_flow_item items[], 859 uint8_t pattern_template_index, 860 const struct rte_flow_action actions[], 861 uint8_t action_template_index, 862 void *user_data, 863 struct rte_flow_error *error); 864 static int 865 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 866 uint32_t queue, 867 const struct rte_flow_op_attr *attr, 868 struct rte_flow *flow, 869 void *user_data, 870 struct rte_flow_error *error); 871 static int 872 mlx5_flow_pull(struct rte_eth_dev *dev, 873 uint32_t queue, 874 struct rte_flow_op_result res[], 875 uint16_t n_res, 876 struct rte_flow_error *error); 877 static int 878 mlx5_flow_push(struct rte_eth_dev *dev, 879 uint32_t queue, 880 struct rte_flow_error *error); 881 882 static const struct rte_flow_ops mlx5_flow_ops = { 883 .validate = mlx5_flow_validate, 884 .create = mlx5_flow_create, 885 .destroy = mlx5_flow_destroy, 886 .flush = mlx5_flow_flush, 887 .isolate = mlx5_flow_isolate, 888 .query = mlx5_flow_query, 889 .dev_dump = mlx5_flow_dev_dump, 890 .get_aged_flows = mlx5_flow_get_aged_flows, 891 .action_handle_create = mlx5_action_handle_create, 892 .action_handle_destroy = mlx5_action_handle_destroy, 893 .action_handle_update = mlx5_action_handle_update, 894 .action_handle_query = mlx5_action_handle_query, 895 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 896 .tunnel_match = mlx5_flow_tunnel_match, 897 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 898 .tunnel_item_release = mlx5_flow_tunnel_item_release, 899 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 900 .flex_item_create = mlx5_flow_flex_item_create, 901 .flex_item_release = mlx5_flow_flex_item_release, 902 .info_get = mlx5_flow_info_get, 903 .configure = mlx5_flow_port_configure, 904 .pattern_template_create = mlx5_flow_pattern_template_create, 905 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 906 .actions_template_create = mlx5_flow_actions_template_create, 907 .actions_template_destroy = mlx5_flow_actions_template_destroy, 908 .template_table_create = mlx5_flow_table_create, 909 .template_table_destroy = mlx5_flow_table_destroy, 910 .async_create = mlx5_flow_async_flow_create, 911 .async_destroy = mlx5_flow_async_flow_destroy, 912 .pull = mlx5_flow_pull, 913 .push = mlx5_flow_push, 914 }; 915 916 /* Tunnel information. */ 917 struct mlx5_flow_tunnel_info { 918 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 919 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 920 }; 921 922 static struct mlx5_flow_tunnel_info tunnels_info[] = { 923 { 924 .tunnel = MLX5_FLOW_LAYER_VXLAN, 925 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 926 }, 927 { 928 .tunnel = MLX5_FLOW_LAYER_GENEVE, 929 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 930 }, 931 { 932 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 933 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 934 }, 935 { 936 .tunnel = MLX5_FLOW_LAYER_GRE, 937 .ptype = RTE_PTYPE_TUNNEL_GRE, 938 }, 939 { 940 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 941 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 942 }, 943 { 944 .tunnel = MLX5_FLOW_LAYER_MPLS, 945 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 946 }, 947 { 948 .tunnel = MLX5_FLOW_LAYER_NVGRE, 949 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 950 }, 951 { 952 .tunnel = MLX5_FLOW_LAYER_IPIP, 953 .ptype = RTE_PTYPE_TUNNEL_IP, 954 }, 955 { 956 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 957 .ptype = RTE_PTYPE_TUNNEL_IP, 958 }, 959 { 960 .tunnel = MLX5_FLOW_LAYER_GTP, 961 .ptype = RTE_PTYPE_TUNNEL_GTPU, 962 }, 963 }; 964 965 966 967 /** 968 * Translate tag ID to register. 969 * 970 * @param[in] dev 971 * Pointer to the Ethernet device structure. 972 * @param[in] feature 973 * The feature that request the register. 974 * @param[in] id 975 * The request register ID. 976 * @param[out] error 977 * Error description in case of any. 978 * 979 * @return 980 * The request register on success, a negative errno 981 * value otherwise and rte_errno is set. 982 */ 983 int 984 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 985 enum mlx5_feature_name feature, 986 uint32_t id, 987 struct rte_flow_error *error) 988 { 989 struct mlx5_priv *priv = dev->data->dev_private; 990 struct mlx5_sh_config *config = &priv->sh->config; 991 enum modify_reg start_reg; 992 bool skip_mtr_reg = false; 993 994 switch (feature) { 995 case MLX5_HAIRPIN_RX: 996 return REG_B; 997 case MLX5_HAIRPIN_TX: 998 return REG_A; 999 case MLX5_METADATA_RX: 1000 switch (config->dv_xmeta_en) { 1001 case MLX5_XMETA_MODE_LEGACY: 1002 return REG_B; 1003 case MLX5_XMETA_MODE_META16: 1004 return REG_C_0; 1005 case MLX5_XMETA_MODE_META32: 1006 return REG_C_1; 1007 } 1008 break; 1009 case MLX5_METADATA_TX: 1010 return REG_A; 1011 case MLX5_METADATA_FDB: 1012 switch (config->dv_xmeta_en) { 1013 case MLX5_XMETA_MODE_LEGACY: 1014 return REG_NON; 1015 case MLX5_XMETA_MODE_META16: 1016 return REG_C_0; 1017 case MLX5_XMETA_MODE_META32: 1018 return REG_C_1; 1019 } 1020 break; 1021 case MLX5_FLOW_MARK: 1022 switch (config->dv_xmeta_en) { 1023 case MLX5_XMETA_MODE_LEGACY: 1024 return REG_NON; 1025 case MLX5_XMETA_MODE_META16: 1026 return REG_C_1; 1027 case MLX5_XMETA_MODE_META32: 1028 return REG_C_0; 1029 } 1030 break; 1031 case MLX5_MTR_ID: 1032 /* 1033 * If meter color and meter id share one register, flow match 1034 * should use the meter color register for match. 1035 */ 1036 if (priv->mtr_reg_share) 1037 return priv->mtr_color_reg; 1038 else 1039 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1040 REG_C_3; 1041 case MLX5_MTR_COLOR: 1042 case MLX5_ASO_FLOW_HIT: 1043 case MLX5_ASO_CONNTRACK: 1044 case MLX5_SAMPLE_ID: 1045 /* All features use the same REG_C. */ 1046 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1047 return priv->mtr_color_reg; 1048 case MLX5_COPY_MARK: 1049 /* 1050 * Metadata COPY_MARK register using is in meter suffix sub 1051 * flow while with meter. It's safe to share the same register. 1052 */ 1053 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1054 case MLX5_APP_TAG: 1055 /* 1056 * If meter is enable, it will engage the register for color 1057 * match and flow match. If meter color match is not using the 1058 * REG_C_2, need to skip the REG_C_x be used by meter color 1059 * match. 1060 * If meter is disable, free to use all available registers. 1061 */ 1062 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1063 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1064 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1065 if (id > (uint32_t)(REG_C_7 - start_reg)) 1066 return rte_flow_error_set(error, EINVAL, 1067 RTE_FLOW_ERROR_TYPE_ITEM, 1068 NULL, "invalid tag id"); 1069 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1070 return rte_flow_error_set(error, ENOTSUP, 1071 RTE_FLOW_ERROR_TYPE_ITEM, 1072 NULL, "unsupported tag id"); 1073 /* 1074 * This case means meter is using the REG_C_x great than 2. 1075 * Take care not to conflict with meter color REG_C_x. 1076 * If the available index REG_C_y >= REG_C_x, skip the 1077 * color register. 1078 */ 1079 if (skip_mtr_reg && priv->sh->flow_mreg_c 1080 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1081 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1082 return rte_flow_error_set(error, EINVAL, 1083 RTE_FLOW_ERROR_TYPE_ITEM, 1084 NULL, "invalid tag id"); 1085 if (priv->sh->flow_mreg_c 1086 [id + 1 + start_reg - REG_C_0] != REG_NON) 1087 return priv->sh->flow_mreg_c 1088 [id + 1 + start_reg - REG_C_0]; 1089 return rte_flow_error_set(error, ENOTSUP, 1090 RTE_FLOW_ERROR_TYPE_ITEM, 1091 NULL, "unsupported tag id"); 1092 } 1093 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1094 } 1095 MLX5_ASSERT(false); 1096 return rte_flow_error_set(error, EINVAL, 1097 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1098 NULL, "invalid feature name"); 1099 } 1100 1101 /** 1102 * Check extensive flow metadata register support. 1103 * 1104 * @param dev 1105 * Pointer to rte_eth_dev structure. 1106 * 1107 * @return 1108 * True if device supports extensive flow metadata register, otherwise false. 1109 */ 1110 bool 1111 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1112 { 1113 struct mlx5_priv *priv = dev->data->dev_private; 1114 1115 /* 1116 * Having available reg_c can be regarded inclusively as supporting 1117 * extensive flow metadata register, which could mean, 1118 * - metadata register copy action by modify header. 1119 * - 16 modify header actions is supported. 1120 * - reg_c's are preserved across different domain (FDB and NIC) on 1121 * packet loopback by flow lookup miss. 1122 */ 1123 return priv->sh->flow_mreg_c[2] != REG_NON; 1124 } 1125 1126 /** 1127 * Get the lowest priority. 1128 * 1129 * @param[in] dev 1130 * Pointer to the Ethernet device structure. 1131 * @param[in] attributes 1132 * Pointer to device flow rule attributes. 1133 * 1134 * @return 1135 * The value of lowest priority of flow. 1136 */ 1137 uint32_t 1138 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1139 const struct rte_flow_attr *attr) 1140 { 1141 struct mlx5_priv *priv = dev->data->dev_private; 1142 1143 if (!attr->group && !attr->transfer) 1144 return priv->sh->flow_max_priority - 2; 1145 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1146 } 1147 1148 /** 1149 * Calculate matcher priority of the flow. 1150 * 1151 * @param[in] dev 1152 * Pointer to the Ethernet device structure. 1153 * @param[in] attr 1154 * Pointer to device flow rule attributes. 1155 * @param[in] subpriority 1156 * The priority based on the items. 1157 * @param[in] external 1158 * Flow is user flow. 1159 * @return 1160 * The matcher priority of the flow. 1161 */ 1162 uint16_t 1163 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1164 const struct rte_flow_attr *attr, 1165 uint32_t subpriority, bool external) 1166 { 1167 uint16_t priority = (uint16_t)attr->priority; 1168 struct mlx5_priv *priv = dev->data->dev_private; 1169 1170 if (!attr->group && !attr->transfer) { 1171 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1172 priority = priv->sh->flow_max_priority - 1; 1173 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1174 } else if (!external && attr->transfer && attr->group == 0 && 1175 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1176 return (priv->sh->flow_max_priority - 1) * 3; 1177 } 1178 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1179 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1180 return priority * 3 + subpriority; 1181 } 1182 1183 /** 1184 * Verify the @p item specifications (spec, last, mask) are compatible with the 1185 * NIC capabilities. 1186 * 1187 * @param[in] item 1188 * Item specification. 1189 * @param[in] mask 1190 * @p item->mask or flow default bit-masks. 1191 * @param[in] nic_mask 1192 * Bit-masks covering supported fields by the NIC to compare with user mask. 1193 * @param[in] size 1194 * Bit-masks size in bytes. 1195 * @param[in] range_accepted 1196 * True if range of values is accepted for specific fields, false otherwise. 1197 * @param[out] error 1198 * Pointer to error structure. 1199 * 1200 * @return 1201 * 0 on success, a negative errno value otherwise and rte_errno is set. 1202 */ 1203 int 1204 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1205 const uint8_t *mask, 1206 const uint8_t *nic_mask, 1207 unsigned int size, 1208 bool range_accepted, 1209 struct rte_flow_error *error) 1210 { 1211 unsigned int i; 1212 1213 MLX5_ASSERT(nic_mask); 1214 for (i = 0; i < size; ++i) 1215 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1216 return rte_flow_error_set(error, ENOTSUP, 1217 RTE_FLOW_ERROR_TYPE_ITEM, 1218 item, 1219 "mask enables non supported" 1220 " bits"); 1221 if (!item->spec && (item->mask || item->last)) 1222 return rte_flow_error_set(error, EINVAL, 1223 RTE_FLOW_ERROR_TYPE_ITEM, item, 1224 "mask/last without a spec is not" 1225 " supported"); 1226 if (item->spec && item->last && !range_accepted) { 1227 uint8_t spec[size]; 1228 uint8_t last[size]; 1229 unsigned int i; 1230 int ret; 1231 1232 for (i = 0; i < size; ++i) { 1233 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1234 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1235 } 1236 ret = memcmp(spec, last, size); 1237 if (ret != 0) 1238 return rte_flow_error_set(error, EINVAL, 1239 RTE_FLOW_ERROR_TYPE_ITEM, 1240 item, 1241 "range is not valid"); 1242 } 1243 return 0; 1244 } 1245 1246 /** 1247 * Adjust the hash fields according to the @p flow information. 1248 * 1249 * @param[in] dev_flow. 1250 * Pointer to the mlx5_flow. 1251 * @param[in] tunnel 1252 * 1 when the hash field is for a tunnel item. 1253 * @param[in] layer_types 1254 * RTE_ETH_RSS_* types. 1255 * @param[in] hash_fields 1256 * Item hash fields. 1257 * 1258 * @return 1259 * The hash fields that should be used. 1260 */ 1261 uint64_t 1262 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1263 int tunnel __rte_unused, uint64_t layer_types, 1264 uint64_t hash_fields) 1265 { 1266 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1267 int rss_request_inner = rss_desc->level >= 2; 1268 1269 /* Check RSS hash level for tunnel. */ 1270 if (tunnel && rss_request_inner) 1271 hash_fields |= IBV_RX_HASH_INNER; 1272 else if (tunnel || rss_request_inner) 1273 return 0; 1274 #endif 1275 /* Check if requested layer matches RSS hash fields. */ 1276 if (!(rss_desc->types & layer_types)) 1277 return 0; 1278 return hash_fields; 1279 } 1280 1281 /** 1282 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1283 * if several tunnel rules are used on this queue, the tunnel ptype will be 1284 * cleared. 1285 * 1286 * @param rxq_ctrl 1287 * Rx queue to update. 1288 */ 1289 static void 1290 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1291 { 1292 unsigned int i; 1293 uint32_t tunnel_ptype = 0; 1294 1295 /* Look up for the ptype to use. */ 1296 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1297 if (!rxq_ctrl->flow_tunnels_n[i]) 1298 continue; 1299 if (!tunnel_ptype) { 1300 tunnel_ptype = tunnels_info[i].ptype; 1301 } else { 1302 tunnel_ptype = 0; 1303 break; 1304 } 1305 } 1306 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1307 } 1308 1309 /** 1310 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1311 * flow. 1312 * 1313 * @param[in] dev 1314 * Pointer to the Ethernet device structure. 1315 * @param[in] dev_handle 1316 * Pointer to device flow handle structure. 1317 */ 1318 void 1319 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1320 struct mlx5_flow_handle *dev_handle) 1321 { 1322 struct mlx5_priv *priv = dev->data->dev_private; 1323 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1324 struct mlx5_ind_table_obj *ind_tbl = NULL; 1325 unsigned int i; 1326 1327 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1328 struct mlx5_hrxq *hrxq; 1329 1330 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1331 dev_handle->rix_hrxq); 1332 if (hrxq) 1333 ind_tbl = hrxq->ind_table; 1334 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1335 struct mlx5_shared_action_rss *shared_rss; 1336 1337 shared_rss = mlx5_ipool_get 1338 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1339 dev_handle->rix_srss); 1340 if (shared_rss) 1341 ind_tbl = shared_rss->ind_tbl; 1342 } 1343 if (!ind_tbl) 1344 return; 1345 for (i = 0; i != ind_tbl->queues_n; ++i) { 1346 int idx = ind_tbl->queues[i]; 1347 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1348 1349 MLX5_ASSERT(rxq_ctrl != NULL); 1350 if (rxq_ctrl == NULL) 1351 continue; 1352 /* 1353 * To support metadata register copy on Tx loopback, 1354 * this must be always enabled (metadata may arive 1355 * from other port - not from local flows only. 1356 */ 1357 if (tunnel) { 1358 unsigned int j; 1359 1360 /* Increase the counter matching the flow. */ 1361 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1362 if ((tunnels_info[j].tunnel & 1363 dev_handle->layers) == 1364 tunnels_info[j].tunnel) { 1365 rxq_ctrl->flow_tunnels_n[j]++; 1366 break; 1367 } 1368 } 1369 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1370 } 1371 } 1372 } 1373 1374 static void 1375 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1376 { 1377 struct mlx5_priv *priv = dev->data->dev_private; 1378 struct mlx5_rxq_ctrl *rxq_ctrl; 1379 1380 if (priv->mark_enabled) 1381 return; 1382 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1383 rxq_ctrl->rxq.mark = 1; 1384 } 1385 priv->mark_enabled = 1; 1386 } 1387 1388 /** 1389 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1390 * 1391 * @param[in] dev 1392 * Pointer to the Ethernet device structure. 1393 * @param[in] flow 1394 * Pointer to flow structure. 1395 */ 1396 static void 1397 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1398 { 1399 struct mlx5_priv *priv = dev->data->dev_private; 1400 uint32_t handle_idx; 1401 struct mlx5_flow_handle *dev_handle; 1402 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1403 1404 MLX5_ASSERT(wks); 1405 if (wks->mark) 1406 flow_rxq_mark_flag_set(dev); 1407 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1408 handle_idx, dev_handle, next) 1409 flow_drv_rxq_flags_set(dev, dev_handle); 1410 } 1411 1412 /** 1413 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1414 * device flow if no other flow uses it with the same kind of request. 1415 * 1416 * @param dev 1417 * Pointer to Ethernet device. 1418 * @param[in] dev_handle 1419 * Pointer to the device flow handle structure. 1420 */ 1421 static void 1422 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1423 struct mlx5_flow_handle *dev_handle) 1424 { 1425 struct mlx5_priv *priv = dev->data->dev_private; 1426 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1427 struct mlx5_ind_table_obj *ind_tbl = NULL; 1428 unsigned int i; 1429 1430 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1431 struct mlx5_hrxq *hrxq; 1432 1433 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1434 dev_handle->rix_hrxq); 1435 if (hrxq) 1436 ind_tbl = hrxq->ind_table; 1437 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1438 struct mlx5_shared_action_rss *shared_rss; 1439 1440 shared_rss = mlx5_ipool_get 1441 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1442 dev_handle->rix_srss); 1443 if (shared_rss) 1444 ind_tbl = shared_rss->ind_tbl; 1445 } 1446 if (!ind_tbl) 1447 return; 1448 MLX5_ASSERT(dev->data->dev_started); 1449 for (i = 0; i != ind_tbl->queues_n; ++i) { 1450 int idx = ind_tbl->queues[i]; 1451 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1452 1453 MLX5_ASSERT(rxq_ctrl != NULL); 1454 if (rxq_ctrl == NULL) 1455 continue; 1456 if (tunnel) { 1457 unsigned int j; 1458 1459 /* Decrease the counter matching the flow. */ 1460 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1461 if ((tunnels_info[j].tunnel & 1462 dev_handle->layers) == 1463 tunnels_info[j].tunnel) { 1464 rxq_ctrl->flow_tunnels_n[j]--; 1465 break; 1466 } 1467 } 1468 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1469 } 1470 } 1471 } 1472 1473 /** 1474 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1475 * @p flow if no other flow uses it with the same kind of request. 1476 * 1477 * @param dev 1478 * Pointer to Ethernet device. 1479 * @param[in] flow 1480 * Pointer to the flow. 1481 */ 1482 static void 1483 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1484 { 1485 struct mlx5_priv *priv = dev->data->dev_private; 1486 uint32_t handle_idx; 1487 struct mlx5_flow_handle *dev_handle; 1488 1489 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1490 handle_idx, dev_handle, next) 1491 flow_drv_rxq_flags_trim(dev, dev_handle); 1492 } 1493 1494 /** 1495 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1496 * 1497 * @param dev 1498 * Pointer to Ethernet device. 1499 */ 1500 static void 1501 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1502 { 1503 struct mlx5_priv *priv = dev->data->dev_private; 1504 unsigned int i; 1505 1506 for (i = 0; i != priv->rxqs_n; ++i) { 1507 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1508 unsigned int j; 1509 1510 if (rxq == NULL || rxq->ctrl == NULL) 1511 continue; 1512 rxq->ctrl->rxq.mark = 0; 1513 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1514 rxq->ctrl->flow_tunnels_n[j] = 0; 1515 rxq->ctrl->rxq.tunnel = 0; 1516 } 1517 priv->mark_enabled = 0; 1518 } 1519 1520 /** 1521 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1522 * 1523 * @param[in] dev 1524 * Pointer to the Ethernet device structure. 1525 */ 1526 void 1527 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1528 { 1529 struct mlx5_priv *priv = dev->data->dev_private; 1530 unsigned int i; 1531 1532 for (i = 0; i != priv->rxqs_n; ++i) { 1533 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1534 struct mlx5_rxq_data *data; 1535 1536 if (rxq == NULL || rxq->ctrl == NULL) 1537 continue; 1538 data = &rxq->ctrl->rxq; 1539 if (!rte_flow_dynf_metadata_avail()) { 1540 data->dynf_meta = 0; 1541 data->flow_meta_mask = 0; 1542 data->flow_meta_offset = -1; 1543 data->flow_meta_port_mask = 0; 1544 } else { 1545 data->dynf_meta = 1; 1546 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1547 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1548 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1549 } 1550 } 1551 } 1552 1553 /* 1554 * return a pointer to the desired action in the list of actions. 1555 * 1556 * @param[in] actions 1557 * The list of actions to search the action in. 1558 * @param[in] action 1559 * The action to find. 1560 * 1561 * @return 1562 * Pointer to the action in the list, if found. NULL otherwise. 1563 */ 1564 const struct rte_flow_action * 1565 mlx5_flow_find_action(const struct rte_flow_action *actions, 1566 enum rte_flow_action_type action) 1567 { 1568 if (actions == NULL) 1569 return NULL; 1570 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1571 if (actions->type == action) 1572 return actions; 1573 return NULL; 1574 } 1575 1576 /* 1577 * Validate the flag action. 1578 * 1579 * @param[in] action_flags 1580 * Bit-fields that holds the actions detected until now. 1581 * @param[in] attr 1582 * Attributes of flow that includes this action. 1583 * @param[out] error 1584 * Pointer to error structure. 1585 * 1586 * @return 1587 * 0 on success, a negative errno value otherwise and rte_errno is set. 1588 */ 1589 int 1590 mlx5_flow_validate_action_flag(uint64_t action_flags, 1591 const struct rte_flow_attr *attr, 1592 struct rte_flow_error *error) 1593 { 1594 if (action_flags & MLX5_FLOW_ACTION_MARK) 1595 return rte_flow_error_set(error, EINVAL, 1596 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1597 "can't mark and flag in same flow"); 1598 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1599 return rte_flow_error_set(error, EINVAL, 1600 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1601 "can't have 2 flag" 1602 " actions in same flow"); 1603 if (attr->egress) 1604 return rte_flow_error_set(error, ENOTSUP, 1605 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1606 "flag action not supported for " 1607 "egress"); 1608 return 0; 1609 } 1610 1611 /* 1612 * Validate the mark action. 1613 * 1614 * @param[in] action 1615 * Pointer to the queue action. 1616 * @param[in] action_flags 1617 * Bit-fields that holds the actions detected until now. 1618 * @param[in] attr 1619 * Attributes of flow that includes this action. 1620 * @param[out] error 1621 * Pointer to error structure. 1622 * 1623 * @return 1624 * 0 on success, a negative errno value otherwise and rte_errno is set. 1625 */ 1626 int 1627 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1628 uint64_t action_flags, 1629 const struct rte_flow_attr *attr, 1630 struct rte_flow_error *error) 1631 { 1632 const struct rte_flow_action_mark *mark = action->conf; 1633 1634 if (!mark) 1635 return rte_flow_error_set(error, EINVAL, 1636 RTE_FLOW_ERROR_TYPE_ACTION, 1637 action, 1638 "configuration cannot be null"); 1639 if (mark->id >= MLX5_FLOW_MARK_MAX) 1640 return rte_flow_error_set(error, EINVAL, 1641 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1642 &mark->id, 1643 "mark id must in 0 <= id < " 1644 RTE_STR(MLX5_FLOW_MARK_MAX)); 1645 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1646 return rte_flow_error_set(error, EINVAL, 1647 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1648 "can't flag and mark in same flow"); 1649 if (action_flags & MLX5_FLOW_ACTION_MARK) 1650 return rte_flow_error_set(error, EINVAL, 1651 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1652 "can't have 2 mark actions in same" 1653 " flow"); 1654 if (attr->egress) 1655 return rte_flow_error_set(error, ENOTSUP, 1656 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1657 "mark action not supported for " 1658 "egress"); 1659 return 0; 1660 } 1661 1662 /* 1663 * Validate the drop action. 1664 * 1665 * @param[in] action_flags 1666 * Bit-fields that holds the actions detected until now. 1667 * @param[in] attr 1668 * Attributes of flow that includes this action. 1669 * @param[out] error 1670 * Pointer to error structure. 1671 * 1672 * @return 1673 * 0 on success, a negative errno value otherwise and rte_errno is set. 1674 */ 1675 int 1676 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1677 const struct rte_flow_attr *attr, 1678 struct rte_flow_error *error) 1679 { 1680 if (attr->egress) 1681 return rte_flow_error_set(error, ENOTSUP, 1682 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1683 "drop action not supported for " 1684 "egress"); 1685 return 0; 1686 } 1687 1688 /* 1689 * Validate the queue action. 1690 * 1691 * @param[in] action 1692 * Pointer to the queue action. 1693 * @param[in] action_flags 1694 * Bit-fields that holds the actions detected until now. 1695 * @param[in] dev 1696 * Pointer to the Ethernet device structure. 1697 * @param[in] attr 1698 * Attributes of flow that includes this action. 1699 * @param[out] error 1700 * Pointer to error structure. 1701 * 1702 * @return 1703 * 0 on success, a negative errno value otherwise and rte_errno is set. 1704 */ 1705 int 1706 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1707 uint64_t action_flags, 1708 struct rte_eth_dev *dev, 1709 const struct rte_flow_attr *attr, 1710 struct rte_flow_error *error) 1711 { 1712 struct mlx5_priv *priv = dev->data->dev_private; 1713 const struct rte_flow_action_queue *queue = action->conf; 1714 1715 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1716 return rte_flow_error_set(error, EINVAL, 1717 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1718 "can't have 2 fate actions in" 1719 " same flow"); 1720 if (!priv->rxqs_n) 1721 return rte_flow_error_set(error, EINVAL, 1722 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1723 NULL, "No Rx queues configured"); 1724 if (queue->index >= priv->rxqs_n) 1725 return rte_flow_error_set(error, EINVAL, 1726 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1727 &queue->index, 1728 "queue index out of range"); 1729 if (mlx5_rxq_get(dev, queue->index) == NULL) 1730 return rte_flow_error_set(error, EINVAL, 1731 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1732 &queue->index, 1733 "queue is not configured"); 1734 if (attr->egress) 1735 return rte_flow_error_set(error, ENOTSUP, 1736 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1737 "queue action not supported for " 1738 "egress"); 1739 return 0; 1740 } 1741 1742 /** 1743 * Validate queue numbers for device RSS. 1744 * 1745 * @param[in] dev 1746 * Configured device. 1747 * @param[in] queues 1748 * Array of queue numbers. 1749 * @param[in] queues_n 1750 * Size of the @p queues array. 1751 * @param[out] error 1752 * On error, filled with a textual error description. 1753 * @param[out] queue 1754 * On error, filled with an offending queue index in @p queues array. 1755 * 1756 * @return 1757 * 0 on success, a negative errno code on error. 1758 */ 1759 static int 1760 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1761 const uint16_t *queues, uint32_t queues_n, 1762 const char **error, uint32_t *queue_idx) 1763 { 1764 const struct mlx5_priv *priv = dev->data->dev_private; 1765 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1766 uint32_t i; 1767 1768 for (i = 0; i != queues_n; ++i) { 1769 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1770 queues[i]); 1771 1772 if (queues[i] >= priv->rxqs_n) { 1773 *error = "queue index out of range"; 1774 *queue_idx = i; 1775 return -EINVAL; 1776 } 1777 if (rxq_ctrl == NULL) { 1778 *error = "queue is not configured"; 1779 *queue_idx = i; 1780 return -EINVAL; 1781 } 1782 if (i == 0) 1783 rxq_type = rxq_ctrl->type; 1784 if (rxq_type != rxq_ctrl->type) { 1785 *error = "combining hairpin and regular RSS queues is not supported"; 1786 *queue_idx = i; 1787 return -ENOTSUP; 1788 } 1789 } 1790 return 0; 1791 } 1792 1793 /* 1794 * Validate the rss action. 1795 * 1796 * @param[in] dev 1797 * Pointer to the Ethernet device structure. 1798 * @param[in] action 1799 * Pointer to the queue action. 1800 * @param[out] error 1801 * Pointer to error structure. 1802 * 1803 * @return 1804 * 0 on success, a negative errno value otherwise and rte_errno is set. 1805 */ 1806 int 1807 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1808 const struct rte_flow_action *action, 1809 struct rte_flow_error *error) 1810 { 1811 struct mlx5_priv *priv = dev->data->dev_private; 1812 const struct rte_flow_action_rss *rss = action->conf; 1813 int ret; 1814 const char *message; 1815 uint32_t queue_idx; 1816 1817 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1818 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1819 return rte_flow_error_set(error, ENOTSUP, 1820 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1821 &rss->func, 1822 "RSS hash function not supported"); 1823 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1824 if (rss->level > 2) 1825 #else 1826 if (rss->level > 1) 1827 #endif 1828 return rte_flow_error_set(error, ENOTSUP, 1829 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1830 &rss->level, 1831 "tunnel RSS is not supported"); 1832 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1833 if (rss->key_len == 0 && rss->key != NULL) 1834 return rte_flow_error_set(error, ENOTSUP, 1835 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1836 &rss->key_len, 1837 "RSS hash key length 0"); 1838 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1839 return rte_flow_error_set(error, ENOTSUP, 1840 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1841 &rss->key_len, 1842 "RSS hash key too small"); 1843 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1844 return rte_flow_error_set(error, ENOTSUP, 1845 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1846 &rss->key_len, 1847 "RSS hash key too large"); 1848 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1849 return rte_flow_error_set(error, ENOTSUP, 1850 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1851 &rss->queue_num, 1852 "number of queues too large"); 1853 if (rss->types & MLX5_RSS_HF_MASK) 1854 return rte_flow_error_set(error, ENOTSUP, 1855 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1856 &rss->types, 1857 "some RSS protocols are not" 1858 " supported"); 1859 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1860 !(rss->types & RTE_ETH_RSS_IP)) 1861 return rte_flow_error_set(error, EINVAL, 1862 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1863 "L3 partial RSS requested but L3 RSS" 1864 " type not specified"); 1865 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1866 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1867 return rte_flow_error_set(error, EINVAL, 1868 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1869 "L4 partial RSS requested but L4 RSS" 1870 " type not specified"); 1871 if (!priv->rxqs_n) 1872 return rte_flow_error_set(error, EINVAL, 1873 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1874 NULL, "No Rx queues configured"); 1875 if (!rss->queue_num) 1876 return rte_flow_error_set(error, EINVAL, 1877 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1878 NULL, "No queues configured"); 1879 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1880 &message, &queue_idx); 1881 if (ret != 0) { 1882 return rte_flow_error_set(error, -ret, 1883 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1884 &rss->queue[queue_idx], message); 1885 } 1886 return 0; 1887 } 1888 1889 /* 1890 * Validate the rss action. 1891 * 1892 * @param[in] action 1893 * Pointer to the queue action. 1894 * @param[in] action_flags 1895 * Bit-fields that holds the actions detected until now. 1896 * @param[in] dev 1897 * Pointer to the Ethernet device structure. 1898 * @param[in] attr 1899 * Attributes of flow that includes this action. 1900 * @param[in] item_flags 1901 * Items that were detected. 1902 * @param[out] error 1903 * Pointer to error structure. 1904 * 1905 * @return 1906 * 0 on success, a negative errno value otherwise and rte_errno is set. 1907 */ 1908 int 1909 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1910 uint64_t action_flags, 1911 struct rte_eth_dev *dev, 1912 const struct rte_flow_attr *attr, 1913 uint64_t item_flags, 1914 struct rte_flow_error *error) 1915 { 1916 const struct rte_flow_action_rss *rss = action->conf; 1917 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1918 int ret; 1919 1920 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1921 return rte_flow_error_set(error, EINVAL, 1922 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1923 "can't have 2 fate actions" 1924 " in same flow"); 1925 ret = mlx5_validate_action_rss(dev, action, error); 1926 if (ret) 1927 return ret; 1928 if (attr->egress) 1929 return rte_flow_error_set(error, ENOTSUP, 1930 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1931 "rss action not supported for " 1932 "egress"); 1933 if (rss->level > 1 && !tunnel) 1934 return rte_flow_error_set(error, EINVAL, 1935 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1936 "inner RSS is not supported for " 1937 "non-tunnel flows"); 1938 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1939 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1940 return rte_flow_error_set(error, EINVAL, 1941 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1942 "RSS on eCPRI is not supported now"); 1943 } 1944 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1945 !(item_flags & 1946 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1947 rss->level > 1) 1948 return rte_flow_error_set(error, EINVAL, 1949 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1950 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1951 return 0; 1952 } 1953 1954 /* 1955 * Validate the default miss action. 1956 * 1957 * @param[in] action_flags 1958 * Bit-fields that holds the actions detected until now. 1959 * @param[out] error 1960 * Pointer to error structure. 1961 * 1962 * @return 1963 * 0 on success, a negative errno value otherwise and rte_errno is set. 1964 */ 1965 int 1966 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1967 const struct rte_flow_attr *attr, 1968 struct rte_flow_error *error) 1969 { 1970 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1971 return rte_flow_error_set(error, EINVAL, 1972 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1973 "can't have 2 fate actions in" 1974 " same flow"); 1975 if (attr->egress) 1976 return rte_flow_error_set(error, ENOTSUP, 1977 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1978 "default miss action not supported " 1979 "for egress"); 1980 if (attr->group) 1981 return rte_flow_error_set(error, ENOTSUP, 1982 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1983 "only group 0 is supported"); 1984 if (attr->transfer) 1985 return rte_flow_error_set(error, ENOTSUP, 1986 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1987 NULL, "transfer is not supported"); 1988 return 0; 1989 } 1990 1991 /* 1992 * Validate the count action. 1993 * 1994 * @param[in] dev 1995 * Pointer to the Ethernet device structure. 1996 * @param[in] attr 1997 * Attributes of flow that includes this action. 1998 * @param[out] error 1999 * Pointer to error structure. 2000 * 2001 * @return 2002 * 0 on success, a negative errno value otherwise and rte_errno is set. 2003 */ 2004 int 2005 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2006 const struct rte_flow_attr *attr, 2007 struct rte_flow_error *error) 2008 { 2009 if (attr->egress) 2010 return rte_flow_error_set(error, ENOTSUP, 2011 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2012 "count action not supported for " 2013 "egress"); 2014 return 0; 2015 } 2016 2017 /* 2018 * Validate the ASO CT action. 2019 * 2020 * @param[in] dev 2021 * Pointer to the Ethernet device structure. 2022 * @param[in] conntrack 2023 * Pointer to the CT action profile. 2024 * @param[out] error 2025 * Pointer to error structure. 2026 * 2027 * @return 2028 * 0 on success, a negative errno value otherwise and rte_errno is set. 2029 */ 2030 int 2031 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2032 const struct rte_flow_action_conntrack *conntrack, 2033 struct rte_flow_error *error) 2034 { 2035 RTE_SET_USED(dev); 2036 2037 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2038 return rte_flow_error_set(error, EINVAL, 2039 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2040 "Invalid CT state"); 2041 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2042 return rte_flow_error_set(error, EINVAL, 2043 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2044 "Invalid last TCP packet flag"); 2045 return 0; 2046 } 2047 2048 /** 2049 * Verify the @p attributes will be correctly understood by the NIC and store 2050 * them in the @p flow if everything is correct. 2051 * 2052 * @param[in] dev 2053 * Pointer to the Ethernet device structure. 2054 * @param[in] attributes 2055 * Pointer to flow attributes 2056 * @param[out] error 2057 * Pointer to error structure. 2058 * 2059 * @return 2060 * 0 on success, a negative errno value otherwise and rte_errno is set. 2061 */ 2062 int 2063 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2064 const struct rte_flow_attr *attributes, 2065 struct rte_flow_error *error) 2066 { 2067 struct mlx5_priv *priv = dev->data->dev_private; 2068 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2069 2070 if (attributes->group) 2071 return rte_flow_error_set(error, ENOTSUP, 2072 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2073 NULL, "groups is not supported"); 2074 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2075 attributes->priority >= priority_max) 2076 return rte_flow_error_set(error, ENOTSUP, 2077 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2078 NULL, "priority out of range"); 2079 if (attributes->egress) 2080 return rte_flow_error_set(error, ENOTSUP, 2081 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2082 "egress is not supported"); 2083 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2084 return rte_flow_error_set(error, ENOTSUP, 2085 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2086 NULL, "transfer is not supported"); 2087 if (!attributes->ingress) 2088 return rte_flow_error_set(error, EINVAL, 2089 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2090 NULL, 2091 "ingress attribute is mandatory"); 2092 return 0; 2093 } 2094 2095 /** 2096 * Validate ICMP6 item. 2097 * 2098 * @param[in] item 2099 * Item specification. 2100 * @param[in] item_flags 2101 * Bit-fields that holds the items detected until now. 2102 * @param[in] ext_vlan_sup 2103 * Whether extended VLAN features are supported or not. 2104 * @param[out] error 2105 * Pointer to error structure. 2106 * 2107 * @return 2108 * 0 on success, a negative errno value otherwise and rte_errno is set. 2109 */ 2110 int 2111 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2112 uint64_t item_flags, 2113 uint8_t target_protocol, 2114 struct rte_flow_error *error) 2115 { 2116 const struct rte_flow_item_icmp6 *mask = item->mask; 2117 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2118 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2119 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2120 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2121 MLX5_FLOW_LAYER_OUTER_L4; 2122 int ret; 2123 2124 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2125 return rte_flow_error_set(error, EINVAL, 2126 RTE_FLOW_ERROR_TYPE_ITEM, item, 2127 "protocol filtering not compatible" 2128 " with ICMP6 layer"); 2129 if (!(item_flags & l3m)) 2130 return rte_flow_error_set(error, EINVAL, 2131 RTE_FLOW_ERROR_TYPE_ITEM, item, 2132 "IPv6 is mandatory to filter on" 2133 " ICMP6"); 2134 if (item_flags & l4m) 2135 return rte_flow_error_set(error, EINVAL, 2136 RTE_FLOW_ERROR_TYPE_ITEM, item, 2137 "multiple L4 layers not supported"); 2138 if (!mask) 2139 mask = &rte_flow_item_icmp6_mask; 2140 ret = mlx5_flow_item_acceptable 2141 (item, (const uint8_t *)mask, 2142 (const uint8_t *)&rte_flow_item_icmp6_mask, 2143 sizeof(struct rte_flow_item_icmp6), 2144 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2145 if (ret < 0) 2146 return ret; 2147 return 0; 2148 } 2149 2150 /** 2151 * Validate ICMP item. 2152 * 2153 * @param[in] item 2154 * Item specification. 2155 * @param[in] item_flags 2156 * Bit-fields that holds the items detected until now. 2157 * @param[out] error 2158 * Pointer to error structure. 2159 * 2160 * @return 2161 * 0 on success, a negative errno value otherwise and rte_errno is set. 2162 */ 2163 int 2164 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2165 uint64_t item_flags, 2166 uint8_t target_protocol, 2167 struct rte_flow_error *error) 2168 { 2169 const struct rte_flow_item_icmp *mask = item->mask; 2170 const struct rte_flow_item_icmp nic_mask = { 2171 .hdr.icmp_type = 0xff, 2172 .hdr.icmp_code = 0xff, 2173 .hdr.icmp_ident = RTE_BE16(0xffff), 2174 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2175 }; 2176 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2177 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2178 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2179 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2180 MLX5_FLOW_LAYER_OUTER_L4; 2181 int ret; 2182 2183 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2184 return rte_flow_error_set(error, EINVAL, 2185 RTE_FLOW_ERROR_TYPE_ITEM, item, 2186 "protocol filtering not compatible" 2187 " with ICMP layer"); 2188 if (!(item_flags & l3m)) 2189 return rte_flow_error_set(error, EINVAL, 2190 RTE_FLOW_ERROR_TYPE_ITEM, item, 2191 "IPv4 is mandatory to filter" 2192 " on ICMP"); 2193 if (item_flags & l4m) 2194 return rte_flow_error_set(error, EINVAL, 2195 RTE_FLOW_ERROR_TYPE_ITEM, item, 2196 "multiple L4 layers not supported"); 2197 if (!mask) 2198 mask = &nic_mask; 2199 ret = mlx5_flow_item_acceptable 2200 (item, (const uint8_t *)mask, 2201 (const uint8_t *)&nic_mask, 2202 sizeof(struct rte_flow_item_icmp), 2203 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2204 if (ret < 0) 2205 return ret; 2206 return 0; 2207 } 2208 2209 /** 2210 * Validate Ethernet item. 2211 * 2212 * @param[in] item 2213 * Item specification. 2214 * @param[in] item_flags 2215 * Bit-fields that holds the items detected until now. 2216 * @param[out] error 2217 * Pointer to error structure. 2218 * 2219 * @return 2220 * 0 on success, a negative errno value otherwise and rte_errno is set. 2221 */ 2222 int 2223 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2224 uint64_t item_flags, bool ext_vlan_sup, 2225 struct rte_flow_error *error) 2226 { 2227 const struct rte_flow_item_eth *mask = item->mask; 2228 const struct rte_flow_item_eth nic_mask = { 2229 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2230 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2231 .type = RTE_BE16(0xffff), 2232 .has_vlan = ext_vlan_sup ? 1 : 0, 2233 }; 2234 int ret; 2235 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2236 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2237 MLX5_FLOW_LAYER_OUTER_L2; 2238 2239 if (item_flags & ethm) 2240 return rte_flow_error_set(error, ENOTSUP, 2241 RTE_FLOW_ERROR_TYPE_ITEM, item, 2242 "multiple L2 layers not supported"); 2243 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2244 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2245 return rte_flow_error_set(error, EINVAL, 2246 RTE_FLOW_ERROR_TYPE_ITEM, item, 2247 "L2 layer should not follow " 2248 "L3 layers"); 2249 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2250 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2251 return rte_flow_error_set(error, EINVAL, 2252 RTE_FLOW_ERROR_TYPE_ITEM, item, 2253 "L2 layer should not follow VLAN"); 2254 if (item_flags & MLX5_FLOW_LAYER_GTP) 2255 return rte_flow_error_set(error, EINVAL, 2256 RTE_FLOW_ERROR_TYPE_ITEM, item, 2257 "L2 layer should not follow GTP"); 2258 if (!mask) 2259 mask = &rte_flow_item_eth_mask; 2260 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2261 (const uint8_t *)&nic_mask, 2262 sizeof(struct rte_flow_item_eth), 2263 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2264 return ret; 2265 } 2266 2267 /** 2268 * Validate VLAN item. 2269 * 2270 * @param[in] item 2271 * Item specification. 2272 * @param[in] item_flags 2273 * Bit-fields that holds the items detected until now. 2274 * @param[in] dev 2275 * Ethernet device flow is being created on. 2276 * @param[out] error 2277 * Pointer to error structure. 2278 * 2279 * @return 2280 * 0 on success, a negative errno value otherwise and rte_errno is set. 2281 */ 2282 int 2283 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2284 uint64_t item_flags, 2285 struct rte_eth_dev *dev, 2286 struct rte_flow_error *error) 2287 { 2288 const struct rte_flow_item_vlan *spec = item->spec; 2289 const struct rte_flow_item_vlan *mask = item->mask; 2290 const struct rte_flow_item_vlan nic_mask = { 2291 .tci = RTE_BE16(UINT16_MAX), 2292 .inner_type = RTE_BE16(UINT16_MAX), 2293 }; 2294 uint16_t vlan_tag = 0; 2295 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2296 int ret; 2297 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2298 MLX5_FLOW_LAYER_INNER_L4) : 2299 (MLX5_FLOW_LAYER_OUTER_L3 | 2300 MLX5_FLOW_LAYER_OUTER_L4); 2301 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2302 MLX5_FLOW_LAYER_OUTER_VLAN; 2303 2304 if (item_flags & vlanm) 2305 return rte_flow_error_set(error, EINVAL, 2306 RTE_FLOW_ERROR_TYPE_ITEM, item, 2307 "multiple VLAN layers not supported"); 2308 else if ((item_flags & l34m) != 0) 2309 return rte_flow_error_set(error, EINVAL, 2310 RTE_FLOW_ERROR_TYPE_ITEM, item, 2311 "VLAN cannot follow L3/L4 layer"); 2312 if (!mask) 2313 mask = &rte_flow_item_vlan_mask; 2314 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2315 (const uint8_t *)&nic_mask, 2316 sizeof(struct rte_flow_item_vlan), 2317 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2318 if (ret) 2319 return ret; 2320 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2321 struct mlx5_priv *priv = dev->data->dev_private; 2322 2323 if (priv->vmwa_context) { 2324 /* 2325 * Non-NULL context means we have a virtual machine 2326 * and SR-IOV enabled, we have to create VLAN interface 2327 * to make hypervisor to setup E-Switch vport 2328 * context correctly. We avoid creating the multiple 2329 * VLAN interfaces, so we cannot support VLAN tag mask. 2330 */ 2331 return rte_flow_error_set(error, EINVAL, 2332 RTE_FLOW_ERROR_TYPE_ITEM, 2333 item, 2334 "VLAN tag mask is not" 2335 " supported in virtual" 2336 " environment"); 2337 } 2338 } 2339 if (spec) { 2340 vlan_tag = spec->tci; 2341 vlan_tag &= mask->tci; 2342 } 2343 /* 2344 * From verbs perspective an empty VLAN is equivalent 2345 * to a packet without VLAN layer. 2346 */ 2347 if (!vlan_tag) 2348 return rte_flow_error_set(error, EINVAL, 2349 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2350 item->spec, 2351 "VLAN cannot be empty"); 2352 return 0; 2353 } 2354 2355 /** 2356 * Validate IPV4 item. 2357 * 2358 * @param[in] item 2359 * Item specification. 2360 * @param[in] item_flags 2361 * Bit-fields that holds the items detected until now. 2362 * @param[in] last_item 2363 * Previous validated item in the pattern items. 2364 * @param[in] ether_type 2365 * Type in the ethernet layer header (including dot1q). 2366 * @param[in] acc_mask 2367 * Acceptable mask, if NULL default internal default mask 2368 * will be used to check whether item fields are supported. 2369 * @param[in] range_accepted 2370 * True if range of values is accepted for specific fields, false otherwise. 2371 * @param[out] error 2372 * Pointer to error structure. 2373 * 2374 * @return 2375 * 0 on success, a negative errno value otherwise and rte_errno is set. 2376 */ 2377 int 2378 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2379 uint64_t item_flags, 2380 uint64_t last_item, 2381 uint16_t ether_type, 2382 const struct rte_flow_item_ipv4 *acc_mask, 2383 bool range_accepted, 2384 struct rte_flow_error *error) 2385 { 2386 const struct rte_flow_item_ipv4 *mask = item->mask; 2387 const struct rte_flow_item_ipv4 *spec = item->spec; 2388 const struct rte_flow_item_ipv4 nic_mask = { 2389 .hdr = { 2390 .src_addr = RTE_BE32(0xffffffff), 2391 .dst_addr = RTE_BE32(0xffffffff), 2392 .type_of_service = 0xff, 2393 .next_proto_id = 0xff, 2394 }, 2395 }; 2396 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2397 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2398 MLX5_FLOW_LAYER_OUTER_L3; 2399 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2400 MLX5_FLOW_LAYER_OUTER_L4; 2401 int ret; 2402 uint8_t next_proto = 0xFF; 2403 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2404 MLX5_FLOW_LAYER_OUTER_VLAN | 2405 MLX5_FLOW_LAYER_INNER_VLAN); 2406 2407 if ((last_item & l2_vlan) && ether_type && 2408 ether_type != RTE_ETHER_TYPE_IPV4) 2409 return rte_flow_error_set(error, EINVAL, 2410 RTE_FLOW_ERROR_TYPE_ITEM, item, 2411 "IPv4 cannot follow L2/VLAN layer " 2412 "which ether type is not IPv4"); 2413 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2414 if (mask && spec) 2415 next_proto = mask->hdr.next_proto_id & 2416 spec->hdr.next_proto_id; 2417 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2418 return rte_flow_error_set(error, EINVAL, 2419 RTE_FLOW_ERROR_TYPE_ITEM, 2420 item, 2421 "multiple tunnel " 2422 "not supported"); 2423 } 2424 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2425 return rte_flow_error_set(error, EINVAL, 2426 RTE_FLOW_ERROR_TYPE_ITEM, item, 2427 "wrong tunnel type - IPv6 specified " 2428 "but IPv4 item provided"); 2429 if (item_flags & l3m) 2430 return rte_flow_error_set(error, ENOTSUP, 2431 RTE_FLOW_ERROR_TYPE_ITEM, item, 2432 "multiple L3 layers not supported"); 2433 else if (item_flags & l4m) 2434 return rte_flow_error_set(error, EINVAL, 2435 RTE_FLOW_ERROR_TYPE_ITEM, item, 2436 "L3 cannot follow an L4 layer."); 2437 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2438 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2439 return rte_flow_error_set(error, EINVAL, 2440 RTE_FLOW_ERROR_TYPE_ITEM, item, 2441 "L3 cannot follow an NVGRE layer."); 2442 if (!mask) 2443 mask = &rte_flow_item_ipv4_mask; 2444 else if (mask->hdr.next_proto_id != 0 && 2445 mask->hdr.next_proto_id != 0xff) 2446 return rte_flow_error_set(error, EINVAL, 2447 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2448 "partial mask is not supported" 2449 " for protocol"); 2450 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2451 acc_mask ? (const uint8_t *)acc_mask 2452 : (const uint8_t *)&nic_mask, 2453 sizeof(struct rte_flow_item_ipv4), 2454 range_accepted, error); 2455 if (ret < 0) 2456 return ret; 2457 return 0; 2458 } 2459 2460 /** 2461 * Validate IPV6 item. 2462 * 2463 * @param[in] item 2464 * Item specification. 2465 * @param[in] item_flags 2466 * Bit-fields that holds the items detected until now. 2467 * @param[in] last_item 2468 * Previous validated item in the pattern items. 2469 * @param[in] ether_type 2470 * Type in the ethernet layer header (including dot1q). 2471 * @param[in] acc_mask 2472 * Acceptable mask, if NULL default internal default mask 2473 * will be used to check whether item fields are supported. 2474 * @param[out] error 2475 * Pointer to error structure. 2476 * 2477 * @return 2478 * 0 on success, a negative errno value otherwise and rte_errno is set. 2479 */ 2480 int 2481 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2482 uint64_t item_flags, 2483 uint64_t last_item, 2484 uint16_t ether_type, 2485 const struct rte_flow_item_ipv6 *acc_mask, 2486 struct rte_flow_error *error) 2487 { 2488 const struct rte_flow_item_ipv6 *mask = item->mask; 2489 const struct rte_flow_item_ipv6 *spec = item->spec; 2490 const struct rte_flow_item_ipv6 nic_mask = { 2491 .hdr = { 2492 .src_addr = 2493 "\xff\xff\xff\xff\xff\xff\xff\xff" 2494 "\xff\xff\xff\xff\xff\xff\xff\xff", 2495 .dst_addr = 2496 "\xff\xff\xff\xff\xff\xff\xff\xff" 2497 "\xff\xff\xff\xff\xff\xff\xff\xff", 2498 .vtc_flow = RTE_BE32(0xffffffff), 2499 .proto = 0xff, 2500 }, 2501 }; 2502 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2503 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2504 MLX5_FLOW_LAYER_OUTER_L3; 2505 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2506 MLX5_FLOW_LAYER_OUTER_L4; 2507 int ret; 2508 uint8_t next_proto = 0xFF; 2509 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2510 MLX5_FLOW_LAYER_OUTER_VLAN | 2511 MLX5_FLOW_LAYER_INNER_VLAN); 2512 2513 if ((last_item & l2_vlan) && ether_type && 2514 ether_type != RTE_ETHER_TYPE_IPV6) 2515 return rte_flow_error_set(error, EINVAL, 2516 RTE_FLOW_ERROR_TYPE_ITEM, item, 2517 "IPv6 cannot follow L2/VLAN layer " 2518 "which ether type is not IPv6"); 2519 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2520 next_proto = spec->hdr.proto; 2521 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2522 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2523 return rte_flow_error_set(error, EINVAL, 2524 RTE_FLOW_ERROR_TYPE_ITEM, 2525 item, 2526 "multiple tunnel " 2527 "not supported"); 2528 } 2529 if (next_proto == IPPROTO_HOPOPTS || 2530 next_proto == IPPROTO_ROUTING || 2531 next_proto == IPPROTO_FRAGMENT || 2532 next_proto == IPPROTO_ESP || 2533 next_proto == IPPROTO_AH || 2534 next_proto == IPPROTO_DSTOPTS) 2535 return rte_flow_error_set(error, EINVAL, 2536 RTE_FLOW_ERROR_TYPE_ITEM, item, 2537 "IPv6 proto (next header) should " 2538 "not be set as extension header"); 2539 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2540 return rte_flow_error_set(error, EINVAL, 2541 RTE_FLOW_ERROR_TYPE_ITEM, item, 2542 "wrong tunnel type - IPv4 specified " 2543 "but IPv6 item provided"); 2544 if (item_flags & l3m) 2545 return rte_flow_error_set(error, ENOTSUP, 2546 RTE_FLOW_ERROR_TYPE_ITEM, item, 2547 "multiple L3 layers not supported"); 2548 else if (item_flags & l4m) 2549 return rte_flow_error_set(error, EINVAL, 2550 RTE_FLOW_ERROR_TYPE_ITEM, item, 2551 "L3 cannot follow an L4 layer."); 2552 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2553 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2554 return rte_flow_error_set(error, EINVAL, 2555 RTE_FLOW_ERROR_TYPE_ITEM, item, 2556 "L3 cannot follow an NVGRE layer."); 2557 if (!mask) 2558 mask = &rte_flow_item_ipv6_mask; 2559 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2560 acc_mask ? (const uint8_t *)acc_mask 2561 : (const uint8_t *)&nic_mask, 2562 sizeof(struct rte_flow_item_ipv6), 2563 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2564 if (ret < 0) 2565 return ret; 2566 return 0; 2567 } 2568 2569 /** 2570 * Validate UDP item. 2571 * 2572 * @param[in] item 2573 * Item specification. 2574 * @param[in] item_flags 2575 * Bit-fields that holds the items detected until now. 2576 * @param[in] target_protocol 2577 * The next protocol in the previous item. 2578 * @param[in] flow_mask 2579 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2580 * @param[out] error 2581 * Pointer to error structure. 2582 * 2583 * @return 2584 * 0 on success, a negative errno value otherwise and rte_errno is set. 2585 */ 2586 int 2587 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2588 uint64_t item_flags, 2589 uint8_t target_protocol, 2590 struct rte_flow_error *error) 2591 { 2592 const struct rte_flow_item_udp *mask = item->mask; 2593 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2594 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2595 MLX5_FLOW_LAYER_OUTER_L3; 2596 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2597 MLX5_FLOW_LAYER_OUTER_L4; 2598 int ret; 2599 2600 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2601 return rte_flow_error_set(error, EINVAL, 2602 RTE_FLOW_ERROR_TYPE_ITEM, item, 2603 "protocol filtering not compatible" 2604 " with UDP layer"); 2605 if (!(item_flags & l3m)) 2606 return rte_flow_error_set(error, EINVAL, 2607 RTE_FLOW_ERROR_TYPE_ITEM, item, 2608 "L3 is mandatory to filter on L4"); 2609 if (item_flags & l4m) 2610 return rte_flow_error_set(error, EINVAL, 2611 RTE_FLOW_ERROR_TYPE_ITEM, item, 2612 "multiple L4 layers not supported"); 2613 if (!mask) 2614 mask = &rte_flow_item_udp_mask; 2615 ret = mlx5_flow_item_acceptable 2616 (item, (const uint8_t *)mask, 2617 (const uint8_t *)&rte_flow_item_udp_mask, 2618 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2619 error); 2620 if (ret < 0) 2621 return ret; 2622 return 0; 2623 } 2624 2625 /** 2626 * Validate TCP item. 2627 * 2628 * @param[in] item 2629 * Item specification. 2630 * @param[in] item_flags 2631 * Bit-fields that holds the items detected until now. 2632 * @param[in] target_protocol 2633 * The next protocol in the previous item. 2634 * @param[out] error 2635 * Pointer to error structure. 2636 * 2637 * @return 2638 * 0 on success, a negative errno value otherwise and rte_errno is set. 2639 */ 2640 int 2641 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2642 uint64_t item_flags, 2643 uint8_t target_protocol, 2644 const struct rte_flow_item_tcp *flow_mask, 2645 struct rte_flow_error *error) 2646 { 2647 const struct rte_flow_item_tcp *mask = item->mask; 2648 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2649 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2650 MLX5_FLOW_LAYER_OUTER_L3; 2651 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2652 MLX5_FLOW_LAYER_OUTER_L4; 2653 int ret; 2654 2655 MLX5_ASSERT(flow_mask); 2656 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2657 return rte_flow_error_set(error, EINVAL, 2658 RTE_FLOW_ERROR_TYPE_ITEM, item, 2659 "protocol filtering not compatible" 2660 " with TCP layer"); 2661 if (!(item_flags & l3m)) 2662 return rte_flow_error_set(error, EINVAL, 2663 RTE_FLOW_ERROR_TYPE_ITEM, item, 2664 "L3 is mandatory to filter on L4"); 2665 if (item_flags & l4m) 2666 return rte_flow_error_set(error, EINVAL, 2667 RTE_FLOW_ERROR_TYPE_ITEM, item, 2668 "multiple L4 layers not supported"); 2669 if (!mask) 2670 mask = &rte_flow_item_tcp_mask; 2671 ret = mlx5_flow_item_acceptable 2672 (item, (const uint8_t *)mask, 2673 (const uint8_t *)flow_mask, 2674 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2675 error); 2676 if (ret < 0) 2677 return ret; 2678 return 0; 2679 } 2680 2681 /** 2682 * Validate VXLAN item. 2683 * 2684 * @param[in] dev 2685 * Pointer to the Ethernet device structure. 2686 * @param[in] udp_dport 2687 * UDP destination port 2688 * @param[in] item 2689 * Item specification. 2690 * @param[in] item_flags 2691 * Bit-fields that holds the items detected until now. 2692 * @param[in] attr 2693 * Flow rule attributes. 2694 * @param[out] error 2695 * Pointer to error structure. 2696 * 2697 * @return 2698 * 0 on success, a negative errno value otherwise and rte_errno is set. 2699 */ 2700 int 2701 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2702 uint16_t udp_dport, 2703 const struct rte_flow_item *item, 2704 uint64_t item_flags, 2705 const struct rte_flow_attr *attr, 2706 struct rte_flow_error *error) 2707 { 2708 const struct rte_flow_item_vxlan *spec = item->spec; 2709 const struct rte_flow_item_vxlan *mask = item->mask; 2710 int ret; 2711 struct mlx5_priv *priv = dev->data->dev_private; 2712 union vni { 2713 uint32_t vlan_id; 2714 uint8_t vni[4]; 2715 } id = { .vlan_id = 0, }; 2716 const struct rte_flow_item_vxlan nic_mask = { 2717 .vni = "\xff\xff\xff", 2718 .rsvd1 = 0xff, 2719 }; 2720 const struct rte_flow_item_vxlan *valid_mask; 2721 2722 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2723 return rte_flow_error_set(error, ENOTSUP, 2724 RTE_FLOW_ERROR_TYPE_ITEM, item, 2725 "multiple tunnel layers not" 2726 " supported"); 2727 valid_mask = &rte_flow_item_vxlan_mask; 2728 /* 2729 * Verify only UDPv4 is present as defined in 2730 * https://tools.ietf.org/html/rfc7348 2731 */ 2732 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2733 return rte_flow_error_set(error, EINVAL, 2734 RTE_FLOW_ERROR_TYPE_ITEM, item, 2735 "no outer UDP layer found"); 2736 if (!mask) 2737 mask = &rte_flow_item_vxlan_mask; 2738 2739 if (priv->sh->steering_format_version != 2740 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2741 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2742 /* FDB domain & NIC domain non-zero group */ 2743 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2744 valid_mask = &nic_mask; 2745 /* Group zero in NIC domain */ 2746 if (!attr->group && !attr->transfer && 2747 priv->sh->tunnel_header_0_1) 2748 valid_mask = &nic_mask; 2749 } 2750 ret = mlx5_flow_item_acceptable 2751 (item, (const uint8_t *)mask, 2752 (const uint8_t *)valid_mask, 2753 sizeof(struct rte_flow_item_vxlan), 2754 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2755 if (ret < 0) 2756 return ret; 2757 if (spec) { 2758 memcpy(&id.vni[1], spec->vni, 3); 2759 memcpy(&id.vni[1], mask->vni, 3); 2760 } 2761 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2762 return rte_flow_error_set(error, ENOTSUP, 2763 RTE_FLOW_ERROR_TYPE_ITEM, item, 2764 "VXLAN tunnel must be fully defined"); 2765 return 0; 2766 } 2767 2768 /** 2769 * Validate VXLAN_GPE item. 2770 * 2771 * @param[in] item 2772 * Item specification. 2773 * @param[in] item_flags 2774 * Bit-fields that holds the items detected until now. 2775 * @param[in] priv 2776 * Pointer to the private data structure. 2777 * @param[in] target_protocol 2778 * The next protocol in the previous item. 2779 * @param[out] error 2780 * Pointer to error structure. 2781 * 2782 * @return 2783 * 0 on success, a negative errno value otherwise and rte_errno is set. 2784 */ 2785 int 2786 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2787 uint64_t item_flags, 2788 struct rte_eth_dev *dev, 2789 struct rte_flow_error *error) 2790 { 2791 struct mlx5_priv *priv = dev->data->dev_private; 2792 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2793 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2794 int ret; 2795 union vni { 2796 uint32_t vlan_id; 2797 uint8_t vni[4]; 2798 } id = { .vlan_id = 0, }; 2799 2800 if (!priv->sh->config.l3_vxlan_en) 2801 return rte_flow_error_set(error, ENOTSUP, 2802 RTE_FLOW_ERROR_TYPE_ITEM, item, 2803 "L3 VXLAN is not enabled by device" 2804 " parameter and/or not configured in" 2805 " firmware"); 2806 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2807 return rte_flow_error_set(error, ENOTSUP, 2808 RTE_FLOW_ERROR_TYPE_ITEM, item, 2809 "multiple tunnel layers not" 2810 " supported"); 2811 /* 2812 * Verify only UDPv4 is present as defined in 2813 * https://tools.ietf.org/html/rfc7348 2814 */ 2815 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2816 return rte_flow_error_set(error, EINVAL, 2817 RTE_FLOW_ERROR_TYPE_ITEM, item, 2818 "no outer UDP layer found"); 2819 if (!mask) 2820 mask = &rte_flow_item_vxlan_gpe_mask; 2821 ret = mlx5_flow_item_acceptable 2822 (item, (const uint8_t *)mask, 2823 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2824 sizeof(struct rte_flow_item_vxlan_gpe), 2825 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2826 if (ret < 0) 2827 return ret; 2828 if (spec) { 2829 if (spec->protocol) 2830 return rte_flow_error_set(error, ENOTSUP, 2831 RTE_FLOW_ERROR_TYPE_ITEM, 2832 item, 2833 "VxLAN-GPE protocol" 2834 " not supported"); 2835 memcpy(&id.vni[1], spec->vni, 3); 2836 memcpy(&id.vni[1], mask->vni, 3); 2837 } 2838 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2839 return rte_flow_error_set(error, ENOTSUP, 2840 RTE_FLOW_ERROR_TYPE_ITEM, item, 2841 "VXLAN-GPE tunnel must be fully" 2842 " defined"); 2843 return 0; 2844 } 2845 /** 2846 * Validate GRE Key item. 2847 * 2848 * @param[in] item 2849 * Item specification. 2850 * @param[in] item_flags 2851 * Bit flags to mark detected items. 2852 * @param[in] gre_item 2853 * Pointer to gre_item 2854 * @param[out] error 2855 * Pointer to error structure. 2856 * 2857 * @return 2858 * 0 on success, a negative errno value otherwise and rte_errno is set. 2859 */ 2860 int 2861 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2862 uint64_t item_flags, 2863 const struct rte_flow_item *gre_item, 2864 struct rte_flow_error *error) 2865 { 2866 const rte_be32_t *mask = item->mask; 2867 int ret = 0; 2868 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2869 const struct rte_flow_item_gre *gre_spec; 2870 const struct rte_flow_item_gre *gre_mask; 2871 2872 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2873 return rte_flow_error_set(error, ENOTSUP, 2874 RTE_FLOW_ERROR_TYPE_ITEM, item, 2875 "Multiple GRE key not support"); 2876 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2877 return rte_flow_error_set(error, ENOTSUP, 2878 RTE_FLOW_ERROR_TYPE_ITEM, item, 2879 "No preceding GRE header"); 2880 if (item_flags & MLX5_FLOW_LAYER_INNER) 2881 return rte_flow_error_set(error, ENOTSUP, 2882 RTE_FLOW_ERROR_TYPE_ITEM, item, 2883 "GRE key following a wrong item"); 2884 gre_mask = gre_item->mask; 2885 if (!gre_mask) 2886 gre_mask = &rte_flow_item_gre_mask; 2887 gre_spec = gre_item->spec; 2888 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2889 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2890 return rte_flow_error_set(error, EINVAL, 2891 RTE_FLOW_ERROR_TYPE_ITEM, item, 2892 "Key bit must be on"); 2893 2894 if (!mask) 2895 mask = &gre_key_default_mask; 2896 ret = mlx5_flow_item_acceptable 2897 (item, (const uint8_t *)mask, 2898 (const uint8_t *)&gre_key_default_mask, 2899 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2900 return ret; 2901 } 2902 2903 /** 2904 * Validate GRE item. 2905 * 2906 * @param[in] item 2907 * Item specification. 2908 * @param[in] item_flags 2909 * Bit flags to mark detected items. 2910 * @param[in] target_protocol 2911 * The next protocol in the previous item. 2912 * @param[out] error 2913 * Pointer to error structure. 2914 * 2915 * @return 2916 * 0 on success, a negative errno value otherwise and rte_errno is set. 2917 */ 2918 int 2919 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2920 uint64_t item_flags, 2921 uint8_t target_protocol, 2922 struct rte_flow_error *error) 2923 { 2924 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2925 const struct rte_flow_item_gre *mask = item->mask; 2926 int ret; 2927 const struct rte_flow_item_gre nic_mask = { 2928 .c_rsvd0_ver = RTE_BE16(0xB000), 2929 .protocol = RTE_BE16(UINT16_MAX), 2930 }; 2931 2932 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2933 return rte_flow_error_set(error, EINVAL, 2934 RTE_FLOW_ERROR_TYPE_ITEM, item, 2935 "protocol filtering not compatible" 2936 " with this GRE layer"); 2937 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2938 return rte_flow_error_set(error, ENOTSUP, 2939 RTE_FLOW_ERROR_TYPE_ITEM, item, 2940 "multiple tunnel layers not" 2941 " supported"); 2942 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2943 return rte_flow_error_set(error, ENOTSUP, 2944 RTE_FLOW_ERROR_TYPE_ITEM, item, 2945 "L3 Layer is missing"); 2946 if (!mask) 2947 mask = &rte_flow_item_gre_mask; 2948 ret = mlx5_flow_item_acceptable 2949 (item, (const uint8_t *)mask, 2950 (const uint8_t *)&nic_mask, 2951 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2952 error); 2953 if (ret < 0) 2954 return ret; 2955 #ifndef HAVE_MLX5DV_DR 2956 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2957 if (spec && (spec->protocol & mask->protocol)) 2958 return rte_flow_error_set(error, ENOTSUP, 2959 RTE_FLOW_ERROR_TYPE_ITEM, item, 2960 "without MPLS support the" 2961 " specification cannot be used for" 2962 " filtering"); 2963 #endif 2964 #endif 2965 return 0; 2966 } 2967 2968 /** 2969 * Validate Geneve item. 2970 * 2971 * @param[in] item 2972 * Item specification. 2973 * @param[in] itemFlags 2974 * Bit-fields that holds the items detected until now. 2975 * @param[in] enPriv 2976 * Pointer to the private data structure. 2977 * @param[out] error 2978 * Pointer to error structure. 2979 * 2980 * @return 2981 * 0 on success, a negative errno value otherwise and rte_errno is set. 2982 */ 2983 2984 int 2985 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2986 uint64_t item_flags, 2987 struct rte_eth_dev *dev, 2988 struct rte_flow_error *error) 2989 { 2990 struct mlx5_priv *priv = dev->data->dev_private; 2991 const struct rte_flow_item_geneve *spec = item->spec; 2992 const struct rte_flow_item_geneve *mask = item->mask; 2993 int ret; 2994 uint16_t gbhdr; 2995 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 2996 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2997 const struct rte_flow_item_geneve nic_mask = { 2998 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2999 .vni = "\xff\xff\xff", 3000 .protocol = RTE_BE16(UINT16_MAX), 3001 }; 3002 3003 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3004 return rte_flow_error_set(error, ENOTSUP, 3005 RTE_FLOW_ERROR_TYPE_ITEM, item, 3006 "L3 Geneve is not enabled by device" 3007 " parameter and/or not configured in" 3008 " firmware"); 3009 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3010 return rte_flow_error_set(error, ENOTSUP, 3011 RTE_FLOW_ERROR_TYPE_ITEM, item, 3012 "multiple tunnel layers not" 3013 " supported"); 3014 /* 3015 * Verify only UDPv4 is present as defined in 3016 * https://tools.ietf.org/html/rfc7348 3017 */ 3018 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3019 return rte_flow_error_set(error, EINVAL, 3020 RTE_FLOW_ERROR_TYPE_ITEM, item, 3021 "no outer UDP layer found"); 3022 if (!mask) 3023 mask = &rte_flow_item_geneve_mask; 3024 ret = mlx5_flow_item_acceptable 3025 (item, (const uint8_t *)mask, 3026 (const uint8_t *)&nic_mask, 3027 sizeof(struct rte_flow_item_geneve), 3028 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3029 if (ret) 3030 return ret; 3031 if (spec) { 3032 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3033 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3034 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3035 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3036 return rte_flow_error_set(error, ENOTSUP, 3037 RTE_FLOW_ERROR_TYPE_ITEM, 3038 item, 3039 "Geneve protocol unsupported" 3040 " fields are being used"); 3041 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3042 return rte_flow_error_set 3043 (error, ENOTSUP, 3044 RTE_FLOW_ERROR_TYPE_ITEM, 3045 item, 3046 "Unsupported Geneve options length"); 3047 } 3048 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3049 return rte_flow_error_set 3050 (error, ENOTSUP, 3051 RTE_FLOW_ERROR_TYPE_ITEM, item, 3052 "Geneve tunnel must be fully defined"); 3053 return 0; 3054 } 3055 3056 /** 3057 * Validate Geneve TLV option item. 3058 * 3059 * @param[in] item 3060 * Item specification. 3061 * @param[in] last_item 3062 * Previous validated item in the pattern items. 3063 * @param[in] geneve_item 3064 * Previous GENEVE item specification. 3065 * @param[in] dev 3066 * Pointer to the rte_eth_dev structure. 3067 * @param[out] error 3068 * Pointer to error structure. 3069 * 3070 * @return 3071 * 0 on success, a negative errno value otherwise and rte_errno is set. 3072 */ 3073 int 3074 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3075 uint64_t last_item, 3076 const struct rte_flow_item *geneve_item, 3077 struct rte_eth_dev *dev, 3078 struct rte_flow_error *error) 3079 { 3080 struct mlx5_priv *priv = dev->data->dev_private; 3081 struct mlx5_dev_ctx_shared *sh = priv->sh; 3082 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3083 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3084 uint8_t data_max_supported = 3085 hca_attr->max_geneve_tlv_option_data_len * 4; 3086 const struct rte_flow_item_geneve *geneve_spec; 3087 const struct rte_flow_item_geneve *geneve_mask; 3088 const struct rte_flow_item_geneve_opt *spec = item->spec; 3089 const struct rte_flow_item_geneve_opt *mask = item->mask; 3090 unsigned int i; 3091 unsigned int data_len; 3092 uint8_t tlv_option_len; 3093 uint16_t optlen_m, optlen_v; 3094 const struct rte_flow_item_geneve_opt full_mask = { 3095 .option_class = RTE_BE16(0xffff), 3096 .option_type = 0xff, 3097 .option_len = 0x1f, 3098 }; 3099 3100 if (!mask) 3101 mask = &rte_flow_item_geneve_opt_mask; 3102 if (!spec) 3103 return rte_flow_error_set 3104 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3105 "Geneve TLV opt class/type/length must be specified"); 3106 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3107 return rte_flow_error_set 3108 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3109 "Geneve TLV opt length exceeds the limit (31)"); 3110 /* Check if class type and length masks are full. */ 3111 if (full_mask.option_class != mask->option_class || 3112 full_mask.option_type != mask->option_type || 3113 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3114 return rte_flow_error_set 3115 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3116 "Geneve TLV opt class/type/length masks must be full"); 3117 /* Check if length is supported */ 3118 if ((uint32_t)spec->option_len > 3119 hca_attr->max_geneve_tlv_option_data_len) 3120 return rte_flow_error_set 3121 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3122 "Geneve TLV opt length not supported"); 3123 if (hca_attr->max_geneve_tlv_options > 1) 3124 DRV_LOG(DEBUG, 3125 "max_geneve_tlv_options supports more than 1 option"); 3126 /* Check GENEVE item preceding. */ 3127 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3128 return rte_flow_error_set 3129 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3130 "Geneve opt item must be preceded with Geneve item"); 3131 geneve_spec = geneve_item->spec; 3132 geneve_mask = geneve_item->mask ? geneve_item->mask : 3133 &rte_flow_item_geneve_mask; 3134 /* Check if GENEVE TLV option size doesn't exceed option length */ 3135 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3136 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3137 tlv_option_len = spec->option_len & mask->option_len; 3138 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3139 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3140 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3141 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3142 if ((optlen_v & optlen_m) <= tlv_option_len) 3143 return rte_flow_error_set 3144 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3145 "GENEVE TLV option length exceeds optlen"); 3146 } 3147 /* Check if length is 0 or data is 0. */ 3148 if (spec->data == NULL || spec->option_len == 0) 3149 return rte_flow_error_set 3150 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3151 "Geneve TLV opt with zero data/length not supported"); 3152 /* Check not all data & mask are 0. */ 3153 data_len = spec->option_len * 4; 3154 if (mask->data == NULL) { 3155 for (i = 0; i < data_len; i++) 3156 if (spec->data[i]) 3157 break; 3158 if (i == data_len) 3159 return rte_flow_error_set(error, ENOTSUP, 3160 RTE_FLOW_ERROR_TYPE_ITEM, item, 3161 "Can't match on Geneve option data 0"); 3162 } else { 3163 for (i = 0; i < data_len; i++) 3164 if (spec->data[i] & mask->data[i]) 3165 break; 3166 if (i == data_len) 3167 return rte_flow_error_set(error, ENOTSUP, 3168 RTE_FLOW_ERROR_TYPE_ITEM, item, 3169 "Can't match on Geneve option data and mask 0"); 3170 /* Check data mask supported. */ 3171 for (i = data_max_supported; i < data_len ; i++) 3172 if (mask->data[i]) 3173 return rte_flow_error_set(error, ENOTSUP, 3174 RTE_FLOW_ERROR_TYPE_ITEM, item, 3175 "Data mask is of unsupported size"); 3176 } 3177 /* Check GENEVE option is supported in NIC. */ 3178 if (!hca_attr->geneve_tlv_opt) 3179 return rte_flow_error_set 3180 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3181 "Geneve TLV opt not supported"); 3182 /* Check if we already have geneve option with different type/class. */ 3183 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3184 geneve_opt_resource = sh->geneve_tlv_option_resource; 3185 if (geneve_opt_resource != NULL) 3186 if (geneve_opt_resource->option_class != spec->option_class || 3187 geneve_opt_resource->option_type != spec->option_type || 3188 geneve_opt_resource->length != spec->option_len) { 3189 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3190 return rte_flow_error_set(error, ENOTSUP, 3191 RTE_FLOW_ERROR_TYPE_ITEM, item, 3192 "Only one Geneve TLV option supported"); 3193 } 3194 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3195 return 0; 3196 } 3197 3198 /** 3199 * Validate MPLS item. 3200 * 3201 * @param[in] dev 3202 * Pointer to the rte_eth_dev structure. 3203 * @param[in] item 3204 * Item specification. 3205 * @param[in] item_flags 3206 * Bit-fields that holds the items detected until now. 3207 * @param[in] prev_layer 3208 * The protocol layer indicated in previous item. 3209 * @param[out] error 3210 * Pointer to error structure. 3211 * 3212 * @return 3213 * 0 on success, a negative errno value otherwise and rte_errno is set. 3214 */ 3215 int 3216 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3217 const struct rte_flow_item *item __rte_unused, 3218 uint64_t item_flags __rte_unused, 3219 uint64_t prev_layer __rte_unused, 3220 struct rte_flow_error *error) 3221 { 3222 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3223 const struct rte_flow_item_mpls *mask = item->mask; 3224 struct mlx5_priv *priv = dev->data->dev_private; 3225 int ret; 3226 3227 if (!priv->sh->dev_cap.mpls_en) 3228 return rte_flow_error_set(error, ENOTSUP, 3229 RTE_FLOW_ERROR_TYPE_ITEM, item, 3230 "MPLS not supported or" 3231 " disabled in firmware" 3232 " configuration."); 3233 /* MPLS over UDP, GRE is allowed */ 3234 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3235 MLX5_FLOW_LAYER_GRE | 3236 MLX5_FLOW_LAYER_GRE_KEY))) 3237 return rte_flow_error_set(error, EINVAL, 3238 RTE_FLOW_ERROR_TYPE_ITEM, item, 3239 "protocol filtering not compatible" 3240 " with MPLS layer"); 3241 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3242 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3243 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3244 return rte_flow_error_set(error, ENOTSUP, 3245 RTE_FLOW_ERROR_TYPE_ITEM, item, 3246 "multiple tunnel layers not" 3247 " supported"); 3248 if (!mask) 3249 mask = &rte_flow_item_mpls_mask; 3250 ret = mlx5_flow_item_acceptable 3251 (item, (const uint8_t *)mask, 3252 (const uint8_t *)&rte_flow_item_mpls_mask, 3253 sizeof(struct rte_flow_item_mpls), 3254 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3255 if (ret < 0) 3256 return ret; 3257 return 0; 3258 #else 3259 return rte_flow_error_set(error, ENOTSUP, 3260 RTE_FLOW_ERROR_TYPE_ITEM, item, 3261 "MPLS is not supported by Verbs, please" 3262 " update."); 3263 #endif 3264 } 3265 3266 /** 3267 * Validate NVGRE item. 3268 * 3269 * @param[in] item 3270 * Item specification. 3271 * @param[in] item_flags 3272 * Bit flags to mark detected items. 3273 * @param[in] target_protocol 3274 * The next protocol in the previous item. 3275 * @param[out] error 3276 * Pointer to error structure. 3277 * 3278 * @return 3279 * 0 on success, a negative errno value otherwise and rte_errno is set. 3280 */ 3281 int 3282 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3283 uint64_t item_flags, 3284 uint8_t target_protocol, 3285 struct rte_flow_error *error) 3286 { 3287 const struct rte_flow_item_nvgre *mask = item->mask; 3288 int ret; 3289 3290 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3291 return rte_flow_error_set(error, EINVAL, 3292 RTE_FLOW_ERROR_TYPE_ITEM, item, 3293 "protocol filtering not compatible" 3294 " with this GRE layer"); 3295 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3296 return rte_flow_error_set(error, ENOTSUP, 3297 RTE_FLOW_ERROR_TYPE_ITEM, item, 3298 "multiple tunnel layers not" 3299 " supported"); 3300 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3301 return rte_flow_error_set(error, ENOTSUP, 3302 RTE_FLOW_ERROR_TYPE_ITEM, item, 3303 "L3 Layer is missing"); 3304 if (!mask) 3305 mask = &rte_flow_item_nvgre_mask; 3306 ret = mlx5_flow_item_acceptable 3307 (item, (const uint8_t *)mask, 3308 (const uint8_t *)&rte_flow_item_nvgre_mask, 3309 sizeof(struct rte_flow_item_nvgre), 3310 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3311 if (ret < 0) 3312 return ret; 3313 return 0; 3314 } 3315 3316 /** 3317 * Validate eCPRI item. 3318 * 3319 * @param[in] item 3320 * Item specification. 3321 * @param[in] item_flags 3322 * Bit-fields that holds the items detected until now. 3323 * @param[in] last_item 3324 * Previous validated item in the pattern items. 3325 * @param[in] ether_type 3326 * Type in the ethernet layer header (including dot1q). 3327 * @param[in] acc_mask 3328 * Acceptable mask, if NULL default internal default mask 3329 * will be used to check whether item fields are supported. 3330 * @param[out] error 3331 * Pointer to error structure. 3332 * 3333 * @return 3334 * 0 on success, a negative errno value otherwise and rte_errno is set. 3335 */ 3336 int 3337 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3338 uint64_t item_flags, 3339 uint64_t last_item, 3340 uint16_t ether_type, 3341 const struct rte_flow_item_ecpri *acc_mask, 3342 struct rte_flow_error *error) 3343 { 3344 const struct rte_flow_item_ecpri *mask = item->mask; 3345 const struct rte_flow_item_ecpri nic_mask = { 3346 .hdr = { 3347 .common = { 3348 .u32 = 3349 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3350 .type = 0xFF, 3351 }).u32), 3352 }, 3353 .dummy[0] = 0xFFFFFFFF, 3354 }, 3355 }; 3356 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3357 MLX5_FLOW_LAYER_OUTER_VLAN); 3358 struct rte_flow_item_ecpri mask_lo; 3359 3360 if (!(last_item & outer_l2_vlan) && 3361 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3362 return rte_flow_error_set(error, EINVAL, 3363 RTE_FLOW_ERROR_TYPE_ITEM, item, 3364 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3365 if ((last_item & outer_l2_vlan) && ether_type && 3366 ether_type != RTE_ETHER_TYPE_ECPRI) 3367 return rte_flow_error_set(error, EINVAL, 3368 RTE_FLOW_ERROR_TYPE_ITEM, item, 3369 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3370 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3371 return rte_flow_error_set(error, EINVAL, 3372 RTE_FLOW_ERROR_TYPE_ITEM, item, 3373 "eCPRI with tunnel is not supported right now"); 3374 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3375 return rte_flow_error_set(error, ENOTSUP, 3376 RTE_FLOW_ERROR_TYPE_ITEM, item, 3377 "multiple L3 layers not supported"); 3378 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3379 return rte_flow_error_set(error, EINVAL, 3380 RTE_FLOW_ERROR_TYPE_ITEM, item, 3381 "eCPRI cannot coexist with a TCP layer"); 3382 /* In specification, eCPRI could be over UDP layer. */ 3383 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3384 return rte_flow_error_set(error, EINVAL, 3385 RTE_FLOW_ERROR_TYPE_ITEM, item, 3386 "eCPRI over UDP layer is not yet supported right now"); 3387 /* Mask for type field in common header could be zero. */ 3388 if (!mask) 3389 mask = &rte_flow_item_ecpri_mask; 3390 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3391 /* Input mask is in big-endian format. */ 3392 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3393 return rte_flow_error_set(error, EINVAL, 3394 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3395 "partial mask is not supported for protocol"); 3396 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3397 return rte_flow_error_set(error, EINVAL, 3398 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3399 "message header mask must be after a type mask"); 3400 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3401 acc_mask ? (const uint8_t *)acc_mask 3402 : (const uint8_t *)&nic_mask, 3403 sizeof(struct rte_flow_item_ecpri), 3404 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3405 } 3406 3407 static int 3408 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3409 const struct rte_flow_attr *attr __rte_unused, 3410 const struct rte_flow_item items[] __rte_unused, 3411 const struct rte_flow_action actions[] __rte_unused, 3412 bool external __rte_unused, 3413 int hairpin __rte_unused, 3414 struct rte_flow_error *error) 3415 { 3416 return rte_flow_error_set(error, ENOTSUP, 3417 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3418 } 3419 3420 static struct mlx5_flow * 3421 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3422 const struct rte_flow_attr *attr __rte_unused, 3423 const struct rte_flow_item items[] __rte_unused, 3424 const struct rte_flow_action actions[] __rte_unused, 3425 struct rte_flow_error *error) 3426 { 3427 rte_flow_error_set(error, ENOTSUP, 3428 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3429 return NULL; 3430 } 3431 3432 static int 3433 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3434 struct mlx5_flow *dev_flow __rte_unused, 3435 const struct rte_flow_attr *attr __rte_unused, 3436 const struct rte_flow_item items[] __rte_unused, 3437 const struct rte_flow_action actions[] __rte_unused, 3438 struct rte_flow_error *error) 3439 { 3440 return rte_flow_error_set(error, ENOTSUP, 3441 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3442 } 3443 3444 static int 3445 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3446 struct rte_flow *flow __rte_unused, 3447 struct rte_flow_error *error) 3448 { 3449 return rte_flow_error_set(error, ENOTSUP, 3450 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3451 } 3452 3453 static void 3454 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3455 struct rte_flow *flow __rte_unused) 3456 { 3457 } 3458 3459 static void 3460 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3461 struct rte_flow *flow __rte_unused) 3462 { 3463 } 3464 3465 static int 3466 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3467 struct rte_flow *flow __rte_unused, 3468 const struct rte_flow_action *actions __rte_unused, 3469 void *data __rte_unused, 3470 struct rte_flow_error *error) 3471 { 3472 return rte_flow_error_set(error, ENOTSUP, 3473 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3474 } 3475 3476 static int 3477 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3478 uint32_t domains __rte_unused, 3479 uint32_t flags __rte_unused) 3480 { 3481 return 0; 3482 } 3483 3484 /* Void driver to protect from null pointer reference. */ 3485 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3486 .validate = flow_null_validate, 3487 .prepare = flow_null_prepare, 3488 .translate = flow_null_translate, 3489 .apply = flow_null_apply, 3490 .remove = flow_null_remove, 3491 .destroy = flow_null_destroy, 3492 .query = flow_null_query, 3493 .sync_domain = flow_null_sync_domain, 3494 }; 3495 3496 /** 3497 * Select flow driver type according to flow attributes and device 3498 * configuration. 3499 * 3500 * @param[in] dev 3501 * Pointer to the dev structure. 3502 * @param[in] attr 3503 * Pointer to the flow attributes. 3504 * 3505 * @return 3506 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3507 */ 3508 static enum mlx5_flow_drv_type 3509 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3510 { 3511 struct mlx5_priv *priv = dev->data->dev_private; 3512 /* The OS can determine first a specific flow type (DV, VERBS) */ 3513 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3514 3515 if (type != MLX5_FLOW_TYPE_MAX) 3516 return type; 3517 /* 3518 * Currently when dv_flow_en == 2, only HW steering engine is 3519 * supported. New engines can also be chosen here if ready. 3520 */ 3521 if (priv->sh->config.dv_flow_en == 2) 3522 return MLX5_FLOW_TYPE_HW; 3523 /* If no OS specific type - continue with DV/VERBS selection */ 3524 if (attr->transfer && priv->sh->config.dv_esw_en) 3525 type = MLX5_FLOW_TYPE_DV; 3526 if (!attr->transfer) 3527 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3528 MLX5_FLOW_TYPE_VERBS; 3529 return type; 3530 } 3531 3532 #define flow_get_drv_ops(type) flow_drv_ops[type] 3533 3534 /** 3535 * Flow driver validation API. This abstracts calling driver specific functions. 3536 * The type of flow driver is determined according to flow attributes. 3537 * 3538 * @param[in] dev 3539 * Pointer to the dev structure. 3540 * @param[in] attr 3541 * Pointer to the flow attributes. 3542 * @param[in] items 3543 * Pointer to the list of items. 3544 * @param[in] actions 3545 * Pointer to the list of actions. 3546 * @param[in] external 3547 * This flow rule is created by request external to PMD. 3548 * @param[in] hairpin 3549 * Number of hairpin TX actions, 0 means classic flow. 3550 * @param[out] error 3551 * Pointer to the error structure. 3552 * 3553 * @return 3554 * 0 on success, a negative errno value otherwise and rte_errno is set. 3555 */ 3556 static inline int 3557 flow_drv_validate(struct rte_eth_dev *dev, 3558 const struct rte_flow_attr *attr, 3559 const struct rte_flow_item items[], 3560 const struct rte_flow_action actions[], 3561 bool external, int hairpin, struct rte_flow_error *error) 3562 { 3563 const struct mlx5_flow_driver_ops *fops; 3564 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3565 3566 fops = flow_get_drv_ops(type); 3567 return fops->validate(dev, attr, items, actions, external, 3568 hairpin, error); 3569 } 3570 3571 /** 3572 * Flow driver preparation API. This abstracts calling driver specific 3573 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3574 * calculates the size of memory required for device flow, allocates the memory, 3575 * initializes the device flow and returns the pointer. 3576 * 3577 * @note 3578 * This function initializes device flow structure such as dv or verbs in 3579 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3580 * rest. For example, adding returning device flow to flow->dev_flow list and 3581 * setting backward reference to the flow should be done out of this function. 3582 * layers field is not filled either. 3583 * 3584 * @param[in] dev 3585 * Pointer to the dev structure. 3586 * @param[in] attr 3587 * Pointer to the flow attributes. 3588 * @param[in] items 3589 * Pointer to the list of items. 3590 * @param[in] actions 3591 * Pointer to the list of actions. 3592 * @param[in] flow_idx 3593 * This memory pool index to the flow. 3594 * @param[out] error 3595 * Pointer to the error structure. 3596 * 3597 * @return 3598 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3599 */ 3600 static inline struct mlx5_flow * 3601 flow_drv_prepare(struct rte_eth_dev *dev, 3602 const struct rte_flow *flow, 3603 const struct rte_flow_attr *attr, 3604 const struct rte_flow_item items[], 3605 const struct rte_flow_action actions[], 3606 uint32_t flow_idx, 3607 struct rte_flow_error *error) 3608 { 3609 const struct mlx5_flow_driver_ops *fops; 3610 enum mlx5_flow_drv_type type = flow->drv_type; 3611 struct mlx5_flow *mlx5_flow = NULL; 3612 3613 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3614 fops = flow_get_drv_ops(type); 3615 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3616 if (mlx5_flow) 3617 mlx5_flow->flow_idx = flow_idx; 3618 return mlx5_flow; 3619 } 3620 3621 /** 3622 * Flow driver translation API. This abstracts calling driver specific 3623 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3624 * translates a generic flow into a driver flow. flow_drv_prepare() must 3625 * precede. 3626 * 3627 * @note 3628 * dev_flow->layers could be filled as a result of parsing during translation 3629 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3630 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3631 * flow->actions could be overwritten even though all the expanded dev_flows 3632 * have the same actions. 3633 * 3634 * @param[in] dev 3635 * Pointer to the rte dev structure. 3636 * @param[in, out] dev_flow 3637 * Pointer to the mlx5 flow. 3638 * @param[in] attr 3639 * Pointer to the flow attributes. 3640 * @param[in] items 3641 * Pointer to the list of items. 3642 * @param[in] actions 3643 * Pointer to the list of actions. 3644 * @param[out] error 3645 * Pointer to the error structure. 3646 * 3647 * @return 3648 * 0 on success, a negative errno value otherwise and rte_errno is set. 3649 */ 3650 static inline int 3651 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3652 const struct rte_flow_attr *attr, 3653 const struct rte_flow_item items[], 3654 const struct rte_flow_action actions[], 3655 struct rte_flow_error *error) 3656 { 3657 const struct mlx5_flow_driver_ops *fops; 3658 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3659 3660 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3661 fops = flow_get_drv_ops(type); 3662 return fops->translate(dev, dev_flow, attr, items, actions, error); 3663 } 3664 3665 /** 3666 * Flow driver apply API. This abstracts calling driver specific functions. 3667 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3668 * translated driver flows on to device. flow_drv_translate() must precede. 3669 * 3670 * @param[in] dev 3671 * Pointer to Ethernet device structure. 3672 * @param[in, out] flow 3673 * Pointer to flow structure. 3674 * @param[out] error 3675 * Pointer to error structure. 3676 * 3677 * @return 3678 * 0 on success, a negative errno value otherwise and rte_errno is set. 3679 */ 3680 static inline int 3681 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3682 struct rte_flow_error *error) 3683 { 3684 const struct mlx5_flow_driver_ops *fops; 3685 enum mlx5_flow_drv_type type = flow->drv_type; 3686 3687 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3688 fops = flow_get_drv_ops(type); 3689 return fops->apply(dev, flow, error); 3690 } 3691 3692 /** 3693 * Flow driver destroy API. This abstracts calling driver specific functions. 3694 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3695 * on device and releases resources of the flow. 3696 * 3697 * @param[in] dev 3698 * Pointer to Ethernet device. 3699 * @param[in, out] flow 3700 * Pointer to flow structure. 3701 */ 3702 static inline void 3703 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3704 { 3705 const struct mlx5_flow_driver_ops *fops; 3706 enum mlx5_flow_drv_type type = flow->drv_type; 3707 3708 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3709 fops = flow_get_drv_ops(type); 3710 fops->destroy(dev, flow); 3711 } 3712 3713 /** 3714 * Flow driver find RSS policy tbl API. This abstracts calling driver 3715 * specific functions. Parent flow (rte_flow) should have driver 3716 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3717 * 3718 * @param[in] dev 3719 * Pointer to Ethernet device. 3720 * @param[in, out] flow 3721 * Pointer to flow structure. 3722 * @param[in] policy 3723 * Pointer to meter policy table. 3724 * @param[in] rss_desc 3725 * Pointer to rss_desc 3726 */ 3727 static struct mlx5_flow_meter_sub_policy * 3728 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3729 struct rte_flow *flow, 3730 struct mlx5_flow_meter_policy *policy, 3731 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3732 { 3733 const struct mlx5_flow_driver_ops *fops; 3734 enum mlx5_flow_drv_type type = flow->drv_type; 3735 3736 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3737 fops = flow_get_drv_ops(type); 3738 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3739 } 3740 3741 /** 3742 * Flow driver color tag rule API. This abstracts calling driver 3743 * specific functions. Parent flow (rte_flow) should have driver 3744 * type (drv_type). It will create the color tag rules in hierarchy meter. 3745 * 3746 * @param[in] dev 3747 * Pointer to Ethernet device. 3748 * @param[in, out] flow 3749 * Pointer to flow structure. 3750 * @param[in] fm 3751 * Pointer to flow meter structure. 3752 * @param[in] src_port 3753 * The src port this extra rule should use. 3754 * @param[in] item 3755 * The src port id match item. 3756 * @param[out] error 3757 * Pointer to error structure. 3758 */ 3759 static int 3760 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3761 struct rte_flow *flow, 3762 struct mlx5_flow_meter_info *fm, 3763 int32_t src_port, 3764 const struct rte_flow_item *item, 3765 struct rte_flow_error *error) 3766 { 3767 const struct mlx5_flow_driver_ops *fops; 3768 enum mlx5_flow_drv_type type = flow->drv_type; 3769 3770 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3771 fops = flow_get_drv_ops(type); 3772 return fops->meter_hierarchy_rule_create(dev, fm, 3773 src_port, item, error); 3774 } 3775 3776 /** 3777 * Get RSS action from the action list. 3778 * 3779 * @param[in] dev 3780 * Pointer to Ethernet device. 3781 * @param[in] actions 3782 * Pointer to the list of actions. 3783 * @param[in] flow 3784 * Parent flow structure pointer. 3785 * 3786 * @return 3787 * Pointer to the RSS action if exist, else return NULL. 3788 */ 3789 static const struct rte_flow_action_rss* 3790 flow_get_rss_action(struct rte_eth_dev *dev, 3791 const struct rte_flow_action actions[]) 3792 { 3793 struct mlx5_priv *priv = dev->data->dev_private; 3794 const struct rte_flow_action_rss *rss = NULL; 3795 struct mlx5_meter_policy_action_container *acg; 3796 struct mlx5_meter_policy_action_container *acy; 3797 3798 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3799 switch (actions->type) { 3800 case RTE_FLOW_ACTION_TYPE_RSS: 3801 rss = actions->conf; 3802 break; 3803 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3804 { 3805 const struct rte_flow_action_sample *sample = 3806 actions->conf; 3807 const struct rte_flow_action *act = sample->actions; 3808 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3809 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3810 rss = act->conf; 3811 break; 3812 } 3813 case RTE_FLOW_ACTION_TYPE_METER: 3814 { 3815 uint32_t mtr_idx; 3816 struct mlx5_flow_meter_info *fm; 3817 struct mlx5_flow_meter_policy *policy; 3818 const struct rte_flow_action_meter *mtr = actions->conf; 3819 3820 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3821 if (fm && !fm->def_policy) { 3822 policy = mlx5_flow_meter_policy_find(dev, 3823 fm->policy_id, NULL); 3824 MLX5_ASSERT(policy); 3825 if (policy->is_hierarchy) { 3826 policy = 3827 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3828 policy); 3829 if (!policy) 3830 return NULL; 3831 } 3832 if (policy->is_rss) { 3833 acg = 3834 &policy->act_cnt[RTE_COLOR_GREEN]; 3835 acy = 3836 &policy->act_cnt[RTE_COLOR_YELLOW]; 3837 if (acg->fate_action == 3838 MLX5_FLOW_FATE_SHARED_RSS) 3839 rss = acg->rss->conf; 3840 else if (acy->fate_action == 3841 MLX5_FLOW_FATE_SHARED_RSS) 3842 rss = acy->rss->conf; 3843 } 3844 } 3845 break; 3846 } 3847 default: 3848 break; 3849 } 3850 } 3851 return rss; 3852 } 3853 3854 /** 3855 * Get ASO age action by index. 3856 * 3857 * @param[in] dev 3858 * Pointer to the Ethernet device structure. 3859 * @param[in] age_idx 3860 * Index to the ASO age action. 3861 * 3862 * @return 3863 * The specified ASO age action. 3864 */ 3865 struct mlx5_aso_age_action* 3866 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3867 { 3868 uint16_t pool_idx = age_idx & UINT16_MAX; 3869 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3870 struct mlx5_priv *priv = dev->data->dev_private; 3871 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3872 struct mlx5_aso_age_pool *pool; 3873 3874 rte_rwlock_read_lock(&mng->resize_rwl); 3875 pool = mng->pools[pool_idx]; 3876 rte_rwlock_read_unlock(&mng->resize_rwl); 3877 return &pool->actions[offset - 1]; 3878 } 3879 3880 /* maps indirect action to translated direct in some actions array */ 3881 struct mlx5_translated_action_handle { 3882 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3883 int index; /**< Index in related array of rte_flow_action. */ 3884 }; 3885 3886 /** 3887 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3888 * direct action if translation possible. 3889 * This functionality used to run same execution path for both direct and 3890 * indirect actions on flow create. All necessary preparations for indirect 3891 * action handling should be performed on *handle* actions list returned 3892 * from this call. 3893 * 3894 * @param[in] dev 3895 * Pointer to Ethernet device. 3896 * @param[in] actions 3897 * List of actions to translate. 3898 * @param[out] handle 3899 * List to store translated indirect action object handles. 3900 * @param[in, out] indir_n 3901 * Size of *handle* array. On return should be updated with number of 3902 * indirect actions retrieved from the *actions* list. 3903 * @param[out] translated_actions 3904 * List of actions where all indirect actions were translated to direct 3905 * if possible. NULL if no translation took place. 3906 * @param[out] error 3907 * Pointer to the error structure. 3908 * 3909 * @return 3910 * 0 on success, a negative errno value otherwise and rte_errno is set. 3911 */ 3912 static int 3913 flow_action_handles_translate(struct rte_eth_dev *dev, 3914 const struct rte_flow_action actions[], 3915 struct mlx5_translated_action_handle *handle, 3916 int *indir_n, 3917 struct rte_flow_action **translated_actions, 3918 struct rte_flow_error *error) 3919 { 3920 struct mlx5_priv *priv = dev->data->dev_private; 3921 struct rte_flow_action *translated = NULL; 3922 size_t actions_size; 3923 int n; 3924 int copied_n = 0; 3925 struct mlx5_translated_action_handle *handle_end = NULL; 3926 3927 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3928 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3929 continue; 3930 if (copied_n == *indir_n) { 3931 return rte_flow_error_set 3932 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3933 NULL, "too many shared actions"); 3934 } 3935 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3936 sizeof(actions[n].conf)); 3937 handle[copied_n].index = n; 3938 copied_n++; 3939 } 3940 n++; 3941 *indir_n = copied_n; 3942 if (!copied_n) 3943 return 0; 3944 actions_size = sizeof(struct rte_flow_action) * n; 3945 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3946 if (!translated) { 3947 rte_errno = ENOMEM; 3948 return -ENOMEM; 3949 } 3950 memcpy(translated, actions, actions_size); 3951 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3952 struct mlx5_shared_action_rss *shared_rss; 3953 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3954 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3955 uint32_t idx = act_idx & 3956 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3957 3958 switch (type) { 3959 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3960 shared_rss = mlx5_ipool_get 3961 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3962 translated[handle->index].type = 3963 RTE_FLOW_ACTION_TYPE_RSS; 3964 translated[handle->index].conf = 3965 &shared_rss->origin; 3966 break; 3967 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3968 translated[handle->index].type = 3969 (enum rte_flow_action_type) 3970 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3971 translated[handle->index].conf = (void *)(uintptr_t)idx; 3972 break; 3973 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3974 if (priv->sh->flow_hit_aso_en) { 3975 translated[handle->index].type = 3976 (enum rte_flow_action_type) 3977 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3978 translated[handle->index].conf = 3979 (void *)(uintptr_t)idx; 3980 break; 3981 } 3982 /* Fall-through */ 3983 case MLX5_INDIRECT_ACTION_TYPE_CT: 3984 if (priv->sh->ct_aso_en) { 3985 translated[handle->index].type = 3986 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3987 translated[handle->index].conf = 3988 (void *)(uintptr_t)idx; 3989 break; 3990 } 3991 /* Fall-through */ 3992 default: 3993 mlx5_free(translated); 3994 return rte_flow_error_set 3995 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3996 NULL, "invalid indirect action type"); 3997 } 3998 } 3999 *translated_actions = translated; 4000 return 0; 4001 } 4002 4003 /** 4004 * Get Shared RSS action from the action list. 4005 * 4006 * @param[in] dev 4007 * Pointer to Ethernet device. 4008 * @param[in] shared 4009 * Pointer to the list of actions. 4010 * @param[in] shared_n 4011 * Actions list length. 4012 * 4013 * @return 4014 * The MLX5 RSS action ID if exists, otherwise return 0. 4015 */ 4016 static uint32_t 4017 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4018 struct mlx5_translated_action_handle *handle, 4019 int shared_n) 4020 { 4021 struct mlx5_translated_action_handle *handle_end; 4022 struct mlx5_priv *priv = dev->data->dev_private; 4023 struct mlx5_shared_action_rss *shared_rss; 4024 4025 4026 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4027 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4028 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4029 uint32_t idx = act_idx & 4030 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4031 switch (type) { 4032 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4033 shared_rss = mlx5_ipool_get 4034 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4035 idx); 4036 __atomic_add_fetch(&shared_rss->refcnt, 1, 4037 __ATOMIC_RELAXED); 4038 return idx; 4039 default: 4040 break; 4041 } 4042 } 4043 return 0; 4044 } 4045 4046 static unsigned int 4047 find_graph_root(uint32_t rss_level) 4048 { 4049 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4050 MLX5_EXPANSION_ROOT_OUTER; 4051 } 4052 4053 /** 4054 * Get layer flags from the prefix flow. 4055 * 4056 * Some flows may be split to several subflows, the prefix subflow gets the 4057 * match items and the suffix sub flow gets the actions. 4058 * Some actions need the user defined match item flags to get the detail for 4059 * the action. 4060 * This function helps the suffix flow to get the item layer flags from prefix 4061 * subflow. 4062 * 4063 * @param[in] dev_flow 4064 * Pointer the created prefix subflow. 4065 * 4066 * @return 4067 * The layers get from prefix subflow. 4068 */ 4069 static inline uint64_t 4070 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4071 { 4072 uint64_t layers = 0; 4073 4074 /* 4075 * Layers bits could be localization, but usually the compiler will 4076 * help to do the optimization work for source code. 4077 * If no decap actions, use the layers directly. 4078 */ 4079 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4080 return dev_flow->handle->layers; 4081 /* Convert L3 layers with decap action. */ 4082 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4083 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4084 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4085 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4086 /* Convert L4 layers with decap action. */ 4087 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4088 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4089 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4090 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4091 return layers; 4092 } 4093 4094 /** 4095 * Get metadata split action information. 4096 * 4097 * @param[in] actions 4098 * Pointer to the list of actions. 4099 * @param[out] qrss 4100 * Pointer to the return pointer. 4101 * @param[out] qrss_type 4102 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4103 * if no QUEUE/RSS is found. 4104 * @param[out] encap_idx 4105 * Pointer to the index of the encap action if exists, otherwise the last 4106 * action index. 4107 * 4108 * @return 4109 * Total number of actions. 4110 */ 4111 static int 4112 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4113 const struct rte_flow_action **qrss, 4114 int *encap_idx) 4115 { 4116 const struct rte_flow_action_raw_encap *raw_encap; 4117 int actions_n = 0; 4118 int raw_decap_idx = -1; 4119 4120 *encap_idx = -1; 4121 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4122 switch (actions->type) { 4123 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4124 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4125 *encap_idx = actions_n; 4126 break; 4127 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4128 raw_decap_idx = actions_n; 4129 break; 4130 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4131 raw_encap = actions->conf; 4132 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4133 *encap_idx = raw_decap_idx != -1 ? 4134 raw_decap_idx : actions_n; 4135 break; 4136 case RTE_FLOW_ACTION_TYPE_QUEUE: 4137 case RTE_FLOW_ACTION_TYPE_RSS: 4138 *qrss = actions; 4139 break; 4140 default: 4141 break; 4142 } 4143 actions_n++; 4144 } 4145 if (*encap_idx == -1) 4146 *encap_idx = actions_n; 4147 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4148 return actions_n + 1; 4149 } 4150 4151 /** 4152 * Check if the action will change packet. 4153 * 4154 * @param dev 4155 * Pointer to Ethernet device. 4156 * @param[in] type 4157 * action type. 4158 * 4159 * @return 4160 * true if action will change packet, false otherwise. 4161 */ 4162 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4163 enum rte_flow_action_type type) 4164 { 4165 struct mlx5_priv *priv = dev->data->dev_private; 4166 4167 switch (type) { 4168 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4169 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4170 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4171 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4172 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4173 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4174 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4175 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4176 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4177 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4178 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4179 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4180 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4181 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4182 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4183 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4184 case RTE_FLOW_ACTION_TYPE_SET_META: 4185 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4186 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4187 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4188 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4189 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4190 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4191 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4192 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4193 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4194 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4195 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4196 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4197 return true; 4198 case RTE_FLOW_ACTION_TYPE_FLAG: 4199 case RTE_FLOW_ACTION_TYPE_MARK: 4200 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4201 return true; 4202 else 4203 return false; 4204 default: 4205 return false; 4206 } 4207 } 4208 4209 /** 4210 * Check meter action from the action list. 4211 * 4212 * @param dev 4213 * Pointer to Ethernet device. 4214 * @param[in] actions 4215 * Pointer to the list of actions. 4216 * @param[out] has_mtr 4217 * Pointer to the meter exist flag. 4218 * @param[out] has_modify 4219 * Pointer to the flag showing there's packet change action. 4220 * @param[out] meter_id 4221 * Pointer to the meter id. 4222 * 4223 * @return 4224 * Total number of actions. 4225 */ 4226 static int 4227 flow_check_meter_action(struct rte_eth_dev *dev, 4228 const struct rte_flow_action actions[], 4229 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4230 { 4231 const struct rte_flow_action_meter *mtr = NULL; 4232 int actions_n = 0; 4233 4234 MLX5_ASSERT(has_mtr); 4235 *has_mtr = false; 4236 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4237 switch (actions->type) { 4238 case RTE_FLOW_ACTION_TYPE_METER: 4239 mtr = actions->conf; 4240 *meter_id = mtr->mtr_id; 4241 *has_mtr = true; 4242 break; 4243 default: 4244 break; 4245 } 4246 if (!*has_mtr) 4247 *has_modify |= flow_check_modify_action_type(dev, 4248 actions->type); 4249 actions_n++; 4250 } 4251 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4252 return actions_n + 1; 4253 } 4254 4255 /** 4256 * Check if the flow should be split due to hairpin. 4257 * The reason for the split is that in current HW we can't 4258 * support encap and push-vlan on Rx, so if a flow contains 4259 * these actions we move it to Tx. 4260 * 4261 * @param dev 4262 * Pointer to Ethernet device. 4263 * @param[in] attr 4264 * Flow rule attributes. 4265 * @param[in] actions 4266 * Associated actions (list terminated by the END action). 4267 * 4268 * @return 4269 * > 0 the number of actions and the flow should be split, 4270 * 0 when no split required. 4271 */ 4272 static int 4273 flow_check_hairpin_split(struct rte_eth_dev *dev, 4274 const struct rte_flow_attr *attr, 4275 const struct rte_flow_action actions[]) 4276 { 4277 int queue_action = 0; 4278 int action_n = 0; 4279 int split = 0; 4280 const struct rte_flow_action_queue *queue; 4281 const struct rte_flow_action_rss *rss; 4282 const struct rte_flow_action_raw_encap *raw_encap; 4283 const struct rte_eth_hairpin_conf *conf; 4284 4285 if (!attr->ingress) 4286 return 0; 4287 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4288 switch (actions->type) { 4289 case RTE_FLOW_ACTION_TYPE_QUEUE: 4290 queue = actions->conf; 4291 if (queue == NULL) 4292 return 0; 4293 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4294 if (conf == NULL || conf->tx_explicit != 0) 4295 return 0; 4296 queue_action = 1; 4297 action_n++; 4298 break; 4299 case RTE_FLOW_ACTION_TYPE_RSS: 4300 rss = actions->conf; 4301 if (rss == NULL || rss->queue_num == 0) 4302 return 0; 4303 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4304 if (conf == NULL || conf->tx_explicit != 0) 4305 return 0; 4306 queue_action = 1; 4307 action_n++; 4308 break; 4309 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4310 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4311 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4312 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4313 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4314 split++; 4315 action_n++; 4316 break; 4317 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4318 raw_encap = actions->conf; 4319 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4320 split++; 4321 action_n++; 4322 break; 4323 default: 4324 action_n++; 4325 break; 4326 } 4327 } 4328 if (split && queue_action) 4329 return action_n; 4330 return 0; 4331 } 4332 4333 /* Declare flow create/destroy prototype in advance. */ 4334 static uint32_t 4335 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4336 const struct rte_flow_attr *attr, 4337 const struct rte_flow_item items[], 4338 const struct rte_flow_action actions[], 4339 bool external, struct rte_flow_error *error); 4340 4341 static void 4342 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4343 uint32_t flow_idx); 4344 4345 int 4346 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4347 struct mlx5_list_entry *entry, void *cb_ctx) 4348 { 4349 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4350 struct mlx5_flow_mreg_copy_resource *mcp_res = 4351 container_of(entry, typeof(*mcp_res), hlist_ent); 4352 4353 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4354 } 4355 4356 struct mlx5_list_entry * 4357 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4358 { 4359 struct rte_eth_dev *dev = tool_ctx; 4360 struct mlx5_priv *priv = dev->data->dev_private; 4361 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4362 struct mlx5_flow_mreg_copy_resource *mcp_res; 4363 struct rte_flow_error *error = ctx->error; 4364 uint32_t idx = 0; 4365 int ret; 4366 uint32_t mark_id = *(uint32_t *)(ctx->data); 4367 struct rte_flow_attr attr = { 4368 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4369 .ingress = 1, 4370 }; 4371 struct mlx5_rte_flow_item_tag tag_spec = { 4372 .data = mark_id, 4373 }; 4374 struct rte_flow_item items[] = { 4375 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4376 }; 4377 struct rte_flow_action_mark ftag = { 4378 .id = mark_id, 4379 }; 4380 struct mlx5_flow_action_copy_mreg cp_mreg = { 4381 .dst = REG_B, 4382 .src = REG_NON, 4383 }; 4384 struct rte_flow_action_jump jump = { 4385 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4386 }; 4387 struct rte_flow_action actions[] = { 4388 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4389 }; 4390 4391 /* Fill the register fields in the flow. */ 4392 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4393 if (ret < 0) 4394 return NULL; 4395 tag_spec.id = ret; 4396 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4397 if (ret < 0) 4398 return NULL; 4399 cp_mreg.src = ret; 4400 /* Provide the full width of FLAG specific value. */ 4401 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4402 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4403 /* Build a new flow. */ 4404 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4405 items[0] = (struct rte_flow_item){ 4406 .type = (enum rte_flow_item_type) 4407 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4408 .spec = &tag_spec, 4409 }; 4410 items[1] = (struct rte_flow_item){ 4411 .type = RTE_FLOW_ITEM_TYPE_END, 4412 }; 4413 actions[0] = (struct rte_flow_action){ 4414 .type = (enum rte_flow_action_type) 4415 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4416 .conf = &ftag, 4417 }; 4418 actions[1] = (struct rte_flow_action){ 4419 .type = (enum rte_flow_action_type) 4420 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4421 .conf = &cp_mreg, 4422 }; 4423 actions[2] = (struct rte_flow_action){ 4424 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4425 .conf = &jump, 4426 }; 4427 actions[3] = (struct rte_flow_action){ 4428 .type = RTE_FLOW_ACTION_TYPE_END, 4429 }; 4430 } else { 4431 /* Default rule, wildcard match. */ 4432 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4433 items[0] = (struct rte_flow_item){ 4434 .type = RTE_FLOW_ITEM_TYPE_END, 4435 }; 4436 actions[0] = (struct rte_flow_action){ 4437 .type = (enum rte_flow_action_type) 4438 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4439 .conf = &cp_mreg, 4440 }; 4441 actions[1] = (struct rte_flow_action){ 4442 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4443 .conf = &jump, 4444 }; 4445 actions[2] = (struct rte_flow_action){ 4446 .type = RTE_FLOW_ACTION_TYPE_END, 4447 }; 4448 } 4449 /* Build a new entry. */ 4450 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4451 if (!mcp_res) { 4452 rte_errno = ENOMEM; 4453 return NULL; 4454 } 4455 mcp_res->idx = idx; 4456 mcp_res->mark_id = mark_id; 4457 /* 4458 * The copy Flows are not included in any list. There 4459 * ones are referenced from other Flows and can not 4460 * be applied, removed, deleted in arbitrary order 4461 * by list traversing. 4462 */ 4463 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4464 &attr, items, actions, false, error); 4465 if (!mcp_res->rix_flow) { 4466 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4467 return NULL; 4468 } 4469 return &mcp_res->hlist_ent; 4470 } 4471 4472 struct mlx5_list_entry * 4473 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4474 void *cb_ctx __rte_unused) 4475 { 4476 struct rte_eth_dev *dev = tool_ctx; 4477 struct mlx5_priv *priv = dev->data->dev_private; 4478 struct mlx5_flow_mreg_copy_resource *mcp_res; 4479 uint32_t idx = 0; 4480 4481 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4482 if (!mcp_res) { 4483 rte_errno = ENOMEM; 4484 return NULL; 4485 } 4486 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4487 mcp_res->idx = idx; 4488 return &mcp_res->hlist_ent; 4489 } 4490 4491 void 4492 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4493 { 4494 struct mlx5_flow_mreg_copy_resource *mcp_res = 4495 container_of(entry, typeof(*mcp_res), hlist_ent); 4496 struct rte_eth_dev *dev = tool_ctx; 4497 struct mlx5_priv *priv = dev->data->dev_private; 4498 4499 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4500 } 4501 4502 /** 4503 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4504 * 4505 * As mark_id is unique, if there's already a registered flow for the mark_id, 4506 * return by increasing the reference counter of the resource. Otherwise, create 4507 * the resource (mcp_res) and flow. 4508 * 4509 * Flow looks like, 4510 * - If ingress port is ANY and reg_c[1] is mark_id, 4511 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4512 * 4513 * For default flow (zero mark_id), flow is like, 4514 * - If ingress port is ANY, 4515 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4516 * 4517 * @param dev 4518 * Pointer to Ethernet device. 4519 * @param mark_id 4520 * ID of MARK action, zero means default flow for META. 4521 * @param[out] error 4522 * Perform verbose error reporting if not NULL. 4523 * 4524 * @return 4525 * Associated resource on success, NULL otherwise and rte_errno is set. 4526 */ 4527 static struct mlx5_flow_mreg_copy_resource * 4528 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4529 struct rte_flow_error *error) 4530 { 4531 struct mlx5_priv *priv = dev->data->dev_private; 4532 struct mlx5_list_entry *entry; 4533 struct mlx5_flow_cb_ctx ctx = { 4534 .dev = dev, 4535 .error = error, 4536 .data = &mark_id, 4537 }; 4538 4539 /* Check if already registered. */ 4540 MLX5_ASSERT(priv->mreg_cp_tbl); 4541 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4542 if (!entry) 4543 return NULL; 4544 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4545 hlist_ent); 4546 } 4547 4548 void 4549 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4550 { 4551 struct mlx5_flow_mreg_copy_resource *mcp_res = 4552 container_of(entry, typeof(*mcp_res), hlist_ent); 4553 struct rte_eth_dev *dev = tool_ctx; 4554 struct mlx5_priv *priv = dev->data->dev_private; 4555 4556 MLX5_ASSERT(mcp_res->rix_flow); 4557 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4558 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4559 } 4560 4561 /** 4562 * Release flow in RX_CP_TBL. 4563 * 4564 * @param dev 4565 * Pointer to Ethernet device. 4566 * @flow 4567 * Parent flow for wich copying is provided. 4568 */ 4569 static void 4570 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4571 struct rte_flow *flow) 4572 { 4573 struct mlx5_flow_mreg_copy_resource *mcp_res; 4574 struct mlx5_priv *priv = dev->data->dev_private; 4575 4576 if (!flow->rix_mreg_copy) 4577 return; 4578 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4579 flow->rix_mreg_copy); 4580 if (!mcp_res || !priv->mreg_cp_tbl) 4581 return; 4582 MLX5_ASSERT(mcp_res->rix_flow); 4583 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4584 flow->rix_mreg_copy = 0; 4585 } 4586 4587 /** 4588 * Remove the default copy action from RX_CP_TBL. 4589 * 4590 * This functions is called in the mlx5_dev_start(). No thread safe 4591 * is guaranteed. 4592 * 4593 * @param dev 4594 * Pointer to Ethernet device. 4595 */ 4596 static void 4597 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4598 { 4599 struct mlx5_list_entry *entry; 4600 struct mlx5_priv *priv = dev->data->dev_private; 4601 struct mlx5_flow_cb_ctx ctx; 4602 uint32_t mark_id; 4603 4604 /* Check if default flow is registered. */ 4605 if (!priv->mreg_cp_tbl) 4606 return; 4607 mark_id = MLX5_DEFAULT_COPY_ID; 4608 ctx.data = &mark_id; 4609 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4610 if (!entry) 4611 return; 4612 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4613 } 4614 4615 /** 4616 * Add the default copy action in in RX_CP_TBL. 4617 * 4618 * This functions is called in the mlx5_dev_start(). No thread safe 4619 * is guaranteed. 4620 * 4621 * @param dev 4622 * Pointer to Ethernet device. 4623 * @param[out] error 4624 * Perform verbose error reporting if not NULL. 4625 * 4626 * @return 4627 * 0 for success, negative value otherwise and rte_errno is set. 4628 */ 4629 static int 4630 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4631 struct rte_flow_error *error) 4632 { 4633 struct mlx5_priv *priv = dev->data->dev_private; 4634 struct mlx5_flow_mreg_copy_resource *mcp_res; 4635 struct mlx5_flow_cb_ctx ctx; 4636 uint32_t mark_id; 4637 4638 /* Check whether extensive metadata feature is engaged. */ 4639 if (!priv->sh->config.dv_flow_en || 4640 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4641 !mlx5_flow_ext_mreg_supported(dev) || 4642 !priv->sh->dv_regc0_mask) 4643 return 0; 4644 /* 4645 * Add default mreg copy flow may be called multiple time, but 4646 * only be called once in stop. Avoid register it twice. 4647 */ 4648 mark_id = MLX5_DEFAULT_COPY_ID; 4649 ctx.data = &mark_id; 4650 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4651 return 0; 4652 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4653 if (!mcp_res) 4654 return -rte_errno; 4655 return 0; 4656 } 4657 4658 /** 4659 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4660 * 4661 * All the flow having Q/RSS action should be split by 4662 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4663 * performs the following, 4664 * - CQE->flow_tag := reg_c[1] (MARK) 4665 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4666 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4667 * but there should be a flow per each MARK ID set by MARK action. 4668 * 4669 * For the aforementioned reason, if there's a MARK action in flow's action 4670 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4671 * the MARK ID to CQE's flow_tag like, 4672 * - If reg_c[1] is mark_id, 4673 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4674 * 4675 * For SET_META action which stores value in reg_c[0], as the destination is 4676 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4677 * MARK ID means the default flow. The default flow looks like, 4678 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4679 * 4680 * @param dev 4681 * Pointer to Ethernet device. 4682 * @param flow 4683 * Pointer to flow structure. 4684 * @param[in] actions 4685 * Pointer to the list of actions. 4686 * @param[out] error 4687 * Perform verbose error reporting if not NULL. 4688 * 4689 * @return 4690 * 0 on success, negative value otherwise and rte_errno is set. 4691 */ 4692 static int 4693 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4694 struct rte_flow *flow, 4695 const struct rte_flow_action *actions, 4696 struct rte_flow_error *error) 4697 { 4698 struct mlx5_priv *priv = dev->data->dev_private; 4699 struct mlx5_sh_config *config = &priv->sh->config; 4700 struct mlx5_flow_mreg_copy_resource *mcp_res; 4701 const struct rte_flow_action_mark *mark; 4702 4703 /* Check whether extensive metadata feature is engaged. */ 4704 if (!config->dv_flow_en || 4705 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4706 !mlx5_flow_ext_mreg_supported(dev) || 4707 !priv->sh->dv_regc0_mask) 4708 return 0; 4709 /* Find MARK action. */ 4710 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4711 switch (actions->type) { 4712 case RTE_FLOW_ACTION_TYPE_FLAG: 4713 mcp_res = flow_mreg_add_copy_action 4714 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4715 if (!mcp_res) 4716 return -rte_errno; 4717 flow->rix_mreg_copy = mcp_res->idx; 4718 return 0; 4719 case RTE_FLOW_ACTION_TYPE_MARK: 4720 mark = (const struct rte_flow_action_mark *) 4721 actions->conf; 4722 mcp_res = 4723 flow_mreg_add_copy_action(dev, mark->id, error); 4724 if (!mcp_res) 4725 return -rte_errno; 4726 flow->rix_mreg_copy = mcp_res->idx; 4727 return 0; 4728 default: 4729 break; 4730 } 4731 } 4732 return 0; 4733 } 4734 4735 #define MLX5_MAX_SPLIT_ACTIONS 24 4736 #define MLX5_MAX_SPLIT_ITEMS 24 4737 4738 /** 4739 * Split the hairpin flow. 4740 * Since HW can't support encap and push-vlan on Rx, we move these 4741 * actions to Tx. 4742 * If the count action is after the encap then we also 4743 * move the count action. in this case the count will also measure 4744 * the outer bytes. 4745 * 4746 * @param dev 4747 * Pointer to Ethernet device. 4748 * @param[in] actions 4749 * Associated actions (list terminated by the END action). 4750 * @param[out] actions_rx 4751 * Rx flow actions. 4752 * @param[out] actions_tx 4753 * Tx flow actions.. 4754 * @param[out] pattern_tx 4755 * The pattern items for the Tx flow. 4756 * @param[out] flow_id 4757 * The flow ID connected to this flow. 4758 * 4759 * @return 4760 * 0 on success. 4761 */ 4762 static int 4763 flow_hairpin_split(struct rte_eth_dev *dev, 4764 const struct rte_flow_action actions[], 4765 struct rte_flow_action actions_rx[], 4766 struct rte_flow_action actions_tx[], 4767 struct rte_flow_item pattern_tx[], 4768 uint32_t flow_id) 4769 { 4770 const struct rte_flow_action_raw_encap *raw_encap; 4771 const struct rte_flow_action_raw_decap *raw_decap; 4772 struct mlx5_rte_flow_action_set_tag *set_tag; 4773 struct rte_flow_action *tag_action; 4774 struct mlx5_rte_flow_item_tag *tag_item; 4775 struct rte_flow_item *item; 4776 char *addr; 4777 int encap = 0; 4778 4779 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4780 switch (actions->type) { 4781 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4782 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4783 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4784 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4785 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4786 rte_memcpy(actions_tx, actions, 4787 sizeof(struct rte_flow_action)); 4788 actions_tx++; 4789 break; 4790 case RTE_FLOW_ACTION_TYPE_COUNT: 4791 if (encap) { 4792 rte_memcpy(actions_tx, actions, 4793 sizeof(struct rte_flow_action)); 4794 actions_tx++; 4795 } else { 4796 rte_memcpy(actions_rx, actions, 4797 sizeof(struct rte_flow_action)); 4798 actions_rx++; 4799 } 4800 break; 4801 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4802 raw_encap = actions->conf; 4803 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4804 memcpy(actions_tx, actions, 4805 sizeof(struct rte_flow_action)); 4806 actions_tx++; 4807 encap = 1; 4808 } else { 4809 rte_memcpy(actions_rx, actions, 4810 sizeof(struct rte_flow_action)); 4811 actions_rx++; 4812 } 4813 break; 4814 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4815 raw_decap = actions->conf; 4816 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4817 memcpy(actions_tx, actions, 4818 sizeof(struct rte_flow_action)); 4819 actions_tx++; 4820 } else { 4821 rte_memcpy(actions_rx, actions, 4822 sizeof(struct rte_flow_action)); 4823 actions_rx++; 4824 } 4825 break; 4826 default: 4827 rte_memcpy(actions_rx, actions, 4828 sizeof(struct rte_flow_action)); 4829 actions_rx++; 4830 break; 4831 } 4832 } 4833 /* Add set meta action and end action for the Rx flow. */ 4834 tag_action = actions_rx; 4835 tag_action->type = (enum rte_flow_action_type) 4836 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4837 actions_rx++; 4838 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4839 actions_rx++; 4840 set_tag = (void *)actions_rx; 4841 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4842 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4843 .data = flow_id, 4844 }; 4845 MLX5_ASSERT(set_tag->id > REG_NON); 4846 tag_action->conf = set_tag; 4847 /* Create Tx item list. */ 4848 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4849 addr = (void *)&pattern_tx[2]; 4850 item = pattern_tx; 4851 item->type = (enum rte_flow_item_type) 4852 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4853 tag_item = (void *)addr; 4854 tag_item->data = flow_id; 4855 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4856 MLX5_ASSERT(set_tag->id > REG_NON); 4857 item->spec = tag_item; 4858 addr += sizeof(struct mlx5_rte_flow_item_tag); 4859 tag_item = (void *)addr; 4860 tag_item->data = UINT32_MAX; 4861 tag_item->id = UINT16_MAX; 4862 item->mask = tag_item; 4863 item->last = NULL; 4864 item++; 4865 item->type = RTE_FLOW_ITEM_TYPE_END; 4866 return 0; 4867 } 4868 4869 /** 4870 * The last stage of splitting chain, just creates the subflow 4871 * without any modification. 4872 * 4873 * @param[in] dev 4874 * Pointer to Ethernet device. 4875 * @param[in] flow 4876 * Parent flow structure pointer. 4877 * @param[in, out] sub_flow 4878 * Pointer to return the created subflow, may be NULL. 4879 * @param[in] attr 4880 * Flow rule attributes. 4881 * @param[in] items 4882 * Pattern specification (list terminated by the END pattern item). 4883 * @param[in] actions 4884 * Associated actions (list terminated by the END action). 4885 * @param[in] flow_split_info 4886 * Pointer to flow split info structure. 4887 * @param[out] error 4888 * Perform verbose error reporting if not NULL. 4889 * @return 4890 * 0 on success, negative value otherwise 4891 */ 4892 static int 4893 flow_create_split_inner(struct rte_eth_dev *dev, 4894 struct rte_flow *flow, 4895 struct mlx5_flow **sub_flow, 4896 const struct rte_flow_attr *attr, 4897 const struct rte_flow_item items[], 4898 const struct rte_flow_action actions[], 4899 struct mlx5_flow_split_info *flow_split_info, 4900 struct rte_flow_error *error) 4901 { 4902 struct mlx5_flow *dev_flow; 4903 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 4904 4905 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4906 flow_split_info->flow_idx, error); 4907 if (!dev_flow) 4908 return -rte_errno; 4909 dev_flow->flow = flow; 4910 dev_flow->external = flow_split_info->external; 4911 dev_flow->skip_scale = flow_split_info->skip_scale; 4912 /* Subflow object was created, we must include one in the list. */ 4913 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4914 dev_flow->handle, next); 4915 /* 4916 * If dev_flow is as one of the suffix flow, some actions in suffix 4917 * flow may need some user defined item layer flags, and pass the 4918 * Metadata rxq mark flag to suffix flow as well. 4919 */ 4920 if (flow_split_info->prefix_layers) 4921 dev_flow->handle->layers = flow_split_info->prefix_layers; 4922 if (flow_split_info->prefix_mark) { 4923 MLX5_ASSERT(wks); 4924 wks->mark = 1; 4925 } 4926 if (sub_flow) 4927 *sub_flow = dev_flow; 4928 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4929 dev_flow->dv.table_id = flow_split_info->table_id; 4930 #endif 4931 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4932 } 4933 4934 /** 4935 * Get the sub policy of a meter. 4936 * 4937 * @param[in] dev 4938 * Pointer to Ethernet device. 4939 * @param[in] flow 4940 * Parent flow structure pointer. 4941 * @param wks 4942 * Pointer to thread flow work space. 4943 * @param[in] attr 4944 * Flow rule attributes. 4945 * @param[in] items 4946 * Pattern specification (list terminated by the END pattern item). 4947 * @param[out] error 4948 * Perform verbose error reporting if not NULL. 4949 * 4950 * @return 4951 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4952 */ 4953 static struct mlx5_flow_meter_sub_policy * 4954 get_meter_sub_policy(struct rte_eth_dev *dev, 4955 struct rte_flow *flow, 4956 struct mlx5_flow_workspace *wks, 4957 const struct rte_flow_attr *attr, 4958 const struct rte_flow_item items[], 4959 struct rte_flow_error *error) 4960 { 4961 struct mlx5_flow_meter_policy *policy; 4962 struct mlx5_flow_meter_policy *final_policy; 4963 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4964 4965 policy = wks->policy; 4966 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4967 if (final_policy->is_rss || final_policy->is_queue) { 4968 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4969 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4970 uint32_t i; 4971 4972 /* 4973 * This is a tmp dev_flow, 4974 * no need to register any matcher for it in translate. 4975 */ 4976 wks->skip_matcher_reg = 1; 4977 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4978 struct mlx5_flow dev_flow = {0}; 4979 struct mlx5_flow_handle dev_handle = { {0} }; 4980 uint8_t fate = final_policy->act_cnt[i].fate_action; 4981 4982 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4983 const struct rte_flow_action_rss *rss_act = 4984 final_policy->act_cnt[i].rss->conf; 4985 struct rte_flow_action rss_actions[2] = { 4986 [0] = { 4987 .type = RTE_FLOW_ACTION_TYPE_RSS, 4988 .conf = rss_act, 4989 }, 4990 [1] = { 4991 .type = RTE_FLOW_ACTION_TYPE_END, 4992 .conf = NULL, 4993 } 4994 }; 4995 4996 dev_flow.handle = &dev_handle; 4997 dev_flow.ingress = attr->ingress; 4998 dev_flow.flow = flow; 4999 dev_flow.external = 0; 5000 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5001 dev_flow.dv.transfer = attr->transfer; 5002 #endif 5003 /** 5004 * Translate RSS action to get rss hash fields. 5005 */ 5006 if (flow_drv_translate(dev, &dev_flow, attr, 5007 items, rss_actions, error)) 5008 goto exit; 5009 rss_desc_v[i] = wks->rss_desc; 5010 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5011 rss_desc_v[i].hash_fields = 5012 dev_flow.hash_fields; 5013 rss_desc_v[i].queue_num = 5014 rss_desc_v[i].hash_fields ? 5015 rss_desc_v[i].queue_num : 1; 5016 rss_desc_v[i].tunnel = 5017 !!(dev_flow.handle->layers & 5018 MLX5_FLOW_LAYER_TUNNEL); 5019 /* Use the RSS queues in the containers. */ 5020 rss_desc_v[i].queue = 5021 (uint16_t *)(uintptr_t)rss_act->queue; 5022 rss_desc[i] = &rss_desc_v[i]; 5023 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5024 /* This is queue action. */ 5025 rss_desc_v[i] = wks->rss_desc; 5026 rss_desc_v[i].key_len = 0; 5027 rss_desc_v[i].hash_fields = 0; 5028 rss_desc_v[i].queue = 5029 &final_policy->act_cnt[i].queue; 5030 rss_desc_v[i].queue_num = 1; 5031 rss_desc[i] = &rss_desc_v[i]; 5032 } else { 5033 rss_desc[i] = NULL; 5034 } 5035 } 5036 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5037 flow, policy, rss_desc); 5038 } else { 5039 enum mlx5_meter_domain mtr_domain = 5040 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5041 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5042 MLX5_MTR_DOMAIN_INGRESS); 5043 sub_policy = policy->sub_policys[mtr_domain][0]; 5044 } 5045 if (!sub_policy) 5046 rte_flow_error_set(error, EINVAL, 5047 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5048 "Failed to get meter sub-policy."); 5049 exit: 5050 return sub_policy; 5051 } 5052 5053 /** 5054 * Split the meter flow. 5055 * 5056 * As meter flow will split to three sub flow, other than meter 5057 * action, the other actions make sense to only meter accepts 5058 * the packet. If it need to be dropped, no other additional 5059 * actions should be take. 5060 * 5061 * One kind of special action which decapsulates the L3 tunnel 5062 * header will be in the prefix sub flow, as not to take the 5063 * L3 tunnel header into account. 5064 * 5065 * @param[in] dev 5066 * Pointer to Ethernet device. 5067 * @param[in] flow 5068 * Parent flow structure pointer. 5069 * @param wks 5070 * Pointer to thread flow work space. 5071 * @param[in] attr 5072 * Flow rule attributes. 5073 * @param[in] items 5074 * Pattern specification (list terminated by the END pattern item). 5075 * @param[out] sfx_items 5076 * Suffix flow match items (list terminated by the END pattern item). 5077 * @param[in] actions 5078 * Associated actions (list terminated by the END action). 5079 * @param[out] actions_sfx 5080 * Suffix flow actions. 5081 * @param[out] actions_pre 5082 * Prefix flow actions. 5083 * @param[out] mtr_flow_id 5084 * Pointer to meter flow id. 5085 * @param[out] error 5086 * Perform verbose error reporting if not NULL. 5087 * 5088 * @return 5089 * 0 on success, a negative errno value otherwise and rte_errno is set. 5090 */ 5091 static int 5092 flow_meter_split_prep(struct rte_eth_dev *dev, 5093 struct rte_flow *flow, 5094 struct mlx5_flow_workspace *wks, 5095 const struct rte_flow_attr *attr, 5096 const struct rte_flow_item items[], 5097 struct rte_flow_item sfx_items[], 5098 const struct rte_flow_action actions[], 5099 struct rte_flow_action actions_sfx[], 5100 struct rte_flow_action actions_pre[], 5101 uint32_t *mtr_flow_id, 5102 struct rte_flow_error *error) 5103 { 5104 struct mlx5_priv *priv = dev->data->dev_private; 5105 struct mlx5_flow_meter_info *fm = wks->fm; 5106 struct rte_flow_action *tag_action = NULL; 5107 struct rte_flow_item *tag_item; 5108 struct mlx5_rte_flow_action_set_tag *set_tag; 5109 const struct rte_flow_action_raw_encap *raw_encap; 5110 const struct rte_flow_action_raw_decap *raw_decap; 5111 struct mlx5_rte_flow_item_tag *tag_item_spec; 5112 struct mlx5_rte_flow_item_tag *tag_item_mask; 5113 uint32_t tag_id = 0; 5114 struct rte_flow_item *vlan_item_dst = NULL; 5115 const struct rte_flow_item *vlan_item_src = NULL; 5116 const struct rte_flow_item *orig_items = items; 5117 struct rte_flow_action *hw_mtr_action; 5118 struct rte_flow_action *action_pre_head = NULL; 5119 int32_t flow_src_port = priv->representor_id; 5120 bool mtr_first; 5121 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5122 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5123 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5124 uint32_t flow_id = 0; 5125 uint32_t flow_id_reversed = 0; 5126 uint8_t flow_id_bits = 0; 5127 int shift; 5128 5129 /* Prepare the suffix subflow items. */ 5130 tag_item = sfx_items++; 5131 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5132 struct mlx5_priv *port_priv; 5133 const struct rte_flow_item_port_id *pid_v; 5134 int item_type = items->type; 5135 5136 switch (item_type) { 5137 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5138 pid_v = items->spec; 5139 MLX5_ASSERT(pid_v); 5140 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5141 if (!port_priv) 5142 return rte_flow_error_set(error, 5143 rte_errno, 5144 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5145 pid_v, 5146 "Failed to get port info."); 5147 flow_src_port = port_priv->representor_id; 5148 if (!fm->def_policy && wks->policy->is_hierarchy && 5149 flow_src_port != priv->representor_id) { 5150 if (flow_drv_mtr_hierarchy_rule_create(dev, 5151 flow, fm, 5152 flow_src_port, 5153 items, 5154 error)) 5155 return -rte_errno; 5156 } 5157 memcpy(sfx_items, items, sizeof(*sfx_items)); 5158 sfx_items++; 5159 break; 5160 case RTE_FLOW_ITEM_TYPE_VLAN: 5161 /* Determine if copy vlan item below. */ 5162 vlan_item_src = items; 5163 vlan_item_dst = sfx_items++; 5164 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5165 break; 5166 default: 5167 break; 5168 } 5169 } 5170 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5171 sfx_items++; 5172 mtr_first = priv->sh->meter_aso_en && 5173 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5174 /* For ASO meter, meter must be before tag in TX direction. */ 5175 if (mtr_first) { 5176 action_pre_head = actions_pre++; 5177 /* Leave space for tag action. */ 5178 tag_action = actions_pre++; 5179 } 5180 /* Prepare the actions for prefix and suffix flow. */ 5181 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5182 struct rte_flow_action *action_cur = NULL; 5183 5184 switch (actions->type) { 5185 case RTE_FLOW_ACTION_TYPE_METER: 5186 if (mtr_first) { 5187 action_cur = action_pre_head; 5188 } else { 5189 /* Leave space for tag action. */ 5190 tag_action = actions_pre++; 5191 action_cur = actions_pre++; 5192 } 5193 break; 5194 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5195 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5196 action_cur = actions_pre++; 5197 break; 5198 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5199 raw_encap = actions->conf; 5200 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5201 action_cur = actions_pre++; 5202 break; 5203 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5204 raw_decap = actions->conf; 5205 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5206 action_cur = actions_pre++; 5207 break; 5208 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5209 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5210 if (vlan_item_dst && vlan_item_src) { 5211 memcpy(vlan_item_dst, vlan_item_src, 5212 sizeof(*vlan_item_dst)); 5213 /* 5214 * Convert to internal match item, it is used 5215 * for vlan push and set vid. 5216 */ 5217 vlan_item_dst->type = (enum rte_flow_item_type) 5218 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5219 } 5220 break; 5221 default: 5222 break; 5223 } 5224 if (!action_cur) 5225 action_cur = (fm->def_policy) ? 5226 actions_sfx++ : actions_pre++; 5227 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5228 } 5229 /* Add end action to the actions. */ 5230 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5231 if (priv->sh->meter_aso_en) { 5232 /** 5233 * For ASO meter, need to add an extra jump action explicitly, 5234 * to jump from meter to policer table. 5235 */ 5236 struct mlx5_flow_meter_sub_policy *sub_policy; 5237 struct mlx5_flow_tbl_data_entry *tbl_data; 5238 5239 if (!fm->def_policy) { 5240 sub_policy = get_meter_sub_policy(dev, flow, wks, 5241 attr, orig_items, 5242 error); 5243 if (!sub_policy) 5244 return -rte_errno; 5245 } else { 5246 enum mlx5_meter_domain mtr_domain = 5247 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5248 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5249 MLX5_MTR_DOMAIN_INGRESS); 5250 5251 sub_policy = 5252 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5253 } 5254 tbl_data = container_of(sub_policy->tbl_rsc, 5255 struct mlx5_flow_tbl_data_entry, tbl); 5256 hw_mtr_action = actions_pre++; 5257 hw_mtr_action->type = (enum rte_flow_action_type) 5258 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5259 hw_mtr_action->conf = tbl_data->jump.action; 5260 } 5261 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5262 actions_pre++; 5263 if (!tag_action) 5264 return rte_flow_error_set(error, ENOMEM, 5265 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5266 NULL, "No tag action space."); 5267 if (!mtr_flow_id) { 5268 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5269 goto exit; 5270 } 5271 /* Only default-policy Meter creates mtr flow id. */ 5272 if (fm->def_policy) { 5273 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5274 if (!tag_id) 5275 return rte_flow_error_set(error, ENOMEM, 5276 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5277 "Failed to allocate meter flow id."); 5278 flow_id = tag_id - 1; 5279 flow_id_bits = (!flow_id) ? 1 : 5280 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5281 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5282 mtr_reg_bits) { 5283 mlx5_ipool_free(fm->flow_ipool, tag_id); 5284 return rte_flow_error_set(error, EINVAL, 5285 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5286 "Meter flow id exceeds max limit."); 5287 } 5288 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5289 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5290 } 5291 /* Build tag actions and items for meter_id/meter flow_id. */ 5292 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5293 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5294 tag_item_mask = tag_item_spec + 1; 5295 /* Both flow_id and meter_id share the same register. */ 5296 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5297 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5298 0, error), 5299 .offset = mtr_id_offset, 5300 .length = mtr_reg_bits, 5301 .data = flow->meter, 5302 }; 5303 /* 5304 * The color Reg bits used by flow_id are growing from 5305 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5306 */ 5307 for (shift = 0; shift < flow_id_bits; shift++) 5308 flow_id_reversed = (flow_id_reversed << 1) | 5309 ((flow_id >> shift) & 0x1); 5310 set_tag->data |= 5311 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5312 tag_item_spec->id = set_tag->id; 5313 tag_item_spec->data = set_tag->data << mtr_id_offset; 5314 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5315 tag_action->type = (enum rte_flow_action_type) 5316 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5317 tag_action->conf = set_tag; 5318 tag_item->type = (enum rte_flow_item_type) 5319 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5320 tag_item->spec = tag_item_spec; 5321 tag_item->last = NULL; 5322 tag_item->mask = tag_item_mask; 5323 exit: 5324 if (mtr_flow_id) 5325 *mtr_flow_id = tag_id; 5326 return 0; 5327 } 5328 5329 /** 5330 * Split action list having QUEUE/RSS for metadata register copy. 5331 * 5332 * Once Q/RSS action is detected in user's action list, the flow action 5333 * should be split in order to copy metadata registers, which will happen in 5334 * RX_CP_TBL like, 5335 * - CQE->flow_tag := reg_c[1] (MARK) 5336 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5337 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5338 * This is because the last action of each flow must be a terminal action 5339 * (QUEUE, RSS or DROP). 5340 * 5341 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5342 * stored and kept in the mlx5_flow structure per each sub_flow. 5343 * 5344 * The Q/RSS action is replaced with, 5345 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5346 * And the following JUMP action is added at the end, 5347 * - JUMP, to RX_CP_TBL. 5348 * 5349 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5350 * flow_create_split_metadata() routine. The flow will look like, 5351 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5352 * 5353 * @param dev 5354 * Pointer to Ethernet device. 5355 * @param[out] split_actions 5356 * Pointer to store split actions to jump to CP_TBL. 5357 * @param[in] actions 5358 * Pointer to the list of original flow actions. 5359 * @param[in] qrss 5360 * Pointer to the Q/RSS action. 5361 * @param[in] actions_n 5362 * Number of original actions. 5363 * @param[in] mtr_sfx 5364 * Check if it is in meter suffix table. 5365 * @param[out] error 5366 * Perform verbose error reporting if not NULL. 5367 * 5368 * @return 5369 * non-zero unique flow_id on success, otherwise 0 and 5370 * error/rte_error are set. 5371 */ 5372 static uint32_t 5373 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5374 struct rte_flow_action *split_actions, 5375 const struct rte_flow_action *actions, 5376 const struct rte_flow_action *qrss, 5377 int actions_n, int mtr_sfx, 5378 struct rte_flow_error *error) 5379 { 5380 struct mlx5_priv *priv = dev->data->dev_private; 5381 struct mlx5_rte_flow_action_set_tag *set_tag; 5382 struct rte_flow_action_jump *jump; 5383 const int qrss_idx = qrss - actions; 5384 uint32_t flow_id = 0; 5385 int ret = 0; 5386 5387 /* 5388 * Given actions will be split 5389 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5390 * - Add jump to mreg CP_TBL. 5391 * As a result, there will be one more action. 5392 */ 5393 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5394 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5395 ++actions_n; 5396 set_tag = (void *)(split_actions + actions_n); 5397 /* 5398 * If we are not the meter suffix flow, add the tag action. 5399 * Since meter suffix flow already has the tag added. 5400 */ 5401 if (!mtr_sfx) { 5402 /* 5403 * Allocate the new subflow ID. This one is unique within 5404 * device and not shared with representors. Otherwise, 5405 * we would have to resolve multi-thread access synch 5406 * issue. Each flow on the shared device is appended 5407 * with source vport identifier, so the resulting 5408 * flows will be unique in the shared (by master and 5409 * representors) domain even if they have coinciding 5410 * IDs. 5411 */ 5412 mlx5_ipool_malloc(priv->sh->ipool 5413 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5414 if (!flow_id) 5415 return rte_flow_error_set(error, ENOMEM, 5416 RTE_FLOW_ERROR_TYPE_ACTION, 5417 NULL, "can't allocate id " 5418 "for split Q/RSS subflow"); 5419 /* Internal SET_TAG action to set flow ID. */ 5420 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5421 .data = flow_id, 5422 }; 5423 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5424 if (ret < 0) 5425 return ret; 5426 set_tag->id = ret; 5427 /* Construct new actions array. */ 5428 /* Replace QUEUE/RSS action. */ 5429 split_actions[qrss_idx] = (struct rte_flow_action){ 5430 .type = (enum rte_flow_action_type) 5431 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5432 .conf = set_tag, 5433 }; 5434 } else { 5435 /* 5436 * If we are the suffix flow of meter, tag already exist. 5437 * Set the QUEUE/RSS action to void. 5438 */ 5439 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5440 } 5441 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5442 jump = (void *)(set_tag + 1); 5443 *jump = (struct rte_flow_action_jump){ 5444 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5445 }; 5446 split_actions[actions_n - 2] = (struct rte_flow_action){ 5447 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5448 .conf = jump, 5449 }; 5450 split_actions[actions_n - 1] = (struct rte_flow_action){ 5451 .type = RTE_FLOW_ACTION_TYPE_END, 5452 }; 5453 return flow_id; 5454 } 5455 5456 /** 5457 * Extend the given action list for Tx metadata copy. 5458 * 5459 * Copy the given action list to the ext_actions and add flow metadata register 5460 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5461 * 5462 * @param[out] ext_actions 5463 * Pointer to the extended action list. 5464 * @param[in] actions 5465 * Pointer to the list of actions. 5466 * @param[in] actions_n 5467 * Number of actions in the list. 5468 * @param[out] error 5469 * Perform verbose error reporting if not NULL. 5470 * @param[in] encap_idx 5471 * The encap action index. 5472 * 5473 * @return 5474 * 0 on success, negative value otherwise 5475 */ 5476 static int 5477 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5478 struct rte_flow_action *ext_actions, 5479 const struct rte_flow_action *actions, 5480 int actions_n, struct rte_flow_error *error, 5481 int encap_idx) 5482 { 5483 struct mlx5_flow_action_copy_mreg *cp_mreg = 5484 (struct mlx5_flow_action_copy_mreg *) 5485 (ext_actions + actions_n + 1); 5486 int ret; 5487 5488 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5489 if (ret < 0) 5490 return ret; 5491 cp_mreg->dst = ret; 5492 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5493 if (ret < 0) 5494 return ret; 5495 cp_mreg->src = ret; 5496 if (encap_idx != 0) 5497 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5498 if (encap_idx == actions_n - 1) { 5499 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5500 .type = (enum rte_flow_action_type) 5501 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5502 .conf = cp_mreg, 5503 }; 5504 ext_actions[actions_n] = (struct rte_flow_action){ 5505 .type = RTE_FLOW_ACTION_TYPE_END, 5506 }; 5507 } else { 5508 ext_actions[encap_idx] = (struct rte_flow_action){ 5509 .type = (enum rte_flow_action_type) 5510 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5511 .conf = cp_mreg, 5512 }; 5513 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5514 sizeof(*ext_actions) * (actions_n - encap_idx)); 5515 } 5516 return 0; 5517 } 5518 5519 /** 5520 * Check the match action from the action list. 5521 * 5522 * @param[in] actions 5523 * Pointer to the list of actions. 5524 * @param[in] attr 5525 * Flow rule attributes. 5526 * @param[in] action 5527 * The action to be check if exist. 5528 * @param[out] match_action_pos 5529 * Pointer to the position of the matched action if exists, otherwise is -1. 5530 * @param[out] qrss_action_pos 5531 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5532 * @param[out] modify_after_mirror 5533 * Pointer to the flag of modify action after FDB mirroring. 5534 * 5535 * @return 5536 * > 0 the total number of actions. 5537 * 0 if not found match action in action list. 5538 */ 5539 static int 5540 flow_check_match_action(const struct rte_flow_action actions[], 5541 const struct rte_flow_attr *attr, 5542 enum rte_flow_action_type action, 5543 int *match_action_pos, int *qrss_action_pos, 5544 int *modify_after_mirror) 5545 { 5546 const struct rte_flow_action_sample *sample; 5547 const struct rte_flow_action_raw_decap *decap; 5548 int actions_n = 0; 5549 uint32_t ratio = 0; 5550 int sub_type = 0; 5551 int flag = 0; 5552 int fdb_mirror = 0; 5553 5554 *match_action_pos = -1; 5555 *qrss_action_pos = -1; 5556 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5557 if (actions->type == action) { 5558 flag = 1; 5559 *match_action_pos = actions_n; 5560 } 5561 switch (actions->type) { 5562 case RTE_FLOW_ACTION_TYPE_QUEUE: 5563 case RTE_FLOW_ACTION_TYPE_RSS: 5564 *qrss_action_pos = actions_n; 5565 break; 5566 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5567 sample = actions->conf; 5568 ratio = sample->ratio; 5569 sub_type = ((const struct rte_flow_action *) 5570 (sample->actions))->type; 5571 if (ratio == 1 && attr->transfer) 5572 fdb_mirror = 1; 5573 break; 5574 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5575 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5576 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5577 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5578 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5579 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5580 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5581 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5582 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5583 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5584 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5585 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5586 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5587 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5588 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5589 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5590 case RTE_FLOW_ACTION_TYPE_FLAG: 5591 case RTE_FLOW_ACTION_TYPE_MARK: 5592 case RTE_FLOW_ACTION_TYPE_SET_META: 5593 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5594 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5595 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5596 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5597 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5598 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5599 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5600 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5601 case RTE_FLOW_ACTION_TYPE_METER: 5602 if (fdb_mirror) 5603 *modify_after_mirror = 1; 5604 break; 5605 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5606 decap = actions->conf; 5607 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5608 ; 5609 actions_n++; 5610 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5611 const struct rte_flow_action_raw_encap *encap = 5612 actions->conf; 5613 if (decap->size <= 5614 MLX5_ENCAPSULATION_DECISION_SIZE && 5615 encap->size > 5616 MLX5_ENCAPSULATION_DECISION_SIZE) 5617 /* L3 encap. */ 5618 break; 5619 } 5620 if (fdb_mirror) 5621 *modify_after_mirror = 1; 5622 break; 5623 default: 5624 break; 5625 } 5626 actions_n++; 5627 } 5628 if (flag && fdb_mirror && !*modify_after_mirror) { 5629 /* FDB mirroring uses the destination array to implement 5630 * instead of FLOW_SAMPLER object. 5631 */ 5632 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5633 flag = 0; 5634 } 5635 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5636 return flag ? actions_n + 1 : 0; 5637 } 5638 5639 #define SAMPLE_SUFFIX_ITEM 2 5640 5641 /** 5642 * Split the sample flow. 5643 * 5644 * As sample flow will split to two sub flow, sample flow with 5645 * sample action, the other actions will move to new suffix flow. 5646 * 5647 * Also add unique tag id with tag action in the sample flow, 5648 * the same tag id will be as match in the suffix flow. 5649 * 5650 * @param dev 5651 * Pointer to Ethernet device. 5652 * @param[in] add_tag 5653 * Add extra tag action flag. 5654 * @param[out] sfx_items 5655 * Suffix flow match items (list terminated by the END pattern item). 5656 * @param[in] actions 5657 * Associated actions (list terminated by the END action). 5658 * @param[out] actions_sfx 5659 * Suffix flow actions. 5660 * @param[out] actions_pre 5661 * Prefix flow actions. 5662 * @param[in] actions_n 5663 * The total number of actions. 5664 * @param[in] sample_action_pos 5665 * The sample action position. 5666 * @param[in] qrss_action_pos 5667 * The Queue/RSS action position. 5668 * @param[in] jump_table 5669 * Add extra jump action flag. 5670 * @param[out] error 5671 * Perform verbose error reporting if not NULL. 5672 * 5673 * @return 5674 * 0 on success, or unique flow_id, a negative errno value 5675 * otherwise and rte_errno is set. 5676 */ 5677 static int 5678 flow_sample_split_prep(struct rte_eth_dev *dev, 5679 int add_tag, 5680 struct rte_flow_item sfx_items[], 5681 const struct rte_flow_action actions[], 5682 struct rte_flow_action actions_sfx[], 5683 struct rte_flow_action actions_pre[], 5684 int actions_n, 5685 int sample_action_pos, 5686 int qrss_action_pos, 5687 int jump_table, 5688 struct rte_flow_error *error) 5689 { 5690 struct mlx5_priv *priv = dev->data->dev_private; 5691 struct mlx5_rte_flow_action_set_tag *set_tag; 5692 struct mlx5_rte_flow_item_tag *tag_spec; 5693 struct mlx5_rte_flow_item_tag *tag_mask; 5694 struct rte_flow_action_jump *jump_action; 5695 uint32_t tag_id = 0; 5696 int index; 5697 int append_index = 0; 5698 int ret; 5699 5700 if (sample_action_pos < 0) 5701 return rte_flow_error_set(error, EINVAL, 5702 RTE_FLOW_ERROR_TYPE_ACTION, 5703 NULL, "invalid position of sample " 5704 "action in list"); 5705 /* Prepare the actions for prefix and suffix flow. */ 5706 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5707 index = qrss_action_pos; 5708 /* Put the preceding the Queue/RSS action into prefix flow. */ 5709 if (index != 0) 5710 memcpy(actions_pre, actions, 5711 sizeof(struct rte_flow_action) * index); 5712 /* Put others preceding the sample action into prefix flow. */ 5713 if (sample_action_pos > index + 1) 5714 memcpy(actions_pre + index, actions + index + 1, 5715 sizeof(struct rte_flow_action) * 5716 (sample_action_pos - index - 1)); 5717 index = sample_action_pos - 1; 5718 /* Put Queue/RSS action into Suffix flow. */ 5719 memcpy(actions_sfx, actions + qrss_action_pos, 5720 sizeof(struct rte_flow_action)); 5721 actions_sfx++; 5722 } else { 5723 index = sample_action_pos; 5724 if (index != 0) 5725 memcpy(actions_pre, actions, 5726 sizeof(struct rte_flow_action) * index); 5727 } 5728 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5729 * For CX6DX and above, metadata registers Cx preserve their value, 5730 * add an extra tag action for NIC-RX and E-Switch Domain. 5731 */ 5732 if (add_tag) { 5733 /* Prepare the prefix tag action. */ 5734 append_index++; 5735 set_tag = (void *)(actions_pre + actions_n + append_index); 5736 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5737 if (ret < 0) 5738 return ret; 5739 mlx5_ipool_malloc(priv->sh->ipool 5740 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5741 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5742 .id = ret, 5743 .data = tag_id, 5744 }; 5745 /* Prepare the suffix subflow items. */ 5746 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5747 tag_spec->data = tag_id; 5748 tag_spec->id = set_tag->id; 5749 tag_mask = tag_spec + 1; 5750 tag_mask->data = UINT32_MAX; 5751 sfx_items[0] = (struct rte_flow_item){ 5752 .type = (enum rte_flow_item_type) 5753 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5754 .spec = tag_spec, 5755 .last = NULL, 5756 .mask = tag_mask, 5757 }; 5758 sfx_items[1] = (struct rte_flow_item){ 5759 .type = (enum rte_flow_item_type) 5760 RTE_FLOW_ITEM_TYPE_END, 5761 }; 5762 /* Prepare the tag action in prefix subflow. */ 5763 actions_pre[index++] = 5764 (struct rte_flow_action){ 5765 .type = (enum rte_flow_action_type) 5766 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5767 .conf = set_tag, 5768 }; 5769 } 5770 memcpy(actions_pre + index, actions + sample_action_pos, 5771 sizeof(struct rte_flow_action)); 5772 index += 1; 5773 /* For the modify action after the sample action in E-Switch mirroring, 5774 * Add the extra jump action in prefix subflow and jump into the next 5775 * table, then do the modify action in the new table. 5776 */ 5777 if (jump_table) { 5778 /* Prepare the prefix jump action. */ 5779 append_index++; 5780 jump_action = (void *)(actions_pre + actions_n + append_index); 5781 jump_action->group = jump_table; 5782 actions_pre[index++] = 5783 (struct rte_flow_action){ 5784 .type = (enum rte_flow_action_type) 5785 RTE_FLOW_ACTION_TYPE_JUMP, 5786 .conf = jump_action, 5787 }; 5788 } 5789 actions_pre[index] = (struct rte_flow_action){ 5790 .type = (enum rte_flow_action_type) 5791 RTE_FLOW_ACTION_TYPE_END, 5792 }; 5793 /* Put the actions after sample into Suffix flow. */ 5794 memcpy(actions_sfx, actions + sample_action_pos + 1, 5795 sizeof(struct rte_flow_action) * 5796 (actions_n - sample_action_pos - 1)); 5797 return tag_id; 5798 } 5799 5800 /** 5801 * The splitting for metadata feature. 5802 * 5803 * - Q/RSS action on NIC Rx should be split in order to pass by 5804 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5805 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5806 * 5807 * - All the actions on NIC Tx should have a mreg copy action to 5808 * copy reg_a from WQE to reg_c[0]. 5809 * 5810 * @param dev 5811 * Pointer to Ethernet device. 5812 * @param[in] flow 5813 * Parent flow structure pointer. 5814 * @param[in] attr 5815 * Flow rule attributes. 5816 * @param[in] items 5817 * Pattern specification (list terminated by the END pattern item). 5818 * @param[in] actions 5819 * Associated actions (list terminated by the END action). 5820 * @param[in] flow_split_info 5821 * Pointer to flow split info structure. 5822 * @param[out] error 5823 * Perform verbose error reporting if not NULL. 5824 * @return 5825 * 0 on success, negative value otherwise 5826 */ 5827 static int 5828 flow_create_split_metadata(struct rte_eth_dev *dev, 5829 struct rte_flow *flow, 5830 const struct rte_flow_attr *attr, 5831 const struct rte_flow_item items[], 5832 const struct rte_flow_action actions[], 5833 struct mlx5_flow_split_info *flow_split_info, 5834 struct rte_flow_error *error) 5835 { 5836 struct mlx5_priv *priv = dev->data->dev_private; 5837 struct mlx5_sh_config *config = &priv->sh->config; 5838 const struct rte_flow_action *qrss = NULL; 5839 struct rte_flow_action *ext_actions = NULL; 5840 struct mlx5_flow *dev_flow = NULL; 5841 uint32_t qrss_id = 0; 5842 int mtr_sfx = 0; 5843 size_t act_size; 5844 int actions_n; 5845 int encap_idx; 5846 int ret; 5847 5848 /* Check whether extensive metadata feature is engaged. */ 5849 if (!config->dv_flow_en || 5850 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5851 !mlx5_flow_ext_mreg_supported(dev)) 5852 return flow_create_split_inner(dev, flow, NULL, attr, items, 5853 actions, flow_split_info, error); 5854 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5855 &encap_idx); 5856 if (qrss) { 5857 /* Exclude hairpin flows from splitting. */ 5858 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5859 const struct rte_flow_action_queue *queue; 5860 5861 queue = qrss->conf; 5862 if (mlx5_rxq_get_type(dev, queue->index) == 5863 MLX5_RXQ_TYPE_HAIRPIN) 5864 qrss = NULL; 5865 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5866 const struct rte_flow_action_rss *rss; 5867 5868 rss = qrss->conf; 5869 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5870 MLX5_RXQ_TYPE_HAIRPIN) 5871 qrss = NULL; 5872 } 5873 } 5874 if (qrss) { 5875 /* Check if it is in meter suffix table. */ 5876 mtr_sfx = attr->group == (attr->transfer ? 5877 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5878 MLX5_FLOW_TABLE_LEVEL_METER); 5879 /* 5880 * Q/RSS action on NIC Rx should be split in order to pass by 5881 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5882 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5883 */ 5884 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5885 sizeof(struct rte_flow_action_set_tag) + 5886 sizeof(struct rte_flow_action_jump); 5887 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5888 SOCKET_ID_ANY); 5889 if (!ext_actions) 5890 return rte_flow_error_set(error, ENOMEM, 5891 RTE_FLOW_ERROR_TYPE_ACTION, 5892 NULL, "no memory to split " 5893 "metadata flow"); 5894 /* 5895 * Create the new actions list with removed Q/RSS action 5896 * and appended set tag and jump to register copy table 5897 * (RX_CP_TBL). We should preallocate unique tag ID here 5898 * in advance, because it is needed for set tag action. 5899 */ 5900 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5901 qrss, actions_n, 5902 mtr_sfx, error); 5903 if (!mtr_sfx && !qrss_id) { 5904 ret = -rte_errno; 5905 goto exit; 5906 } 5907 } else if (attr->egress && !attr->transfer) { 5908 /* 5909 * All the actions on NIC Tx should have a metadata register 5910 * copy action to copy reg_a from WQE to reg_c[meta] 5911 */ 5912 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5913 sizeof(struct mlx5_flow_action_copy_mreg); 5914 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5915 SOCKET_ID_ANY); 5916 if (!ext_actions) 5917 return rte_flow_error_set(error, ENOMEM, 5918 RTE_FLOW_ERROR_TYPE_ACTION, 5919 NULL, "no memory to split " 5920 "metadata flow"); 5921 /* Create the action list appended with copy register. */ 5922 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5923 actions_n, error, encap_idx); 5924 if (ret < 0) 5925 goto exit; 5926 } 5927 /* Add the unmodified original or prefix subflow. */ 5928 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5929 items, ext_actions ? ext_actions : 5930 actions, flow_split_info, error); 5931 if (ret < 0) 5932 goto exit; 5933 MLX5_ASSERT(dev_flow); 5934 if (qrss) { 5935 const struct rte_flow_attr q_attr = { 5936 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5937 .ingress = 1, 5938 }; 5939 /* Internal PMD action to set register. */ 5940 struct mlx5_rte_flow_item_tag q_tag_spec = { 5941 .data = qrss_id, 5942 .id = REG_NON, 5943 }; 5944 struct rte_flow_item q_items[] = { 5945 { 5946 .type = (enum rte_flow_item_type) 5947 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5948 .spec = &q_tag_spec, 5949 .last = NULL, 5950 .mask = NULL, 5951 }, 5952 { 5953 .type = RTE_FLOW_ITEM_TYPE_END, 5954 }, 5955 }; 5956 struct rte_flow_action q_actions[] = { 5957 { 5958 .type = qrss->type, 5959 .conf = qrss->conf, 5960 }, 5961 { 5962 .type = RTE_FLOW_ACTION_TYPE_END, 5963 }, 5964 }; 5965 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5966 5967 /* 5968 * Configure the tag item only if there is no meter subflow. 5969 * Since tag is already marked in the meter suffix subflow 5970 * we can just use the meter suffix items as is. 5971 */ 5972 if (qrss_id) { 5973 /* Not meter subflow. */ 5974 MLX5_ASSERT(!mtr_sfx); 5975 /* 5976 * Put unique id in prefix flow due to it is destroyed 5977 * after suffix flow and id will be freed after there 5978 * is no actual flows with this id and identifier 5979 * reallocation becomes possible (for example, for 5980 * other flows in other threads). 5981 */ 5982 dev_flow->handle->split_flow_id = qrss_id; 5983 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5984 error); 5985 if (ret < 0) 5986 goto exit; 5987 q_tag_spec.id = ret; 5988 } 5989 dev_flow = NULL; 5990 /* Add suffix subflow to execute Q/RSS. */ 5991 flow_split_info->prefix_layers = layers; 5992 flow_split_info->prefix_mark = 0; 5993 flow_split_info->table_id = 0; 5994 ret = flow_create_split_inner(dev, flow, &dev_flow, 5995 &q_attr, mtr_sfx ? items : 5996 q_items, q_actions, 5997 flow_split_info, error); 5998 if (ret < 0) 5999 goto exit; 6000 /* qrss ID should be freed if failed. */ 6001 qrss_id = 0; 6002 MLX5_ASSERT(dev_flow); 6003 } 6004 6005 exit: 6006 /* 6007 * We do not destroy the partially created sub_flows in case of error. 6008 * These ones are included into parent flow list and will be destroyed 6009 * by flow_drv_destroy. 6010 */ 6011 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6012 qrss_id); 6013 mlx5_free(ext_actions); 6014 return ret; 6015 } 6016 6017 /** 6018 * Create meter internal drop flow with the original pattern. 6019 * 6020 * @param dev 6021 * Pointer to Ethernet device. 6022 * @param[in] flow 6023 * Parent flow structure pointer. 6024 * @param[in] attr 6025 * Flow rule attributes. 6026 * @param[in] items 6027 * Pattern specification (list terminated by the END pattern item). 6028 * @param[in] flow_split_info 6029 * Pointer to flow split info structure. 6030 * @param[in] fm 6031 * Pointer to flow meter structure. 6032 * @param[out] error 6033 * Perform verbose error reporting if not NULL. 6034 * @return 6035 * 0 on success, negative value otherwise 6036 */ 6037 static uint32_t 6038 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6039 struct rte_flow *flow, 6040 const struct rte_flow_attr *attr, 6041 const struct rte_flow_item items[], 6042 struct mlx5_flow_split_info *flow_split_info, 6043 struct mlx5_flow_meter_info *fm, 6044 struct rte_flow_error *error) 6045 { 6046 struct mlx5_flow *dev_flow = NULL; 6047 struct rte_flow_attr drop_attr = *attr; 6048 struct rte_flow_action drop_actions[3]; 6049 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6050 6051 MLX5_ASSERT(fm->drop_cnt); 6052 drop_actions[0].type = 6053 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6054 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6055 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6056 drop_actions[1].conf = NULL; 6057 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6058 drop_actions[2].conf = NULL; 6059 drop_split_info.external = false; 6060 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6061 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6062 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6063 return flow_create_split_inner(dev, flow, &dev_flow, 6064 &drop_attr, items, drop_actions, 6065 &drop_split_info, error); 6066 } 6067 6068 /** 6069 * The splitting for meter feature. 6070 * 6071 * - The meter flow will be split to two flows as prefix and 6072 * suffix flow. The packets make sense only it pass the prefix 6073 * meter action. 6074 * 6075 * - Reg_C_5 is used for the packet to match betweend prefix and 6076 * suffix flow. 6077 * 6078 * @param dev 6079 * Pointer to Ethernet device. 6080 * @param[in] flow 6081 * Parent flow structure pointer. 6082 * @param[in] attr 6083 * Flow rule attributes. 6084 * @param[in] items 6085 * Pattern specification (list terminated by the END pattern item). 6086 * @param[in] actions 6087 * Associated actions (list terminated by the END action). 6088 * @param[in] flow_split_info 6089 * Pointer to flow split info structure. 6090 * @param[out] error 6091 * Perform verbose error reporting if not NULL. 6092 * @return 6093 * 0 on success, negative value otherwise 6094 */ 6095 static int 6096 flow_create_split_meter(struct rte_eth_dev *dev, 6097 struct rte_flow *flow, 6098 const struct rte_flow_attr *attr, 6099 const struct rte_flow_item items[], 6100 const struct rte_flow_action actions[], 6101 struct mlx5_flow_split_info *flow_split_info, 6102 struct rte_flow_error *error) 6103 { 6104 struct mlx5_priv *priv = dev->data->dev_private; 6105 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6106 struct rte_flow_action *sfx_actions = NULL; 6107 struct rte_flow_action *pre_actions = NULL; 6108 struct rte_flow_item *sfx_items = NULL; 6109 struct mlx5_flow *dev_flow = NULL; 6110 struct rte_flow_attr sfx_attr = *attr; 6111 struct mlx5_flow_meter_info *fm = NULL; 6112 uint8_t skip_scale_restore; 6113 bool has_mtr = false; 6114 bool has_modify = false; 6115 bool set_mtr_reg = true; 6116 bool is_mtr_hierarchy = false; 6117 uint32_t meter_id = 0; 6118 uint32_t mtr_idx = 0; 6119 uint32_t mtr_flow_id = 0; 6120 size_t act_size; 6121 size_t item_size; 6122 int actions_n = 0; 6123 int ret = 0; 6124 6125 if (priv->mtr_en) 6126 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6127 &has_modify, &meter_id); 6128 if (has_mtr) { 6129 if (flow->meter) { 6130 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6131 if (!fm) 6132 return rte_flow_error_set(error, EINVAL, 6133 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6134 NULL, "Meter not found."); 6135 } else { 6136 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6137 if (!fm) 6138 return rte_flow_error_set(error, EINVAL, 6139 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6140 NULL, "Meter not found."); 6141 ret = mlx5_flow_meter_attach(priv, fm, 6142 &sfx_attr, error); 6143 if (ret) 6144 return -rte_errno; 6145 flow->meter = mtr_idx; 6146 } 6147 MLX5_ASSERT(wks); 6148 wks->fm = fm; 6149 if (!fm->def_policy) { 6150 wks->policy = mlx5_flow_meter_policy_find(dev, 6151 fm->policy_id, 6152 NULL); 6153 MLX5_ASSERT(wks->policy); 6154 if (wks->policy->is_hierarchy) { 6155 wks->final_policy = 6156 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6157 wks->policy); 6158 if (!wks->final_policy) 6159 return rte_flow_error_set(error, 6160 EINVAL, 6161 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6162 "Failed to find terminal policy of hierarchy."); 6163 is_mtr_hierarchy = true; 6164 } 6165 } 6166 /* 6167 * If it isn't default-policy Meter, and 6168 * 1. There's no action in flow to change 6169 * packet (modify/encap/decap etc.), OR 6170 * 2. No drop count needed for this meter. 6171 * 3. It's not meter hierarchy. 6172 * Then no need to use regC to save meter id anymore. 6173 */ 6174 if (!fm->def_policy && !is_mtr_hierarchy && 6175 (!has_modify || !fm->drop_cnt)) 6176 set_mtr_reg = false; 6177 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6178 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6179 sizeof(struct mlx5_rte_flow_action_set_tag); 6180 /* Suffix items: tag, vlan, port id, end. */ 6181 #define METER_SUFFIX_ITEM 4 6182 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6183 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6184 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6185 0, SOCKET_ID_ANY); 6186 if (!sfx_actions) 6187 return rte_flow_error_set(error, ENOMEM, 6188 RTE_FLOW_ERROR_TYPE_ACTION, 6189 NULL, "no memory to split " 6190 "meter flow"); 6191 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6192 act_size); 6193 /* There's no suffix flow for meter of non-default policy. */ 6194 if (!fm->def_policy) 6195 pre_actions = sfx_actions + 1; 6196 else 6197 pre_actions = sfx_actions + actions_n; 6198 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6199 items, sfx_items, actions, 6200 sfx_actions, pre_actions, 6201 (set_mtr_reg ? &mtr_flow_id : NULL), 6202 error); 6203 if (ret) { 6204 ret = -rte_errno; 6205 goto exit; 6206 } 6207 /* Add the prefix subflow. */ 6208 skip_scale_restore = flow_split_info->skip_scale; 6209 flow_split_info->skip_scale |= 6210 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6211 ret = flow_create_split_inner(dev, flow, &dev_flow, 6212 attr, items, pre_actions, 6213 flow_split_info, error); 6214 flow_split_info->skip_scale = skip_scale_restore; 6215 if (ret) { 6216 if (mtr_flow_id) 6217 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6218 ret = -rte_errno; 6219 goto exit; 6220 } 6221 if (mtr_flow_id) { 6222 dev_flow->handle->split_flow_id = mtr_flow_id; 6223 dev_flow->handle->is_meter_flow_id = 1; 6224 } 6225 if (!fm->def_policy) { 6226 if (!set_mtr_reg && fm->drop_cnt) 6227 ret = 6228 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6229 &sfx_attr, items, 6230 flow_split_info, 6231 fm, error); 6232 goto exit; 6233 } 6234 /* Setting the sfx group atrr. */ 6235 sfx_attr.group = sfx_attr.transfer ? 6236 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6237 MLX5_FLOW_TABLE_LEVEL_METER; 6238 flow_split_info->prefix_layers = 6239 flow_get_prefix_layer_flags(dev_flow); 6240 flow_split_info->prefix_mark |= wks->mark; 6241 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6242 } 6243 /* Add the prefix subflow. */ 6244 ret = flow_create_split_metadata(dev, flow, 6245 &sfx_attr, sfx_items ? 6246 sfx_items : items, 6247 sfx_actions ? sfx_actions : actions, 6248 flow_split_info, error); 6249 exit: 6250 if (sfx_actions) 6251 mlx5_free(sfx_actions); 6252 return ret; 6253 } 6254 6255 /** 6256 * The splitting for sample feature. 6257 * 6258 * Once Sample action is detected in the action list, the flow actions should 6259 * be split into prefix sub flow and suffix sub flow. 6260 * 6261 * The original items remain in the prefix sub flow, all actions preceding the 6262 * sample action and the sample action itself will be copied to the prefix 6263 * sub flow, the actions following the sample action will be copied to the 6264 * suffix sub flow, Queue action always be located in the suffix sub flow. 6265 * 6266 * In order to make the packet from prefix sub flow matches with suffix sub 6267 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6268 * flow uses tag item with the unique flow id. 6269 * 6270 * @param dev 6271 * Pointer to Ethernet device. 6272 * @param[in] flow 6273 * Parent flow structure pointer. 6274 * @param[in] attr 6275 * Flow rule attributes. 6276 * @param[in] items 6277 * Pattern specification (list terminated by the END pattern item). 6278 * @param[in] actions 6279 * Associated actions (list terminated by the END action). 6280 * @param[in] flow_split_info 6281 * Pointer to flow split info structure. 6282 * @param[out] error 6283 * Perform verbose error reporting if not NULL. 6284 * @return 6285 * 0 on success, negative value otherwise 6286 */ 6287 static int 6288 flow_create_split_sample(struct rte_eth_dev *dev, 6289 struct rte_flow *flow, 6290 const struct rte_flow_attr *attr, 6291 const struct rte_flow_item items[], 6292 const struct rte_flow_action actions[], 6293 struct mlx5_flow_split_info *flow_split_info, 6294 struct rte_flow_error *error) 6295 { 6296 struct mlx5_priv *priv = dev->data->dev_private; 6297 struct rte_flow_action *sfx_actions = NULL; 6298 struct rte_flow_action *pre_actions = NULL; 6299 struct rte_flow_item *sfx_items = NULL; 6300 struct mlx5_flow *dev_flow = NULL; 6301 struct rte_flow_attr sfx_attr = *attr; 6302 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6303 struct mlx5_flow_dv_sample_resource *sample_res; 6304 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6305 struct mlx5_flow_tbl_resource *sfx_tbl; 6306 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6307 #endif 6308 size_t act_size; 6309 size_t item_size; 6310 uint32_t fdb_tx = 0; 6311 int32_t tag_id = 0; 6312 int actions_n = 0; 6313 int sample_action_pos; 6314 int qrss_action_pos; 6315 int add_tag = 0; 6316 int modify_after_mirror = 0; 6317 uint16_t jump_table = 0; 6318 const uint32_t next_ft_step = 1; 6319 int ret = 0; 6320 6321 if (priv->sampler_en) 6322 actions_n = flow_check_match_action(actions, attr, 6323 RTE_FLOW_ACTION_TYPE_SAMPLE, 6324 &sample_action_pos, &qrss_action_pos, 6325 &modify_after_mirror); 6326 if (actions_n) { 6327 /* The prefix actions must includes sample, tag, end. */ 6328 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6329 + sizeof(struct mlx5_rte_flow_action_set_tag); 6330 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6331 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6332 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6333 item_size), 0, SOCKET_ID_ANY); 6334 if (!sfx_actions) 6335 return rte_flow_error_set(error, ENOMEM, 6336 RTE_FLOW_ERROR_TYPE_ACTION, 6337 NULL, "no memory to split " 6338 "sample flow"); 6339 /* The representor_id is UINT16_MAX for uplink. */ 6340 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6341 /* 6342 * When reg_c_preserve is set, metadata registers Cx preserve 6343 * their value even through packet duplication. 6344 */ 6345 add_tag = (!fdb_tx || 6346 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6347 if (add_tag) 6348 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6349 + act_size); 6350 if (modify_after_mirror) 6351 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6352 next_ft_step; 6353 pre_actions = sfx_actions + actions_n; 6354 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6355 actions, sfx_actions, 6356 pre_actions, actions_n, 6357 sample_action_pos, 6358 qrss_action_pos, jump_table, 6359 error); 6360 if (tag_id < 0 || (add_tag && !tag_id)) { 6361 ret = -rte_errno; 6362 goto exit; 6363 } 6364 if (modify_after_mirror) 6365 flow_split_info->skip_scale = 6366 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6367 /* Add the prefix subflow. */ 6368 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6369 items, pre_actions, 6370 flow_split_info, error); 6371 if (ret) { 6372 ret = -rte_errno; 6373 goto exit; 6374 } 6375 dev_flow->handle->split_flow_id = tag_id; 6376 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6377 if (!modify_after_mirror) { 6378 /* Set the sfx group attr. */ 6379 sample_res = (struct mlx5_flow_dv_sample_resource *) 6380 dev_flow->dv.sample_res; 6381 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6382 sample_res->normal_path_tbl; 6383 sfx_tbl_data = container_of(sfx_tbl, 6384 struct mlx5_flow_tbl_data_entry, 6385 tbl); 6386 sfx_attr.group = sfx_attr.transfer ? 6387 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6388 } else { 6389 MLX5_ASSERT(attr->transfer); 6390 sfx_attr.group = jump_table; 6391 } 6392 flow_split_info->prefix_layers = 6393 flow_get_prefix_layer_flags(dev_flow); 6394 MLX5_ASSERT(wks); 6395 flow_split_info->prefix_mark |= wks->mark; 6396 /* Suffix group level already be scaled with factor, set 6397 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6398 * again in translation. 6399 */ 6400 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6401 #endif 6402 } 6403 /* Add the suffix subflow. */ 6404 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6405 sfx_items ? sfx_items : items, 6406 sfx_actions ? sfx_actions : actions, 6407 flow_split_info, error); 6408 exit: 6409 if (sfx_actions) 6410 mlx5_free(sfx_actions); 6411 return ret; 6412 } 6413 6414 /** 6415 * Split the flow to subflow set. The splitters might be linked 6416 * in the chain, like this: 6417 * flow_create_split_outer() calls: 6418 * flow_create_split_meter() calls: 6419 * flow_create_split_metadata(meter_subflow_0) calls: 6420 * flow_create_split_inner(metadata_subflow_0) 6421 * flow_create_split_inner(metadata_subflow_1) 6422 * flow_create_split_inner(metadata_subflow_2) 6423 * flow_create_split_metadata(meter_subflow_1) calls: 6424 * flow_create_split_inner(metadata_subflow_0) 6425 * flow_create_split_inner(metadata_subflow_1) 6426 * flow_create_split_inner(metadata_subflow_2) 6427 * 6428 * This provide flexible way to add new levels of flow splitting. 6429 * The all of successfully created subflows are included to the 6430 * parent flow dev_flow list. 6431 * 6432 * @param dev 6433 * Pointer to Ethernet device. 6434 * @param[in] flow 6435 * Parent flow structure pointer. 6436 * @param[in] attr 6437 * Flow rule attributes. 6438 * @param[in] items 6439 * Pattern specification (list terminated by the END pattern item). 6440 * @param[in] actions 6441 * Associated actions (list terminated by the END action). 6442 * @param[in] flow_split_info 6443 * Pointer to flow split info structure. 6444 * @param[out] error 6445 * Perform verbose error reporting if not NULL. 6446 * @return 6447 * 0 on success, negative value otherwise 6448 */ 6449 static int 6450 flow_create_split_outer(struct rte_eth_dev *dev, 6451 struct rte_flow *flow, 6452 const struct rte_flow_attr *attr, 6453 const struct rte_flow_item items[], 6454 const struct rte_flow_action actions[], 6455 struct mlx5_flow_split_info *flow_split_info, 6456 struct rte_flow_error *error) 6457 { 6458 int ret; 6459 6460 ret = flow_create_split_sample(dev, flow, attr, items, 6461 actions, flow_split_info, error); 6462 MLX5_ASSERT(ret <= 0); 6463 return ret; 6464 } 6465 6466 static inline struct mlx5_flow_tunnel * 6467 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6468 { 6469 struct mlx5_flow_tunnel *tunnel; 6470 6471 #pragma GCC diagnostic push 6472 #pragma GCC diagnostic ignored "-Wcast-qual" 6473 tunnel = (typeof(tunnel))flow->tunnel; 6474 #pragma GCC diagnostic pop 6475 6476 return tunnel; 6477 } 6478 6479 /** 6480 * Adjust flow RSS workspace if needed. 6481 * 6482 * @param wks 6483 * Pointer to thread flow work space. 6484 * @param rss_desc 6485 * Pointer to RSS descriptor. 6486 * @param[in] nrssq_num 6487 * New RSS queue number. 6488 * 6489 * @return 6490 * 0 on success, -1 otherwise and rte_errno is set. 6491 */ 6492 static int 6493 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6494 struct mlx5_flow_rss_desc *rss_desc, 6495 uint32_t nrssq_num) 6496 { 6497 if (likely(nrssq_num <= wks->rssq_num)) 6498 return 0; 6499 rss_desc->queue = realloc(rss_desc->queue, 6500 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6501 if (!rss_desc->queue) { 6502 rte_errno = ENOMEM; 6503 return -1; 6504 } 6505 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6506 return 0; 6507 } 6508 6509 /** 6510 * Create a flow and add it to @p list. 6511 * 6512 * @param dev 6513 * Pointer to Ethernet device. 6514 * @param list 6515 * Pointer to a TAILQ flow list. If this parameter NULL, 6516 * no list insertion occurred, flow is just created, 6517 * this is caller's responsibility to track the 6518 * created flow. 6519 * @param[in] attr 6520 * Flow rule attributes. 6521 * @param[in] items 6522 * Pattern specification (list terminated by the END pattern item). 6523 * @param[in] actions 6524 * Associated actions (list terminated by the END action). 6525 * @param[in] external 6526 * This flow rule is created by request external to PMD. 6527 * @param[out] error 6528 * Perform verbose error reporting if not NULL. 6529 * 6530 * @return 6531 * A flow index on success, 0 otherwise and rte_errno is set. 6532 */ 6533 static uint32_t 6534 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6535 const struct rte_flow_attr *attr, 6536 const struct rte_flow_item items[], 6537 const struct rte_flow_action original_actions[], 6538 bool external, struct rte_flow_error *error) 6539 { 6540 struct mlx5_priv *priv = dev->data->dev_private; 6541 struct rte_flow *flow = NULL; 6542 struct mlx5_flow *dev_flow; 6543 const struct rte_flow_action_rss *rss = NULL; 6544 struct mlx5_translated_action_handle 6545 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6546 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6547 union { 6548 struct mlx5_flow_expand_rss buf; 6549 uint8_t buffer[4096]; 6550 } expand_buffer; 6551 union { 6552 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6553 uint8_t buffer[2048]; 6554 } actions_rx; 6555 union { 6556 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6557 uint8_t buffer[2048]; 6558 } actions_hairpin_tx; 6559 union { 6560 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6561 uint8_t buffer[2048]; 6562 } items_tx; 6563 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6564 struct mlx5_flow_rss_desc *rss_desc; 6565 const struct rte_flow_action *p_actions_rx; 6566 uint32_t i; 6567 uint32_t idx = 0; 6568 int hairpin_flow; 6569 struct rte_flow_attr attr_tx = { .priority = 0 }; 6570 const struct rte_flow_action *actions; 6571 struct rte_flow_action *translated_actions = NULL; 6572 struct mlx5_flow_tunnel *tunnel; 6573 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6574 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6575 struct mlx5_flow_split_info flow_split_info = { 6576 .external = !!external, 6577 .skip_scale = 0, 6578 .flow_idx = 0, 6579 .prefix_mark = 0, 6580 .prefix_layers = 0, 6581 .table_id = 0 6582 }; 6583 int ret; 6584 6585 MLX5_ASSERT(wks); 6586 rss_desc = &wks->rss_desc; 6587 ret = flow_action_handles_translate(dev, original_actions, 6588 indir_actions, 6589 &indir_actions_n, 6590 &translated_actions, error); 6591 if (ret < 0) { 6592 MLX5_ASSERT(translated_actions == NULL); 6593 return 0; 6594 } 6595 actions = translated_actions ? translated_actions : original_actions; 6596 p_actions_rx = actions; 6597 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6598 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6599 external, hairpin_flow, error); 6600 if (ret < 0) 6601 goto error_before_hairpin_split; 6602 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6603 if (!flow) { 6604 rte_errno = ENOMEM; 6605 goto error_before_hairpin_split; 6606 } 6607 if (hairpin_flow > 0) { 6608 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6609 rte_errno = EINVAL; 6610 goto error_before_hairpin_split; 6611 } 6612 flow_hairpin_split(dev, actions, actions_rx.actions, 6613 actions_hairpin_tx.actions, items_tx.items, 6614 idx); 6615 p_actions_rx = actions_rx.actions; 6616 } 6617 flow_split_info.flow_idx = idx; 6618 flow->drv_type = flow_get_drv_type(dev, attr); 6619 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6620 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6621 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6622 /* RSS Action only works on NIC RX domain */ 6623 if (attr->ingress && !attr->transfer) 6624 rss = flow_get_rss_action(dev, p_actions_rx); 6625 if (rss) { 6626 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6627 return 0; 6628 /* 6629 * The following information is required by 6630 * mlx5_flow_hashfields_adjust() in advance. 6631 */ 6632 rss_desc->level = rss->level; 6633 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6634 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6635 } 6636 flow->dev_handles = 0; 6637 if (rss && rss->types) { 6638 unsigned int graph_root; 6639 6640 graph_root = find_graph_root(rss->level); 6641 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6642 items, rss->types, 6643 mlx5_support_expansion, graph_root); 6644 MLX5_ASSERT(ret > 0 && 6645 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6646 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6647 for (i = 0; i < buf->entries; ++i) 6648 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6649 } 6650 } else { 6651 buf->entries = 1; 6652 buf->entry[0].pattern = (void *)(uintptr_t)items; 6653 } 6654 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6655 indir_actions_n); 6656 for (i = 0; i < buf->entries; ++i) { 6657 /* Initialize flow split data. */ 6658 flow_split_info.prefix_layers = 0; 6659 flow_split_info.prefix_mark = 0; 6660 flow_split_info.skip_scale = 0; 6661 /* 6662 * The splitter may create multiple dev_flows, 6663 * depending on configuration. In the simplest 6664 * case it just creates unmodified original flow. 6665 */ 6666 ret = flow_create_split_outer(dev, flow, attr, 6667 buf->entry[i].pattern, 6668 p_actions_rx, &flow_split_info, 6669 error); 6670 if (ret < 0) 6671 goto error; 6672 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6673 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6674 p_actions_rx, 6675 idx, 6676 wks->flows[0].tunnel, 6677 &default_miss_ctx, 6678 error); 6679 if (ret < 0) { 6680 mlx5_free(default_miss_ctx.queue); 6681 goto error; 6682 } 6683 } 6684 } 6685 /* Create the tx flow. */ 6686 if (hairpin_flow) { 6687 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6688 attr_tx.ingress = 0; 6689 attr_tx.egress = 1; 6690 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6691 actions_hairpin_tx.actions, 6692 idx, error); 6693 if (!dev_flow) 6694 goto error; 6695 dev_flow->flow = flow; 6696 dev_flow->external = 0; 6697 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6698 dev_flow->handle, next); 6699 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6700 items_tx.items, 6701 actions_hairpin_tx.actions, error); 6702 if (ret < 0) 6703 goto error; 6704 } 6705 /* 6706 * Update the metadata register copy table. If extensive 6707 * metadata feature is enabled and registers are supported 6708 * we might create the extra rte_flow for each unique 6709 * MARK/FLAG action ID. 6710 * 6711 * The table is updated for ingress Flows only, because 6712 * the egress Flows belong to the different device and 6713 * copy table should be updated in peer NIC Rx domain. 6714 */ 6715 if (attr->ingress && 6716 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6717 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6718 if (ret) 6719 goto error; 6720 } 6721 /* 6722 * If the flow is external (from application) OR device is started, 6723 * OR mreg discover, then apply immediately. 6724 */ 6725 if (external || dev->data->dev_started || 6726 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6727 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6728 ret = flow_drv_apply(dev, flow, error); 6729 if (ret < 0) 6730 goto error; 6731 } 6732 flow->type = type; 6733 flow_rxq_flags_set(dev, flow); 6734 rte_free(translated_actions); 6735 tunnel = flow_tunnel_from_rule(wks->flows); 6736 if (tunnel) { 6737 flow->tunnel = 1; 6738 flow->tunnel_id = tunnel->tunnel_id; 6739 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6740 mlx5_free(default_miss_ctx.queue); 6741 } 6742 mlx5_flow_pop_thread_workspace(); 6743 return idx; 6744 error: 6745 MLX5_ASSERT(flow); 6746 ret = rte_errno; /* Save rte_errno before cleanup. */ 6747 flow_mreg_del_copy_action(dev, flow); 6748 flow_drv_destroy(dev, flow); 6749 if (rss_desc->shared_rss) 6750 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6751 mlx5_ipool_get 6752 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6753 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6754 mlx5_ipool_free(priv->flows[type], idx); 6755 rte_errno = ret; /* Restore rte_errno. */ 6756 ret = rte_errno; 6757 rte_errno = ret; 6758 mlx5_flow_pop_thread_workspace(); 6759 error_before_hairpin_split: 6760 rte_free(translated_actions); 6761 return 0; 6762 } 6763 6764 /** 6765 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6766 * incoming packets to table 1. 6767 * 6768 * Other flow rules, requested for group n, will be created in 6769 * e-switch table n+1. 6770 * Jump action to e-switch group n will be created to group n+1. 6771 * 6772 * Used when working in switchdev mode, to utilise advantages of table 1 6773 * and above. 6774 * 6775 * @param dev 6776 * Pointer to Ethernet device. 6777 * 6778 * @return 6779 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6780 */ 6781 struct rte_flow * 6782 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6783 { 6784 const struct rte_flow_attr attr = { 6785 .group = 0, 6786 .priority = 0, 6787 .ingress = 1, 6788 .egress = 0, 6789 .transfer = 1, 6790 }; 6791 const struct rte_flow_item pattern = { 6792 .type = RTE_FLOW_ITEM_TYPE_END, 6793 }; 6794 struct rte_flow_action_jump jump = { 6795 .group = 1, 6796 }; 6797 const struct rte_flow_action actions[] = { 6798 { 6799 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6800 .conf = &jump, 6801 }, 6802 { 6803 .type = RTE_FLOW_ACTION_TYPE_END, 6804 }, 6805 }; 6806 struct rte_flow_error error; 6807 6808 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6809 &attr, &pattern, 6810 actions, false, &error); 6811 } 6812 6813 /** 6814 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6815 * and sq number, directs all packets to peer vport. 6816 * 6817 * @param dev 6818 * Pointer to Ethernet device. 6819 * @param txq 6820 * Txq index. 6821 * 6822 * @return 6823 * Flow ID on success, 0 otherwise and rte_errno is set. 6824 */ 6825 uint32_t 6826 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6827 { 6828 struct rte_flow_attr attr = { 6829 .group = 0, 6830 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6831 .ingress = 1, 6832 .egress = 0, 6833 .transfer = 1, 6834 }; 6835 struct rte_flow_item_port_id port_spec = { 6836 .id = MLX5_PORT_ESW_MGR, 6837 }; 6838 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6839 .queue = txq, 6840 }; 6841 struct rte_flow_item pattern[] = { 6842 { 6843 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6844 .spec = &port_spec, 6845 }, 6846 { 6847 .type = (enum rte_flow_item_type) 6848 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6849 .spec = &txq_spec, 6850 }, 6851 { 6852 .type = RTE_FLOW_ITEM_TYPE_END, 6853 }, 6854 }; 6855 struct rte_flow_action_jump jump = { 6856 .group = 1, 6857 }; 6858 struct rte_flow_action_port_id port = { 6859 .id = dev->data->port_id, 6860 }; 6861 struct rte_flow_action actions[] = { 6862 { 6863 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6864 .conf = &jump, 6865 }, 6866 { 6867 .type = RTE_FLOW_ACTION_TYPE_END, 6868 }, 6869 }; 6870 struct rte_flow_error error; 6871 6872 /* 6873 * Creates group 0, highest priority jump flow. 6874 * Matches txq to bypass kernel packets. 6875 */ 6876 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6877 false, &error) == 0) 6878 return 0; 6879 /* Create group 1, lowest priority redirect flow for txq. */ 6880 attr.group = 1; 6881 actions[0].conf = &port; 6882 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6883 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6884 actions, false, &error); 6885 } 6886 6887 /** 6888 * Validate a flow supported by the NIC. 6889 * 6890 * @see rte_flow_validate() 6891 * @see rte_flow_ops 6892 */ 6893 int 6894 mlx5_flow_validate(struct rte_eth_dev *dev, 6895 const struct rte_flow_attr *attr, 6896 const struct rte_flow_item items[], 6897 const struct rte_flow_action original_actions[], 6898 struct rte_flow_error *error) 6899 { 6900 int hairpin_flow; 6901 struct mlx5_translated_action_handle 6902 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6903 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6904 const struct rte_flow_action *actions; 6905 struct rte_flow_action *translated_actions = NULL; 6906 int ret = flow_action_handles_translate(dev, original_actions, 6907 indir_actions, 6908 &indir_actions_n, 6909 &translated_actions, error); 6910 6911 if (ret) 6912 return ret; 6913 actions = translated_actions ? translated_actions : original_actions; 6914 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6915 ret = flow_drv_validate(dev, attr, items, actions, 6916 true, hairpin_flow, error); 6917 rte_free(translated_actions); 6918 return ret; 6919 } 6920 6921 /** 6922 * Create a flow. 6923 * 6924 * @see rte_flow_create() 6925 * @see rte_flow_ops 6926 */ 6927 struct rte_flow * 6928 mlx5_flow_create(struct rte_eth_dev *dev, 6929 const struct rte_flow_attr *attr, 6930 const struct rte_flow_item items[], 6931 const struct rte_flow_action actions[], 6932 struct rte_flow_error *error) 6933 { 6934 struct mlx5_priv *priv = dev->data->dev_private; 6935 6936 if (priv->sh->config.dv_flow_en == 2) { 6937 rte_flow_error_set(error, ENOTSUP, 6938 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6939 NULL, 6940 "Flow non-Q creation not supported"); 6941 return NULL; 6942 } 6943 /* 6944 * If the device is not started yet, it is not allowed to created a 6945 * flow from application. PMD default flows and traffic control flows 6946 * are not affected. 6947 */ 6948 if (unlikely(!dev->data->dev_started)) { 6949 DRV_LOG(DEBUG, "port %u is not started when " 6950 "inserting a flow", dev->data->port_id); 6951 rte_flow_error_set(error, ENODEV, 6952 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6953 NULL, 6954 "port not started"); 6955 return NULL; 6956 } 6957 6958 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6959 attr, items, actions, 6960 true, error); 6961 } 6962 6963 /** 6964 * Destroy a flow in a list. 6965 * 6966 * @param dev 6967 * Pointer to Ethernet device. 6968 * @param[in] flow_idx 6969 * Index of flow to destroy. 6970 */ 6971 static void 6972 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6973 uint32_t flow_idx) 6974 { 6975 struct mlx5_priv *priv = dev->data->dev_private; 6976 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6977 6978 if (!flow) 6979 return; 6980 MLX5_ASSERT(flow->type == type); 6981 /* 6982 * Update RX queue flags only if port is started, otherwise it is 6983 * already clean. 6984 */ 6985 if (dev->data->dev_started) 6986 flow_rxq_flags_trim(dev, flow); 6987 flow_drv_destroy(dev, flow); 6988 if (flow->tunnel) { 6989 struct mlx5_flow_tunnel *tunnel; 6990 6991 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6992 RTE_VERIFY(tunnel); 6993 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6994 mlx5_flow_tunnel_free(dev, tunnel); 6995 } 6996 flow_mreg_del_copy_action(dev, flow); 6997 mlx5_ipool_free(priv->flows[type], flow_idx); 6998 } 6999 7000 /** 7001 * Destroy all flows. 7002 * 7003 * @param dev 7004 * Pointer to Ethernet device. 7005 * @param type 7006 * Flow type to be flushed. 7007 * @param active 7008 * If flushing is called actively. 7009 */ 7010 void 7011 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7012 bool active) 7013 { 7014 struct mlx5_priv *priv = dev->data->dev_private; 7015 uint32_t num_flushed = 0, fidx = 1; 7016 struct rte_flow *flow; 7017 7018 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7019 if (priv->sh->config.dv_flow_en == 2 && 7020 type == MLX5_FLOW_TYPE_GEN) { 7021 flow_hw_q_flow_flush(dev, NULL); 7022 return; 7023 } 7024 #endif 7025 7026 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7027 flow_list_destroy(dev, type, fidx); 7028 num_flushed++; 7029 } 7030 if (active) { 7031 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7032 dev->data->port_id, num_flushed); 7033 } 7034 } 7035 7036 /** 7037 * Stop all default actions for flows. 7038 * 7039 * @param dev 7040 * Pointer to Ethernet device. 7041 */ 7042 void 7043 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7044 { 7045 flow_mreg_del_default_copy_action(dev); 7046 flow_rxq_flags_clear(dev); 7047 } 7048 7049 /** 7050 * Start all default actions for flows. 7051 * 7052 * @param dev 7053 * Pointer to Ethernet device. 7054 * @return 7055 * 0 on success, a negative errno value otherwise and rte_errno is set. 7056 */ 7057 int 7058 mlx5_flow_start_default(struct rte_eth_dev *dev) 7059 { 7060 struct rte_flow_error error; 7061 7062 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7063 return flow_mreg_add_default_copy_action(dev, &error); 7064 } 7065 7066 /** 7067 * Release key of thread specific flow workspace data. 7068 */ 7069 void 7070 flow_release_workspace(void *data) 7071 { 7072 struct mlx5_flow_workspace *wks = data; 7073 struct mlx5_flow_workspace *next; 7074 7075 while (wks) { 7076 next = wks->next; 7077 free(wks->rss_desc.queue); 7078 free(wks); 7079 wks = next; 7080 } 7081 } 7082 7083 /** 7084 * Get thread specific current flow workspace. 7085 * 7086 * @return pointer to thread specific flow workspace data, NULL on error. 7087 */ 7088 struct mlx5_flow_workspace* 7089 mlx5_flow_get_thread_workspace(void) 7090 { 7091 struct mlx5_flow_workspace *data; 7092 7093 data = mlx5_flow_os_get_specific_workspace(); 7094 MLX5_ASSERT(data && data->inuse); 7095 if (!data || !data->inuse) 7096 DRV_LOG(ERR, "flow workspace not initialized."); 7097 return data; 7098 } 7099 7100 /** 7101 * Allocate and init new flow workspace. 7102 * 7103 * @return pointer to flow workspace data, NULL on error. 7104 */ 7105 static struct mlx5_flow_workspace* 7106 flow_alloc_thread_workspace(void) 7107 { 7108 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7109 7110 if (!data) { 7111 DRV_LOG(ERR, "Failed to allocate flow workspace " 7112 "memory."); 7113 return NULL; 7114 } 7115 data->rss_desc.queue = calloc(1, 7116 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7117 if (!data->rss_desc.queue) 7118 goto err; 7119 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7120 return data; 7121 err: 7122 free(data->rss_desc.queue); 7123 free(data); 7124 return NULL; 7125 } 7126 7127 /** 7128 * Get new thread specific flow workspace. 7129 * 7130 * If current workspace inuse, create new one and set as current. 7131 * 7132 * @return pointer to thread specific flow workspace data, NULL on error. 7133 */ 7134 static struct mlx5_flow_workspace* 7135 mlx5_flow_push_thread_workspace(void) 7136 { 7137 struct mlx5_flow_workspace *curr; 7138 struct mlx5_flow_workspace *data; 7139 7140 curr = mlx5_flow_os_get_specific_workspace(); 7141 if (!curr) { 7142 data = flow_alloc_thread_workspace(); 7143 if (!data) 7144 return NULL; 7145 } else if (!curr->inuse) { 7146 data = curr; 7147 } else if (curr->next) { 7148 data = curr->next; 7149 } else { 7150 data = flow_alloc_thread_workspace(); 7151 if (!data) 7152 return NULL; 7153 curr->next = data; 7154 data->prev = curr; 7155 } 7156 data->inuse = 1; 7157 data->flow_idx = 0; 7158 /* Set as current workspace */ 7159 if (mlx5_flow_os_set_specific_workspace(data)) 7160 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7161 return data; 7162 } 7163 7164 /** 7165 * Close current thread specific flow workspace. 7166 * 7167 * If previous workspace available, set it as current. 7168 * 7169 * @return pointer to thread specific flow workspace data, NULL on error. 7170 */ 7171 static void 7172 mlx5_flow_pop_thread_workspace(void) 7173 { 7174 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7175 7176 if (!data) 7177 return; 7178 if (!data->inuse) { 7179 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7180 return; 7181 } 7182 data->inuse = 0; 7183 if (!data->prev) 7184 return; 7185 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7186 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7187 } 7188 7189 /** 7190 * Verify the flow list is empty 7191 * 7192 * @param dev 7193 * Pointer to Ethernet device. 7194 * 7195 * @return the number of flows not released. 7196 */ 7197 int 7198 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7199 { 7200 struct mlx5_priv *priv = dev->data->dev_private; 7201 struct rte_flow *flow; 7202 uint32_t idx = 0; 7203 int ret = 0, i; 7204 7205 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7206 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7207 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7208 dev->data->port_id, (void *)flow); 7209 ret++; 7210 } 7211 } 7212 return ret; 7213 } 7214 7215 /** 7216 * Enable default hairpin egress flow. 7217 * 7218 * @param dev 7219 * Pointer to Ethernet device. 7220 * @param queue 7221 * The queue index. 7222 * 7223 * @return 7224 * 0 on success, a negative errno value otherwise and rte_errno is set. 7225 */ 7226 int 7227 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7228 uint32_t queue) 7229 { 7230 const struct rte_flow_attr attr = { 7231 .egress = 1, 7232 .priority = 0, 7233 }; 7234 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7235 .queue = queue, 7236 }; 7237 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7238 .queue = UINT32_MAX, 7239 }; 7240 struct rte_flow_item items[] = { 7241 { 7242 .type = (enum rte_flow_item_type) 7243 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7244 .spec = &queue_spec, 7245 .last = NULL, 7246 .mask = &queue_mask, 7247 }, 7248 { 7249 .type = RTE_FLOW_ITEM_TYPE_END, 7250 }, 7251 }; 7252 struct rte_flow_action_jump jump = { 7253 .group = MLX5_HAIRPIN_TX_TABLE, 7254 }; 7255 struct rte_flow_action actions[2]; 7256 uint32_t flow_idx; 7257 struct rte_flow_error error; 7258 7259 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7260 actions[0].conf = &jump; 7261 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7262 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7263 &attr, items, actions, false, &error); 7264 if (!flow_idx) { 7265 DRV_LOG(DEBUG, 7266 "Failed to create ctrl flow: rte_errno(%d)," 7267 " type(%d), message(%s)", 7268 rte_errno, error.type, 7269 error.message ? error.message : " (no stated reason)"); 7270 return -rte_errno; 7271 } 7272 return 0; 7273 } 7274 7275 /** 7276 * Enable a control flow configured from the control plane. 7277 * 7278 * @param dev 7279 * Pointer to Ethernet device. 7280 * @param eth_spec 7281 * An Ethernet flow spec to apply. 7282 * @param eth_mask 7283 * An Ethernet flow mask to apply. 7284 * @param vlan_spec 7285 * A VLAN flow spec to apply. 7286 * @param vlan_mask 7287 * A VLAN flow mask to apply. 7288 * 7289 * @return 7290 * 0 on success, a negative errno value otherwise and rte_errno is set. 7291 */ 7292 int 7293 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7294 struct rte_flow_item_eth *eth_spec, 7295 struct rte_flow_item_eth *eth_mask, 7296 struct rte_flow_item_vlan *vlan_spec, 7297 struct rte_flow_item_vlan *vlan_mask) 7298 { 7299 struct mlx5_priv *priv = dev->data->dev_private; 7300 const struct rte_flow_attr attr = { 7301 .ingress = 1, 7302 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7303 }; 7304 struct rte_flow_item items[] = { 7305 { 7306 .type = RTE_FLOW_ITEM_TYPE_ETH, 7307 .spec = eth_spec, 7308 .last = NULL, 7309 .mask = eth_mask, 7310 }, 7311 { 7312 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7313 RTE_FLOW_ITEM_TYPE_END, 7314 .spec = vlan_spec, 7315 .last = NULL, 7316 .mask = vlan_mask, 7317 }, 7318 { 7319 .type = RTE_FLOW_ITEM_TYPE_END, 7320 }, 7321 }; 7322 uint16_t queue[priv->reta_idx_n]; 7323 struct rte_flow_action_rss action_rss = { 7324 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7325 .level = 0, 7326 .types = priv->rss_conf.rss_hf, 7327 .key_len = priv->rss_conf.rss_key_len, 7328 .queue_num = priv->reta_idx_n, 7329 .key = priv->rss_conf.rss_key, 7330 .queue = queue, 7331 }; 7332 struct rte_flow_action actions[] = { 7333 { 7334 .type = RTE_FLOW_ACTION_TYPE_RSS, 7335 .conf = &action_rss, 7336 }, 7337 { 7338 .type = RTE_FLOW_ACTION_TYPE_END, 7339 }, 7340 }; 7341 uint32_t flow_idx; 7342 struct rte_flow_error error; 7343 unsigned int i; 7344 7345 if (!priv->reta_idx_n || !priv->rxqs_n) { 7346 return 0; 7347 } 7348 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7349 action_rss.types = 0; 7350 for (i = 0; i != priv->reta_idx_n; ++i) 7351 queue[i] = (*priv->reta_idx)[i]; 7352 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7353 &attr, items, actions, false, &error); 7354 if (!flow_idx) 7355 return -rte_errno; 7356 return 0; 7357 } 7358 7359 /** 7360 * Enable a flow control configured from the control plane. 7361 * 7362 * @param dev 7363 * Pointer to Ethernet device. 7364 * @param eth_spec 7365 * An Ethernet flow spec to apply. 7366 * @param eth_mask 7367 * An Ethernet flow mask to apply. 7368 * 7369 * @return 7370 * 0 on success, a negative errno value otherwise and rte_errno is set. 7371 */ 7372 int 7373 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7374 struct rte_flow_item_eth *eth_spec, 7375 struct rte_flow_item_eth *eth_mask) 7376 { 7377 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7378 } 7379 7380 /** 7381 * Create default miss flow rule matching lacp traffic 7382 * 7383 * @param dev 7384 * Pointer to Ethernet device. 7385 * @param eth_spec 7386 * An Ethernet flow spec to apply. 7387 * 7388 * @return 7389 * 0 on success, a negative errno value otherwise and rte_errno is set. 7390 */ 7391 int 7392 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7393 { 7394 /* 7395 * The LACP matching is done by only using ether type since using 7396 * a multicast dst mac causes kernel to give low priority to this flow. 7397 */ 7398 static const struct rte_flow_item_eth lacp_spec = { 7399 .type = RTE_BE16(0x8809), 7400 }; 7401 static const struct rte_flow_item_eth lacp_mask = { 7402 .type = 0xffff, 7403 }; 7404 const struct rte_flow_attr attr = { 7405 .ingress = 1, 7406 }; 7407 struct rte_flow_item items[] = { 7408 { 7409 .type = RTE_FLOW_ITEM_TYPE_ETH, 7410 .spec = &lacp_spec, 7411 .mask = &lacp_mask, 7412 }, 7413 { 7414 .type = RTE_FLOW_ITEM_TYPE_END, 7415 }, 7416 }; 7417 struct rte_flow_action actions[] = { 7418 { 7419 .type = (enum rte_flow_action_type) 7420 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7421 }, 7422 { 7423 .type = RTE_FLOW_ACTION_TYPE_END, 7424 }, 7425 }; 7426 struct rte_flow_error error; 7427 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7428 &attr, items, actions, 7429 false, &error); 7430 7431 if (!flow_idx) 7432 return -rte_errno; 7433 return 0; 7434 } 7435 7436 /** 7437 * Destroy a flow. 7438 * 7439 * @see rte_flow_destroy() 7440 * @see rte_flow_ops 7441 */ 7442 int 7443 mlx5_flow_destroy(struct rte_eth_dev *dev, 7444 struct rte_flow *flow, 7445 struct rte_flow_error *error __rte_unused) 7446 { 7447 struct mlx5_priv *priv = dev->data->dev_private; 7448 7449 if (priv->sh->config.dv_flow_en == 2) 7450 return rte_flow_error_set(error, ENOTSUP, 7451 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7452 NULL, 7453 "Flow non-Q destruction not supported"); 7454 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7455 (uintptr_t)(void *)flow); 7456 return 0; 7457 } 7458 7459 /** 7460 * Destroy all flows. 7461 * 7462 * @see rte_flow_flush() 7463 * @see rte_flow_ops 7464 */ 7465 int 7466 mlx5_flow_flush(struct rte_eth_dev *dev, 7467 struct rte_flow_error *error __rte_unused) 7468 { 7469 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7470 return 0; 7471 } 7472 7473 /** 7474 * Isolated mode. 7475 * 7476 * @see rte_flow_isolate() 7477 * @see rte_flow_ops 7478 */ 7479 int 7480 mlx5_flow_isolate(struct rte_eth_dev *dev, 7481 int enable, 7482 struct rte_flow_error *error) 7483 { 7484 struct mlx5_priv *priv = dev->data->dev_private; 7485 7486 if (dev->data->dev_started) { 7487 rte_flow_error_set(error, EBUSY, 7488 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7489 NULL, 7490 "port must be stopped first"); 7491 return -rte_errno; 7492 } 7493 priv->isolated = !!enable; 7494 if (enable) 7495 dev->dev_ops = &mlx5_dev_ops_isolate; 7496 else 7497 dev->dev_ops = &mlx5_dev_ops; 7498 7499 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7500 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7501 7502 return 0; 7503 } 7504 7505 /** 7506 * Query a flow. 7507 * 7508 * @see rte_flow_query() 7509 * @see rte_flow_ops 7510 */ 7511 static int 7512 flow_drv_query(struct rte_eth_dev *dev, 7513 uint32_t flow_idx, 7514 const struct rte_flow_action *actions, 7515 void *data, 7516 struct rte_flow_error *error) 7517 { 7518 struct mlx5_priv *priv = dev->data->dev_private; 7519 const struct mlx5_flow_driver_ops *fops; 7520 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7521 flow_idx); 7522 enum mlx5_flow_drv_type ftype; 7523 7524 if (!flow) { 7525 return rte_flow_error_set(error, ENOENT, 7526 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7527 NULL, 7528 "invalid flow handle"); 7529 } 7530 ftype = flow->drv_type; 7531 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7532 fops = flow_get_drv_ops(ftype); 7533 7534 return fops->query(dev, flow, actions, data, error); 7535 } 7536 7537 /** 7538 * Query a flow. 7539 * 7540 * @see rte_flow_query() 7541 * @see rte_flow_ops 7542 */ 7543 int 7544 mlx5_flow_query(struct rte_eth_dev *dev, 7545 struct rte_flow *flow, 7546 const struct rte_flow_action *actions, 7547 void *data, 7548 struct rte_flow_error *error) 7549 { 7550 int ret; 7551 struct mlx5_priv *priv = dev->data->dev_private; 7552 7553 if (priv->sh->config.dv_flow_en == 2) 7554 return rte_flow_error_set(error, ENOTSUP, 7555 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7556 NULL, 7557 "Flow non-Q query not supported"); 7558 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7559 error); 7560 if (ret < 0) 7561 return ret; 7562 return 0; 7563 } 7564 7565 /** 7566 * Get rte_flow callbacks. 7567 * 7568 * @param dev 7569 * Pointer to Ethernet device structure. 7570 * @param ops 7571 * Pointer to operation-specific structure. 7572 * 7573 * @return 0 7574 */ 7575 int 7576 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7577 const struct rte_flow_ops **ops) 7578 { 7579 *ops = &mlx5_flow_ops; 7580 return 0; 7581 } 7582 7583 /** 7584 * Validate meter policy actions. 7585 * Dispatcher for action type specific validation. 7586 * 7587 * @param[in] dev 7588 * Pointer to the Ethernet device structure. 7589 * @param[in] action 7590 * The meter policy action object to validate. 7591 * @param[in] attr 7592 * Attributes of flow to determine steering domain. 7593 * @param[out] is_rss 7594 * Is RSS or not. 7595 * @param[out] domain_bitmap 7596 * Domain bitmap. 7597 * @param[out] is_def_policy 7598 * Is default policy or not. 7599 * @param[out] error 7600 * Perform verbose error reporting if not NULL. Initialized in case of 7601 * error only. 7602 * 7603 * @return 7604 * 0 on success, otherwise negative errno value. 7605 */ 7606 int 7607 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7608 const struct rte_flow_action *actions[RTE_COLORS], 7609 struct rte_flow_attr *attr, 7610 bool *is_rss, 7611 uint8_t *domain_bitmap, 7612 uint8_t *policy_mode, 7613 struct rte_mtr_error *error) 7614 { 7615 const struct mlx5_flow_driver_ops *fops; 7616 7617 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7618 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7619 domain_bitmap, policy_mode, error); 7620 } 7621 7622 /** 7623 * Destroy the meter table set. 7624 * 7625 * @param[in] dev 7626 * Pointer to Ethernet device. 7627 * @param[in] mtr_policy 7628 * Meter policy struct. 7629 */ 7630 void 7631 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7632 struct mlx5_flow_meter_policy *mtr_policy) 7633 { 7634 const struct mlx5_flow_driver_ops *fops; 7635 7636 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7637 fops->destroy_mtr_acts(dev, mtr_policy); 7638 } 7639 7640 /** 7641 * Create policy action, lock free, 7642 * (mutex should be acquired by caller). 7643 * Dispatcher for action type specific call. 7644 * 7645 * @param[in] dev 7646 * Pointer to the Ethernet device structure. 7647 * @param[in] mtr_policy 7648 * Meter policy struct. 7649 * @param[in] action 7650 * Action specification used to create meter actions. 7651 * @param[out] error 7652 * Perform verbose error reporting if not NULL. Initialized in case of 7653 * error only. 7654 * 7655 * @return 7656 * 0 on success, otherwise negative errno value. 7657 */ 7658 int 7659 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7660 struct mlx5_flow_meter_policy *mtr_policy, 7661 const struct rte_flow_action *actions[RTE_COLORS], 7662 struct rte_mtr_error *error) 7663 { 7664 const struct mlx5_flow_driver_ops *fops; 7665 7666 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7667 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7668 } 7669 7670 /** 7671 * Create policy rules, lock free, 7672 * (mutex should be acquired by caller). 7673 * Dispatcher for action type specific call. 7674 * 7675 * @param[in] dev 7676 * Pointer to the Ethernet device structure. 7677 * @param[in] mtr_policy 7678 * Meter policy struct. 7679 * 7680 * @return 7681 * 0 on success, -1 otherwise. 7682 */ 7683 int 7684 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7685 struct mlx5_flow_meter_policy *mtr_policy) 7686 { 7687 const struct mlx5_flow_driver_ops *fops; 7688 7689 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7690 return fops->create_policy_rules(dev, mtr_policy); 7691 } 7692 7693 /** 7694 * Destroy policy rules, lock free, 7695 * (mutex should be acquired by caller). 7696 * Dispatcher for action type specific call. 7697 * 7698 * @param[in] dev 7699 * Pointer to the Ethernet device structure. 7700 * @param[in] mtr_policy 7701 * Meter policy struct. 7702 */ 7703 void 7704 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7705 struct mlx5_flow_meter_policy *mtr_policy) 7706 { 7707 const struct mlx5_flow_driver_ops *fops; 7708 7709 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7710 fops->destroy_policy_rules(dev, mtr_policy); 7711 } 7712 7713 /** 7714 * Destroy the default policy table set. 7715 * 7716 * @param[in] dev 7717 * Pointer to Ethernet device. 7718 */ 7719 void 7720 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7721 { 7722 const struct mlx5_flow_driver_ops *fops; 7723 7724 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7725 fops->destroy_def_policy(dev); 7726 } 7727 7728 /** 7729 * Destroy the default policy table set. 7730 * 7731 * @param[in] dev 7732 * Pointer to Ethernet device. 7733 * 7734 * @return 7735 * 0 on success, -1 otherwise. 7736 */ 7737 int 7738 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7739 { 7740 const struct mlx5_flow_driver_ops *fops; 7741 7742 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7743 return fops->create_def_policy(dev); 7744 } 7745 7746 /** 7747 * Create the needed meter and suffix tables. 7748 * 7749 * @param[in] dev 7750 * Pointer to Ethernet device. 7751 * 7752 * @return 7753 * 0 on success, -1 otherwise. 7754 */ 7755 int 7756 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7757 struct mlx5_flow_meter_info *fm, 7758 uint32_t mtr_idx, 7759 uint8_t domain_bitmap) 7760 { 7761 const struct mlx5_flow_driver_ops *fops; 7762 7763 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7764 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7765 } 7766 7767 /** 7768 * Destroy the meter table set. 7769 * 7770 * @param[in] dev 7771 * Pointer to Ethernet device. 7772 * @param[in] tbl 7773 * Pointer to the meter table set. 7774 */ 7775 void 7776 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7777 struct mlx5_flow_meter_info *fm) 7778 { 7779 const struct mlx5_flow_driver_ops *fops; 7780 7781 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7782 fops->destroy_mtr_tbls(dev, fm); 7783 } 7784 7785 /** 7786 * Destroy the global meter drop table. 7787 * 7788 * @param[in] dev 7789 * Pointer to Ethernet device. 7790 */ 7791 void 7792 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7793 { 7794 const struct mlx5_flow_driver_ops *fops; 7795 7796 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7797 fops->destroy_mtr_drop_tbls(dev); 7798 } 7799 7800 /** 7801 * Destroy the sub policy table with RX queue. 7802 * 7803 * @param[in] dev 7804 * Pointer to Ethernet device. 7805 * @param[in] mtr_policy 7806 * Pointer to meter policy table. 7807 */ 7808 void 7809 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7810 struct mlx5_flow_meter_policy *mtr_policy) 7811 { 7812 const struct mlx5_flow_driver_ops *fops; 7813 7814 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7815 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7816 } 7817 7818 /** 7819 * Allocate the needed aso flow meter id. 7820 * 7821 * @param[in] dev 7822 * Pointer to Ethernet device. 7823 * 7824 * @return 7825 * Index to aso flow meter on success, NULL otherwise. 7826 */ 7827 uint32_t 7828 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7829 { 7830 const struct mlx5_flow_driver_ops *fops; 7831 7832 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7833 return fops->create_meter(dev); 7834 } 7835 7836 /** 7837 * Free the aso flow meter id. 7838 * 7839 * @param[in] dev 7840 * Pointer to Ethernet device. 7841 * @param[in] mtr_idx 7842 * Index to aso flow meter to be free. 7843 * 7844 * @return 7845 * 0 on success. 7846 */ 7847 void 7848 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7849 { 7850 const struct mlx5_flow_driver_ops *fops; 7851 7852 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7853 fops->free_meter(dev, mtr_idx); 7854 } 7855 7856 /** 7857 * Allocate a counter. 7858 * 7859 * @param[in] dev 7860 * Pointer to Ethernet device structure. 7861 * 7862 * @return 7863 * Index to allocated counter on success, 0 otherwise. 7864 */ 7865 uint32_t 7866 mlx5_counter_alloc(struct rte_eth_dev *dev) 7867 { 7868 const struct mlx5_flow_driver_ops *fops; 7869 struct rte_flow_attr attr = { .transfer = 0 }; 7870 7871 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7872 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7873 return fops->counter_alloc(dev); 7874 } 7875 DRV_LOG(ERR, 7876 "port %u counter allocate is not supported.", 7877 dev->data->port_id); 7878 return 0; 7879 } 7880 7881 /** 7882 * Free a counter. 7883 * 7884 * @param[in] dev 7885 * Pointer to Ethernet device structure. 7886 * @param[in] cnt 7887 * Index to counter to be free. 7888 */ 7889 void 7890 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7891 { 7892 const struct mlx5_flow_driver_ops *fops; 7893 struct rte_flow_attr attr = { .transfer = 0 }; 7894 7895 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7896 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7897 fops->counter_free(dev, cnt); 7898 return; 7899 } 7900 DRV_LOG(ERR, 7901 "port %u counter free is not supported.", 7902 dev->data->port_id); 7903 } 7904 7905 /** 7906 * Query counter statistics. 7907 * 7908 * @param[in] dev 7909 * Pointer to Ethernet device structure. 7910 * @param[in] cnt 7911 * Index to counter to query. 7912 * @param[in] clear 7913 * Set to clear counter statistics. 7914 * @param[out] pkts 7915 * The counter hits packets number to save. 7916 * @param[out] bytes 7917 * The counter hits bytes number to save. 7918 * 7919 * @return 7920 * 0 on success, a negative errno value otherwise. 7921 */ 7922 int 7923 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7924 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 7925 { 7926 const struct mlx5_flow_driver_ops *fops; 7927 struct rte_flow_attr attr = { .transfer = 0 }; 7928 7929 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7930 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7931 return fops->counter_query(dev, cnt, clear, pkts, 7932 bytes, action); 7933 } 7934 DRV_LOG(ERR, 7935 "port %u counter query is not supported.", 7936 dev->data->port_id); 7937 return -ENOTSUP; 7938 } 7939 7940 /** 7941 * Get information about HWS pre-configurable resources. 7942 * 7943 * @param[in] dev 7944 * Pointer to the rte_eth_dev structure. 7945 * @param[out] port_info 7946 * Pointer to port information. 7947 * @param[out] queue_info 7948 * Pointer to queue information. 7949 * @param[out] error 7950 * Pointer to error structure. 7951 * 7952 * @return 7953 * 0 on success, a negative errno value otherwise and rte_errno is set. 7954 */ 7955 static int 7956 mlx5_flow_info_get(struct rte_eth_dev *dev, 7957 struct rte_flow_port_info *port_info, 7958 struct rte_flow_queue_info *queue_info, 7959 struct rte_flow_error *error) 7960 { 7961 const struct mlx5_flow_driver_ops *fops; 7962 7963 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7964 return rte_flow_error_set(error, ENOTSUP, 7965 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7966 NULL, 7967 "info get with incorrect steering mode"); 7968 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7969 return fops->info_get(dev, port_info, queue_info, error); 7970 } 7971 7972 /** 7973 * Configure port HWS resources. 7974 * 7975 * @param[in] dev 7976 * Pointer to the rte_eth_dev structure. 7977 * @param[in] port_attr 7978 * Port configuration attributes. 7979 * @param[in] nb_queue 7980 * Number of queue. 7981 * @param[in] queue_attr 7982 * Array that holds attributes for each flow queue. 7983 * @param[out] error 7984 * Pointer to error structure. 7985 * 7986 * @return 7987 * 0 on success, a negative errno value otherwise and rte_errno is set. 7988 */ 7989 static int 7990 mlx5_flow_port_configure(struct rte_eth_dev *dev, 7991 const struct rte_flow_port_attr *port_attr, 7992 uint16_t nb_queue, 7993 const struct rte_flow_queue_attr *queue_attr[], 7994 struct rte_flow_error *error) 7995 { 7996 const struct mlx5_flow_driver_ops *fops; 7997 7998 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7999 return rte_flow_error_set(error, ENOTSUP, 8000 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8001 NULL, 8002 "port configure with incorrect steering mode"); 8003 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8004 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8005 } 8006 8007 /** 8008 * Create flow item template. 8009 * 8010 * @param[in] dev 8011 * Pointer to the rte_eth_dev structure. 8012 * @param[in] attr 8013 * Pointer to the item template attributes. 8014 * @param[in] items 8015 * The template item pattern. 8016 * @param[out] error 8017 * Pointer to error structure. 8018 * 8019 * @return 8020 * 0 on success, a negative errno value otherwise and rte_errno is set. 8021 */ 8022 static struct rte_flow_pattern_template * 8023 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8024 const struct rte_flow_pattern_template_attr *attr, 8025 const struct rte_flow_item items[], 8026 struct rte_flow_error *error) 8027 { 8028 const struct mlx5_flow_driver_ops *fops; 8029 8030 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8031 rte_flow_error_set(error, ENOTSUP, 8032 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8033 NULL, 8034 "pattern create with incorrect steering mode"); 8035 return NULL; 8036 } 8037 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8038 return fops->pattern_template_create(dev, attr, items, error); 8039 } 8040 8041 /** 8042 * Destroy flow item template. 8043 * 8044 * @param[in] dev 8045 * Pointer to the rte_eth_dev structure. 8046 * @param[in] template 8047 * Pointer to the item template to be destroyed. 8048 * @param[out] error 8049 * Pointer to error structure. 8050 * 8051 * @return 8052 * 0 on success, a negative errno value otherwise and rte_errno is set. 8053 */ 8054 static int 8055 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8056 struct rte_flow_pattern_template *template, 8057 struct rte_flow_error *error) 8058 { 8059 const struct mlx5_flow_driver_ops *fops; 8060 8061 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8062 return rte_flow_error_set(error, ENOTSUP, 8063 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8064 NULL, 8065 "pattern destroy with incorrect steering mode"); 8066 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8067 return fops->pattern_template_destroy(dev, template, error); 8068 } 8069 8070 /** 8071 * Create flow item template. 8072 * 8073 * @param[in] dev 8074 * Pointer to the rte_eth_dev structure. 8075 * @param[in] attr 8076 * Pointer to the action template attributes. 8077 * @param[in] actions 8078 * Associated actions (list terminated by the END action). 8079 * @param[in] masks 8080 * List of actions that marks which of the action's member is constant. 8081 * @param[out] error 8082 * Pointer to error structure. 8083 * 8084 * @return 8085 * 0 on success, a negative errno value otherwise and rte_errno is set. 8086 */ 8087 static struct rte_flow_actions_template * 8088 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8089 const struct rte_flow_actions_template_attr *attr, 8090 const struct rte_flow_action actions[], 8091 const struct rte_flow_action masks[], 8092 struct rte_flow_error *error) 8093 { 8094 const struct mlx5_flow_driver_ops *fops; 8095 8096 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8097 rte_flow_error_set(error, ENOTSUP, 8098 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8099 NULL, 8100 "action create with incorrect steering mode"); 8101 return NULL; 8102 } 8103 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8104 return fops->actions_template_create(dev, attr, actions, masks, error); 8105 } 8106 8107 /** 8108 * Destroy flow action template. 8109 * 8110 * @param[in] dev 8111 * Pointer to the rte_eth_dev structure. 8112 * @param[in] template 8113 * Pointer to the action template to be destroyed. 8114 * @param[out] error 8115 * Pointer to error structure. 8116 * 8117 * @return 8118 * 0 on success, a negative errno value otherwise and rte_errno is set. 8119 */ 8120 static int 8121 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8122 struct rte_flow_actions_template *template, 8123 struct rte_flow_error *error) 8124 { 8125 const struct mlx5_flow_driver_ops *fops; 8126 8127 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8128 return rte_flow_error_set(error, ENOTSUP, 8129 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8130 NULL, 8131 "action destroy with incorrect steering mode"); 8132 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8133 return fops->actions_template_destroy(dev, template, error); 8134 } 8135 8136 /** 8137 * Create flow table. 8138 * 8139 * @param[in] dev 8140 * Pointer to the rte_eth_dev structure. 8141 * @param[in] attr 8142 * Pointer to the table attributes. 8143 * @param[in] item_templates 8144 * Item template array to be binded to the table. 8145 * @param[in] nb_item_templates 8146 * Number of item template. 8147 * @param[in] action_templates 8148 * Action template array to be binded to the table. 8149 * @param[in] nb_action_templates 8150 * Number of action template. 8151 * @param[out] error 8152 * Pointer to error structure. 8153 * 8154 * @return 8155 * Table on success, NULL otherwise and rte_errno is set. 8156 */ 8157 static struct rte_flow_template_table * 8158 mlx5_flow_table_create(struct rte_eth_dev *dev, 8159 const struct rte_flow_template_table_attr *attr, 8160 struct rte_flow_pattern_template *item_templates[], 8161 uint8_t nb_item_templates, 8162 struct rte_flow_actions_template *action_templates[], 8163 uint8_t nb_action_templates, 8164 struct rte_flow_error *error) 8165 { 8166 const struct mlx5_flow_driver_ops *fops; 8167 8168 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8169 rte_flow_error_set(error, ENOTSUP, 8170 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8171 NULL, 8172 "table create with incorrect steering mode"); 8173 return NULL; 8174 } 8175 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8176 return fops->template_table_create(dev, 8177 attr, 8178 item_templates, 8179 nb_item_templates, 8180 action_templates, 8181 nb_action_templates, 8182 error); 8183 } 8184 8185 /** 8186 * PMD destroy flow table. 8187 * 8188 * @param[in] dev 8189 * Pointer to the rte_eth_dev structure. 8190 * @param[in] table 8191 * Pointer to the table to be destroyed. 8192 * @param[out] error 8193 * Pointer to error structure. 8194 * 8195 * @return 8196 * 0 on success, a negative errno value otherwise and rte_errno is set. 8197 */ 8198 static int 8199 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8200 struct rte_flow_template_table *table, 8201 struct rte_flow_error *error) 8202 { 8203 const struct mlx5_flow_driver_ops *fops; 8204 8205 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8206 return rte_flow_error_set(error, ENOTSUP, 8207 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8208 NULL, 8209 "table destroy with incorrect steering mode"); 8210 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8211 return fops->template_table_destroy(dev, table, error); 8212 } 8213 8214 /** 8215 * Enqueue flow creation. 8216 * 8217 * @param[in] dev 8218 * Pointer to the rte_eth_dev structure. 8219 * @param[in] queue_id 8220 * The queue to create the flow. 8221 * @param[in] attr 8222 * Pointer to the flow operation attributes. 8223 * @param[in] items 8224 * Items with flow spec value. 8225 * @param[in] pattern_template_index 8226 * The item pattern flow follows from the table. 8227 * @param[in] actions 8228 * Action with flow spec value. 8229 * @param[in] action_template_index 8230 * The action pattern flow follows from the table. 8231 * @param[in] user_data 8232 * Pointer to the user_data. 8233 * @param[out] error 8234 * Pointer to error structure. 8235 * 8236 * @return 8237 * Flow pointer on success, NULL otherwise and rte_errno is set. 8238 */ 8239 static struct rte_flow * 8240 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8241 uint32_t queue_id, 8242 const struct rte_flow_op_attr *attr, 8243 struct rte_flow_template_table *table, 8244 const struct rte_flow_item items[], 8245 uint8_t pattern_template_index, 8246 const struct rte_flow_action actions[], 8247 uint8_t action_template_index, 8248 void *user_data, 8249 struct rte_flow_error *error) 8250 { 8251 const struct mlx5_flow_driver_ops *fops; 8252 8253 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8254 rte_flow_error_set(error, ENOTSUP, 8255 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8256 NULL, 8257 "flow_q create with incorrect steering mode"); 8258 return NULL; 8259 } 8260 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8261 return fops->async_flow_create(dev, queue_id, attr, table, 8262 items, pattern_template_index, 8263 actions, action_template_index, 8264 user_data, error); 8265 } 8266 8267 /** 8268 * Enqueue flow destruction. 8269 * 8270 * @param[in] dev 8271 * Pointer to the rte_eth_dev structure. 8272 * @param[in] queue 8273 * The queue to destroy the flow. 8274 * @param[in] attr 8275 * Pointer to the flow operation attributes. 8276 * @param[in] flow 8277 * Pointer to the flow to be destroyed. 8278 * @param[in] user_data 8279 * Pointer to the user_data. 8280 * @param[out] error 8281 * Pointer to error structure. 8282 * 8283 * @return 8284 * 0 on success, negative value otherwise and rte_errno is set. 8285 */ 8286 static int 8287 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8288 uint32_t queue, 8289 const struct rte_flow_op_attr *attr, 8290 struct rte_flow *flow, 8291 void *user_data, 8292 struct rte_flow_error *error) 8293 { 8294 const struct mlx5_flow_driver_ops *fops; 8295 8296 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8297 return rte_flow_error_set(error, ENOTSUP, 8298 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8299 NULL, 8300 "flow_q destroy with incorrect steering mode"); 8301 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8302 return fops->async_flow_destroy(dev, queue, attr, flow, 8303 user_data, error); 8304 } 8305 8306 /** 8307 * Pull the enqueued flows. 8308 * 8309 * @param[in] dev 8310 * Pointer to the rte_eth_dev structure. 8311 * @param[in] queue 8312 * The queue to pull the result. 8313 * @param[in/out] res 8314 * Array to save the results. 8315 * @param[in] n_res 8316 * Available result with the array. 8317 * @param[out] error 8318 * Pointer to error structure. 8319 * 8320 * @return 8321 * Result number on success, negative value otherwise and rte_errno is set. 8322 */ 8323 static int 8324 mlx5_flow_pull(struct rte_eth_dev *dev, 8325 uint32_t queue, 8326 struct rte_flow_op_result res[], 8327 uint16_t n_res, 8328 struct rte_flow_error *error) 8329 { 8330 const struct mlx5_flow_driver_ops *fops; 8331 8332 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8333 return rte_flow_error_set(error, ENOTSUP, 8334 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8335 NULL, 8336 "flow_q pull with incorrect steering mode"); 8337 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8338 return fops->pull(dev, queue, res, n_res, error); 8339 } 8340 8341 /** 8342 * Push the enqueued flows. 8343 * 8344 * @param[in] dev 8345 * Pointer to the rte_eth_dev structure. 8346 * @param[in] queue 8347 * The queue to push the flows. 8348 * @param[out] error 8349 * Pointer to error structure. 8350 * 8351 * @return 8352 * 0 on success, negative value otherwise and rte_errno is set. 8353 */ 8354 static int 8355 mlx5_flow_push(struct rte_eth_dev *dev, 8356 uint32_t queue, 8357 struct rte_flow_error *error) 8358 { 8359 const struct mlx5_flow_driver_ops *fops; 8360 8361 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8362 return rte_flow_error_set(error, ENOTSUP, 8363 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8364 NULL, 8365 "flow_q push with incorrect steering mode"); 8366 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8367 return fops->push(dev, queue, error); 8368 } 8369 8370 /** 8371 * Allocate a new memory for the counter values wrapped by all the needed 8372 * management. 8373 * 8374 * @param[in] sh 8375 * Pointer to mlx5_dev_ctx_shared object. 8376 * 8377 * @return 8378 * 0 on success, a negative errno value otherwise. 8379 */ 8380 static int 8381 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8382 { 8383 struct mlx5_counter_stats_mem_mng *mem_mng; 8384 volatile struct flow_counter_stats *raw_data; 8385 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8386 int size = (sizeof(struct flow_counter_stats) * 8387 MLX5_COUNTERS_PER_POOL + 8388 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8389 sizeof(struct mlx5_counter_stats_mem_mng); 8390 size_t pgsize = rte_mem_page_size(); 8391 uint8_t *mem; 8392 int ret; 8393 int i; 8394 8395 if (pgsize == (size_t)-1) { 8396 DRV_LOG(ERR, "Failed to get mem page size"); 8397 rte_errno = ENOMEM; 8398 return -ENOMEM; 8399 } 8400 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8401 if (!mem) { 8402 rte_errno = ENOMEM; 8403 return -ENOMEM; 8404 } 8405 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8406 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8407 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8408 sh->cdev->pdn, mem, size, 8409 &mem_mng->wm); 8410 if (ret) { 8411 rte_errno = errno; 8412 mlx5_free(mem); 8413 return -rte_errno; 8414 } 8415 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8416 raw_data = (volatile struct flow_counter_stats *)mem; 8417 for (i = 0; i < raws_n; ++i) { 8418 mem_mng->raws[i].mem_mng = mem_mng; 8419 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8420 } 8421 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8422 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 8423 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8424 next); 8425 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 8426 sh->cmng.mem_mng = mem_mng; 8427 return 0; 8428 } 8429 8430 /** 8431 * Set the statistic memory to the new counter pool. 8432 * 8433 * @param[in] sh 8434 * Pointer to mlx5_dev_ctx_shared object. 8435 * @param[in] pool 8436 * Pointer to the pool to set the statistic memory. 8437 * 8438 * @return 8439 * 0 on success, a negative errno value otherwise. 8440 */ 8441 static int 8442 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 8443 struct mlx5_flow_counter_pool *pool) 8444 { 8445 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8446 /* Resize statistic memory once used out. */ 8447 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 8448 mlx5_flow_create_counter_stat_mem_mng(sh)) { 8449 DRV_LOG(ERR, "Cannot resize counter stat mem."); 8450 return -1; 8451 } 8452 rte_spinlock_lock(&pool->sl); 8453 pool->raw = cmng->mem_mng->raws + pool->index % 8454 MLX5_CNT_CONTAINER_RESIZE; 8455 rte_spinlock_unlock(&pool->sl); 8456 pool->raw_hw = NULL; 8457 return 0; 8458 } 8459 8460 #define MLX5_POOL_QUERY_FREQ_US 1000000 8461 8462 /** 8463 * Set the periodic procedure for triggering asynchronous batch queries for all 8464 * the counter pools. 8465 * 8466 * @param[in] sh 8467 * Pointer to mlx5_dev_ctx_shared object. 8468 */ 8469 void 8470 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 8471 { 8472 uint32_t pools_n, us; 8473 8474 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 8475 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 8476 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 8477 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 8478 sh->cmng.query_thread_on = 0; 8479 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 8480 } else { 8481 sh->cmng.query_thread_on = 1; 8482 } 8483 } 8484 8485 /** 8486 * The periodic procedure for triggering asynchronous batch queries for all the 8487 * counter pools. This function is probably called by the host thread. 8488 * 8489 * @param[in] arg 8490 * The parameter for the alarm process. 8491 */ 8492 void 8493 mlx5_flow_query_alarm(void *arg) 8494 { 8495 struct mlx5_dev_ctx_shared *sh = arg; 8496 int ret; 8497 uint16_t pool_index = sh->cmng.pool_index; 8498 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8499 struct mlx5_flow_counter_pool *pool; 8500 uint16_t n_valid; 8501 8502 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 8503 goto set_alarm; 8504 rte_spinlock_lock(&cmng->pool_update_sl); 8505 pool = cmng->pools[pool_index]; 8506 n_valid = cmng->n_valid; 8507 rte_spinlock_unlock(&cmng->pool_update_sl); 8508 /* Set the statistic memory to the new created pool. */ 8509 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 8510 goto set_alarm; 8511 if (pool->raw_hw) 8512 /* There is a pool query in progress. */ 8513 goto set_alarm; 8514 pool->raw_hw = 8515 LIST_FIRST(&sh->cmng.free_stat_raws); 8516 if (!pool->raw_hw) 8517 /* No free counter statistics raw memory. */ 8518 goto set_alarm; 8519 /* 8520 * Identify the counters released between query trigger and query 8521 * handle more efficiently. The counter released in this gap period 8522 * should wait for a new round of query as the new arrived packets 8523 * will not be taken into account. 8524 */ 8525 pool->query_gen++; 8526 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 8527 MLX5_COUNTERS_PER_POOL, 8528 NULL, NULL, 8529 pool->raw_hw->mem_mng->wm.lkey, 8530 (void *)(uintptr_t) 8531 pool->raw_hw->data, 8532 sh->devx_comp, 8533 (uint64_t)(uintptr_t)pool); 8534 if (ret) { 8535 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 8536 " %d", pool->min_dcs->id); 8537 pool->raw_hw = NULL; 8538 goto set_alarm; 8539 } 8540 LIST_REMOVE(pool->raw_hw, next); 8541 sh->cmng.pending_queries++; 8542 pool_index++; 8543 if (pool_index >= n_valid) 8544 pool_index = 0; 8545 set_alarm: 8546 sh->cmng.pool_index = pool_index; 8547 mlx5_set_query_alarm(sh); 8548 } 8549 8550 /** 8551 * Check and callback event for new aged flow in the counter pool 8552 * 8553 * @param[in] sh 8554 * Pointer to mlx5_dev_ctx_shared object. 8555 * @param[in] pool 8556 * Pointer to Current counter pool. 8557 */ 8558 static void 8559 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8560 struct mlx5_flow_counter_pool *pool) 8561 { 8562 struct mlx5_priv *priv; 8563 struct mlx5_flow_counter *cnt; 8564 struct mlx5_age_info *age_info; 8565 struct mlx5_age_param *age_param; 8566 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8567 struct mlx5_counter_stats_raw *prev = pool->raw; 8568 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8569 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8570 uint16_t expected = AGE_CANDIDATE; 8571 uint32_t i; 8572 8573 pool->time_of_last_age_check = curr_time; 8574 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8575 cnt = MLX5_POOL_GET_CNT(pool, i); 8576 age_param = MLX5_CNT_TO_AGE(cnt); 8577 if (__atomic_load_n(&age_param->state, 8578 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8579 continue; 8580 if (cur->data[i].hits != prev->data[i].hits) { 8581 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8582 __ATOMIC_RELAXED); 8583 continue; 8584 } 8585 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8586 time_delta, 8587 __ATOMIC_RELAXED) <= age_param->timeout) 8588 continue; 8589 /** 8590 * Hold the lock first, or if between the 8591 * state AGE_TMOUT and tailq operation the 8592 * release happened, the release procedure 8593 * may delete a non-existent tailq node. 8594 */ 8595 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8596 age_info = GET_PORT_AGE_INFO(priv); 8597 rte_spinlock_lock(&age_info->aged_sl); 8598 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8599 AGE_TMOUT, false, 8600 __ATOMIC_RELAXED, 8601 __ATOMIC_RELAXED)) { 8602 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8603 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8604 } 8605 rte_spinlock_unlock(&age_info->aged_sl); 8606 } 8607 mlx5_age_event_prepare(sh); 8608 } 8609 8610 /** 8611 * Handler for the HW respond about ready values from an asynchronous batch 8612 * query. This function is probably called by the host thread. 8613 * 8614 * @param[in] sh 8615 * The pointer to the shared device context. 8616 * @param[in] async_id 8617 * The Devx async ID. 8618 * @param[in] status 8619 * The status of the completion. 8620 */ 8621 void 8622 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8623 uint64_t async_id, int status) 8624 { 8625 struct mlx5_flow_counter_pool *pool = 8626 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8627 struct mlx5_counter_stats_raw *raw_to_free; 8628 uint8_t query_gen = pool->query_gen ^ 1; 8629 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8630 enum mlx5_counter_type cnt_type = 8631 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8632 MLX5_COUNTER_TYPE_ORIGIN; 8633 8634 if (unlikely(status)) { 8635 raw_to_free = pool->raw_hw; 8636 } else { 8637 raw_to_free = pool->raw; 8638 if (pool->is_aged) 8639 mlx5_flow_aging_check(sh, pool); 8640 rte_spinlock_lock(&pool->sl); 8641 pool->raw = pool->raw_hw; 8642 rte_spinlock_unlock(&pool->sl); 8643 /* Be sure the new raw counters data is updated in memory. */ 8644 rte_io_wmb(); 8645 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8646 rte_spinlock_lock(&cmng->csl[cnt_type]); 8647 TAILQ_CONCAT(&cmng->counters[cnt_type], 8648 &pool->counters[query_gen], next); 8649 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8650 } 8651 } 8652 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8653 pool->raw_hw = NULL; 8654 sh->cmng.pending_queries--; 8655 } 8656 8657 static int 8658 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8659 const struct flow_grp_info *grp_info, 8660 struct rte_flow_error *error) 8661 { 8662 if (grp_info->transfer && grp_info->external && 8663 grp_info->fdb_def_rule) { 8664 if (group == UINT32_MAX) 8665 return rte_flow_error_set 8666 (error, EINVAL, 8667 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8668 NULL, 8669 "group index not supported"); 8670 *table = group + 1; 8671 } else { 8672 *table = group; 8673 } 8674 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8675 return 0; 8676 } 8677 8678 /** 8679 * Translate the rte_flow group index to HW table value. 8680 * 8681 * If tunnel offload is disabled, all group ids converted to flow table 8682 * id using the standard method. 8683 * If tunnel offload is enabled, group id can be converted using the 8684 * standard or tunnel conversion method. Group conversion method 8685 * selection depends on flags in `grp_info` parameter: 8686 * - Internal (grp_info.external == 0) groups conversion uses the 8687 * standard method. 8688 * - Group ids in JUMP action converted with the tunnel conversion. 8689 * - Group id in rule attribute conversion depends on a rule type and 8690 * group id value: 8691 * ** non zero group attributes converted with the tunnel method 8692 * ** zero group attribute in non-tunnel rule is converted using the 8693 * standard method - there's only one root table 8694 * ** zero group attribute in steer tunnel rule is converted with the 8695 * standard method - single root table 8696 * ** zero group attribute in match tunnel rule is a special OvS 8697 * case: that value is used for portability reasons. That group 8698 * id is converted with the tunnel conversion method. 8699 * 8700 * @param[in] dev 8701 * Port device 8702 * @param[in] tunnel 8703 * PMD tunnel offload object 8704 * @param[in] group 8705 * rte_flow group index value. 8706 * @param[out] table 8707 * HW table value. 8708 * @param[in] grp_info 8709 * flags used for conversion 8710 * @param[out] error 8711 * Pointer to error structure. 8712 * 8713 * @return 8714 * 0 on success, a negative errno value otherwise and rte_errno is set. 8715 */ 8716 int 8717 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8718 const struct mlx5_flow_tunnel *tunnel, 8719 uint32_t group, uint32_t *table, 8720 const struct flow_grp_info *grp_info, 8721 struct rte_flow_error *error) 8722 { 8723 int ret; 8724 bool standard_translation; 8725 8726 if (!grp_info->skip_scale && grp_info->external && 8727 group < MLX5_MAX_TABLES_EXTERNAL) 8728 group *= MLX5_FLOW_TABLE_FACTOR; 8729 if (is_tunnel_offload_active(dev)) { 8730 standard_translation = !grp_info->external || 8731 grp_info->std_tbl_fix; 8732 } else { 8733 standard_translation = true; 8734 } 8735 DRV_LOG(DEBUG, 8736 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8737 dev->data->port_id, group, grp_info->transfer, 8738 grp_info->external, grp_info->fdb_def_rule, 8739 standard_translation ? "STANDARD" : "TUNNEL"); 8740 if (standard_translation) 8741 ret = flow_group_to_table(dev->data->port_id, group, table, 8742 grp_info, error); 8743 else 8744 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8745 table, error); 8746 8747 return ret; 8748 } 8749 8750 /** 8751 * Discover availability of metadata reg_c's. 8752 * 8753 * Iteratively use test flows to check availability. 8754 * 8755 * @param[in] dev 8756 * Pointer to the Ethernet device structure. 8757 * 8758 * @return 8759 * 0 on success, a negative errno value otherwise and rte_errno is set. 8760 */ 8761 int 8762 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8763 { 8764 struct mlx5_priv *priv = dev->data->dev_private; 8765 enum modify_reg idx; 8766 int n = 0; 8767 8768 /* reg_c[0] and reg_c[1] are reserved. */ 8769 priv->sh->flow_mreg_c[n++] = REG_C_0; 8770 priv->sh->flow_mreg_c[n++] = REG_C_1; 8771 /* Discover availability of other reg_c's. */ 8772 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8773 struct rte_flow_attr attr = { 8774 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8775 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8776 .ingress = 1, 8777 }; 8778 struct rte_flow_item items[] = { 8779 [0] = { 8780 .type = RTE_FLOW_ITEM_TYPE_END, 8781 }, 8782 }; 8783 struct rte_flow_action actions[] = { 8784 [0] = { 8785 .type = (enum rte_flow_action_type) 8786 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8787 .conf = &(struct mlx5_flow_action_copy_mreg){ 8788 .src = REG_C_1, 8789 .dst = idx, 8790 }, 8791 }, 8792 [1] = { 8793 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8794 .conf = &(struct rte_flow_action_jump){ 8795 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8796 }, 8797 }, 8798 [2] = { 8799 .type = RTE_FLOW_ACTION_TYPE_END, 8800 }, 8801 }; 8802 uint32_t flow_idx; 8803 struct rte_flow *flow; 8804 struct rte_flow_error error; 8805 8806 if (!priv->sh->config.dv_flow_en) 8807 break; 8808 /* Create internal flow, validation skips copy action. */ 8809 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8810 items, actions, false, &error); 8811 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8812 flow_idx); 8813 if (!flow) 8814 continue; 8815 priv->sh->flow_mreg_c[n++] = idx; 8816 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8817 } 8818 for (; n < MLX5_MREG_C_NUM; ++n) 8819 priv->sh->flow_mreg_c[n] = REG_NON; 8820 priv->sh->metadata_regc_check_flag = 1; 8821 return 0; 8822 } 8823 8824 int 8825 save_dump_file(const uint8_t *data, uint32_t size, 8826 uint32_t type, uint64_t id, void *arg, FILE *file) 8827 { 8828 char line[BUF_SIZE]; 8829 uint32_t out = 0; 8830 uint32_t k; 8831 uint32_t actions_num; 8832 struct rte_flow_query_count *count; 8833 8834 memset(line, 0, BUF_SIZE); 8835 switch (type) { 8836 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8837 actions_num = *(uint32_t *)(arg); 8838 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 8839 type, id, actions_num); 8840 break; 8841 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8842 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 8843 type, id); 8844 break; 8845 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8846 count = (struct rte_flow_query_count *)arg; 8847 fprintf(file, 8848 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 8849 type, id, count->hits, count->bytes); 8850 return 0; 8851 default: 8852 return -1; 8853 } 8854 8855 for (k = 0; k < size; k++) { 8856 /* Make sure we do not overrun the line buffer length. */ 8857 if (out >= BUF_SIZE - 4) { 8858 line[out] = '\0'; 8859 break; 8860 } 8861 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8862 (data[k]) & 0xff); 8863 } 8864 fprintf(file, "%s\n", line); 8865 return 0; 8866 } 8867 8868 int 8869 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8870 struct rte_flow_query_count *count, struct rte_flow_error *error) 8871 { 8872 struct rte_flow_action action[2]; 8873 enum mlx5_flow_drv_type ftype; 8874 const struct mlx5_flow_driver_ops *fops; 8875 8876 if (!flow) { 8877 return rte_flow_error_set(error, ENOENT, 8878 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8879 NULL, 8880 "invalid flow handle"); 8881 } 8882 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8883 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8884 if (flow->counter) { 8885 memset(count, 0, sizeof(struct rte_flow_query_count)); 8886 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8887 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8888 ftype < MLX5_FLOW_TYPE_MAX); 8889 fops = flow_get_drv_ops(ftype); 8890 return fops->query(dev, flow, action, count, error); 8891 } 8892 return -1; 8893 } 8894 8895 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8896 /** 8897 * Dump flow ipool data to file 8898 * 8899 * @param[in] dev 8900 * The pointer to Ethernet device. 8901 * @param[in] file 8902 * A pointer to a file for output. 8903 * @param[out] error 8904 * Perform verbose error reporting if not NULL. PMDs initialize this 8905 * structure in case of error only. 8906 * @return 8907 * 0 on success, a negative value otherwise. 8908 */ 8909 int 8910 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8911 struct rte_flow *flow, FILE *file, 8912 struct rte_flow_error *error) 8913 { 8914 struct mlx5_priv *priv = dev->data->dev_private; 8915 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8916 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8917 uint32_t handle_idx; 8918 struct mlx5_flow_handle *dh; 8919 struct rte_flow_query_count count; 8920 uint32_t actions_num; 8921 const uint8_t *data; 8922 size_t size; 8923 uint64_t id; 8924 uint32_t type; 8925 void *action = NULL; 8926 8927 if (!flow) { 8928 return rte_flow_error_set(error, ENOENT, 8929 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8930 NULL, 8931 "invalid flow handle"); 8932 } 8933 handle_idx = flow->dev_handles; 8934 /* query counter */ 8935 if (flow->counter && 8936 (!mlx5_counter_query(dev, flow->counter, false, 8937 &count.hits, &count.bytes, &action)) && action) { 8938 id = (uint64_t)(uintptr_t)action; 8939 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8940 save_dump_file(NULL, 0, type, 8941 id, (void *)&count, file); 8942 } 8943 8944 while (handle_idx) { 8945 dh = mlx5_ipool_get(priv->sh->ipool 8946 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8947 if (!dh) 8948 continue; 8949 handle_idx = dh->next.next; 8950 8951 /* Get modify_hdr and encap_decap buf from ipools. */ 8952 encap_decap = NULL; 8953 modify_hdr = dh->dvh.modify_hdr; 8954 8955 if (dh->dvh.rix_encap_decap) { 8956 encap_decap = mlx5_ipool_get(priv->sh->ipool 8957 [MLX5_IPOOL_DECAP_ENCAP], 8958 dh->dvh.rix_encap_decap); 8959 } 8960 if (modify_hdr) { 8961 data = (const uint8_t *)modify_hdr->actions; 8962 size = (size_t)(modify_hdr->actions_num) * 8; 8963 id = (uint64_t)(uintptr_t)modify_hdr->action; 8964 actions_num = modify_hdr->actions_num; 8965 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8966 save_dump_file(data, size, type, id, 8967 (void *)(&actions_num), file); 8968 } 8969 if (encap_decap) { 8970 data = encap_decap->buf; 8971 size = encap_decap->size; 8972 id = (uint64_t)(uintptr_t)encap_decap->action; 8973 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8974 save_dump_file(data, size, type, 8975 id, NULL, file); 8976 } 8977 } 8978 return 0; 8979 } 8980 8981 /** 8982 * Dump all flow's encap_decap/modify_hdr/counter data to file 8983 * 8984 * @param[in] dev 8985 * The pointer to Ethernet device. 8986 * @param[in] file 8987 * A pointer to a file for output. 8988 * @param[out] error 8989 * Perform verbose error reporting if not NULL. PMDs initialize this 8990 * structure in case of error only. 8991 * @return 8992 * 0 on success, a negative value otherwise. 8993 */ 8994 static int 8995 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 8996 FILE *file, struct rte_flow_error *error __rte_unused) 8997 { 8998 struct mlx5_priv *priv = dev->data->dev_private; 8999 struct mlx5_dev_ctx_shared *sh = priv->sh; 9000 struct mlx5_hlist *h; 9001 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9002 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9003 struct rte_flow_query_count count; 9004 uint32_t actions_num; 9005 const uint8_t *data; 9006 size_t size; 9007 uint64_t id; 9008 uint32_t type; 9009 uint32_t i; 9010 uint32_t j; 9011 struct mlx5_list_inconst *l_inconst; 9012 struct mlx5_list_entry *e; 9013 int lcore_index; 9014 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 9015 uint32_t max; 9016 void *action; 9017 9018 /* encap_decap hlist is lcore_share, get global core cache. */ 9019 i = MLX5_LIST_GLOBAL; 9020 h = sh->encaps_decaps; 9021 if (h) { 9022 for (j = 0; j <= h->mask; j++) { 9023 l_inconst = &h->buckets[j].l; 9024 if (!l_inconst || !l_inconst->cache[i]) 9025 continue; 9026 9027 e = LIST_FIRST(&l_inconst->cache[i]->h); 9028 while (e) { 9029 encap_decap = 9030 (struct mlx5_flow_dv_encap_decap_resource *)e; 9031 data = encap_decap->buf; 9032 size = encap_decap->size; 9033 id = (uint64_t)(uintptr_t)encap_decap->action; 9034 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9035 save_dump_file(data, size, type, 9036 id, NULL, file); 9037 e = LIST_NEXT(e, next); 9038 } 9039 } 9040 } 9041 9042 /* get modify_hdr */ 9043 h = sh->modify_cmds; 9044 if (h) { 9045 lcore_index = rte_lcore_index(rte_lcore_id()); 9046 if (unlikely(lcore_index == -1)) { 9047 lcore_index = MLX5_LIST_NLCORE; 9048 rte_spinlock_lock(&h->l_const.lcore_lock); 9049 } 9050 i = lcore_index; 9051 9052 for (j = 0; j <= h->mask; j++) { 9053 l_inconst = &h->buckets[j].l; 9054 if (!l_inconst || !l_inconst->cache[i]) 9055 continue; 9056 9057 e = LIST_FIRST(&l_inconst->cache[i]->h); 9058 while (e) { 9059 modify_hdr = 9060 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9061 data = (const uint8_t *)modify_hdr->actions; 9062 size = (size_t)(modify_hdr->actions_num) * 8; 9063 actions_num = modify_hdr->actions_num; 9064 id = (uint64_t)(uintptr_t)modify_hdr->action; 9065 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9066 save_dump_file(data, size, type, id, 9067 (void *)(&actions_num), file); 9068 e = LIST_NEXT(e, next); 9069 } 9070 } 9071 9072 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9073 rte_spinlock_unlock(&h->l_const.lcore_lock); 9074 } 9075 9076 /* get counter */ 9077 MLX5_ASSERT(cmng->n_valid <= cmng->n); 9078 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9079 for (j = 1; j <= max; j++) { 9080 action = NULL; 9081 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9082 &count.bytes, &action)) && action) { 9083 id = (uint64_t)(uintptr_t)action; 9084 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9085 save_dump_file(NULL, 0, type, 9086 id, (void *)&count, file); 9087 } 9088 } 9089 return 0; 9090 } 9091 #endif 9092 9093 /** 9094 * Dump flow raw hw data to file 9095 * 9096 * @param[in] dev 9097 * The pointer to Ethernet device. 9098 * @param[in] file 9099 * A pointer to a file for output. 9100 * @param[out] error 9101 * Perform verbose error reporting if not NULL. PMDs initialize this 9102 * structure in case of error only. 9103 * @return 9104 * 0 on success, a negative value otherwise. 9105 */ 9106 int 9107 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9108 FILE *file, 9109 struct rte_flow_error *error __rte_unused) 9110 { 9111 struct mlx5_priv *priv = dev->data->dev_private; 9112 struct mlx5_dev_ctx_shared *sh = priv->sh; 9113 uint32_t handle_idx; 9114 int ret; 9115 struct mlx5_flow_handle *dh; 9116 struct rte_flow *flow; 9117 9118 if (!sh->config.dv_flow_en) { 9119 if (fputs("device dv flow disabled\n", file) <= 0) 9120 return -errno; 9121 return -ENOTSUP; 9122 } 9123 9124 /* dump all */ 9125 if (!flow_idx) { 9126 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9127 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9128 return -EINVAL; 9129 #endif 9130 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9131 sh->rx_domain, 9132 sh->tx_domain, file); 9133 } 9134 /* dump one */ 9135 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9136 (uintptr_t)(void *)flow_idx); 9137 if (!flow) 9138 return -EINVAL; 9139 9140 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9141 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9142 #endif 9143 handle_idx = flow->dev_handles; 9144 while (handle_idx) { 9145 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9146 handle_idx); 9147 if (!dh) 9148 return -ENOENT; 9149 if (dh->drv_flow) { 9150 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9151 file); 9152 if (ret) 9153 return -ENOENT; 9154 } 9155 handle_idx = dh->next.next; 9156 } 9157 return 0; 9158 } 9159 9160 /** 9161 * Get aged-out flows. 9162 * 9163 * @param[in] dev 9164 * Pointer to the Ethernet device structure. 9165 * @param[in] context 9166 * The address of an array of pointers to the aged-out flows contexts. 9167 * @param[in] nb_countexts 9168 * The length of context array pointers. 9169 * @param[out] error 9170 * Perform verbose error reporting if not NULL. Initialized in case of 9171 * error only. 9172 * 9173 * @return 9174 * how many contexts get in success, otherwise negative errno value. 9175 * if nb_contexts is 0, return the amount of all aged contexts. 9176 * if nb_contexts is not 0 , return the amount of aged flows reported 9177 * in the context array. 9178 */ 9179 int 9180 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9181 uint32_t nb_contexts, struct rte_flow_error *error) 9182 { 9183 const struct mlx5_flow_driver_ops *fops; 9184 struct rte_flow_attr attr = { .transfer = 0 }; 9185 9186 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 9187 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 9188 return fops->get_aged_flows(dev, contexts, nb_contexts, 9189 error); 9190 } 9191 DRV_LOG(ERR, 9192 "port %u get aged flows is not supported.", 9193 dev->data->port_id); 9194 return -ENOTSUP; 9195 } 9196 9197 /* Wrapper for driver action_validate op callback */ 9198 static int 9199 flow_drv_action_validate(struct rte_eth_dev *dev, 9200 const struct rte_flow_indir_action_conf *conf, 9201 const struct rte_flow_action *action, 9202 const struct mlx5_flow_driver_ops *fops, 9203 struct rte_flow_error *error) 9204 { 9205 static const char err_msg[] = "indirect action validation unsupported"; 9206 9207 if (!fops->action_validate) { 9208 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9209 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9210 NULL, err_msg); 9211 return -rte_errno; 9212 } 9213 return fops->action_validate(dev, conf, action, error); 9214 } 9215 9216 /** 9217 * Destroys the shared action by handle. 9218 * 9219 * @param dev 9220 * Pointer to Ethernet device structure. 9221 * @param[in] handle 9222 * Handle for the indirect action object to be destroyed. 9223 * @param[out] error 9224 * Perform verbose error reporting if not NULL. PMDs initialize this 9225 * structure in case of error only. 9226 * 9227 * @return 9228 * 0 on success, a negative errno value otherwise and rte_errno is set. 9229 * 9230 * @note: wrapper for driver action_create op callback. 9231 */ 9232 static int 9233 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9234 struct rte_flow_action_handle *handle, 9235 struct rte_flow_error *error) 9236 { 9237 static const char err_msg[] = "indirect action destruction unsupported"; 9238 struct rte_flow_attr attr = { .transfer = 0 }; 9239 const struct mlx5_flow_driver_ops *fops = 9240 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9241 9242 if (!fops->action_destroy) { 9243 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9244 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9245 NULL, err_msg); 9246 return -rte_errno; 9247 } 9248 return fops->action_destroy(dev, handle, error); 9249 } 9250 9251 /* Wrapper for driver action_destroy op callback */ 9252 static int 9253 flow_drv_action_update(struct rte_eth_dev *dev, 9254 struct rte_flow_action_handle *handle, 9255 const void *update, 9256 const struct mlx5_flow_driver_ops *fops, 9257 struct rte_flow_error *error) 9258 { 9259 static const char err_msg[] = "indirect action update unsupported"; 9260 9261 if (!fops->action_update) { 9262 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9263 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9264 NULL, err_msg); 9265 return -rte_errno; 9266 } 9267 return fops->action_update(dev, handle, update, error); 9268 } 9269 9270 /* Wrapper for driver action_destroy op callback */ 9271 static int 9272 flow_drv_action_query(struct rte_eth_dev *dev, 9273 const struct rte_flow_action_handle *handle, 9274 void *data, 9275 const struct mlx5_flow_driver_ops *fops, 9276 struct rte_flow_error *error) 9277 { 9278 static const char err_msg[] = "indirect action query unsupported"; 9279 9280 if (!fops->action_query) { 9281 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9282 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9283 NULL, err_msg); 9284 return -rte_errno; 9285 } 9286 return fops->action_query(dev, handle, data, error); 9287 } 9288 9289 /** 9290 * Create indirect action for reuse in multiple flow rules. 9291 * 9292 * @param dev 9293 * Pointer to Ethernet device structure. 9294 * @param conf 9295 * Pointer to indirect action object configuration. 9296 * @param[in] action 9297 * Action configuration for indirect action object creation. 9298 * @param[out] error 9299 * Perform verbose error reporting if not NULL. PMDs initialize this 9300 * structure in case of error only. 9301 * @return 9302 * A valid handle in case of success, NULL otherwise and rte_errno is set. 9303 */ 9304 static struct rte_flow_action_handle * 9305 mlx5_action_handle_create(struct rte_eth_dev *dev, 9306 const struct rte_flow_indir_action_conf *conf, 9307 const struct rte_flow_action *action, 9308 struct rte_flow_error *error) 9309 { 9310 static const char err_msg[] = "indirect action creation unsupported"; 9311 struct rte_flow_attr attr = { .transfer = 0 }; 9312 const struct mlx5_flow_driver_ops *fops = 9313 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9314 9315 if (flow_drv_action_validate(dev, conf, action, fops, error)) 9316 return NULL; 9317 if (!fops->action_create) { 9318 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9319 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9320 NULL, err_msg); 9321 return NULL; 9322 } 9323 return fops->action_create(dev, conf, action, error); 9324 } 9325 9326 /** 9327 * Updates inplace the indirect action configuration pointed by *handle* 9328 * with the configuration provided as *update* argument. 9329 * The update of the indirect action configuration effects all flow rules 9330 * reusing the action via handle. 9331 * 9332 * @param dev 9333 * Pointer to Ethernet device structure. 9334 * @param[in] handle 9335 * Handle for the indirect action to be updated. 9336 * @param[in] update 9337 * Action specification used to modify the action pointed by handle. 9338 * *update* could be of same type with the action pointed by the *handle* 9339 * handle argument, or some other structures like a wrapper, depending on 9340 * the indirect action type. 9341 * @param[out] error 9342 * Perform verbose error reporting if not NULL. PMDs initialize this 9343 * structure in case of error only. 9344 * 9345 * @return 9346 * 0 on success, a negative errno value otherwise and rte_errno is set. 9347 */ 9348 static int 9349 mlx5_action_handle_update(struct rte_eth_dev *dev, 9350 struct rte_flow_action_handle *handle, 9351 const void *update, 9352 struct rte_flow_error *error) 9353 { 9354 struct rte_flow_attr attr = { .transfer = 0 }; 9355 const struct mlx5_flow_driver_ops *fops = 9356 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9357 int ret; 9358 9359 ret = flow_drv_action_validate(dev, NULL, 9360 (const struct rte_flow_action *)update, fops, error); 9361 if (ret) 9362 return ret; 9363 return flow_drv_action_update(dev, handle, update, fops, 9364 error); 9365 } 9366 9367 /** 9368 * Query the indirect action by handle. 9369 * 9370 * This function allows retrieving action-specific data such as counters. 9371 * Data is gathered by special action which may be present/referenced in 9372 * more than one flow rule definition. 9373 * 9374 * see @RTE_FLOW_ACTION_TYPE_COUNT 9375 * 9376 * @param dev 9377 * Pointer to Ethernet device structure. 9378 * @param[in] handle 9379 * Handle for the indirect action to query. 9380 * @param[in, out] data 9381 * Pointer to storage for the associated query data type. 9382 * @param[out] error 9383 * Perform verbose error reporting if not NULL. PMDs initialize this 9384 * structure in case of error only. 9385 * 9386 * @return 9387 * 0 on success, a negative errno value otherwise and rte_errno is set. 9388 */ 9389 static int 9390 mlx5_action_handle_query(struct rte_eth_dev *dev, 9391 const struct rte_flow_action_handle *handle, 9392 void *data, 9393 struct rte_flow_error *error) 9394 { 9395 struct rte_flow_attr attr = { .transfer = 0 }; 9396 const struct mlx5_flow_driver_ops *fops = 9397 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9398 9399 return flow_drv_action_query(dev, handle, data, fops, error); 9400 } 9401 9402 /** 9403 * Destroy all indirect actions (shared RSS). 9404 * 9405 * @param dev 9406 * Pointer to Ethernet device. 9407 * 9408 * @return 9409 * 0 on success, a negative errno value otherwise and rte_errno is set. 9410 */ 9411 int 9412 mlx5_action_handle_flush(struct rte_eth_dev *dev) 9413 { 9414 struct rte_flow_error error; 9415 struct mlx5_priv *priv = dev->data->dev_private; 9416 struct mlx5_shared_action_rss *shared_rss; 9417 int ret = 0; 9418 uint32_t idx; 9419 9420 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 9421 priv->rss_shared_actions, idx, shared_rss, next) { 9422 ret |= mlx5_action_handle_destroy(dev, 9423 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 9424 } 9425 return ret; 9426 } 9427 9428 /** 9429 * Validate existing indirect actions against current device configuration 9430 * and attach them to device resources. 9431 * 9432 * @param dev 9433 * Pointer to Ethernet device. 9434 * 9435 * @return 9436 * 0 on success, a negative errno value otherwise and rte_errno is set. 9437 */ 9438 int 9439 mlx5_action_handle_attach(struct rte_eth_dev *dev) 9440 { 9441 struct mlx5_priv *priv = dev->data->dev_private; 9442 int ret = 0; 9443 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9444 9445 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9446 const char *message; 9447 uint32_t queue_idx; 9448 9449 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 9450 ind_tbl->queues_n, 9451 &message, &queue_idx); 9452 if (ret != 0) { 9453 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 9454 dev->data->port_id, ind_tbl->queues[queue_idx], 9455 message); 9456 break; 9457 } 9458 } 9459 if (ret != 0) 9460 return ret; 9461 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9462 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 9463 if (ret != 0) { 9464 DRV_LOG(ERR, "Port %u could not attach " 9465 "indirection table obj %p", 9466 dev->data->port_id, (void *)ind_tbl); 9467 goto error; 9468 } 9469 } 9470 9471 return 0; 9472 error: 9473 ind_tbl_last = ind_tbl; 9474 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9475 if (ind_tbl == ind_tbl_last) 9476 break; 9477 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 9478 DRV_LOG(CRIT, "Port %u could not detach " 9479 "indirection table obj %p on rollback", 9480 dev->data->port_id, (void *)ind_tbl); 9481 } 9482 return ret; 9483 } 9484 9485 /** 9486 * Detach indirect actions of the device from its resources. 9487 * 9488 * @param dev 9489 * Pointer to Ethernet device. 9490 * 9491 * @return 9492 * 0 on success, a negative errno value otherwise and rte_errno is set. 9493 */ 9494 int 9495 mlx5_action_handle_detach(struct rte_eth_dev *dev) 9496 { 9497 struct mlx5_priv *priv = dev->data->dev_private; 9498 int ret = 0; 9499 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9500 9501 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9502 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 9503 if (ret != 0) { 9504 DRV_LOG(ERR, "Port %u could not detach " 9505 "indirection table obj %p", 9506 dev->data->port_id, (void *)ind_tbl); 9507 goto error; 9508 } 9509 } 9510 return 0; 9511 error: 9512 ind_tbl_last = ind_tbl; 9513 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9514 if (ind_tbl == ind_tbl_last) 9515 break; 9516 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 9517 DRV_LOG(CRIT, "Port %u could not attach " 9518 "indirection table obj %p on rollback", 9519 dev->data->port_id, (void *)ind_tbl); 9520 } 9521 return ret; 9522 } 9523 9524 #ifndef HAVE_MLX5DV_DR 9525 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 9526 #else 9527 #define MLX5_DOMAIN_SYNC_FLOW \ 9528 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 9529 #endif 9530 9531 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 9532 { 9533 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 9534 const struct mlx5_flow_driver_ops *fops; 9535 int ret; 9536 struct rte_flow_attr attr = { .transfer = 0 }; 9537 9538 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9539 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9540 if (ret > 0) 9541 ret = -ret; 9542 return ret; 9543 } 9544 9545 const struct mlx5_flow_tunnel * 9546 mlx5_get_tof(const struct rte_flow_item *item, 9547 const struct rte_flow_action *action, 9548 enum mlx5_tof_rule_type *rule_type) 9549 { 9550 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9551 if (item->type == (typeof(item->type)) 9552 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9553 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9554 return flow_items_to_tunnel(item); 9555 } 9556 } 9557 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9558 if (action->type == (typeof(action->type)) 9559 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9560 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9561 return flow_actions_to_tunnel(action); 9562 } 9563 } 9564 return NULL; 9565 } 9566 9567 /** 9568 * tunnel offload functionality is defined for DV environment only 9569 */ 9570 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9571 __extension__ 9572 union tunnel_offload_mark { 9573 uint32_t val; 9574 struct { 9575 uint32_t app_reserve:8; 9576 uint32_t table_id:15; 9577 uint32_t transfer:1; 9578 uint32_t _unused_:8; 9579 }; 9580 }; 9581 9582 static bool 9583 mlx5_access_tunnel_offload_db 9584 (struct rte_eth_dev *dev, 9585 bool (*match)(struct rte_eth_dev *, 9586 struct mlx5_flow_tunnel *, const void *), 9587 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9588 void (*miss)(struct rte_eth_dev *, void *), 9589 void *ctx, bool lock_op); 9590 9591 static int 9592 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9593 struct rte_flow *flow, 9594 const struct rte_flow_attr *attr, 9595 const struct rte_flow_action *app_actions, 9596 uint32_t flow_idx, 9597 const struct mlx5_flow_tunnel *tunnel, 9598 struct tunnel_default_miss_ctx *ctx, 9599 struct rte_flow_error *error) 9600 { 9601 struct mlx5_priv *priv = dev->data->dev_private; 9602 struct mlx5_flow *dev_flow; 9603 struct rte_flow_attr miss_attr = *attr; 9604 const struct rte_flow_item miss_items[2] = { 9605 { 9606 .type = RTE_FLOW_ITEM_TYPE_ETH, 9607 .spec = NULL, 9608 .last = NULL, 9609 .mask = NULL 9610 }, 9611 { 9612 .type = RTE_FLOW_ITEM_TYPE_END, 9613 .spec = NULL, 9614 .last = NULL, 9615 .mask = NULL 9616 } 9617 }; 9618 union tunnel_offload_mark mark_id; 9619 struct rte_flow_action_mark miss_mark; 9620 struct rte_flow_action miss_actions[3] = { 9621 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9622 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9623 }; 9624 const struct rte_flow_action_jump *jump_data; 9625 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9626 struct flow_grp_info grp_info = { 9627 .external = 1, 9628 .transfer = attr->transfer, 9629 .fdb_def_rule = !!priv->fdb_def_rule, 9630 .std_tbl_fix = 0, 9631 }; 9632 int ret; 9633 9634 if (!attr->transfer) { 9635 uint32_t q_size; 9636 9637 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9638 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9639 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9640 0, SOCKET_ID_ANY); 9641 if (!ctx->queue) 9642 return rte_flow_error_set 9643 (error, ENOMEM, 9644 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9645 NULL, "invalid default miss RSS"); 9646 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9647 ctx->action_rss.level = 0, 9648 ctx->action_rss.types = priv->rss_conf.rss_hf, 9649 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9650 ctx->action_rss.queue_num = priv->reta_idx_n, 9651 ctx->action_rss.key = priv->rss_conf.rss_key, 9652 ctx->action_rss.queue = ctx->queue; 9653 if (!priv->reta_idx_n || !priv->rxqs_n) 9654 return rte_flow_error_set 9655 (error, EINVAL, 9656 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9657 NULL, "invalid port configuration"); 9658 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9659 ctx->action_rss.types = 0; 9660 for (i = 0; i != priv->reta_idx_n; ++i) 9661 ctx->queue[i] = (*priv->reta_idx)[i]; 9662 } else { 9663 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9664 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9665 } 9666 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9667 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9668 jump_data = app_actions->conf; 9669 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9670 miss_attr.group = jump_data->group; 9671 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9672 &flow_table, &grp_info, error); 9673 if (ret) 9674 return rte_flow_error_set(error, EINVAL, 9675 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9676 NULL, "invalid tunnel id"); 9677 mark_id.app_reserve = 0; 9678 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9679 mark_id.transfer = !!attr->transfer; 9680 mark_id._unused_ = 0; 9681 miss_mark.id = mark_id.val; 9682 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9683 miss_items, miss_actions, flow_idx, error); 9684 if (!dev_flow) 9685 return -rte_errno; 9686 dev_flow->flow = flow; 9687 dev_flow->external = true; 9688 dev_flow->tunnel = tunnel; 9689 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9690 /* Subflow object was created, we must include one in the list. */ 9691 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9692 dev_flow->handle, next); 9693 DRV_LOG(DEBUG, 9694 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9695 dev->data->port_id, tunnel->app_tunnel.type, 9696 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9697 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9698 miss_actions, error); 9699 if (!ret) 9700 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9701 error); 9702 9703 return ret; 9704 } 9705 9706 static const struct mlx5_flow_tbl_data_entry * 9707 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9708 { 9709 struct mlx5_priv *priv = dev->data->dev_private; 9710 struct mlx5_dev_ctx_shared *sh = priv->sh; 9711 struct mlx5_list_entry *he; 9712 union tunnel_offload_mark mbits = { .val = mark }; 9713 union mlx5_flow_tbl_key table_key = { 9714 { 9715 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9716 .id = 0, 9717 .reserved = 0, 9718 .dummy = 0, 9719 .is_fdb = !!mbits.transfer, 9720 .is_egress = 0, 9721 } 9722 }; 9723 struct mlx5_flow_cb_ctx ctx = { 9724 .data = &table_key.v64, 9725 }; 9726 9727 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9728 return he ? 9729 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9730 } 9731 9732 static void 9733 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9734 struct mlx5_list_entry *entry) 9735 { 9736 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9737 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9738 9739 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9740 tunnel_flow_tbl_to_id(tte->flow_table)); 9741 mlx5_free(tte); 9742 } 9743 9744 static int 9745 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9746 struct mlx5_list_entry *entry, void *cb_ctx) 9747 { 9748 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9749 union tunnel_tbl_key tbl = { 9750 .val = *(uint64_t *)(ctx->data), 9751 }; 9752 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9753 9754 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9755 } 9756 9757 static struct mlx5_list_entry * 9758 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9759 { 9760 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9761 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9762 struct tunnel_tbl_entry *tte; 9763 union tunnel_tbl_key tbl = { 9764 .val = *(uint64_t *)(ctx->data), 9765 }; 9766 9767 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 9768 sizeof(*tte), 0, 9769 SOCKET_ID_ANY); 9770 if (!tte) 9771 goto err; 9772 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9773 &tte->flow_table); 9774 if (tte->flow_table >= MLX5_MAX_TABLES) { 9775 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 9776 tte->flow_table); 9777 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9778 tte->flow_table); 9779 goto err; 9780 } else if (!tte->flow_table) { 9781 goto err; 9782 } 9783 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 9784 tte->tunnel_id = tbl.tunnel_id; 9785 tte->group = tbl.group; 9786 return &tte->hash; 9787 err: 9788 if (tte) 9789 mlx5_free(tte); 9790 return NULL; 9791 } 9792 9793 static struct mlx5_list_entry * 9794 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 9795 struct mlx5_list_entry *oentry, 9796 void *cb_ctx __rte_unused) 9797 { 9798 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 9799 0, SOCKET_ID_ANY); 9800 9801 if (!tte) 9802 return NULL; 9803 memcpy(tte, oentry, sizeof(*tte)); 9804 return &tte->hash; 9805 } 9806 9807 static void 9808 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 9809 struct mlx5_list_entry *entry) 9810 { 9811 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9812 9813 mlx5_free(tte); 9814 } 9815 9816 static uint32_t 9817 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 9818 const struct mlx5_flow_tunnel *tunnel, 9819 uint32_t group, uint32_t *table, 9820 struct rte_flow_error *error) 9821 { 9822 struct mlx5_list_entry *he; 9823 struct tunnel_tbl_entry *tte; 9824 union tunnel_tbl_key key = { 9825 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 9826 .group = group 9827 }; 9828 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9829 struct mlx5_hlist *group_hash; 9830 struct mlx5_flow_cb_ctx ctx = { 9831 .data = &key.val, 9832 }; 9833 9834 group_hash = tunnel ? tunnel->groups : thub->groups; 9835 he = mlx5_hlist_register(group_hash, key.val, &ctx); 9836 if (!he) 9837 return rte_flow_error_set(error, EINVAL, 9838 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9839 NULL, 9840 "tunnel group index not supported"); 9841 tte = container_of(he, typeof(*tte), hash); 9842 *table = tte->flow_table; 9843 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 9844 dev->data->port_id, key.tunnel_id, group, *table); 9845 return 0; 9846 } 9847 9848 static void 9849 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 9850 struct mlx5_flow_tunnel *tunnel) 9851 { 9852 struct mlx5_priv *priv = dev->data->dev_private; 9853 struct mlx5_indexed_pool *ipool; 9854 9855 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 9856 dev->data->port_id, tunnel->tunnel_id); 9857 LIST_REMOVE(tunnel, chain); 9858 mlx5_hlist_destroy(tunnel->groups); 9859 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9860 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9861 } 9862 9863 static bool 9864 mlx5_access_tunnel_offload_db 9865 (struct rte_eth_dev *dev, 9866 bool (*match)(struct rte_eth_dev *, 9867 struct mlx5_flow_tunnel *, const void *), 9868 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9869 void (*miss)(struct rte_eth_dev *, void *), 9870 void *ctx, bool lock_op) 9871 { 9872 bool verdict = false; 9873 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9874 struct mlx5_flow_tunnel *tunnel; 9875 9876 rte_spinlock_lock(&thub->sl); 9877 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 9878 verdict = match(dev, tunnel, (const void *)ctx); 9879 if (verdict) 9880 break; 9881 } 9882 if (!lock_op) 9883 rte_spinlock_unlock(&thub->sl); 9884 if (verdict && hit) 9885 hit(dev, tunnel, ctx); 9886 if (!verdict && miss) 9887 miss(dev, ctx); 9888 if (lock_op) 9889 rte_spinlock_unlock(&thub->sl); 9890 9891 return verdict; 9892 } 9893 9894 struct tunnel_db_find_tunnel_id_ctx { 9895 uint32_t tunnel_id; 9896 struct mlx5_flow_tunnel *tunnel; 9897 }; 9898 9899 static bool 9900 find_tunnel_id_match(struct rte_eth_dev *dev, 9901 struct mlx5_flow_tunnel *tunnel, const void *x) 9902 { 9903 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9904 9905 RTE_SET_USED(dev); 9906 return tunnel->tunnel_id == ctx->tunnel_id; 9907 } 9908 9909 static void 9910 find_tunnel_id_hit(struct rte_eth_dev *dev, 9911 struct mlx5_flow_tunnel *tunnel, void *x) 9912 { 9913 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9914 RTE_SET_USED(dev); 9915 ctx->tunnel = tunnel; 9916 } 9917 9918 static struct mlx5_flow_tunnel * 9919 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 9920 { 9921 struct tunnel_db_find_tunnel_id_ctx ctx = { 9922 .tunnel_id = id, 9923 }; 9924 9925 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 9926 find_tunnel_id_hit, NULL, &ctx, true); 9927 9928 return ctx.tunnel; 9929 } 9930 9931 static struct mlx5_flow_tunnel * 9932 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 9933 const struct rte_flow_tunnel *app_tunnel) 9934 { 9935 struct mlx5_priv *priv = dev->data->dev_private; 9936 struct mlx5_indexed_pool *ipool; 9937 struct mlx5_flow_tunnel *tunnel; 9938 uint32_t id; 9939 9940 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9941 tunnel = mlx5_ipool_zmalloc(ipool, &id); 9942 if (!tunnel) 9943 return NULL; 9944 if (id >= MLX5_MAX_TUNNELS) { 9945 mlx5_ipool_free(ipool, id); 9946 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 9947 return NULL; 9948 } 9949 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 9950 priv->sh, 9951 mlx5_flow_tunnel_grp2tbl_create_cb, 9952 mlx5_flow_tunnel_grp2tbl_match_cb, 9953 mlx5_flow_tunnel_grp2tbl_remove_cb, 9954 mlx5_flow_tunnel_grp2tbl_clone_cb, 9955 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9956 if (!tunnel->groups) { 9957 mlx5_ipool_free(ipool, id); 9958 return NULL; 9959 } 9960 /* initiate new PMD tunnel */ 9961 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 9962 tunnel->tunnel_id = id; 9963 tunnel->action.type = (typeof(tunnel->action.type)) 9964 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 9965 tunnel->action.conf = tunnel; 9966 tunnel->item.type = (typeof(tunnel->item.type)) 9967 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 9968 tunnel->item.spec = tunnel; 9969 tunnel->item.last = NULL; 9970 tunnel->item.mask = NULL; 9971 9972 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 9973 dev->data->port_id, tunnel->tunnel_id); 9974 9975 return tunnel; 9976 } 9977 9978 struct tunnel_db_get_tunnel_ctx { 9979 const struct rte_flow_tunnel *app_tunnel; 9980 struct mlx5_flow_tunnel *tunnel; 9981 }; 9982 9983 static bool get_tunnel_match(struct rte_eth_dev *dev, 9984 struct mlx5_flow_tunnel *tunnel, const void *x) 9985 { 9986 const struct tunnel_db_get_tunnel_ctx *ctx = x; 9987 9988 RTE_SET_USED(dev); 9989 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 9990 sizeof(*ctx->app_tunnel)); 9991 } 9992 9993 static void get_tunnel_hit(struct rte_eth_dev *dev, 9994 struct mlx5_flow_tunnel *tunnel, void *x) 9995 { 9996 /* called under tunnel spinlock protection */ 9997 struct tunnel_db_get_tunnel_ctx *ctx = x; 9998 9999 RTE_SET_USED(dev); 10000 tunnel->refctn++; 10001 ctx->tunnel = tunnel; 10002 } 10003 10004 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10005 { 10006 /* called under tunnel spinlock protection */ 10007 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10008 struct tunnel_db_get_tunnel_ctx *ctx = x; 10009 10010 rte_spinlock_unlock(&thub->sl); 10011 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10012 rte_spinlock_lock(&thub->sl); 10013 if (ctx->tunnel) { 10014 ctx->tunnel->refctn = 1; 10015 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10016 } 10017 } 10018 10019 10020 static int 10021 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10022 const struct rte_flow_tunnel *app_tunnel, 10023 struct mlx5_flow_tunnel **tunnel) 10024 { 10025 struct tunnel_db_get_tunnel_ctx ctx = { 10026 .app_tunnel = app_tunnel, 10027 }; 10028 10029 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10030 get_tunnel_miss, &ctx, true); 10031 *tunnel = ctx.tunnel; 10032 return ctx.tunnel ? 0 : -ENOMEM; 10033 } 10034 10035 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10036 { 10037 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10038 10039 if (!thub) 10040 return; 10041 if (!LIST_EMPTY(&thub->tunnels)) 10042 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10043 mlx5_hlist_destroy(thub->groups); 10044 mlx5_free(thub); 10045 } 10046 10047 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10048 { 10049 int err; 10050 struct mlx5_flow_tunnel_hub *thub; 10051 10052 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10053 0, SOCKET_ID_ANY); 10054 if (!thub) 10055 return -ENOMEM; 10056 LIST_INIT(&thub->tunnels); 10057 rte_spinlock_init(&thub->sl); 10058 thub->groups = mlx5_hlist_create("flow groups", 64, 10059 false, true, sh, 10060 mlx5_flow_tunnel_grp2tbl_create_cb, 10061 mlx5_flow_tunnel_grp2tbl_match_cb, 10062 mlx5_flow_tunnel_grp2tbl_remove_cb, 10063 mlx5_flow_tunnel_grp2tbl_clone_cb, 10064 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10065 if (!thub->groups) { 10066 err = -rte_errno; 10067 goto err; 10068 } 10069 sh->tunnel_hub = thub; 10070 10071 return 0; 10072 10073 err: 10074 if (thub->groups) 10075 mlx5_hlist_destroy(thub->groups); 10076 if (thub) 10077 mlx5_free(thub); 10078 return err; 10079 } 10080 10081 static inline int 10082 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10083 struct rte_flow_tunnel *tunnel, 10084 struct rte_flow_error *error) 10085 { 10086 struct mlx5_priv *priv = dev->data->dev_private; 10087 10088 if (!priv->sh->config.dv_flow_en) 10089 return rte_flow_error_set(error, ENOTSUP, 10090 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10091 "flow DV interface is off"); 10092 if (!is_tunnel_offload_active(dev)) 10093 return rte_flow_error_set(error, ENOTSUP, 10094 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10095 "tunnel offload was not activated"); 10096 if (!tunnel) 10097 return rte_flow_error_set(error, EINVAL, 10098 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10099 "no application tunnel"); 10100 switch (tunnel->type) { 10101 default: 10102 return rte_flow_error_set(error, EINVAL, 10103 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10104 "unsupported tunnel type"); 10105 case RTE_FLOW_ITEM_TYPE_VXLAN: 10106 case RTE_FLOW_ITEM_TYPE_GRE: 10107 case RTE_FLOW_ITEM_TYPE_NVGRE: 10108 case RTE_FLOW_ITEM_TYPE_GENEVE: 10109 break; 10110 } 10111 return 0; 10112 } 10113 10114 static int 10115 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10116 struct rte_flow_tunnel *app_tunnel, 10117 struct rte_flow_action **actions, 10118 uint32_t *num_of_actions, 10119 struct rte_flow_error *error) 10120 { 10121 struct mlx5_flow_tunnel *tunnel; 10122 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10123 10124 if (ret) 10125 return ret; 10126 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10127 if (ret < 0) { 10128 return rte_flow_error_set(error, ret, 10129 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10130 "failed to initialize pmd tunnel"); 10131 } 10132 *actions = &tunnel->action; 10133 *num_of_actions = 1; 10134 return 0; 10135 } 10136 10137 static int 10138 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10139 struct rte_flow_tunnel *app_tunnel, 10140 struct rte_flow_item **items, 10141 uint32_t *num_of_items, 10142 struct rte_flow_error *error) 10143 { 10144 struct mlx5_flow_tunnel *tunnel; 10145 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10146 10147 if (ret) 10148 return ret; 10149 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10150 if (ret < 0) { 10151 return rte_flow_error_set(error, ret, 10152 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10153 "failed to initialize pmd tunnel"); 10154 } 10155 *items = &tunnel->item; 10156 *num_of_items = 1; 10157 return 0; 10158 } 10159 10160 struct tunnel_db_element_release_ctx { 10161 struct rte_flow_item *items; 10162 struct rte_flow_action *actions; 10163 uint32_t num_elements; 10164 struct rte_flow_error *error; 10165 int ret; 10166 }; 10167 10168 static bool 10169 tunnel_element_release_match(struct rte_eth_dev *dev, 10170 struct mlx5_flow_tunnel *tunnel, const void *x) 10171 { 10172 const struct tunnel_db_element_release_ctx *ctx = x; 10173 10174 RTE_SET_USED(dev); 10175 if (ctx->num_elements != 1) 10176 return false; 10177 else if (ctx->items) 10178 return ctx->items == &tunnel->item; 10179 else if (ctx->actions) 10180 return ctx->actions == &tunnel->action; 10181 10182 return false; 10183 } 10184 10185 static void 10186 tunnel_element_release_hit(struct rte_eth_dev *dev, 10187 struct mlx5_flow_tunnel *tunnel, void *x) 10188 { 10189 struct tunnel_db_element_release_ctx *ctx = x; 10190 ctx->ret = 0; 10191 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10192 mlx5_flow_tunnel_free(dev, tunnel); 10193 } 10194 10195 static void 10196 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10197 { 10198 struct tunnel_db_element_release_ctx *ctx = x; 10199 RTE_SET_USED(dev); 10200 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10201 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10202 "invalid argument"); 10203 } 10204 10205 static int 10206 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10207 struct rte_flow_item *pmd_items, 10208 uint32_t num_items, struct rte_flow_error *err) 10209 { 10210 struct tunnel_db_element_release_ctx ctx = { 10211 .items = pmd_items, 10212 .actions = NULL, 10213 .num_elements = num_items, 10214 .error = err, 10215 }; 10216 10217 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10218 tunnel_element_release_hit, 10219 tunnel_element_release_miss, &ctx, false); 10220 10221 return ctx.ret; 10222 } 10223 10224 static int 10225 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10226 struct rte_flow_action *pmd_actions, 10227 uint32_t num_actions, struct rte_flow_error *err) 10228 { 10229 struct tunnel_db_element_release_ctx ctx = { 10230 .items = NULL, 10231 .actions = pmd_actions, 10232 .num_elements = num_actions, 10233 .error = err, 10234 }; 10235 10236 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10237 tunnel_element_release_hit, 10238 tunnel_element_release_miss, &ctx, false); 10239 10240 return ctx.ret; 10241 } 10242 10243 static int 10244 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10245 struct rte_mbuf *m, 10246 struct rte_flow_restore_info *info, 10247 struct rte_flow_error *err) 10248 { 10249 uint64_t ol_flags = m->ol_flags; 10250 const struct mlx5_flow_tbl_data_entry *tble; 10251 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10252 10253 if (!is_tunnel_offload_active(dev)) { 10254 info->flags = 0; 10255 return 0; 10256 } 10257 10258 if ((ol_flags & mask) != mask) 10259 goto err; 10260 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10261 if (!tble) { 10262 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10263 dev->data->port_id, m->hash.fdir.hi); 10264 goto err; 10265 } 10266 MLX5_ASSERT(tble->tunnel); 10267 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10268 info->group_id = tble->group_id; 10269 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10270 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10271 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10272 10273 return 0; 10274 10275 err: 10276 return rte_flow_error_set(err, EINVAL, 10277 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10278 "failed to get restore info"); 10279 } 10280 10281 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 10282 static int 10283 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 10284 __rte_unused struct rte_flow_tunnel *app_tunnel, 10285 __rte_unused struct rte_flow_action **actions, 10286 __rte_unused uint32_t *num_of_actions, 10287 __rte_unused struct rte_flow_error *error) 10288 { 10289 return -ENOTSUP; 10290 } 10291 10292 static int 10293 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 10294 __rte_unused struct rte_flow_tunnel *app_tunnel, 10295 __rte_unused struct rte_flow_item **items, 10296 __rte_unused uint32_t *num_of_items, 10297 __rte_unused struct rte_flow_error *error) 10298 { 10299 return -ENOTSUP; 10300 } 10301 10302 static int 10303 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 10304 __rte_unused struct rte_flow_item *pmd_items, 10305 __rte_unused uint32_t num_items, 10306 __rte_unused struct rte_flow_error *err) 10307 { 10308 return -ENOTSUP; 10309 } 10310 10311 static int 10312 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 10313 __rte_unused struct rte_flow_action *pmd_action, 10314 __rte_unused uint32_t num_actions, 10315 __rte_unused struct rte_flow_error *err) 10316 { 10317 return -ENOTSUP; 10318 } 10319 10320 static int 10321 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 10322 __rte_unused struct rte_mbuf *m, 10323 __rte_unused struct rte_flow_restore_info *i, 10324 __rte_unused struct rte_flow_error *err) 10325 { 10326 return -ENOTSUP; 10327 } 10328 10329 static int 10330 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 10331 __rte_unused struct rte_flow *flow, 10332 __rte_unused const struct rte_flow_attr *attr, 10333 __rte_unused const struct rte_flow_action *actions, 10334 __rte_unused uint32_t flow_idx, 10335 __rte_unused const struct mlx5_flow_tunnel *tunnel, 10336 __rte_unused struct tunnel_default_miss_ctx *ctx, 10337 __rte_unused struct rte_flow_error *error) 10338 { 10339 return -ENOTSUP; 10340 } 10341 10342 static struct mlx5_flow_tunnel * 10343 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 10344 __rte_unused uint32_t id) 10345 { 10346 return NULL; 10347 } 10348 10349 static void 10350 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 10351 __rte_unused struct mlx5_flow_tunnel *tunnel) 10352 { 10353 } 10354 10355 static uint32_t 10356 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10357 __rte_unused const struct mlx5_flow_tunnel *t, 10358 __rte_unused uint32_t group, 10359 __rte_unused uint32_t *table, 10360 struct rte_flow_error *error) 10361 { 10362 return rte_flow_error_set(error, ENOTSUP, 10363 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10364 "tunnel offload requires DV support"); 10365 } 10366 10367 void 10368 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10369 __rte_unused uint16_t port_id) 10370 { 10371 } 10372 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10373 10374 /* Flex flow item API */ 10375 static struct rte_flow_item_flex_handle * 10376 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10377 const struct rte_flow_item_flex_conf *conf, 10378 struct rte_flow_error *error) 10379 { 10380 static const char err_msg[] = "flex item creation unsupported"; 10381 struct rte_flow_attr attr = { .transfer = 0 }; 10382 const struct mlx5_flow_driver_ops *fops = 10383 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10384 10385 if (!fops->item_create) { 10386 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10387 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10388 NULL, err_msg); 10389 return NULL; 10390 } 10391 return fops->item_create(dev, conf, error); 10392 } 10393 10394 static int 10395 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 10396 const struct rte_flow_item_flex_handle *handle, 10397 struct rte_flow_error *error) 10398 { 10399 static const char err_msg[] = "flex item release unsupported"; 10400 struct rte_flow_attr attr = { .transfer = 0 }; 10401 const struct mlx5_flow_driver_ops *fops = 10402 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10403 10404 if (!fops->item_release) { 10405 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10406 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10407 NULL, err_msg); 10408 return -rte_errno; 10409 } 10410 return fops->item_release(dev, handle, error); 10411 } 10412 10413 static void 10414 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 10415 { 10416 int ret; 10417 struct rte_flow_error error; 10418 10419 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10420 char *item_name; 10421 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 10422 sizeof(item_name), 10423 (void *)(uintptr_t)item->type, &error); 10424 if (ret > 0) 10425 printf("%s ", item_name); 10426 else 10427 printf("%d\n", (int)item->type); 10428 } 10429 printf("END\n"); 10430 } 10431 10432 static int 10433 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 10434 { 10435 const struct rte_flow_item_udp *spec = udp_item->spec; 10436 const struct rte_flow_item_udp *mask = udp_item->mask; 10437 uint16_t udp_dport = 0; 10438 10439 if (spec != NULL) { 10440 if (!mask) 10441 mask = &rte_flow_item_udp_mask; 10442 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 10443 mask->hdr.dst_port); 10444 } 10445 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 10446 } 10447 10448 static const struct mlx5_flow_expand_node * 10449 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 10450 unsigned int item_idx, 10451 const struct mlx5_flow_expand_node graph[], 10452 const struct mlx5_flow_expand_node *node) 10453 { 10454 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 10455 10456 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 10457 node != NULL && 10458 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 10459 /* 10460 * The expansion node is VXLAN and it is also the last 10461 * expandable item in the pattern, so need to continue 10462 * expansion of the inner tunnel. 10463 */ 10464 MLX5_ASSERT(item_idx > 0); 10465 prev_item = pattern + item_idx - 1; 10466 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 10467 if (mlx5_flow_is_std_vxlan_port(prev_item)) 10468 return &graph[MLX5_EXPANSION_STD_VXLAN]; 10469 return &graph[MLX5_EXPANSION_L3_VXLAN]; 10470 } 10471 return node; 10472 } 10473 10474 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 10475 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 10476 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 10477 }; 10478 10479 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 10480 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 10481 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 10482 { 9, 10, 11 }, { 12, 13, 14 }, 10483 }; 10484 10485 /** 10486 * Discover the number of available flow priorities. 10487 * 10488 * @param dev 10489 * Ethernet device. 10490 * 10491 * @return 10492 * On success, number of available flow priorities. 10493 * On failure, a negative errno-style code and rte_errno is set. 10494 */ 10495 int 10496 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 10497 { 10498 static const uint16_t vprio[] = {8, 16}; 10499 const struct mlx5_priv *priv = dev->data->dev_private; 10500 const struct mlx5_flow_driver_ops *fops; 10501 enum mlx5_flow_drv_type type; 10502 int ret; 10503 10504 type = mlx5_flow_os_get_type(); 10505 if (type == MLX5_FLOW_TYPE_MAX) { 10506 type = MLX5_FLOW_TYPE_VERBS; 10507 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 10508 type = MLX5_FLOW_TYPE_DV; 10509 } 10510 fops = flow_get_drv_ops(type); 10511 if (fops->discover_priorities == NULL) { 10512 DRV_LOG(ERR, "Priority discovery not supported"); 10513 rte_errno = ENOTSUP; 10514 return -rte_errno; 10515 } 10516 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 10517 if (ret < 0) 10518 return ret; 10519 switch (ret) { 10520 case 8: 10521 ret = RTE_DIM(priority_map_3); 10522 break; 10523 case 16: 10524 ret = RTE_DIM(priority_map_5); 10525 break; 10526 default: 10527 rte_errno = ENOTSUP; 10528 DRV_LOG(ERR, 10529 "port %u maximum priority: %d expected 8/16", 10530 dev->data->port_id, ret); 10531 return -rte_errno; 10532 } 10533 DRV_LOG(INFO, "port %u supported flow priorities:" 10534 " 0-%d for ingress or egress root table," 10535 " 0-%d for non-root table or transfer root table.", 10536 dev->data->port_id, ret - 2, 10537 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10538 return ret; 10539 } 10540 10541 /** 10542 * Adjust flow priority based on the highest layer and the request priority. 10543 * 10544 * @param[in] dev 10545 * Pointer to the Ethernet device structure. 10546 * @param[in] priority 10547 * The rule base priority. 10548 * @param[in] subpriority 10549 * The priority based on the items. 10550 * 10551 * @return 10552 * The new priority. 10553 */ 10554 uint32_t 10555 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10556 uint32_t subpriority) 10557 { 10558 uint32_t res = 0; 10559 struct mlx5_priv *priv = dev->data->dev_private; 10560 10561 switch (priv->sh->flow_max_priority) { 10562 case RTE_DIM(priority_map_3): 10563 res = priority_map_3[priority][subpriority]; 10564 break; 10565 case RTE_DIM(priority_map_5): 10566 res = priority_map_5[priority][subpriority]; 10567 break; 10568 } 10569 return res; 10570 } 10571