1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see RTE_ETH_RSS_* definitions). 102 */ 103 uint64_t node_flags; 104 /**< 105 * Bit-fields that define how the node is used in the expansion. 106 * (see MLX5_EXPANSION_NODE_* definitions). 107 */ 108 }; 109 110 /* Optional expand field. The expansion alg will not go deeper. */ 111 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 112 113 /* The node is not added implicitly as expansion to the flow pattern. 114 * If the node type does not match the flow pattern item type, the 115 * expansion alg will go deeper to its next items. 116 * In the current implementation, the list of next nodes indexes can 117 * have up to one node with this flag set and it has to be the last 118 * node index (before the list terminator). 119 */ 120 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 121 122 /** Object returned by mlx5_flow_expand_rss(). */ 123 struct mlx5_flow_expand_rss { 124 uint32_t entries; 125 /**< Number of entries @p patterns and @p priorities. */ 126 struct { 127 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 128 uint32_t priority; /**< Priority offset for each expansion. */ 129 } entry[]; 130 }; 131 132 static void 133 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 134 135 static const struct mlx5_flow_expand_node * 136 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 137 unsigned int item_idx, 138 const struct mlx5_flow_expand_node graph[], 139 const struct mlx5_flow_expand_node *node); 140 141 static bool 142 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 143 { 144 switch (item->type) { 145 case RTE_FLOW_ITEM_TYPE_ETH: 146 case RTE_FLOW_ITEM_TYPE_VLAN: 147 case RTE_FLOW_ITEM_TYPE_IPV4: 148 case RTE_FLOW_ITEM_TYPE_IPV6: 149 case RTE_FLOW_ITEM_TYPE_UDP: 150 case RTE_FLOW_ITEM_TYPE_TCP: 151 case RTE_FLOW_ITEM_TYPE_VXLAN: 152 case RTE_FLOW_ITEM_TYPE_NVGRE: 153 case RTE_FLOW_ITEM_TYPE_GRE: 154 case RTE_FLOW_ITEM_TYPE_GENEVE: 155 case RTE_FLOW_ITEM_TYPE_MPLS: 156 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 157 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 158 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 159 case RTE_FLOW_ITEM_TYPE_GTP: 160 return true; 161 default: 162 break; 163 } 164 return false; 165 } 166 167 static enum rte_flow_item_type 168 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 169 { 170 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 171 uint16_t ether_type = 0; 172 uint16_t ether_type_m; 173 uint8_t ip_next_proto = 0; 174 uint8_t ip_next_proto_m; 175 176 if (item == NULL || item->spec == NULL) 177 return ret; 178 switch (item->type) { 179 case RTE_FLOW_ITEM_TYPE_ETH: 180 if (item->mask) 181 ether_type_m = ((const struct rte_flow_item_eth *) 182 (item->mask))->type; 183 else 184 ether_type_m = rte_flow_item_eth_mask.type; 185 if (ether_type_m != RTE_BE16(0xFFFF)) 186 break; 187 ether_type = ((const struct rte_flow_item_eth *) 188 (item->spec))->type; 189 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 190 ret = RTE_FLOW_ITEM_TYPE_IPV4; 191 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 192 ret = RTE_FLOW_ITEM_TYPE_IPV6; 193 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 194 ret = RTE_FLOW_ITEM_TYPE_VLAN; 195 else 196 ret = RTE_FLOW_ITEM_TYPE_END; 197 break; 198 case RTE_FLOW_ITEM_TYPE_VLAN: 199 if (item->mask) 200 ether_type_m = ((const struct rte_flow_item_vlan *) 201 (item->mask))->inner_type; 202 else 203 ether_type_m = rte_flow_item_vlan_mask.inner_type; 204 if (ether_type_m != RTE_BE16(0xFFFF)) 205 break; 206 ether_type = ((const struct rte_flow_item_vlan *) 207 (item->spec))->inner_type; 208 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 209 ret = RTE_FLOW_ITEM_TYPE_IPV4; 210 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 211 ret = RTE_FLOW_ITEM_TYPE_IPV6; 212 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 213 ret = RTE_FLOW_ITEM_TYPE_VLAN; 214 else 215 ret = RTE_FLOW_ITEM_TYPE_END; 216 break; 217 case RTE_FLOW_ITEM_TYPE_IPV4: 218 if (item->mask) 219 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 220 (item->mask))->hdr.next_proto_id; 221 else 222 ip_next_proto_m = 223 rte_flow_item_ipv4_mask.hdr.next_proto_id; 224 if (ip_next_proto_m != 0xFF) 225 break; 226 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 227 (item->spec))->hdr.next_proto_id; 228 if (ip_next_proto == IPPROTO_UDP) 229 ret = RTE_FLOW_ITEM_TYPE_UDP; 230 else if (ip_next_proto == IPPROTO_TCP) 231 ret = RTE_FLOW_ITEM_TYPE_TCP; 232 else if (ip_next_proto == IPPROTO_IP) 233 ret = RTE_FLOW_ITEM_TYPE_IPV4; 234 else if (ip_next_proto == IPPROTO_IPV6) 235 ret = RTE_FLOW_ITEM_TYPE_IPV6; 236 else 237 ret = RTE_FLOW_ITEM_TYPE_END; 238 break; 239 case RTE_FLOW_ITEM_TYPE_IPV6: 240 if (item->mask) 241 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 242 (item->mask))->hdr.proto; 243 else 244 ip_next_proto_m = 245 rte_flow_item_ipv6_mask.hdr.proto; 246 if (ip_next_proto_m != 0xFF) 247 break; 248 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 249 (item->spec))->hdr.proto; 250 if (ip_next_proto == IPPROTO_UDP) 251 ret = RTE_FLOW_ITEM_TYPE_UDP; 252 else if (ip_next_proto == IPPROTO_TCP) 253 ret = RTE_FLOW_ITEM_TYPE_TCP; 254 else if (ip_next_proto == IPPROTO_IP) 255 ret = RTE_FLOW_ITEM_TYPE_IPV4; 256 else if (ip_next_proto == IPPROTO_IPV6) 257 ret = RTE_FLOW_ITEM_TYPE_IPV6; 258 else 259 ret = RTE_FLOW_ITEM_TYPE_END; 260 break; 261 default: 262 ret = RTE_FLOW_ITEM_TYPE_VOID; 263 break; 264 } 265 return ret; 266 } 267 268 static const int * 269 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 270 const int *next_node) 271 { 272 const struct mlx5_flow_expand_node *node = NULL; 273 const int *next = next_node; 274 275 while (next && *next) { 276 /* 277 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 278 * flag set, because they were not found in the flow pattern. 279 */ 280 node = &graph[*next]; 281 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 282 break; 283 next = node->next; 284 } 285 return next; 286 } 287 288 #define MLX5_RSS_EXP_ELT_N 16 289 290 /** 291 * Expand RSS flows into several possible flows according to the RSS hash 292 * fields requested and the driver capabilities. 293 * 294 * @param[out] buf 295 * Buffer to store the result expansion. 296 * @param[in] size 297 * Buffer size in bytes. If 0, @p buf can be NULL. 298 * @param[in] pattern 299 * User flow pattern. 300 * @param[in] types 301 * RSS types to expand (see RTE_ETH_RSS_* definitions). 302 * @param[in] graph 303 * Input graph to expand @p pattern according to @p types. 304 * @param[in] graph_root_index 305 * Index of root node in @p graph, typically 0. 306 * 307 * @return 308 * A positive value representing the size of @p buf in bytes regardless of 309 * @p size on success, a negative errno value otherwise and rte_errno is 310 * set, the following errors are defined: 311 * 312 * -E2BIG: graph-depth @p graph is too deep. 313 * -EINVAL: @p size has not enough space for expanded pattern. 314 */ 315 static int 316 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 317 const struct rte_flow_item *pattern, uint64_t types, 318 const struct mlx5_flow_expand_node graph[], 319 int graph_root_index) 320 { 321 const struct rte_flow_item *item; 322 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 323 const int *next_node; 324 const int *stack[MLX5_RSS_EXP_ELT_N]; 325 int stack_pos = 0; 326 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 327 unsigned int i, item_idx, last_expand_item_idx = 0; 328 size_t lsize; 329 size_t user_pattern_size = 0; 330 void *addr = NULL; 331 const struct mlx5_flow_expand_node *next = NULL; 332 struct rte_flow_item missed_item; 333 int missed = 0; 334 int elt = 0; 335 const struct rte_flow_item *last_expand_item = NULL; 336 337 memset(&missed_item, 0, sizeof(missed_item)); 338 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 339 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 340 if (lsize > size) 341 return -EINVAL; 342 buf->entry[0].priority = 0; 343 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 344 buf->entries = 0; 345 addr = buf->entry[0].pattern; 346 for (item = pattern, item_idx = 0; 347 item->type != RTE_FLOW_ITEM_TYPE_END; 348 item++, item_idx++) { 349 if (!mlx5_flow_is_rss_expandable_item(item)) { 350 user_pattern_size += sizeof(*item); 351 continue; 352 } 353 last_expand_item = item; 354 last_expand_item_idx = item_idx; 355 i = 0; 356 while (node->next && node->next[i]) { 357 next = &graph[node->next[i]]; 358 if (next->type == item->type) 359 break; 360 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 361 node = next; 362 i = 0; 363 } else { 364 ++i; 365 } 366 } 367 if (next) 368 node = next; 369 user_pattern_size += sizeof(*item); 370 } 371 user_pattern_size += sizeof(*item); /* Handle END item. */ 372 lsize += user_pattern_size; 373 if (lsize > size) 374 return -EINVAL; 375 /* Copy the user pattern in the first entry of the buffer. */ 376 rte_memcpy(addr, pattern, user_pattern_size); 377 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 378 buf->entries = 1; 379 /* Start expanding. */ 380 memset(flow_items, 0, sizeof(flow_items)); 381 user_pattern_size -= sizeof(*item); 382 /* 383 * Check if the last valid item has spec set, need complete pattern, 384 * and the pattern can be used for expansion. 385 */ 386 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 387 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 388 /* Item type END indicates expansion is not required. */ 389 return lsize; 390 } 391 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 392 next = NULL; 393 missed = 1; 394 for (i = 0; node->next && node->next[i]; ++i) { 395 next = &graph[node->next[i]]; 396 if (next->type == missed_item.type) { 397 flow_items[0].type = missed_item.type; 398 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 399 break; 400 } 401 next = NULL; 402 } 403 } 404 if (next && missed) { 405 elt = 2; /* missed item + item end. */ 406 node = next; 407 lsize += elt * sizeof(*item) + user_pattern_size; 408 if (lsize > size) 409 return -EINVAL; 410 if (node->rss_types & types) { 411 buf->entry[buf->entries].priority = 1; 412 buf->entry[buf->entries].pattern = addr; 413 buf->entries++; 414 rte_memcpy(addr, buf->entry[0].pattern, 415 user_pattern_size); 416 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 417 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 418 addr = (void *)(((uintptr_t)addr) + 419 elt * sizeof(*item)); 420 } 421 } else if (last_expand_item != NULL) { 422 node = mlx5_flow_expand_rss_adjust_node(pattern, 423 last_expand_item_idx, graph, node); 424 } 425 memset(flow_items, 0, sizeof(flow_items)); 426 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 427 node->next); 428 stack[stack_pos] = next_node; 429 node = next_node ? &graph[*next_node] : NULL; 430 while (node) { 431 flow_items[stack_pos].type = node->type; 432 if (node->rss_types & types) { 433 size_t n; 434 /* 435 * compute the number of items to copy from the 436 * expansion and copy it. 437 * When the stack_pos is 0, there are 1 element in it, 438 * plus the addition END item. 439 */ 440 elt = stack_pos + 2; 441 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 442 lsize += elt * sizeof(*item) + user_pattern_size; 443 if (lsize > size) 444 return -EINVAL; 445 n = elt * sizeof(*item); 446 buf->entry[buf->entries].priority = 447 stack_pos + 1 + missed; 448 buf->entry[buf->entries].pattern = addr; 449 buf->entries++; 450 rte_memcpy(addr, buf->entry[0].pattern, 451 user_pattern_size); 452 addr = (void *)(((uintptr_t)addr) + 453 user_pattern_size); 454 rte_memcpy(addr, &missed_item, 455 missed * sizeof(*item)); 456 addr = (void *)(((uintptr_t)addr) + 457 missed * sizeof(*item)); 458 rte_memcpy(addr, flow_items, n); 459 addr = (void *)(((uintptr_t)addr) + n); 460 } 461 /* Go deeper. */ 462 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 463 node->next) { 464 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 465 node->next); 466 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 467 rte_errno = E2BIG; 468 return -rte_errno; 469 } 470 stack[stack_pos] = next_node; 471 } else if (*(next_node + 1)) { 472 /* Follow up with the next possibility. */ 473 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 474 ++next_node); 475 } else if (!stack_pos) { 476 /* 477 * Completing the traverse over the different paths. 478 * The next_node is advanced to the terminator. 479 */ 480 ++next_node; 481 } else { 482 /* Move to the next path. */ 483 while (stack_pos) { 484 next_node = stack[--stack_pos]; 485 next_node++; 486 if (*next_node) 487 break; 488 } 489 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 490 next_node); 491 stack[stack_pos] = next_node; 492 } 493 node = next_node && *next_node ? &graph[*next_node] : NULL; 494 }; 495 return lsize; 496 } 497 498 enum mlx5_expansion { 499 MLX5_EXPANSION_ROOT, 500 MLX5_EXPANSION_ROOT_OUTER, 501 MLX5_EXPANSION_OUTER_ETH, 502 MLX5_EXPANSION_OUTER_VLAN, 503 MLX5_EXPANSION_OUTER_IPV4, 504 MLX5_EXPANSION_OUTER_IPV4_UDP, 505 MLX5_EXPANSION_OUTER_IPV4_TCP, 506 MLX5_EXPANSION_OUTER_IPV6, 507 MLX5_EXPANSION_OUTER_IPV6_UDP, 508 MLX5_EXPANSION_OUTER_IPV6_TCP, 509 MLX5_EXPANSION_VXLAN, 510 MLX5_EXPANSION_STD_VXLAN, 511 MLX5_EXPANSION_L3_VXLAN, 512 MLX5_EXPANSION_VXLAN_GPE, 513 MLX5_EXPANSION_GRE, 514 MLX5_EXPANSION_NVGRE, 515 MLX5_EXPANSION_GRE_KEY, 516 MLX5_EXPANSION_MPLS, 517 MLX5_EXPANSION_ETH, 518 MLX5_EXPANSION_VLAN, 519 MLX5_EXPANSION_IPV4, 520 MLX5_EXPANSION_IPV4_UDP, 521 MLX5_EXPANSION_IPV4_TCP, 522 MLX5_EXPANSION_IPV6, 523 MLX5_EXPANSION_IPV6_UDP, 524 MLX5_EXPANSION_IPV6_TCP, 525 MLX5_EXPANSION_IPV6_FRAG_EXT, 526 MLX5_EXPANSION_GTP 527 }; 528 529 /** Supported expansion of items. */ 530 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 531 [MLX5_EXPANSION_ROOT] = { 532 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 533 MLX5_EXPANSION_IPV4, 534 MLX5_EXPANSION_IPV6), 535 .type = RTE_FLOW_ITEM_TYPE_END, 536 }, 537 [MLX5_EXPANSION_ROOT_OUTER] = { 538 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 539 MLX5_EXPANSION_OUTER_IPV4, 540 MLX5_EXPANSION_OUTER_IPV6), 541 .type = RTE_FLOW_ITEM_TYPE_END, 542 }, 543 [MLX5_EXPANSION_OUTER_ETH] = { 544 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 545 .type = RTE_FLOW_ITEM_TYPE_ETH, 546 .rss_types = 0, 547 }, 548 [MLX5_EXPANSION_OUTER_VLAN] = { 549 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 550 MLX5_EXPANSION_OUTER_IPV6), 551 .type = RTE_FLOW_ITEM_TYPE_VLAN, 552 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 553 }, 554 [MLX5_EXPANSION_OUTER_IPV4] = { 555 .next = MLX5_FLOW_EXPAND_RSS_NEXT 556 (MLX5_EXPANSION_OUTER_IPV4_UDP, 557 MLX5_EXPANSION_OUTER_IPV4_TCP, 558 MLX5_EXPANSION_GRE, 559 MLX5_EXPANSION_NVGRE, 560 MLX5_EXPANSION_IPV4, 561 MLX5_EXPANSION_IPV6), 562 .type = RTE_FLOW_ITEM_TYPE_IPV4, 563 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 564 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 565 }, 566 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 567 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 568 MLX5_EXPANSION_VXLAN_GPE, 569 MLX5_EXPANSION_MPLS, 570 MLX5_EXPANSION_GTP), 571 .type = RTE_FLOW_ITEM_TYPE_UDP, 572 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 573 }, 574 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 575 .type = RTE_FLOW_ITEM_TYPE_TCP, 576 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 577 }, 578 [MLX5_EXPANSION_OUTER_IPV6] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT 580 (MLX5_EXPANSION_OUTER_IPV6_UDP, 581 MLX5_EXPANSION_OUTER_IPV6_TCP, 582 MLX5_EXPANSION_IPV4, 583 MLX5_EXPANSION_IPV6, 584 MLX5_EXPANSION_GRE, 585 MLX5_EXPANSION_NVGRE), 586 .type = RTE_FLOW_ITEM_TYPE_IPV6, 587 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 588 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 589 }, 590 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 592 MLX5_EXPANSION_VXLAN_GPE, 593 MLX5_EXPANSION_MPLS, 594 MLX5_EXPANSION_GTP), 595 .type = RTE_FLOW_ITEM_TYPE_UDP, 596 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 597 }, 598 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 599 .type = RTE_FLOW_ITEM_TYPE_TCP, 600 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 601 }, 602 [MLX5_EXPANSION_VXLAN] = { 603 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 604 MLX5_EXPANSION_IPV4, 605 MLX5_EXPANSION_IPV6), 606 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 607 }, 608 [MLX5_EXPANSION_STD_VXLAN] = { 609 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 610 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 611 }, 612 [MLX5_EXPANSION_L3_VXLAN] = { 613 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 614 MLX5_EXPANSION_IPV6), 615 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 616 }, 617 [MLX5_EXPANSION_VXLAN_GPE] = { 618 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 619 MLX5_EXPANSION_IPV4, 620 MLX5_EXPANSION_IPV6), 621 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 622 }, 623 [MLX5_EXPANSION_GRE] = { 624 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 625 MLX5_EXPANSION_IPV6, 626 MLX5_EXPANSION_GRE_KEY, 627 MLX5_EXPANSION_MPLS), 628 .type = RTE_FLOW_ITEM_TYPE_GRE, 629 }, 630 [MLX5_EXPANSION_GRE_KEY] = { 631 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 632 MLX5_EXPANSION_IPV6, 633 MLX5_EXPANSION_MPLS), 634 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 635 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 636 }, 637 [MLX5_EXPANSION_NVGRE] = { 638 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 639 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 640 }, 641 [MLX5_EXPANSION_MPLS] = { 642 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 643 MLX5_EXPANSION_IPV6, 644 MLX5_EXPANSION_ETH), 645 .type = RTE_FLOW_ITEM_TYPE_MPLS, 646 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 647 }, 648 [MLX5_EXPANSION_ETH] = { 649 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 650 .type = RTE_FLOW_ITEM_TYPE_ETH, 651 }, 652 [MLX5_EXPANSION_VLAN] = { 653 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 654 MLX5_EXPANSION_IPV6), 655 .type = RTE_FLOW_ITEM_TYPE_VLAN, 656 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 657 }, 658 [MLX5_EXPANSION_IPV4] = { 659 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 660 MLX5_EXPANSION_IPV4_TCP), 661 .type = RTE_FLOW_ITEM_TYPE_IPV4, 662 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 663 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 664 }, 665 [MLX5_EXPANSION_IPV4_UDP] = { 666 .type = RTE_FLOW_ITEM_TYPE_UDP, 667 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 668 }, 669 [MLX5_EXPANSION_IPV4_TCP] = { 670 .type = RTE_FLOW_ITEM_TYPE_TCP, 671 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 672 }, 673 [MLX5_EXPANSION_IPV6] = { 674 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 675 MLX5_EXPANSION_IPV6_TCP, 676 MLX5_EXPANSION_IPV6_FRAG_EXT), 677 .type = RTE_FLOW_ITEM_TYPE_IPV6, 678 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 679 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 680 }, 681 [MLX5_EXPANSION_IPV6_UDP] = { 682 .type = RTE_FLOW_ITEM_TYPE_UDP, 683 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 684 }, 685 [MLX5_EXPANSION_IPV6_TCP] = { 686 .type = RTE_FLOW_ITEM_TYPE_TCP, 687 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 688 }, 689 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 690 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 691 }, 692 [MLX5_EXPANSION_GTP] = { 693 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 694 MLX5_EXPANSION_IPV6), 695 .type = RTE_FLOW_ITEM_TYPE_GTP, 696 }, 697 }; 698 699 static struct rte_flow_action_handle * 700 mlx5_action_handle_create(struct rte_eth_dev *dev, 701 const struct rte_flow_indir_action_conf *conf, 702 const struct rte_flow_action *action, 703 struct rte_flow_error *error); 704 static int mlx5_action_handle_destroy 705 (struct rte_eth_dev *dev, 706 struct rte_flow_action_handle *handle, 707 struct rte_flow_error *error); 708 static int mlx5_action_handle_update 709 (struct rte_eth_dev *dev, 710 struct rte_flow_action_handle *handle, 711 const void *update, 712 struct rte_flow_error *error); 713 static int mlx5_action_handle_query 714 (struct rte_eth_dev *dev, 715 const struct rte_flow_action_handle *handle, 716 void *data, 717 struct rte_flow_error *error); 718 static int 719 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 720 struct rte_flow_tunnel *app_tunnel, 721 struct rte_flow_action **actions, 722 uint32_t *num_of_actions, 723 struct rte_flow_error *error); 724 static int 725 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 726 struct rte_flow_tunnel *app_tunnel, 727 struct rte_flow_item **items, 728 uint32_t *num_of_items, 729 struct rte_flow_error *error); 730 static int 731 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 732 struct rte_flow_item *pmd_items, 733 uint32_t num_items, struct rte_flow_error *err); 734 static int 735 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 736 struct rte_flow_action *pmd_actions, 737 uint32_t num_actions, 738 struct rte_flow_error *err); 739 static int 740 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 741 struct rte_mbuf *m, 742 struct rte_flow_restore_info *info, 743 struct rte_flow_error *err); 744 745 static const struct rte_flow_ops mlx5_flow_ops = { 746 .validate = mlx5_flow_validate, 747 .create = mlx5_flow_create, 748 .destroy = mlx5_flow_destroy, 749 .flush = mlx5_flow_flush, 750 .isolate = mlx5_flow_isolate, 751 .query = mlx5_flow_query, 752 .dev_dump = mlx5_flow_dev_dump, 753 .get_aged_flows = mlx5_flow_get_aged_flows, 754 .action_handle_create = mlx5_action_handle_create, 755 .action_handle_destroy = mlx5_action_handle_destroy, 756 .action_handle_update = mlx5_action_handle_update, 757 .action_handle_query = mlx5_action_handle_query, 758 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 759 .tunnel_match = mlx5_flow_tunnel_match, 760 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 761 .tunnel_item_release = mlx5_flow_tunnel_item_release, 762 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 763 }; 764 765 /* Tunnel information. */ 766 struct mlx5_flow_tunnel_info { 767 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 768 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 769 }; 770 771 static struct mlx5_flow_tunnel_info tunnels_info[] = { 772 { 773 .tunnel = MLX5_FLOW_LAYER_VXLAN, 774 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 775 }, 776 { 777 .tunnel = MLX5_FLOW_LAYER_GENEVE, 778 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 779 }, 780 { 781 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 782 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 783 }, 784 { 785 .tunnel = MLX5_FLOW_LAYER_GRE, 786 .ptype = RTE_PTYPE_TUNNEL_GRE, 787 }, 788 { 789 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 790 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 791 }, 792 { 793 .tunnel = MLX5_FLOW_LAYER_MPLS, 794 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 795 }, 796 { 797 .tunnel = MLX5_FLOW_LAYER_NVGRE, 798 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 799 }, 800 { 801 .tunnel = MLX5_FLOW_LAYER_IPIP, 802 .ptype = RTE_PTYPE_TUNNEL_IP, 803 }, 804 { 805 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 806 .ptype = RTE_PTYPE_TUNNEL_IP, 807 }, 808 { 809 .tunnel = MLX5_FLOW_LAYER_GTP, 810 .ptype = RTE_PTYPE_TUNNEL_GTPU, 811 }, 812 }; 813 814 815 816 /** 817 * Translate tag ID to register. 818 * 819 * @param[in] dev 820 * Pointer to the Ethernet device structure. 821 * @param[in] feature 822 * The feature that request the register. 823 * @param[in] id 824 * The request register ID. 825 * @param[out] error 826 * Error description in case of any. 827 * 828 * @return 829 * The request register on success, a negative errno 830 * value otherwise and rte_errno is set. 831 */ 832 int 833 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 834 enum mlx5_feature_name feature, 835 uint32_t id, 836 struct rte_flow_error *error) 837 { 838 struct mlx5_priv *priv = dev->data->dev_private; 839 struct mlx5_dev_config *config = &priv->config; 840 enum modify_reg start_reg; 841 bool skip_mtr_reg = false; 842 843 switch (feature) { 844 case MLX5_HAIRPIN_RX: 845 return REG_B; 846 case MLX5_HAIRPIN_TX: 847 return REG_A; 848 case MLX5_METADATA_RX: 849 switch (config->dv_xmeta_en) { 850 case MLX5_XMETA_MODE_LEGACY: 851 return REG_B; 852 case MLX5_XMETA_MODE_META16: 853 return REG_C_0; 854 case MLX5_XMETA_MODE_META32: 855 return REG_C_1; 856 } 857 break; 858 case MLX5_METADATA_TX: 859 return REG_A; 860 case MLX5_METADATA_FDB: 861 switch (config->dv_xmeta_en) { 862 case MLX5_XMETA_MODE_LEGACY: 863 return REG_NON; 864 case MLX5_XMETA_MODE_META16: 865 return REG_C_0; 866 case MLX5_XMETA_MODE_META32: 867 return REG_C_1; 868 } 869 break; 870 case MLX5_FLOW_MARK: 871 switch (config->dv_xmeta_en) { 872 case MLX5_XMETA_MODE_LEGACY: 873 return REG_NON; 874 case MLX5_XMETA_MODE_META16: 875 return REG_C_1; 876 case MLX5_XMETA_MODE_META32: 877 return REG_C_0; 878 } 879 break; 880 case MLX5_MTR_ID: 881 /* 882 * If meter color and meter id share one register, flow match 883 * should use the meter color register for match. 884 */ 885 if (priv->mtr_reg_share) 886 return priv->mtr_color_reg; 887 else 888 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 889 REG_C_3; 890 case MLX5_MTR_COLOR: 891 case MLX5_ASO_FLOW_HIT: 892 case MLX5_ASO_CONNTRACK: 893 /* All features use the same REG_C. */ 894 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 895 return priv->mtr_color_reg; 896 case MLX5_COPY_MARK: 897 /* 898 * Metadata COPY_MARK register using is in meter suffix sub 899 * flow while with meter. It's safe to share the same register. 900 */ 901 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 902 case MLX5_APP_TAG: 903 /* 904 * If meter is enable, it will engage the register for color 905 * match and flow match. If meter color match is not using the 906 * REG_C_2, need to skip the REG_C_x be used by meter color 907 * match. 908 * If meter is disable, free to use all available registers. 909 */ 910 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 911 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 912 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 913 if (id > (uint32_t)(REG_C_7 - start_reg)) 914 return rte_flow_error_set(error, EINVAL, 915 RTE_FLOW_ERROR_TYPE_ITEM, 916 NULL, "invalid tag id"); 917 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 918 return rte_flow_error_set(error, ENOTSUP, 919 RTE_FLOW_ERROR_TYPE_ITEM, 920 NULL, "unsupported tag id"); 921 /* 922 * This case means meter is using the REG_C_x great than 2. 923 * Take care not to conflict with meter color REG_C_x. 924 * If the available index REG_C_y >= REG_C_x, skip the 925 * color register. 926 */ 927 if (skip_mtr_reg && config->flow_mreg_c 928 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 929 if (id >= (uint32_t)(REG_C_7 - start_reg)) 930 return rte_flow_error_set(error, EINVAL, 931 RTE_FLOW_ERROR_TYPE_ITEM, 932 NULL, "invalid tag id"); 933 if (config->flow_mreg_c 934 [id + 1 + start_reg - REG_C_0] != REG_NON) 935 return config->flow_mreg_c 936 [id + 1 + start_reg - REG_C_0]; 937 return rte_flow_error_set(error, ENOTSUP, 938 RTE_FLOW_ERROR_TYPE_ITEM, 939 NULL, "unsupported tag id"); 940 } 941 return config->flow_mreg_c[id + start_reg - REG_C_0]; 942 } 943 MLX5_ASSERT(false); 944 return rte_flow_error_set(error, EINVAL, 945 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 946 NULL, "invalid feature name"); 947 } 948 949 /** 950 * Check extensive flow metadata register support. 951 * 952 * @param dev 953 * Pointer to rte_eth_dev structure. 954 * 955 * @return 956 * True if device supports extensive flow metadata register, otherwise false. 957 */ 958 bool 959 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 960 { 961 struct mlx5_priv *priv = dev->data->dev_private; 962 struct mlx5_dev_config *config = &priv->config; 963 964 /* 965 * Having available reg_c can be regarded inclusively as supporting 966 * extensive flow metadata register, which could mean, 967 * - metadata register copy action by modify header. 968 * - 16 modify header actions is supported. 969 * - reg_c's are preserved across different domain (FDB and NIC) on 970 * packet loopback by flow lookup miss. 971 */ 972 return config->flow_mreg_c[2] != REG_NON; 973 } 974 975 /** 976 * Get the lowest priority. 977 * 978 * @param[in] dev 979 * Pointer to the Ethernet device structure. 980 * @param[in] attributes 981 * Pointer to device flow rule attributes. 982 * 983 * @return 984 * The value of lowest priority of flow. 985 */ 986 uint32_t 987 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 988 const struct rte_flow_attr *attr) 989 { 990 struct mlx5_priv *priv = dev->data->dev_private; 991 992 if (!attr->group && !attr->transfer) 993 return priv->config.flow_prio - 2; 994 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 995 } 996 997 /** 998 * Calculate matcher priority of the flow. 999 * 1000 * @param[in] dev 1001 * Pointer to the Ethernet device structure. 1002 * @param[in] attr 1003 * Pointer to device flow rule attributes. 1004 * @param[in] subpriority 1005 * The priority based on the items. 1006 * @param[in] external 1007 * Flow is user flow. 1008 * @return 1009 * The matcher priority of the flow. 1010 */ 1011 uint16_t 1012 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1013 const struct rte_flow_attr *attr, 1014 uint32_t subpriority, bool external) 1015 { 1016 uint16_t priority = (uint16_t)attr->priority; 1017 struct mlx5_priv *priv = dev->data->dev_private; 1018 1019 if (!attr->group && !attr->transfer) { 1020 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1021 priority = priv->config.flow_prio - 1; 1022 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1023 } else if (!external && attr->transfer && attr->group == 0 && 1024 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1025 return (priv->config.flow_prio - 1) * 3; 1026 } 1027 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1028 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1029 return priority * 3 + subpriority; 1030 } 1031 1032 /** 1033 * Verify the @p item specifications (spec, last, mask) are compatible with the 1034 * NIC capabilities. 1035 * 1036 * @param[in] item 1037 * Item specification. 1038 * @param[in] mask 1039 * @p item->mask or flow default bit-masks. 1040 * @param[in] nic_mask 1041 * Bit-masks covering supported fields by the NIC to compare with user mask. 1042 * @param[in] size 1043 * Bit-masks size in bytes. 1044 * @param[in] range_accepted 1045 * True if range of values is accepted for specific fields, false otherwise. 1046 * @param[out] error 1047 * Pointer to error structure. 1048 * 1049 * @return 1050 * 0 on success, a negative errno value otherwise and rte_errno is set. 1051 */ 1052 int 1053 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1054 const uint8_t *mask, 1055 const uint8_t *nic_mask, 1056 unsigned int size, 1057 bool range_accepted, 1058 struct rte_flow_error *error) 1059 { 1060 unsigned int i; 1061 1062 MLX5_ASSERT(nic_mask); 1063 for (i = 0; i < size; ++i) 1064 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1065 return rte_flow_error_set(error, ENOTSUP, 1066 RTE_FLOW_ERROR_TYPE_ITEM, 1067 item, 1068 "mask enables non supported" 1069 " bits"); 1070 if (!item->spec && (item->mask || item->last)) 1071 return rte_flow_error_set(error, EINVAL, 1072 RTE_FLOW_ERROR_TYPE_ITEM, item, 1073 "mask/last without a spec is not" 1074 " supported"); 1075 if (item->spec && item->last && !range_accepted) { 1076 uint8_t spec[size]; 1077 uint8_t last[size]; 1078 unsigned int i; 1079 int ret; 1080 1081 for (i = 0; i < size; ++i) { 1082 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1083 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1084 } 1085 ret = memcmp(spec, last, size); 1086 if (ret != 0) 1087 return rte_flow_error_set(error, EINVAL, 1088 RTE_FLOW_ERROR_TYPE_ITEM, 1089 item, 1090 "range is not valid"); 1091 } 1092 return 0; 1093 } 1094 1095 /** 1096 * Adjust the hash fields according to the @p flow information. 1097 * 1098 * @param[in] dev_flow. 1099 * Pointer to the mlx5_flow. 1100 * @param[in] tunnel 1101 * 1 when the hash field is for a tunnel item. 1102 * @param[in] layer_types 1103 * RTE_ETH_RSS_* types. 1104 * @param[in] hash_fields 1105 * Item hash fields. 1106 * 1107 * @return 1108 * The hash fields that should be used. 1109 */ 1110 uint64_t 1111 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1112 int tunnel __rte_unused, uint64_t layer_types, 1113 uint64_t hash_fields) 1114 { 1115 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1116 int rss_request_inner = rss_desc->level >= 2; 1117 1118 /* Check RSS hash level for tunnel. */ 1119 if (tunnel && rss_request_inner) 1120 hash_fields |= IBV_RX_HASH_INNER; 1121 else if (tunnel || rss_request_inner) 1122 return 0; 1123 #endif 1124 /* Check if requested layer matches RSS hash fields. */ 1125 if (!(rss_desc->types & layer_types)) 1126 return 0; 1127 return hash_fields; 1128 } 1129 1130 /** 1131 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1132 * if several tunnel rules are used on this queue, the tunnel ptype will be 1133 * cleared. 1134 * 1135 * @param rxq_ctrl 1136 * Rx queue to update. 1137 */ 1138 static void 1139 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1140 { 1141 unsigned int i; 1142 uint32_t tunnel_ptype = 0; 1143 1144 /* Look up for the ptype to use. */ 1145 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1146 if (!rxq_ctrl->flow_tunnels_n[i]) 1147 continue; 1148 if (!tunnel_ptype) { 1149 tunnel_ptype = tunnels_info[i].ptype; 1150 } else { 1151 tunnel_ptype = 0; 1152 break; 1153 } 1154 } 1155 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1156 } 1157 1158 /** 1159 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1160 * flow. 1161 * 1162 * @param[in] dev 1163 * Pointer to the Ethernet device structure. 1164 * @param[in] dev_handle 1165 * Pointer to device flow handle structure. 1166 */ 1167 void 1168 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1169 struct mlx5_flow_handle *dev_handle) 1170 { 1171 struct mlx5_priv *priv = dev->data->dev_private; 1172 const int mark = dev_handle->mark; 1173 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1174 struct mlx5_ind_table_obj *ind_tbl = NULL; 1175 unsigned int i; 1176 1177 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1178 struct mlx5_hrxq *hrxq; 1179 1180 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1181 dev_handle->rix_hrxq); 1182 if (hrxq) 1183 ind_tbl = hrxq->ind_table; 1184 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1185 struct mlx5_shared_action_rss *shared_rss; 1186 1187 shared_rss = mlx5_ipool_get 1188 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1189 dev_handle->rix_srss); 1190 if (shared_rss) 1191 ind_tbl = shared_rss->ind_tbl; 1192 } 1193 if (!ind_tbl) 1194 return; 1195 for (i = 0; i != ind_tbl->queues_n; ++i) { 1196 int idx = ind_tbl->queues[i]; 1197 struct mlx5_rxq_ctrl *rxq_ctrl = 1198 container_of((*priv->rxqs)[idx], 1199 struct mlx5_rxq_ctrl, rxq); 1200 1201 /* 1202 * To support metadata register copy on Tx loopback, 1203 * this must be always enabled (metadata may arive 1204 * from other port - not from local flows only. 1205 */ 1206 if (priv->config.dv_flow_en && 1207 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1208 mlx5_flow_ext_mreg_supported(dev)) { 1209 rxq_ctrl->rxq.mark = 1; 1210 rxq_ctrl->flow_mark_n = 1; 1211 } else if (mark) { 1212 rxq_ctrl->rxq.mark = 1; 1213 rxq_ctrl->flow_mark_n++; 1214 } 1215 if (tunnel) { 1216 unsigned int j; 1217 1218 /* Increase the counter matching the flow. */ 1219 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1220 if ((tunnels_info[j].tunnel & 1221 dev_handle->layers) == 1222 tunnels_info[j].tunnel) { 1223 rxq_ctrl->flow_tunnels_n[j]++; 1224 break; 1225 } 1226 } 1227 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1228 } 1229 } 1230 } 1231 1232 /** 1233 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1234 * 1235 * @param[in] dev 1236 * Pointer to the Ethernet device structure. 1237 * @param[in] flow 1238 * Pointer to flow structure. 1239 */ 1240 static void 1241 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1242 { 1243 struct mlx5_priv *priv = dev->data->dev_private; 1244 uint32_t handle_idx; 1245 struct mlx5_flow_handle *dev_handle; 1246 1247 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1248 handle_idx, dev_handle, next) 1249 flow_drv_rxq_flags_set(dev, dev_handle); 1250 } 1251 1252 /** 1253 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1254 * device flow if no other flow uses it with the same kind of request. 1255 * 1256 * @param dev 1257 * Pointer to Ethernet device. 1258 * @param[in] dev_handle 1259 * Pointer to the device flow handle structure. 1260 */ 1261 static void 1262 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1263 struct mlx5_flow_handle *dev_handle) 1264 { 1265 struct mlx5_priv *priv = dev->data->dev_private; 1266 const int mark = dev_handle->mark; 1267 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1268 struct mlx5_ind_table_obj *ind_tbl = NULL; 1269 unsigned int i; 1270 1271 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1272 struct mlx5_hrxq *hrxq; 1273 1274 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1275 dev_handle->rix_hrxq); 1276 if (hrxq) 1277 ind_tbl = hrxq->ind_table; 1278 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1279 struct mlx5_shared_action_rss *shared_rss; 1280 1281 shared_rss = mlx5_ipool_get 1282 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1283 dev_handle->rix_srss); 1284 if (shared_rss) 1285 ind_tbl = shared_rss->ind_tbl; 1286 } 1287 if (!ind_tbl) 1288 return; 1289 MLX5_ASSERT(dev->data->dev_started); 1290 for (i = 0; i != ind_tbl->queues_n; ++i) { 1291 int idx = ind_tbl->queues[i]; 1292 struct mlx5_rxq_ctrl *rxq_ctrl = 1293 container_of((*priv->rxqs)[idx], 1294 struct mlx5_rxq_ctrl, rxq); 1295 1296 if (priv->config.dv_flow_en && 1297 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1298 mlx5_flow_ext_mreg_supported(dev)) { 1299 rxq_ctrl->rxq.mark = 1; 1300 rxq_ctrl->flow_mark_n = 1; 1301 } else if (mark) { 1302 rxq_ctrl->flow_mark_n--; 1303 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1304 } 1305 if (tunnel) { 1306 unsigned int j; 1307 1308 /* Decrease the counter matching the flow. */ 1309 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1310 if ((tunnels_info[j].tunnel & 1311 dev_handle->layers) == 1312 tunnels_info[j].tunnel) { 1313 rxq_ctrl->flow_tunnels_n[j]--; 1314 break; 1315 } 1316 } 1317 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1318 } 1319 } 1320 } 1321 1322 /** 1323 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1324 * @p flow if no other flow uses it with the same kind of request. 1325 * 1326 * @param dev 1327 * Pointer to Ethernet device. 1328 * @param[in] flow 1329 * Pointer to the flow. 1330 */ 1331 static void 1332 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1333 { 1334 struct mlx5_priv *priv = dev->data->dev_private; 1335 uint32_t handle_idx; 1336 struct mlx5_flow_handle *dev_handle; 1337 1338 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1339 handle_idx, dev_handle, next) 1340 flow_drv_rxq_flags_trim(dev, dev_handle); 1341 } 1342 1343 /** 1344 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1345 * 1346 * @param dev 1347 * Pointer to Ethernet device. 1348 */ 1349 static void 1350 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1351 { 1352 struct mlx5_priv *priv = dev->data->dev_private; 1353 unsigned int i; 1354 1355 for (i = 0; i != priv->rxqs_n; ++i) { 1356 struct mlx5_rxq_ctrl *rxq_ctrl; 1357 unsigned int j; 1358 1359 if (!(*priv->rxqs)[i]) 1360 continue; 1361 rxq_ctrl = container_of((*priv->rxqs)[i], 1362 struct mlx5_rxq_ctrl, rxq); 1363 rxq_ctrl->flow_mark_n = 0; 1364 rxq_ctrl->rxq.mark = 0; 1365 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1366 rxq_ctrl->flow_tunnels_n[j] = 0; 1367 rxq_ctrl->rxq.tunnel = 0; 1368 } 1369 } 1370 1371 /** 1372 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1373 * 1374 * @param[in] dev 1375 * Pointer to the Ethernet device structure. 1376 */ 1377 void 1378 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1379 { 1380 struct mlx5_priv *priv = dev->data->dev_private; 1381 struct mlx5_rxq_data *data; 1382 unsigned int i; 1383 1384 for (i = 0; i != priv->rxqs_n; ++i) { 1385 if (!(*priv->rxqs)[i]) 1386 continue; 1387 data = (*priv->rxqs)[i]; 1388 if (!rte_flow_dynf_metadata_avail()) { 1389 data->dynf_meta = 0; 1390 data->flow_meta_mask = 0; 1391 data->flow_meta_offset = -1; 1392 data->flow_meta_port_mask = 0; 1393 } else { 1394 data->dynf_meta = 1; 1395 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1396 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1397 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1398 } 1399 } 1400 } 1401 1402 /* 1403 * return a pointer to the desired action in the list of actions. 1404 * 1405 * @param[in] actions 1406 * The list of actions to search the action in. 1407 * @param[in] action 1408 * The action to find. 1409 * 1410 * @return 1411 * Pointer to the action in the list, if found. NULL otherwise. 1412 */ 1413 const struct rte_flow_action * 1414 mlx5_flow_find_action(const struct rte_flow_action *actions, 1415 enum rte_flow_action_type action) 1416 { 1417 if (actions == NULL) 1418 return NULL; 1419 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1420 if (actions->type == action) 1421 return actions; 1422 return NULL; 1423 } 1424 1425 /* 1426 * Validate the flag action. 1427 * 1428 * @param[in] action_flags 1429 * Bit-fields that holds the actions detected until now. 1430 * @param[in] attr 1431 * Attributes of flow that includes this action. 1432 * @param[out] error 1433 * Pointer to error structure. 1434 * 1435 * @return 1436 * 0 on success, a negative errno value otherwise and rte_errno is set. 1437 */ 1438 int 1439 mlx5_flow_validate_action_flag(uint64_t action_flags, 1440 const struct rte_flow_attr *attr, 1441 struct rte_flow_error *error) 1442 { 1443 if (action_flags & MLX5_FLOW_ACTION_MARK) 1444 return rte_flow_error_set(error, EINVAL, 1445 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1446 "can't mark and flag in same flow"); 1447 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1448 return rte_flow_error_set(error, EINVAL, 1449 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1450 "can't have 2 flag" 1451 " actions in same flow"); 1452 if (attr->egress) 1453 return rte_flow_error_set(error, ENOTSUP, 1454 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1455 "flag action not supported for " 1456 "egress"); 1457 return 0; 1458 } 1459 1460 /* 1461 * Validate the mark action. 1462 * 1463 * @param[in] action 1464 * Pointer to the queue action. 1465 * @param[in] action_flags 1466 * Bit-fields that holds the actions detected until now. 1467 * @param[in] attr 1468 * Attributes of flow that includes this action. 1469 * @param[out] error 1470 * Pointer to error structure. 1471 * 1472 * @return 1473 * 0 on success, a negative errno value otherwise and rte_errno is set. 1474 */ 1475 int 1476 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1477 uint64_t action_flags, 1478 const struct rte_flow_attr *attr, 1479 struct rte_flow_error *error) 1480 { 1481 const struct rte_flow_action_mark *mark = action->conf; 1482 1483 if (!mark) 1484 return rte_flow_error_set(error, EINVAL, 1485 RTE_FLOW_ERROR_TYPE_ACTION, 1486 action, 1487 "configuration cannot be null"); 1488 if (mark->id >= MLX5_FLOW_MARK_MAX) 1489 return rte_flow_error_set(error, EINVAL, 1490 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1491 &mark->id, 1492 "mark id must in 0 <= id < " 1493 RTE_STR(MLX5_FLOW_MARK_MAX)); 1494 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1495 return rte_flow_error_set(error, EINVAL, 1496 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1497 "can't flag and mark in same flow"); 1498 if (action_flags & MLX5_FLOW_ACTION_MARK) 1499 return rte_flow_error_set(error, EINVAL, 1500 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1501 "can't have 2 mark actions in same" 1502 " flow"); 1503 if (attr->egress) 1504 return rte_flow_error_set(error, ENOTSUP, 1505 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1506 "mark action not supported for " 1507 "egress"); 1508 return 0; 1509 } 1510 1511 /* 1512 * Validate the drop action. 1513 * 1514 * @param[in] action_flags 1515 * Bit-fields that holds the actions detected until now. 1516 * @param[in] attr 1517 * Attributes of flow that includes this action. 1518 * @param[out] error 1519 * Pointer to error structure. 1520 * 1521 * @return 1522 * 0 on success, a negative errno value otherwise and rte_errno is set. 1523 */ 1524 int 1525 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1526 const struct rte_flow_attr *attr, 1527 struct rte_flow_error *error) 1528 { 1529 if (attr->egress) 1530 return rte_flow_error_set(error, ENOTSUP, 1531 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1532 "drop action not supported for " 1533 "egress"); 1534 return 0; 1535 } 1536 1537 /* 1538 * Validate the queue action. 1539 * 1540 * @param[in] action 1541 * Pointer to the queue action. 1542 * @param[in] action_flags 1543 * Bit-fields that holds the actions detected until now. 1544 * @param[in] dev 1545 * Pointer to the Ethernet device structure. 1546 * @param[in] attr 1547 * Attributes of flow that includes this action. 1548 * @param[out] error 1549 * Pointer to error structure. 1550 * 1551 * @return 1552 * 0 on success, a negative errno value otherwise and rte_errno is set. 1553 */ 1554 int 1555 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1556 uint64_t action_flags, 1557 struct rte_eth_dev *dev, 1558 const struct rte_flow_attr *attr, 1559 struct rte_flow_error *error) 1560 { 1561 struct mlx5_priv *priv = dev->data->dev_private; 1562 const struct rte_flow_action_queue *queue = action->conf; 1563 1564 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1565 return rte_flow_error_set(error, EINVAL, 1566 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1567 "can't have 2 fate actions in" 1568 " same flow"); 1569 if (!priv->rxqs_n) 1570 return rte_flow_error_set(error, EINVAL, 1571 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1572 NULL, "No Rx queues configured"); 1573 if (queue->index >= priv->rxqs_n) 1574 return rte_flow_error_set(error, EINVAL, 1575 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1576 &queue->index, 1577 "queue index out of range"); 1578 if (!(*priv->rxqs)[queue->index]) 1579 return rte_flow_error_set(error, EINVAL, 1580 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1581 &queue->index, 1582 "queue is not configured"); 1583 if (attr->egress) 1584 return rte_flow_error_set(error, ENOTSUP, 1585 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1586 "queue action not supported for " 1587 "egress"); 1588 return 0; 1589 } 1590 1591 /* 1592 * Validate the rss action. 1593 * 1594 * @param[in] dev 1595 * Pointer to the Ethernet device structure. 1596 * @param[in] action 1597 * Pointer to the queue action. 1598 * @param[out] error 1599 * Pointer to error structure. 1600 * 1601 * @return 1602 * 0 on success, a negative errno value otherwise and rte_errno is set. 1603 */ 1604 int 1605 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1606 const struct rte_flow_action *action, 1607 struct rte_flow_error *error) 1608 { 1609 struct mlx5_priv *priv = dev->data->dev_private; 1610 const struct rte_flow_action_rss *rss = action->conf; 1611 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1612 unsigned int i; 1613 1614 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1615 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1616 return rte_flow_error_set(error, ENOTSUP, 1617 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1618 &rss->func, 1619 "RSS hash function not supported"); 1620 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1621 if (rss->level > 2) 1622 #else 1623 if (rss->level > 1) 1624 #endif 1625 return rte_flow_error_set(error, ENOTSUP, 1626 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1627 &rss->level, 1628 "tunnel RSS is not supported"); 1629 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1630 if (rss->key_len == 0 && rss->key != NULL) 1631 return rte_flow_error_set(error, ENOTSUP, 1632 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1633 &rss->key_len, 1634 "RSS hash key length 0"); 1635 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1636 return rte_flow_error_set(error, ENOTSUP, 1637 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1638 &rss->key_len, 1639 "RSS hash key too small"); 1640 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1641 return rte_flow_error_set(error, ENOTSUP, 1642 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1643 &rss->key_len, 1644 "RSS hash key too large"); 1645 if (rss->queue_num > priv->config.ind_table_max_size) 1646 return rte_flow_error_set(error, ENOTSUP, 1647 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1648 &rss->queue_num, 1649 "number of queues too large"); 1650 if (rss->types & MLX5_RSS_HF_MASK) 1651 return rte_flow_error_set(error, ENOTSUP, 1652 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1653 &rss->types, 1654 "some RSS protocols are not" 1655 " supported"); 1656 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1657 !(rss->types & RTE_ETH_RSS_IP)) 1658 return rte_flow_error_set(error, EINVAL, 1659 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1660 "L3 partial RSS requested but L3 RSS" 1661 " type not specified"); 1662 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1663 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1664 return rte_flow_error_set(error, EINVAL, 1665 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1666 "L4 partial RSS requested but L4 RSS" 1667 " type not specified"); 1668 if (!priv->rxqs_n) 1669 return rte_flow_error_set(error, EINVAL, 1670 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1671 NULL, "No Rx queues configured"); 1672 if (!rss->queue_num) 1673 return rte_flow_error_set(error, EINVAL, 1674 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1675 NULL, "No queues configured"); 1676 for (i = 0; i != rss->queue_num; ++i) { 1677 struct mlx5_rxq_ctrl *rxq_ctrl; 1678 1679 if (rss->queue[i] >= priv->rxqs_n) 1680 return rte_flow_error_set 1681 (error, EINVAL, 1682 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1683 &rss->queue[i], "queue index out of range"); 1684 if (!(*priv->rxqs)[rss->queue[i]]) 1685 return rte_flow_error_set 1686 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1687 &rss->queue[i], "queue is not configured"); 1688 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1689 struct mlx5_rxq_ctrl, rxq); 1690 if (i == 0) 1691 rxq_type = rxq_ctrl->type; 1692 if (rxq_type != rxq_ctrl->type) 1693 return rte_flow_error_set 1694 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1695 &rss->queue[i], 1696 "combining hairpin and regular RSS queues is not supported"); 1697 } 1698 return 0; 1699 } 1700 1701 /* 1702 * Validate the rss action. 1703 * 1704 * @param[in] action 1705 * Pointer to the queue action. 1706 * @param[in] action_flags 1707 * Bit-fields that holds the actions detected until now. 1708 * @param[in] dev 1709 * Pointer to the Ethernet device structure. 1710 * @param[in] attr 1711 * Attributes of flow that includes this action. 1712 * @param[in] item_flags 1713 * Items that were detected. 1714 * @param[out] error 1715 * Pointer to error structure. 1716 * 1717 * @return 1718 * 0 on success, a negative errno value otherwise and rte_errno is set. 1719 */ 1720 int 1721 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1722 uint64_t action_flags, 1723 struct rte_eth_dev *dev, 1724 const struct rte_flow_attr *attr, 1725 uint64_t item_flags, 1726 struct rte_flow_error *error) 1727 { 1728 const struct rte_flow_action_rss *rss = action->conf; 1729 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1730 int ret; 1731 1732 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1733 return rte_flow_error_set(error, EINVAL, 1734 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1735 "can't have 2 fate actions" 1736 " in same flow"); 1737 ret = mlx5_validate_action_rss(dev, action, error); 1738 if (ret) 1739 return ret; 1740 if (attr->egress) 1741 return rte_flow_error_set(error, ENOTSUP, 1742 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1743 "rss action not supported for " 1744 "egress"); 1745 if (rss->level > 1 && !tunnel) 1746 return rte_flow_error_set(error, EINVAL, 1747 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1748 "inner RSS is not supported for " 1749 "non-tunnel flows"); 1750 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1751 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1752 return rte_flow_error_set(error, EINVAL, 1753 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1754 "RSS on eCPRI is not supported now"); 1755 } 1756 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1757 !(item_flags & 1758 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1759 rss->level > 1) 1760 return rte_flow_error_set(error, EINVAL, 1761 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1762 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1763 return 0; 1764 } 1765 1766 /* 1767 * Validate the default miss action. 1768 * 1769 * @param[in] action_flags 1770 * Bit-fields that holds the actions detected until now. 1771 * @param[out] error 1772 * Pointer to error structure. 1773 * 1774 * @return 1775 * 0 on success, a negative errno value otherwise and rte_errno is set. 1776 */ 1777 int 1778 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1779 const struct rte_flow_attr *attr, 1780 struct rte_flow_error *error) 1781 { 1782 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1783 return rte_flow_error_set(error, EINVAL, 1784 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1785 "can't have 2 fate actions in" 1786 " same flow"); 1787 if (attr->egress) 1788 return rte_flow_error_set(error, ENOTSUP, 1789 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1790 "default miss action not supported " 1791 "for egress"); 1792 if (attr->group) 1793 return rte_flow_error_set(error, ENOTSUP, 1794 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1795 "only group 0 is supported"); 1796 if (attr->transfer) 1797 return rte_flow_error_set(error, ENOTSUP, 1798 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1799 NULL, "transfer is not supported"); 1800 return 0; 1801 } 1802 1803 /* 1804 * Validate the count action. 1805 * 1806 * @param[in] dev 1807 * Pointer to the Ethernet device structure. 1808 * @param[in] attr 1809 * Attributes of flow that includes this action. 1810 * @param[out] error 1811 * Pointer to error structure. 1812 * 1813 * @return 1814 * 0 on success, a negative errno value otherwise and rte_errno is set. 1815 */ 1816 int 1817 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1818 const struct rte_flow_attr *attr, 1819 struct rte_flow_error *error) 1820 { 1821 if (attr->egress) 1822 return rte_flow_error_set(error, ENOTSUP, 1823 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1824 "count action not supported for " 1825 "egress"); 1826 return 0; 1827 } 1828 1829 /* 1830 * Validate the ASO CT action. 1831 * 1832 * @param[in] dev 1833 * Pointer to the Ethernet device structure. 1834 * @param[in] conntrack 1835 * Pointer to the CT action profile. 1836 * @param[out] error 1837 * Pointer to error structure. 1838 * 1839 * @return 1840 * 0 on success, a negative errno value otherwise and rte_errno is set. 1841 */ 1842 int 1843 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1844 const struct rte_flow_action_conntrack *conntrack, 1845 struct rte_flow_error *error) 1846 { 1847 RTE_SET_USED(dev); 1848 1849 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1850 return rte_flow_error_set(error, EINVAL, 1851 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1852 "Invalid CT state"); 1853 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1854 return rte_flow_error_set(error, EINVAL, 1855 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1856 "Invalid last TCP packet flag"); 1857 return 0; 1858 } 1859 1860 /** 1861 * Verify the @p attributes will be correctly understood by the NIC and store 1862 * them in the @p flow if everything is correct. 1863 * 1864 * @param[in] dev 1865 * Pointer to the Ethernet device structure. 1866 * @param[in] attributes 1867 * Pointer to flow attributes 1868 * @param[out] error 1869 * Pointer to error structure. 1870 * 1871 * @return 1872 * 0 on success, a negative errno value otherwise and rte_errno is set. 1873 */ 1874 int 1875 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1876 const struct rte_flow_attr *attributes, 1877 struct rte_flow_error *error) 1878 { 1879 struct mlx5_priv *priv = dev->data->dev_private; 1880 uint32_t priority_max = priv->config.flow_prio - 1; 1881 1882 if (attributes->group) 1883 return rte_flow_error_set(error, ENOTSUP, 1884 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1885 NULL, "groups is not supported"); 1886 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1887 attributes->priority >= priority_max) 1888 return rte_flow_error_set(error, ENOTSUP, 1889 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1890 NULL, "priority out of range"); 1891 if (attributes->egress) 1892 return rte_flow_error_set(error, ENOTSUP, 1893 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1894 "egress is not supported"); 1895 if (attributes->transfer && !priv->config.dv_esw_en) 1896 return rte_flow_error_set(error, ENOTSUP, 1897 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1898 NULL, "transfer is not supported"); 1899 if (!attributes->ingress) 1900 return rte_flow_error_set(error, EINVAL, 1901 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1902 NULL, 1903 "ingress attribute is mandatory"); 1904 return 0; 1905 } 1906 1907 /** 1908 * Validate ICMP6 item. 1909 * 1910 * @param[in] item 1911 * Item specification. 1912 * @param[in] item_flags 1913 * Bit-fields that holds the items detected until now. 1914 * @param[in] ext_vlan_sup 1915 * Whether extended VLAN features are supported or not. 1916 * @param[out] error 1917 * Pointer to error structure. 1918 * 1919 * @return 1920 * 0 on success, a negative errno value otherwise and rte_errno is set. 1921 */ 1922 int 1923 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1924 uint64_t item_flags, 1925 uint8_t target_protocol, 1926 struct rte_flow_error *error) 1927 { 1928 const struct rte_flow_item_icmp6 *mask = item->mask; 1929 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1930 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1931 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1932 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1933 MLX5_FLOW_LAYER_OUTER_L4; 1934 int ret; 1935 1936 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1937 return rte_flow_error_set(error, EINVAL, 1938 RTE_FLOW_ERROR_TYPE_ITEM, item, 1939 "protocol filtering not compatible" 1940 " with ICMP6 layer"); 1941 if (!(item_flags & l3m)) 1942 return rte_flow_error_set(error, EINVAL, 1943 RTE_FLOW_ERROR_TYPE_ITEM, item, 1944 "IPv6 is mandatory to filter on" 1945 " ICMP6"); 1946 if (item_flags & l4m) 1947 return rte_flow_error_set(error, EINVAL, 1948 RTE_FLOW_ERROR_TYPE_ITEM, item, 1949 "multiple L4 layers not supported"); 1950 if (!mask) 1951 mask = &rte_flow_item_icmp6_mask; 1952 ret = mlx5_flow_item_acceptable 1953 (item, (const uint8_t *)mask, 1954 (const uint8_t *)&rte_flow_item_icmp6_mask, 1955 sizeof(struct rte_flow_item_icmp6), 1956 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1957 if (ret < 0) 1958 return ret; 1959 return 0; 1960 } 1961 1962 /** 1963 * Validate ICMP item. 1964 * 1965 * @param[in] item 1966 * Item specification. 1967 * @param[in] item_flags 1968 * Bit-fields that holds the items detected until now. 1969 * @param[out] error 1970 * Pointer to error structure. 1971 * 1972 * @return 1973 * 0 on success, a negative errno value otherwise and rte_errno is set. 1974 */ 1975 int 1976 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1977 uint64_t item_flags, 1978 uint8_t target_protocol, 1979 struct rte_flow_error *error) 1980 { 1981 const struct rte_flow_item_icmp *mask = item->mask; 1982 const struct rte_flow_item_icmp nic_mask = { 1983 .hdr.icmp_type = 0xff, 1984 .hdr.icmp_code = 0xff, 1985 .hdr.icmp_ident = RTE_BE16(0xffff), 1986 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1987 }; 1988 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1989 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1990 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1991 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1992 MLX5_FLOW_LAYER_OUTER_L4; 1993 int ret; 1994 1995 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1996 return rte_flow_error_set(error, EINVAL, 1997 RTE_FLOW_ERROR_TYPE_ITEM, item, 1998 "protocol filtering not compatible" 1999 " with ICMP layer"); 2000 if (!(item_flags & l3m)) 2001 return rte_flow_error_set(error, EINVAL, 2002 RTE_FLOW_ERROR_TYPE_ITEM, item, 2003 "IPv4 is mandatory to filter" 2004 " on ICMP"); 2005 if (item_flags & l4m) 2006 return rte_flow_error_set(error, EINVAL, 2007 RTE_FLOW_ERROR_TYPE_ITEM, item, 2008 "multiple L4 layers not supported"); 2009 if (!mask) 2010 mask = &nic_mask; 2011 ret = mlx5_flow_item_acceptable 2012 (item, (const uint8_t *)mask, 2013 (const uint8_t *)&nic_mask, 2014 sizeof(struct rte_flow_item_icmp), 2015 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2016 if (ret < 0) 2017 return ret; 2018 return 0; 2019 } 2020 2021 /** 2022 * Validate Ethernet item. 2023 * 2024 * @param[in] item 2025 * Item specification. 2026 * @param[in] item_flags 2027 * Bit-fields that holds the items detected until now. 2028 * @param[out] error 2029 * Pointer to error structure. 2030 * 2031 * @return 2032 * 0 on success, a negative errno value otherwise and rte_errno is set. 2033 */ 2034 int 2035 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2036 uint64_t item_flags, bool ext_vlan_sup, 2037 struct rte_flow_error *error) 2038 { 2039 const struct rte_flow_item_eth *mask = item->mask; 2040 const struct rte_flow_item_eth nic_mask = { 2041 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2042 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2043 .type = RTE_BE16(0xffff), 2044 .has_vlan = ext_vlan_sup ? 1 : 0, 2045 }; 2046 int ret; 2047 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2048 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2049 MLX5_FLOW_LAYER_OUTER_L2; 2050 2051 if (item_flags & ethm) 2052 return rte_flow_error_set(error, ENOTSUP, 2053 RTE_FLOW_ERROR_TYPE_ITEM, item, 2054 "multiple L2 layers not supported"); 2055 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2056 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2057 return rte_flow_error_set(error, EINVAL, 2058 RTE_FLOW_ERROR_TYPE_ITEM, item, 2059 "L2 layer should not follow " 2060 "L3 layers"); 2061 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2062 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2063 return rte_flow_error_set(error, EINVAL, 2064 RTE_FLOW_ERROR_TYPE_ITEM, item, 2065 "L2 layer should not follow VLAN"); 2066 if (item_flags & MLX5_FLOW_LAYER_GTP) 2067 return rte_flow_error_set(error, EINVAL, 2068 RTE_FLOW_ERROR_TYPE_ITEM, item, 2069 "L2 layer should not follow GTP"); 2070 if (!mask) 2071 mask = &rte_flow_item_eth_mask; 2072 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2073 (const uint8_t *)&nic_mask, 2074 sizeof(struct rte_flow_item_eth), 2075 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2076 return ret; 2077 } 2078 2079 /** 2080 * Validate VLAN item. 2081 * 2082 * @param[in] item 2083 * Item specification. 2084 * @param[in] item_flags 2085 * Bit-fields that holds the items detected until now. 2086 * @param[in] dev 2087 * Ethernet device flow is being created on. 2088 * @param[out] error 2089 * Pointer to error structure. 2090 * 2091 * @return 2092 * 0 on success, a negative errno value otherwise and rte_errno is set. 2093 */ 2094 int 2095 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2096 uint64_t item_flags, 2097 struct rte_eth_dev *dev, 2098 struct rte_flow_error *error) 2099 { 2100 const struct rte_flow_item_vlan *spec = item->spec; 2101 const struct rte_flow_item_vlan *mask = item->mask; 2102 const struct rte_flow_item_vlan nic_mask = { 2103 .tci = RTE_BE16(UINT16_MAX), 2104 .inner_type = RTE_BE16(UINT16_MAX), 2105 }; 2106 uint16_t vlan_tag = 0; 2107 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2108 int ret; 2109 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2110 MLX5_FLOW_LAYER_INNER_L4) : 2111 (MLX5_FLOW_LAYER_OUTER_L3 | 2112 MLX5_FLOW_LAYER_OUTER_L4); 2113 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2114 MLX5_FLOW_LAYER_OUTER_VLAN; 2115 2116 if (item_flags & vlanm) 2117 return rte_flow_error_set(error, EINVAL, 2118 RTE_FLOW_ERROR_TYPE_ITEM, item, 2119 "multiple VLAN layers not supported"); 2120 else if ((item_flags & l34m) != 0) 2121 return rte_flow_error_set(error, EINVAL, 2122 RTE_FLOW_ERROR_TYPE_ITEM, item, 2123 "VLAN cannot follow L3/L4 layer"); 2124 if (!mask) 2125 mask = &rte_flow_item_vlan_mask; 2126 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2127 (const uint8_t *)&nic_mask, 2128 sizeof(struct rte_flow_item_vlan), 2129 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2130 if (ret) 2131 return ret; 2132 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2133 struct mlx5_priv *priv = dev->data->dev_private; 2134 2135 if (priv->vmwa_context) { 2136 /* 2137 * Non-NULL context means we have a virtual machine 2138 * and SR-IOV enabled, we have to create VLAN interface 2139 * to make hypervisor to setup E-Switch vport 2140 * context correctly. We avoid creating the multiple 2141 * VLAN interfaces, so we cannot support VLAN tag mask. 2142 */ 2143 return rte_flow_error_set(error, EINVAL, 2144 RTE_FLOW_ERROR_TYPE_ITEM, 2145 item, 2146 "VLAN tag mask is not" 2147 " supported in virtual" 2148 " environment"); 2149 } 2150 } 2151 if (spec) { 2152 vlan_tag = spec->tci; 2153 vlan_tag &= mask->tci; 2154 } 2155 /* 2156 * From verbs perspective an empty VLAN is equivalent 2157 * to a packet without VLAN layer. 2158 */ 2159 if (!vlan_tag) 2160 return rte_flow_error_set(error, EINVAL, 2161 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2162 item->spec, 2163 "VLAN cannot be empty"); 2164 return 0; 2165 } 2166 2167 /** 2168 * Validate IPV4 item. 2169 * 2170 * @param[in] item 2171 * Item specification. 2172 * @param[in] item_flags 2173 * Bit-fields that holds the items detected until now. 2174 * @param[in] last_item 2175 * Previous validated item in the pattern items. 2176 * @param[in] ether_type 2177 * Type in the ethernet layer header (including dot1q). 2178 * @param[in] acc_mask 2179 * Acceptable mask, if NULL default internal default mask 2180 * will be used to check whether item fields are supported. 2181 * @param[in] range_accepted 2182 * True if range of values is accepted for specific fields, false otherwise. 2183 * @param[out] error 2184 * Pointer to error structure. 2185 * 2186 * @return 2187 * 0 on success, a negative errno value otherwise and rte_errno is set. 2188 */ 2189 int 2190 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2191 uint64_t item_flags, 2192 uint64_t last_item, 2193 uint16_t ether_type, 2194 const struct rte_flow_item_ipv4 *acc_mask, 2195 bool range_accepted, 2196 struct rte_flow_error *error) 2197 { 2198 const struct rte_flow_item_ipv4 *mask = item->mask; 2199 const struct rte_flow_item_ipv4 *spec = item->spec; 2200 const struct rte_flow_item_ipv4 nic_mask = { 2201 .hdr = { 2202 .src_addr = RTE_BE32(0xffffffff), 2203 .dst_addr = RTE_BE32(0xffffffff), 2204 .type_of_service = 0xff, 2205 .next_proto_id = 0xff, 2206 }, 2207 }; 2208 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2209 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2210 MLX5_FLOW_LAYER_OUTER_L3; 2211 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2212 MLX5_FLOW_LAYER_OUTER_L4; 2213 int ret; 2214 uint8_t next_proto = 0xFF; 2215 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2216 MLX5_FLOW_LAYER_OUTER_VLAN | 2217 MLX5_FLOW_LAYER_INNER_VLAN); 2218 2219 if ((last_item & l2_vlan) && ether_type && 2220 ether_type != RTE_ETHER_TYPE_IPV4) 2221 return rte_flow_error_set(error, EINVAL, 2222 RTE_FLOW_ERROR_TYPE_ITEM, item, 2223 "IPv4 cannot follow L2/VLAN layer " 2224 "which ether type is not IPv4"); 2225 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2226 if (mask && spec) 2227 next_proto = mask->hdr.next_proto_id & 2228 spec->hdr.next_proto_id; 2229 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2230 return rte_flow_error_set(error, EINVAL, 2231 RTE_FLOW_ERROR_TYPE_ITEM, 2232 item, 2233 "multiple tunnel " 2234 "not supported"); 2235 } 2236 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2237 return rte_flow_error_set(error, EINVAL, 2238 RTE_FLOW_ERROR_TYPE_ITEM, item, 2239 "wrong tunnel type - IPv6 specified " 2240 "but IPv4 item provided"); 2241 if (item_flags & l3m) 2242 return rte_flow_error_set(error, ENOTSUP, 2243 RTE_FLOW_ERROR_TYPE_ITEM, item, 2244 "multiple L3 layers not supported"); 2245 else if (item_flags & l4m) 2246 return rte_flow_error_set(error, EINVAL, 2247 RTE_FLOW_ERROR_TYPE_ITEM, item, 2248 "L3 cannot follow an L4 layer."); 2249 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2250 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2251 return rte_flow_error_set(error, EINVAL, 2252 RTE_FLOW_ERROR_TYPE_ITEM, item, 2253 "L3 cannot follow an NVGRE layer."); 2254 if (!mask) 2255 mask = &rte_flow_item_ipv4_mask; 2256 else if (mask->hdr.next_proto_id != 0 && 2257 mask->hdr.next_proto_id != 0xff) 2258 return rte_flow_error_set(error, EINVAL, 2259 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2260 "partial mask is not supported" 2261 " for protocol"); 2262 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2263 acc_mask ? (const uint8_t *)acc_mask 2264 : (const uint8_t *)&nic_mask, 2265 sizeof(struct rte_flow_item_ipv4), 2266 range_accepted, error); 2267 if (ret < 0) 2268 return ret; 2269 return 0; 2270 } 2271 2272 /** 2273 * Validate IPV6 item. 2274 * 2275 * @param[in] item 2276 * Item specification. 2277 * @param[in] item_flags 2278 * Bit-fields that holds the items detected until now. 2279 * @param[in] last_item 2280 * Previous validated item in the pattern items. 2281 * @param[in] ether_type 2282 * Type in the ethernet layer header (including dot1q). 2283 * @param[in] acc_mask 2284 * Acceptable mask, if NULL default internal default mask 2285 * will be used to check whether item fields are supported. 2286 * @param[out] error 2287 * Pointer to error structure. 2288 * 2289 * @return 2290 * 0 on success, a negative errno value otherwise and rte_errno is set. 2291 */ 2292 int 2293 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2294 uint64_t item_flags, 2295 uint64_t last_item, 2296 uint16_t ether_type, 2297 const struct rte_flow_item_ipv6 *acc_mask, 2298 struct rte_flow_error *error) 2299 { 2300 const struct rte_flow_item_ipv6 *mask = item->mask; 2301 const struct rte_flow_item_ipv6 *spec = item->spec; 2302 const struct rte_flow_item_ipv6 nic_mask = { 2303 .hdr = { 2304 .src_addr = 2305 "\xff\xff\xff\xff\xff\xff\xff\xff" 2306 "\xff\xff\xff\xff\xff\xff\xff\xff", 2307 .dst_addr = 2308 "\xff\xff\xff\xff\xff\xff\xff\xff" 2309 "\xff\xff\xff\xff\xff\xff\xff\xff", 2310 .vtc_flow = RTE_BE32(0xffffffff), 2311 .proto = 0xff, 2312 }, 2313 }; 2314 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2315 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2316 MLX5_FLOW_LAYER_OUTER_L3; 2317 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2318 MLX5_FLOW_LAYER_OUTER_L4; 2319 int ret; 2320 uint8_t next_proto = 0xFF; 2321 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2322 MLX5_FLOW_LAYER_OUTER_VLAN | 2323 MLX5_FLOW_LAYER_INNER_VLAN); 2324 2325 if ((last_item & l2_vlan) && ether_type && 2326 ether_type != RTE_ETHER_TYPE_IPV6) 2327 return rte_flow_error_set(error, EINVAL, 2328 RTE_FLOW_ERROR_TYPE_ITEM, item, 2329 "IPv6 cannot follow L2/VLAN layer " 2330 "which ether type is not IPv6"); 2331 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2332 next_proto = spec->hdr.proto; 2333 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2334 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2335 return rte_flow_error_set(error, EINVAL, 2336 RTE_FLOW_ERROR_TYPE_ITEM, 2337 item, 2338 "multiple tunnel " 2339 "not supported"); 2340 } 2341 if (next_proto == IPPROTO_HOPOPTS || 2342 next_proto == IPPROTO_ROUTING || 2343 next_proto == IPPROTO_FRAGMENT || 2344 next_proto == IPPROTO_ESP || 2345 next_proto == IPPROTO_AH || 2346 next_proto == IPPROTO_DSTOPTS) 2347 return rte_flow_error_set(error, EINVAL, 2348 RTE_FLOW_ERROR_TYPE_ITEM, item, 2349 "IPv6 proto (next header) should " 2350 "not be set as extension header"); 2351 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2352 return rte_flow_error_set(error, EINVAL, 2353 RTE_FLOW_ERROR_TYPE_ITEM, item, 2354 "wrong tunnel type - IPv4 specified " 2355 "but IPv6 item provided"); 2356 if (item_flags & l3m) 2357 return rte_flow_error_set(error, ENOTSUP, 2358 RTE_FLOW_ERROR_TYPE_ITEM, item, 2359 "multiple L3 layers not supported"); 2360 else if (item_flags & l4m) 2361 return rte_flow_error_set(error, EINVAL, 2362 RTE_FLOW_ERROR_TYPE_ITEM, item, 2363 "L3 cannot follow an L4 layer."); 2364 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2365 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2366 return rte_flow_error_set(error, EINVAL, 2367 RTE_FLOW_ERROR_TYPE_ITEM, item, 2368 "L3 cannot follow an NVGRE layer."); 2369 if (!mask) 2370 mask = &rte_flow_item_ipv6_mask; 2371 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2372 acc_mask ? (const uint8_t *)acc_mask 2373 : (const uint8_t *)&nic_mask, 2374 sizeof(struct rte_flow_item_ipv6), 2375 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2376 if (ret < 0) 2377 return ret; 2378 return 0; 2379 } 2380 2381 /** 2382 * Validate UDP item. 2383 * 2384 * @param[in] item 2385 * Item specification. 2386 * @param[in] item_flags 2387 * Bit-fields that holds the items detected until now. 2388 * @param[in] target_protocol 2389 * The next protocol in the previous item. 2390 * @param[in] flow_mask 2391 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2392 * @param[out] error 2393 * Pointer to error structure. 2394 * 2395 * @return 2396 * 0 on success, a negative errno value otherwise and rte_errno is set. 2397 */ 2398 int 2399 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2400 uint64_t item_flags, 2401 uint8_t target_protocol, 2402 struct rte_flow_error *error) 2403 { 2404 const struct rte_flow_item_udp *mask = item->mask; 2405 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2406 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2407 MLX5_FLOW_LAYER_OUTER_L3; 2408 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2409 MLX5_FLOW_LAYER_OUTER_L4; 2410 int ret; 2411 2412 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2413 return rte_flow_error_set(error, EINVAL, 2414 RTE_FLOW_ERROR_TYPE_ITEM, item, 2415 "protocol filtering not compatible" 2416 " with UDP layer"); 2417 if (!(item_flags & l3m)) 2418 return rte_flow_error_set(error, EINVAL, 2419 RTE_FLOW_ERROR_TYPE_ITEM, item, 2420 "L3 is mandatory to filter on L4"); 2421 if (item_flags & l4m) 2422 return rte_flow_error_set(error, EINVAL, 2423 RTE_FLOW_ERROR_TYPE_ITEM, item, 2424 "multiple L4 layers not supported"); 2425 if (!mask) 2426 mask = &rte_flow_item_udp_mask; 2427 ret = mlx5_flow_item_acceptable 2428 (item, (const uint8_t *)mask, 2429 (const uint8_t *)&rte_flow_item_udp_mask, 2430 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2431 error); 2432 if (ret < 0) 2433 return ret; 2434 return 0; 2435 } 2436 2437 /** 2438 * Validate TCP item. 2439 * 2440 * @param[in] item 2441 * Item specification. 2442 * @param[in] item_flags 2443 * Bit-fields that holds the items detected until now. 2444 * @param[in] target_protocol 2445 * The next protocol in the previous item. 2446 * @param[out] error 2447 * Pointer to error structure. 2448 * 2449 * @return 2450 * 0 on success, a negative errno value otherwise and rte_errno is set. 2451 */ 2452 int 2453 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2454 uint64_t item_flags, 2455 uint8_t target_protocol, 2456 const struct rte_flow_item_tcp *flow_mask, 2457 struct rte_flow_error *error) 2458 { 2459 const struct rte_flow_item_tcp *mask = item->mask; 2460 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2461 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2462 MLX5_FLOW_LAYER_OUTER_L3; 2463 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2464 MLX5_FLOW_LAYER_OUTER_L4; 2465 int ret; 2466 2467 MLX5_ASSERT(flow_mask); 2468 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2469 return rte_flow_error_set(error, EINVAL, 2470 RTE_FLOW_ERROR_TYPE_ITEM, item, 2471 "protocol filtering not compatible" 2472 " with TCP layer"); 2473 if (!(item_flags & l3m)) 2474 return rte_flow_error_set(error, EINVAL, 2475 RTE_FLOW_ERROR_TYPE_ITEM, item, 2476 "L3 is mandatory to filter on L4"); 2477 if (item_flags & l4m) 2478 return rte_flow_error_set(error, EINVAL, 2479 RTE_FLOW_ERROR_TYPE_ITEM, item, 2480 "multiple L4 layers not supported"); 2481 if (!mask) 2482 mask = &rte_flow_item_tcp_mask; 2483 ret = mlx5_flow_item_acceptable 2484 (item, (const uint8_t *)mask, 2485 (const uint8_t *)flow_mask, 2486 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2487 error); 2488 if (ret < 0) 2489 return ret; 2490 return 0; 2491 } 2492 2493 /** 2494 * Validate VXLAN item. 2495 * 2496 * @param[in] dev 2497 * Pointer to the Ethernet device structure. 2498 * @param[in] udp_dport 2499 * UDP destination port 2500 * @param[in] item 2501 * Item specification. 2502 * @param[in] item_flags 2503 * Bit-fields that holds the items detected until now. 2504 * @param[in] attr 2505 * Flow rule attributes. 2506 * @param[out] error 2507 * Pointer to error structure. 2508 * 2509 * @return 2510 * 0 on success, a negative errno value otherwise and rte_errno is set. 2511 */ 2512 int 2513 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2514 uint16_t udp_dport, 2515 const struct rte_flow_item *item, 2516 uint64_t item_flags, 2517 const struct rte_flow_attr *attr, 2518 struct rte_flow_error *error) 2519 { 2520 const struct rte_flow_item_vxlan *spec = item->spec; 2521 const struct rte_flow_item_vxlan *mask = item->mask; 2522 int ret; 2523 struct mlx5_priv *priv = dev->data->dev_private; 2524 union vni { 2525 uint32_t vlan_id; 2526 uint8_t vni[4]; 2527 } id = { .vlan_id = 0, }; 2528 const struct rte_flow_item_vxlan nic_mask = { 2529 .vni = "\xff\xff\xff", 2530 .rsvd1 = 0xff, 2531 }; 2532 const struct rte_flow_item_vxlan *valid_mask; 2533 2534 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2535 return rte_flow_error_set(error, ENOTSUP, 2536 RTE_FLOW_ERROR_TYPE_ITEM, item, 2537 "multiple tunnel layers not" 2538 " supported"); 2539 valid_mask = &rte_flow_item_vxlan_mask; 2540 /* 2541 * Verify only UDPv4 is present as defined in 2542 * https://tools.ietf.org/html/rfc7348 2543 */ 2544 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2545 return rte_flow_error_set(error, EINVAL, 2546 RTE_FLOW_ERROR_TYPE_ITEM, item, 2547 "no outer UDP layer found"); 2548 if (!mask) 2549 mask = &rte_flow_item_vxlan_mask; 2550 2551 if (priv->sh->steering_format_version != 2552 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2553 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2554 /* FDB domain & NIC domain non-zero group */ 2555 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2556 valid_mask = &nic_mask; 2557 /* Group zero in NIC domain */ 2558 if (!attr->group && !attr->transfer && 2559 priv->sh->tunnel_header_0_1) 2560 valid_mask = &nic_mask; 2561 } 2562 ret = mlx5_flow_item_acceptable 2563 (item, (const uint8_t *)mask, 2564 (const uint8_t *)valid_mask, 2565 sizeof(struct rte_flow_item_vxlan), 2566 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2567 if (ret < 0) 2568 return ret; 2569 if (spec) { 2570 memcpy(&id.vni[1], spec->vni, 3); 2571 memcpy(&id.vni[1], mask->vni, 3); 2572 } 2573 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2574 return rte_flow_error_set(error, ENOTSUP, 2575 RTE_FLOW_ERROR_TYPE_ITEM, item, 2576 "VXLAN tunnel must be fully defined"); 2577 return 0; 2578 } 2579 2580 /** 2581 * Validate VXLAN_GPE item. 2582 * 2583 * @param[in] item 2584 * Item specification. 2585 * @param[in] item_flags 2586 * Bit-fields that holds the items detected until now. 2587 * @param[in] priv 2588 * Pointer to the private data structure. 2589 * @param[in] target_protocol 2590 * The next protocol in the previous item. 2591 * @param[out] error 2592 * Pointer to error structure. 2593 * 2594 * @return 2595 * 0 on success, a negative errno value otherwise and rte_errno is set. 2596 */ 2597 int 2598 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2599 uint64_t item_flags, 2600 struct rte_eth_dev *dev, 2601 struct rte_flow_error *error) 2602 { 2603 struct mlx5_priv *priv = dev->data->dev_private; 2604 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2605 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2606 int ret; 2607 union vni { 2608 uint32_t vlan_id; 2609 uint8_t vni[4]; 2610 } id = { .vlan_id = 0, }; 2611 2612 if (!priv->config.l3_vxlan_en) 2613 return rte_flow_error_set(error, ENOTSUP, 2614 RTE_FLOW_ERROR_TYPE_ITEM, item, 2615 "L3 VXLAN is not enabled by device" 2616 " parameter and/or not configured in" 2617 " firmware"); 2618 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2619 return rte_flow_error_set(error, ENOTSUP, 2620 RTE_FLOW_ERROR_TYPE_ITEM, item, 2621 "multiple tunnel layers not" 2622 " supported"); 2623 /* 2624 * Verify only UDPv4 is present as defined in 2625 * https://tools.ietf.org/html/rfc7348 2626 */ 2627 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2628 return rte_flow_error_set(error, EINVAL, 2629 RTE_FLOW_ERROR_TYPE_ITEM, item, 2630 "no outer UDP layer found"); 2631 if (!mask) 2632 mask = &rte_flow_item_vxlan_gpe_mask; 2633 ret = mlx5_flow_item_acceptable 2634 (item, (const uint8_t *)mask, 2635 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2636 sizeof(struct rte_flow_item_vxlan_gpe), 2637 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2638 if (ret < 0) 2639 return ret; 2640 if (spec) { 2641 if (spec->protocol) 2642 return rte_flow_error_set(error, ENOTSUP, 2643 RTE_FLOW_ERROR_TYPE_ITEM, 2644 item, 2645 "VxLAN-GPE protocol" 2646 " not supported"); 2647 memcpy(&id.vni[1], spec->vni, 3); 2648 memcpy(&id.vni[1], mask->vni, 3); 2649 } 2650 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2651 return rte_flow_error_set(error, ENOTSUP, 2652 RTE_FLOW_ERROR_TYPE_ITEM, item, 2653 "VXLAN-GPE tunnel must be fully" 2654 " defined"); 2655 return 0; 2656 } 2657 /** 2658 * Validate GRE Key item. 2659 * 2660 * @param[in] item 2661 * Item specification. 2662 * @param[in] item_flags 2663 * Bit flags to mark detected items. 2664 * @param[in] gre_item 2665 * Pointer to gre_item 2666 * @param[out] error 2667 * Pointer to error structure. 2668 * 2669 * @return 2670 * 0 on success, a negative errno value otherwise and rte_errno is set. 2671 */ 2672 int 2673 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2674 uint64_t item_flags, 2675 const struct rte_flow_item *gre_item, 2676 struct rte_flow_error *error) 2677 { 2678 const rte_be32_t *mask = item->mask; 2679 int ret = 0; 2680 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2681 const struct rte_flow_item_gre *gre_spec; 2682 const struct rte_flow_item_gre *gre_mask; 2683 2684 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2685 return rte_flow_error_set(error, ENOTSUP, 2686 RTE_FLOW_ERROR_TYPE_ITEM, item, 2687 "Multiple GRE key not support"); 2688 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2689 return rte_flow_error_set(error, ENOTSUP, 2690 RTE_FLOW_ERROR_TYPE_ITEM, item, 2691 "No preceding GRE header"); 2692 if (item_flags & MLX5_FLOW_LAYER_INNER) 2693 return rte_flow_error_set(error, ENOTSUP, 2694 RTE_FLOW_ERROR_TYPE_ITEM, item, 2695 "GRE key following a wrong item"); 2696 gre_mask = gre_item->mask; 2697 if (!gre_mask) 2698 gre_mask = &rte_flow_item_gre_mask; 2699 gre_spec = gre_item->spec; 2700 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2701 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2702 return rte_flow_error_set(error, EINVAL, 2703 RTE_FLOW_ERROR_TYPE_ITEM, item, 2704 "Key bit must be on"); 2705 2706 if (!mask) 2707 mask = &gre_key_default_mask; 2708 ret = mlx5_flow_item_acceptable 2709 (item, (const uint8_t *)mask, 2710 (const uint8_t *)&gre_key_default_mask, 2711 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2712 return ret; 2713 } 2714 2715 /** 2716 * Validate GRE item. 2717 * 2718 * @param[in] item 2719 * Item specification. 2720 * @param[in] item_flags 2721 * Bit flags to mark detected items. 2722 * @param[in] target_protocol 2723 * The next protocol in the previous item. 2724 * @param[out] error 2725 * Pointer to error structure. 2726 * 2727 * @return 2728 * 0 on success, a negative errno value otherwise and rte_errno is set. 2729 */ 2730 int 2731 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2732 uint64_t item_flags, 2733 uint8_t target_protocol, 2734 struct rte_flow_error *error) 2735 { 2736 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2737 const struct rte_flow_item_gre *mask = item->mask; 2738 int ret; 2739 const struct rte_flow_item_gre nic_mask = { 2740 .c_rsvd0_ver = RTE_BE16(0xB000), 2741 .protocol = RTE_BE16(UINT16_MAX), 2742 }; 2743 2744 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2745 return rte_flow_error_set(error, EINVAL, 2746 RTE_FLOW_ERROR_TYPE_ITEM, item, 2747 "protocol filtering not compatible" 2748 " with this GRE layer"); 2749 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2750 return rte_flow_error_set(error, ENOTSUP, 2751 RTE_FLOW_ERROR_TYPE_ITEM, item, 2752 "multiple tunnel layers not" 2753 " supported"); 2754 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2755 return rte_flow_error_set(error, ENOTSUP, 2756 RTE_FLOW_ERROR_TYPE_ITEM, item, 2757 "L3 Layer is missing"); 2758 if (!mask) 2759 mask = &rte_flow_item_gre_mask; 2760 ret = mlx5_flow_item_acceptable 2761 (item, (const uint8_t *)mask, 2762 (const uint8_t *)&nic_mask, 2763 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2764 error); 2765 if (ret < 0) 2766 return ret; 2767 #ifndef HAVE_MLX5DV_DR 2768 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2769 if (spec && (spec->protocol & mask->protocol)) 2770 return rte_flow_error_set(error, ENOTSUP, 2771 RTE_FLOW_ERROR_TYPE_ITEM, item, 2772 "without MPLS support the" 2773 " specification cannot be used for" 2774 " filtering"); 2775 #endif 2776 #endif 2777 return 0; 2778 } 2779 2780 /** 2781 * Validate Geneve item. 2782 * 2783 * @param[in] item 2784 * Item specification. 2785 * @param[in] itemFlags 2786 * Bit-fields that holds the items detected until now. 2787 * @param[in] enPriv 2788 * Pointer to the private data structure. 2789 * @param[out] error 2790 * Pointer to error structure. 2791 * 2792 * @return 2793 * 0 on success, a negative errno value otherwise and rte_errno is set. 2794 */ 2795 2796 int 2797 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2798 uint64_t item_flags, 2799 struct rte_eth_dev *dev, 2800 struct rte_flow_error *error) 2801 { 2802 struct mlx5_priv *priv = dev->data->dev_private; 2803 const struct rte_flow_item_geneve *spec = item->spec; 2804 const struct rte_flow_item_geneve *mask = item->mask; 2805 int ret; 2806 uint16_t gbhdr; 2807 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2808 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2809 const struct rte_flow_item_geneve nic_mask = { 2810 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2811 .vni = "\xff\xff\xff", 2812 .protocol = RTE_BE16(UINT16_MAX), 2813 }; 2814 2815 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2816 return rte_flow_error_set(error, ENOTSUP, 2817 RTE_FLOW_ERROR_TYPE_ITEM, item, 2818 "L3 Geneve is not enabled by device" 2819 " parameter and/or not configured in" 2820 " firmware"); 2821 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2822 return rte_flow_error_set(error, ENOTSUP, 2823 RTE_FLOW_ERROR_TYPE_ITEM, item, 2824 "multiple tunnel layers not" 2825 " supported"); 2826 /* 2827 * Verify only UDPv4 is present as defined in 2828 * https://tools.ietf.org/html/rfc7348 2829 */ 2830 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2831 return rte_flow_error_set(error, EINVAL, 2832 RTE_FLOW_ERROR_TYPE_ITEM, item, 2833 "no outer UDP layer found"); 2834 if (!mask) 2835 mask = &rte_flow_item_geneve_mask; 2836 ret = mlx5_flow_item_acceptable 2837 (item, (const uint8_t *)mask, 2838 (const uint8_t *)&nic_mask, 2839 sizeof(struct rte_flow_item_geneve), 2840 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2841 if (ret) 2842 return ret; 2843 if (spec) { 2844 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2845 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2846 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2847 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2848 return rte_flow_error_set(error, ENOTSUP, 2849 RTE_FLOW_ERROR_TYPE_ITEM, 2850 item, 2851 "Geneve protocol unsupported" 2852 " fields are being used"); 2853 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2854 return rte_flow_error_set 2855 (error, ENOTSUP, 2856 RTE_FLOW_ERROR_TYPE_ITEM, 2857 item, 2858 "Unsupported Geneve options length"); 2859 } 2860 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2861 return rte_flow_error_set 2862 (error, ENOTSUP, 2863 RTE_FLOW_ERROR_TYPE_ITEM, item, 2864 "Geneve tunnel must be fully defined"); 2865 return 0; 2866 } 2867 2868 /** 2869 * Validate Geneve TLV option item. 2870 * 2871 * @param[in] item 2872 * Item specification. 2873 * @param[in] last_item 2874 * Previous validated item in the pattern items. 2875 * @param[in] geneve_item 2876 * Previous GENEVE item specification. 2877 * @param[in] dev 2878 * Pointer to the rte_eth_dev structure. 2879 * @param[out] error 2880 * Pointer to error structure. 2881 * 2882 * @return 2883 * 0 on success, a negative errno value otherwise and rte_errno is set. 2884 */ 2885 int 2886 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2887 uint64_t last_item, 2888 const struct rte_flow_item *geneve_item, 2889 struct rte_eth_dev *dev, 2890 struct rte_flow_error *error) 2891 { 2892 struct mlx5_priv *priv = dev->data->dev_private; 2893 struct mlx5_dev_ctx_shared *sh = priv->sh; 2894 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2895 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2896 uint8_t data_max_supported = 2897 hca_attr->max_geneve_tlv_option_data_len * 4; 2898 struct mlx5_dev_config *config = &priv->config; 2899 const struct rte_flow_item_geneve *geneve_spec; 2900 const struct rte_flow_item_geneve *geneve_mask; 2901 const struct rte_flow_item_geneve_opt *spec = item->spec; 2902 const struct rte_flow_item_geneve_opt *mask = item->mask; 2903 unsigned int i; 2904 unsigned int data_len; 2905 uint8_t tlv_option_len; 2906 uint16_t optlen_m, optlen_v; 2907 const struct rte_flow_item_geneve_opt full_mask = { 2908 .option_class = RTE_BE16(0xffff), 2909 .option_type = 0xff, 2910 .option_len = 0x1f, 2911 }; 2912 2913 if (!mask) 2914 mask = &rte_flow_item_geneve_opt_mask; 2915 if (!spec) 2916 return rte_flow_error_set 2917 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2918 "Geneve TLV opt class/type/length must be specified"); 2919 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2920 return rte_flow_error_set 2921 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2922 "Geneve TLV opt length exceeeds the limit (31)"); 2923 /* Check if class type and length masks are full. */ 2924 if (full_mask.option_class != mask->option_class || 2925 full_mask.option_type != mask->option_type || 2926 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2927 return rte_flow_error_set 2928 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2929 "Geneve TLV opt class/type/length masks must be full"); 2930 /* Check if length is supported */ 2931 if ((uint32_t)spec->option_len > 2932 config->hca_attr.max_geneve_tlv_option_data_len) 2933 return rte_flow_error_set 2934 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2935 "Geneve TLV opt length not supported"); 2936 if (config->hca_attr.max_geneve_tlv_options > 1) 2937 DRV_LOG(DEBUG, 2938 "max_geneve_tlv_options supports more than 1 option"); 2939 /* Check GENEVE item preceding. */ 2940 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2941 return rte_flow_error_set 2942 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2943 "Geneve opt item must be preceded with Geneve item"); 2944 geneve_spec = geneve_item->spec; 2945 geneve_mask = geneve_item->mask ? geneve_item->mask : 2946 &rte_flow_item_geneve_mask; 2947 /* Check if GENEVE TLV option size doesn't exceed option length */ 2948 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2949 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2950 tlv_option_len = spec->option_len & mask->option_len; 2951 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2952 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2953 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2954 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2955 if ((optlen_v & optlen_m) <= tlv_option_len) 2956 return rte_flow_error_set 2957 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2958 "GENEVE TLV option length exceeds optlen"); 2959 } 2960 /* Check if length is 0 or data is 0. */ 2961 if (spec->data == NULL || spec->option_len == 0) 2962 return rte_flow_error_set 2963 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2964 "Geneve TLV opt with zero data/length not supported"); 2965 /* Check not all data & mask are 0. */ 2966 data_len = spec->option_len * 4; 2967 if (mask->data == NULL) { 2968 for (i = 0; i < data_len; i++) 2969 if (spec->data[i]) 2970 break; 2971 if (i == data_len) 2972 return rte_flow_error_set(error, ENOTSUP, 2973 RTE_FLOW_ERROR_TYPE_ITEM, item, 2974 "Can't match on Geneve option data 0"); 2975 } else { 2976 for (i = 0; i < data_len; i++) 2977 if (spec->data[i] & mask->data[i]) 2978 break; 2979 if (i == data_len) 2980 return rte_flow_error_set(error, ENOTSUP, 2981 RTE_FLOW_ERROR_TYPE_ITEM, item, 2982 "Can't match on Geneve option data and mask 0"); 2983 /* Check data mask supported. */ 2984 for (i = data_max_supported; i < data_len ; i++) 2985 if (mask->data[i]) 2986 return rte_flow_error_set(error, ENOTSUP, 2987 RTE_FLOW_ERROR_TYPE_ITEM, item, 2988 "Data mask is of unsupported size"); 2989 } 2990 /* Check GENEVE option is supported in NIC. */ 2991 if (!config->hca_attr.geneve_tlv_opt) 2992 return rte_flow_error_set 2993 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2994 "Geneve TLV opt not supported"); 2995 /* Check if we already have geneve option with different type/class. */ 2996 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2997 geneve_opt_resource = sh->geneve_tlv_option_resource; 2998 if (geneve_opt_resource != NULL) 2999 if (geneve_opt_resource->option_class != spec->option_class || 3000 geneve_opt_resource->option_type != spec->option_type || 3001 geneve_opt_resource->length != spec->option_len) { 3002 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3003 return rte_flow_error_set(error, ENOTSUP, 3004 RTE_FLOW_ERROR_TYPE_ITEM, item, 3005 "Only one Geneve TLV option supported"); 3006 } 3007 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3008 return 0; 3009 } 3010 3011 /** 3012 * Validate MPLS item. 3013 * 3014 * @param[in] dev 3015 * Pointer to the rte_eth_dev structure. 3016 * @param[in] item 3017 * Item specification. 3018 * @param[in] item_flags 3019 * Bit-fields that holds the items detected until now. 3020 * @param[in] prev_layer 3021 * The protocol layer indicated in previous item. 3022 * @param[out] error 3023 * Pointer to error structure. 3024 * 3025 * @return 3026 * 0 on success, a negative errno value otherwise and rte_errno is set. 3027 */ 3028 int 3029 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3030 const struct rte_flow_item *item __rte_unused, 3031 uint64_t item_flags __rte_unused, 3032 uint64_t prev_layer __rte_unused, 3033 struct rte_flow_error *error) 3034 { 3035 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3036 const struct rte_flow_item_mpls *mask = item->mask; 3037 struct mlx5_priv *priv = dev->data->dev_private; 3038 int ret; 3039 3040 if (!priv->config.mpls_en) 3041 return rte_flow_error_set(error, ENOTSUP, 3042 RTE_FLOW_ERROR_TYPE_ITEM, item, 3043 "MPLS not supported or" 3044 " disabled in firmware" 3045 " configuration."); 3046 /* MPLS over UDP, GRE is allowed */ 3047 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3048 MLX5_FLOW_LAYER_GRE | 3049 MLX5_FLOW_LAYER_GRE_KEY))) 3050 return rte_flow_error_set(error, EINVAL, 3051 RTE_FLOW_ERROR_TYPE_ITEM, item, 3052 "protocol filtering not compatible" 3053 " with MPLS layer"); 3054 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3055 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3056 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3057 return rte_flow_error_set(error, ENOTSUP, 3058 RTE_FLOW_ERROR_TYPE_ITEM, item, 3059 "multiple tunnel layers not" 3060 " supported"); 3061 if (!mask) 3062 mask = &rte_flow_item_mpls_mask; 3063 ret = mlx5_flow_item_acceptable 3064 (item, (const uint8_t *)mask, 3065 (const uint8_t *)&rte_flow_item_mpls_mask, 3066 sizeof(struct rte_flow_item_mpls), 3067 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3068 if (ret < 0) 3069 return ret; 3070 return 0; 3071 #else 3072 return rte_flow_error_set(error, ENOTSUP, 3073 RTE_FLOW_ERROR_TYPE_ITEM, item, 3074 "MPLS is not supported by Verbs, please" 3075 " update."); 3076 #endif 3077 } 3078 3079 /** 3080 * Validate NVGRE item. 3081 * 3082 * @param[in] item 3083 * Item specification. 3084 * @param[in] item_flags 3085 * Bit flags to mark detected items. 3086 * @param[in] target_protocol 3087 * The next protocol in the previous item. 3088 * @param[out] error 3089 * Pointer to error structure. 3090 * 3091 * @return 3092 * 0 on success, a negative errno value otherwise and rte_errno is set. 3093 */ 3094 int 3095 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3096 uint64_t item_flags, 3097 uint8_t target_protocol, 3098 struct rte_flow_error *error) 3099 { 3100 const struct rte_flow_item_nvgre *mask = item->mask; 3101 int ret; 3102 3103 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3104 return rte_flow_error_set(error, EINVAL, 3105 RTE_FLOW_ERROR_TYPE_ITEM, item, 3106 "protocol filtering not compatible" 3107 " with this GRE layer"); 3108 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3109 return rte_flow_error_set(error, ENOTSUP, 3110 RTE_FLOW_ERROR_TYPE_ITEM, item, 3111 "multiple tunnel layers not" 3112 " supported"); 3113 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3114 return rte_flow_error_set(error, ENOTSUP, 3115 RTE_FLOW_ERROR_TYPE_ITEM, item, 3116 "L3 Layer is missing"); 3117 if (!mask) 3118 mask = &rte_flow_item_nvgre_mask; 3119 ret = mlx5_flow_item_acceptable 3120 (item, (const uint8_t *)mask, 3121 (const uint8_t *)&rte_flow_item_nvgre_mask, 3122 sizeof(struct rte_flow_item_nvgre), 3123 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3124 if (ret < 0) 3125 return ret; 3126 return 0; 3127 } 3128 3129 /** 3130 * Validate eCPRI item. 3131 * 3132 * @param[in] item 3133 * Item specification. 3134 * @param[in] item_flags 3135 * Bit-fields that holds the items detected until now. 3136 * @param[in] last_item 3137 * Previous validated item in the pattern items. 3138 * @param[in] ether_type 3139 * Type in the ethernet layer header (including dot1q). 3140 * @param[in] acc_mask 3141 * Acceptable mask, if NULL default internal default mask 3142 * will be used to check whether item fields are supported. 3143 * @param[out] error 3144 * Pointer to error structure. 3145 * 3146 * @return 3147 * 0 on success, a negative errno value otherwise and rte_errno is set. 3148 */ 3149 int 3150 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3151 uint64_t item_flags, 3152 uint64_t last_item, 3153 uint16_t ether_type, 3154 const struct rte_flow_item_ecpri *acc_mask, 3155 struct rte_flow_error *error) 3156 { 3157 const struct rte_flow_item_ecpri *mask = item->mask; 3158 const struct rte_flow_item_ecpri nic_mask = { 3159 .hdr = { 3160 .common = { 3161 .u32 = 3162 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3163 .type = 0xFF, 3164 }).u32), 3165 }, 3166 .dummy[0] = 0xFFFFFFFF, 3167 }, 3168 }; 3169 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3170 MLX5_FLOW_LAYER_OUTER_VLAN); 3171 struct rte_flow_item_ecpri mask_lo; 3172 3173 if (!(last_item & outer_l2_vlan) && 3174 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3175 return rte_flow_error_set(error, EINVAL, 3176 RTE_FLOW_ERROR_TYPE_ITEM, item, 3177 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3178 if ((last_item & outer_l2_vlan) && ether_type && 3179 ether_type != RTE_ETHER_TYPE_ECPRI) 3180 return rte_flow_error_set(error, EINVAL, 3181 RTE_FLOW_ERROR_TYPE_ITEM, item, 3182 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3183 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3184 return rte_flow_error_set(error, EINVAL, 3185 RTE_FLOW_ERROR_TYPE_ITEM, item, 3186 "eCPRI with tunnel is not supported right now"); 3187 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3188 return rte_flow_error_set(error, ENOTSUP, 3189 RTE_FLOW_ERROR_TYPE_ITEM, item, 3190 "multiple L3 layers not supported"); 3191 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3192 return rte_flow_error_set(error, EINVAL, 3193 RTE_FLOW_ERROR_TYPE_ITEM, item, 3194 "eCPRI cannot coexist with a TCP layer"); 3195 /* In specification, eCPRI could be over UDP layer. */ 3196 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3197 return rte_flow_error_set(error, EINVAL, 3198 RTE_FLOW_ERROR_TYPE_ITEM, item, 3199 "eCPRI over UDP layer is not yet supported right now"); 3200 /* Mask for type field in common header could be zero. */ 3201 if (!mask) 3202 mask = &rte_flow_item_ecpri_mask; 3203 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3204 /* Input mask is in big-endian format. */ 3205 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3206 return rte_flow_error_set(error, EINVAL, 3207 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3208 "partial mask is not supported for protocol"); 3209 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3210 return rte_flow_error_set(error, EINVAL, 3211 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3212 "message header mask must be after a type mask"); 3213 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3214 acc_mask ? (const uint8_t *)acc_mask 3215 : (const uint8_t *)&nic_mask, 3216 sizeof(struct rte_flow_item_ecpri), 3217 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3218 } 3219 3220 static int 3221 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3222 const struct rte_flow_attr *attr __rte_unused, 3223 const struct rte_flow_item items[] __rte_unused, 3224 const struct rte_flow_action actions[] __rte_unused, 3225 bool external __rte_unused, 3226 int hairpin __rte_unused, 3227 struct rte_flow_error *error) 3228 { 3229 return rte_flow_error_set(error, ENOTSUP, 3230 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3231 } 3232 3233 static struct mlx5_flow * 3234 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3235 const struct rte_flow_attr *attr __rte_unused, 3236 const struct rte_flow_item items[] __rte_unused, 3237 const struct rte_flow_action actions[] __rte_unused, 3238 struct rte_flow_error *error) 3239 { 3240 rte_flow_error_set(error, ENOTSUP, 3241 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3242 return NULL; 3243 } 3244 3245 static int 3246 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3247 struct mlx5_flow *dev_flow __rte_unused, 3248 const struct rte_flow_attr *attr __rte_unused, 3249 const struct rte_flow_item items[] __rte_unused, 3250 const struct rte_flow_action actions[] __rte_unused, 3251 struct rte_flow_error *error) 3252 { 3253 return rte_flow_error_set(error, ENOTSUP, 3254 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3255 } 3256 3257 static int 3258 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3259 struct rte_flow *flow __rte_unused, 3260 struct rte_flow_error *error) 3261 { 3262 return rte_flow_error_set(error, ENOTSUP, 3263 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3264 } 3265 3266 static void 3267 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3268 struct rte_flow *flow __rte_unused) 3269 { 3270 } 3271 3272 static void 3273 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3274 struct rte_flow *flow __rte_unused) 3275 { 3276 } 3277 3278 static int 3279 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3280 struct rte_flow *flow __rte_unused, 3281 const struct rte_flow_action *actions __rte_unused, 3282 void *data __rte_unused, 3283 struct rte_flow_error *error) 3284 { 3285 return rte_flow_error_set(error, ENOTSUP, 3286 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3287 } 3288 3289 static int 3290 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3291 uint32_t domains __rte_unused, 3292 uint32_t flags __rte_unused) 3293 { 3294 return 0; 3295 } 3296 3297 /* Void driver to protect from null pointer reference. */ 3298 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3299 .validate = flow_null_validate, 3300 .prepare = flow_null_prepare, 3301 .translate = flow_null_translate, 3302 .apply = flow_null_apply, 3303 .remove = flow_null_remove, 3304 .destroy = flow_null_destroy, 3305 .query = flow_null_query, 3306 .sync_domain = flow_null_sync_domain, 3307 }; 3308 3309 /** 3310 * Select flow driver type according to flow attributes and device 3311 * configuration. 3312 * 3313 * @param[in] dev 3314 * Pointer to the dev structure. 3315 * @param[in] attr 3316 * Pointer to the flow attributes. 3317 * 3318 * @return 3319 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3320 */ 3321 static enum mlx5_flow_drv_type 3322 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3323 { 3324 struct mlx5_priv *priv = dev->data->dev_private; 3325 /* The OS can determine first a specific flow type (DV, VERBS) */ 3326 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3327 3328 if (type != MLX5_FLOW_TYPE_MAX) 3329 return type; 3330 /* If no OS specific type - continue with DV/VERBS selection */ 3331 if (attr->transfer && priv->config.dv_esw_en) 3332 type = MLX5_FLOW_TYPE_DV; 3333 if (!attr->transfer) 3334 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3335 MLX5_FLOW_TYPE_VERBS; 3336 return type; 3337 } 3338 3339 #define flow_get_drv_ops(type) flow_drv_ops[type] 3340 3341 /** 3342 * Flow driver validation API. This abstracts calling driver specific functions. 3343 * The type of flow driver is determined according to flow attributes. 3344 * 3345 * @param[in] dev 3346 * Pointer to the dev structure. 3347 * @param[in] attr 3348 * Pointer to the flow attributes. 3349 * @param[in] items 3350 * Pointer to the list of items. 3351 * @param[in] actions 3352 * Pointer to the list of actions. 3353 * @param[in] external 3354 * This flow rule is created by request external to PMD. 3355 * @param[in] hairpin 3356 * Number of hairpin TX actions, 0 means classic flow. 3357 * @param[out] error 3358 * Pointer to the error structure. 3359 * 3360 * @return 3361 * 0 on success, a negative errno value otherwise and rte_errno is set. 3362 */ 3363 static inline int 3364 flow_drv_validate(struct rte_eth_dev *dev, 3365 const struct rte_flow_attr *attr, 3366 const struct rte_flow_item items[], 3367 const struct rte_flow_action actions[], 3368 bool external, int hairpin, struct rte_flow_error *error) 3369 { 3370 const struct mlx5_flow_driver_ops *fops; 3371 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3372 3373 fops = flow_get_drv_ops(type); 3374 return fops->validate(dev, attr, items, actions, external, 3375 hairpin, error); 3376 } 3377 3378 /** 3379 * Flow driver preparation API. This abstracts calling driver specific 3380 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3381 * calculates the size of memory required for device flow, allocates the memory, 3382 * initializes the device flow and returns the pointer. 3383 * 3384 * @note 3385 * This function initializes device flow structure such as dv or verbs in 3386 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3387 * rest. For example, adding returning device flow to flow->dev_flow list and 3388 * setting backward reference to the flow should be done out of this function. 3389 * layers field is not filled either. 3390 * 3391 * @param[in] dev 3392 * Pointer to the dev structure. 3393 * @param[in] attr 3394 * Pointer to the flow attributes. 3395 * @param[in] items 3396 * Pointer to the list of items. 3397 * @param[in] actions 3398 * Pointer to the list of actions. 3399 * @param[in] flow_idx 3400 * This memory pool index to the flow. 3401 * @param[out] error 3402 * Pointer to the error structure. 3403 * 3404 * @return 3405 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3406 */ 3407 static inline struct mlx5_flow * 3408 flow_drv_prepare(struct rte_eth_dev *dev, 3409 const struct rte_flow *flow, 3410 const struct rte_flow_attr *attr, 3411 const struct rte_flow_item items[], 3412 const struct rte_flow_action actions[], 3413 uint32_t flow_idx, 3414 struct rte_flow_error *error) 3415 { 3416 const struct mlx5_flow_driver_ops *fops; 3417 enum mlx5_flow_drv_type type = flow->drv_type; 3418 struct mlx5_flow *mlx5_flow = NULL; 3419 3420 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3421 fops = flow_get_drv_ops(type); 3422 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3423 if (mlx5_flow) 3424 mlx5_flow->flow_idx = flow_idx; 3425 return mlx5_flow; 3426 } 3427 3428 /** 3429 * Flow driver translation API. This abstracts calling driver specific 3430 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3431 * translates a generic flow into a driver flow. flow_drv_prepare() must 3432 * precede. 3433 * 3434 * @note 3435 * dev_flow->layers could be filled as a result of parsing during translation 3436 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3437 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3438 * flow->actions could be overwritten even though all the expanded dev_flows 3439 * have the same actions. 3440 * 3441 * @param[in] dev 3442 * Pointer to the rte dev structure. 3443 * @param[in, out] dev_flow 3444 * Pointer to the mlx5 flow. 3445 * @param[in] attr 3446 * Pointer to the flow attributes. 3447 * @param[in] items 3448 * Pointer to the list of items. 3449 * @param[in] actions 3450 * Pointer to the list of actions. 3451 * @param[out] error 3452 * Pointer to the error structure. 3453 * 3454 * @return 3455 * 0 on success, a negative errno value otherwise and rte_errno is set. 3456 */ 3457 static inline int 3458 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3459 const struct rte_flow_attr *attr, 3460 const struct rte_flow_item items[], 3461 const struct rte_flow_action actions[], 3462 struct rte_flow_error *error) 3463 { 3464 const struct mlx5_flow_driver_ops *fops; 3465 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3466 3467 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3468 fops = flow_get_drv_ops(type); 3469 return fops->translate(dev, dev_flow, attr, items, actions, error); 3470 } 3471 3472 /** 3473 * Flow driver apply API. This abstracts calling driver specific functions. 3474 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3475 * translated driver flows on to device. flow_drv_translate() must precede. 3476 * 3477 * @param[in] dev 3478 * Pointer to Ethernet device structure. 3479 * @param[in, out] flow 3480 * Pointer to flow structure. 3481 * @param[out] error 3482 * Pointer to error structure. 3483 * 3484 * @return 3485 * 0 on success, a negative errno value otherwise and rte_errno is set. 3486 */ 3487 static inline int 3488 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3489 struct rte_flow_error *error) 3490 { 3491 const struct mlx5_flow_driver_ops *fops; 3492 enum mlx5_flow_drv_type type = flow->drv_type; 3493 3494 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3495 fops = flow_get_drv_ops(type); 3496 return fops->apply(dev, flow, error); 3497 } 3498 3499 /** 3500 * Flow driver destroy API. This abstracts calling driver specific functions. 3501 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3502 * on device and releases resources of the flow. 3503 * 3504 * @param[in] dev 3505 * Pointer to Ethernet device. 3506 * @param[in, out] flow 3507 * Pointer to flow structure. 3508 */ 3509 static inline void 3510 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3511 { 3512 const struct mlx5_flow_driver_ops *fops; 3513 enum mlx5_flow_drv_type type = flow->drv_type; 3514 3515 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3516 fops = flow_get_drv_ops(type); 3517 fops->destroy(dev, flow); 3518 } 3519 3520 /** 3521 * Flow driver find RSS policy tbl API. This abstracts calling driver 3522 * specific functions. Parent flow (rte_flow) should have driver 3523 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3524 * 3525 * @param[in] dev 3526 * Pointer to Ethernet device. 3527 * @param[in, out] flow 3528 * Pointer to flow structure. 3529 * @param[in] policy 3530 * Pointer to meter policy table. 3531 * @param[in] rss_desc 3532 * Pointer to rss_desc 3533 */ 3534 static struct mlx5_flow_meter_sub_policy * 3535 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3536 struct rte_flow *flow, 3537 struct mlx5_flow_meter_policy *policy, 3538 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3539 { 3540 const struct mlx5_flow_driver_ops *fops; 3541 enum mlx5_flow_drv_type type = flow->drv_type; 3542 3543 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3544 fops = flow_get_drv_ops(type); 3545 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3546 } 3547 3548 /** 3549 * Flow driver color tag rule API. This abstracts calling driver 3550 * specific functions. Parent flow (rte_flow) should have driver 3551 * type (drv_type). It will create the color tag rules in hierarchy meter. 3552 * 3553 * @param[in] dev 3554 * Pointer to Ethernet device. 3555 * @param[in, out] flow 3556 * Pointer to flow structure. 3557 * @param[in] fm 3558 * Pointer to flow meter structure. 3559 * @param[in] src_port 3560 * The src port this extra rule should use. 3561 * @param[in] item 3562 * The src port id match item. 3563 * @param[out] error 3564 * Pointer to error structure. 3565 */ 3566 static int 3567 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3568 struct rte_flow *flow, 3569 struct mlx5_flow_meter_info *fm, 3570 int32_t src_port, 3571 const struct rte_flow_item *item, 3572 struct rte_flow_error *error) 3573 { 3574 const struct mlx5_flow_driver_ops *fops; 3575 enum mlx5_flow_drv_type type = flow->drv_type; 3576 3577 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3578 fops = flow_get_drv_ops(type); 3579 return fops->meter_hierarchy_rule_create(dev, fm, 3580 src_port, item, error); 3581 } 3582 3583 /** 3584 * Get RSS action from the action list. 3585 * 3586 * @param[in] dev 3587 * Pointer to Ethernet device. 3588 * @param[in] actions 3589 * Pointer to the list of actions. 3590 * @param[in] flow 3591 * Parent flow structure pointer. 3592 * 3593 * @return 3594 * Pointer to the RSS action if exist, else return NULL. 3595 */ 3596 static const struct rte_flow_action_rss* 3597 flow_get_rss_action(struct rte_eth_dev *dev, 3598 const struct rte_flow_action actions[]) 3599 { 3600 struct mlx5_priv *priv = dev->data->dev_private; 3601 const struct rte_flow_action_rss *rss = NULL; 3602 struct mlx5_meter_policy_action_container *acg; 3603 struct mlx5_meter_policy_action_container *acy; 3604 3605 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3606 switch (actions->type) { 3607 case RTE_FLOW_ACTION_TYPE_RSS: 3608 rss = actions->conf; 3609 break; 3610 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3611 { 3612 const struct rte_flow_action_sample *sample = 3613 actions->conf; 3614 const struct rte_flow_action *act = sample->actions; 3615 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3616 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3617 rss = act->conf; 3618 break; 3619 } 3620 case RTE_FLOW_ACTION_TYPE_METER: 3621 { 3622 uint32_t mtr_idx; 3623 struct mlx5_flow_meter_info *fm; 3624 struct mlx5_flow_meter_policy *policy; 3625 const struct rte_flow_action_meter *mtr = actions->conf; 3626 3627 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3628 if (fm && !fm->def_policy) { 3629 policy = mlx5_flow_meter_policy_find(dev, 3630 fm->policy_id, NULL); 3631 MLX5_ASSERT(policy); 3632 if (policy->is_hierarchy) { 3633 policy = 3634 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3635 policy); 3636 if (!policy) 3637 return NULL; 3638 } 3639 if (policy->is_rss) { 3640 acg = 3641 &policy->act_cnt[RTE_COLOR_GREEN]; 3642 acy = 3643 &policy->act_cnt[RTE_COLOR_YELLOW]; 3644 if (acg->fate_action == 3645 MLX5_FLOW_FATE_SHARED_RSS) 3646 rss = acg->rss->conf; 3647 else if (acy->fate_action == 3648 MLX5_FLOW_FATE_SHARED_RSS) 3649 rss = acy->rss->conf; 3650 } 3651 } 3652 break; 3653 } 3654 default: 3655 break; 3656 } 3657 } 3658 return rss; 3659 } 3660 3661 /** 3662 * Get ASO age action by index. 3663 * 3664 * @param[in] dev 3665 * Pointer to the Ethernet device structure. 3666 * @param[in] age_idx 3667 * Index to the ASO age action. 3668 * 3669 * @return 3670 * The specified ASO age action. 3671 */ 3672 struct mlx5_aso_age_action* 3673 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3674 { 3675 uint16_t pool_idx = age_idx & UINT16_MAX; 3676 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3677 struct mlx5_priv *priv = dev->data->dev_private; 3678 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3679 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3680 3681 return &pool->actions[offset - 1]; 3682 } 3683 3684 /* maps indirect action to translated direct in some actions array */ 3685 struct mlx5_translated_action_handle { 3686 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3687 int index; /**< Index in related array of rte_flow_action. */ 3688 }; 3689 3690 /** 3691 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3692 * direct action if translation possible. 3693 * This functionality used to run same execution path for both direct and 3694 * indirect actions on flow create. All necessary preparations for indirect 3695 * action handling should be performed on *handle* actions list returned 3696 * from this call. 3697 * 3698 * @param[in] dev 3699 * Pointer to Ethernet device. 3700 * @param[in] actions 3701 * List of actions to translate. 3702 * @param[out] handle 3703 * List to store translated indirect action object handles. 3704 * @param[in, out] indir_n 3705 * Size of *handle* array. On return should be updated with number of 3706 * indirect actions retrieved from the *actions* list. 3707 * @param[out] translated_actions 3708 * List of actions where all indirect actions were translated to direct 3709 * if possible. NULL if no translation took place. 3710 * @param[out] error 3711 * Pointer to the error structure. 3712 * 3713 * @return 3714 * 0 on success, a negative errno value otherwise and rte_errno is set. 3715 */ 3716 static int 3717 flow_action_handles_translate(struct rte_eth_dev *dev, 3718 const struct rte_flow_action actions[], 3719 struct mlx5_translated_action_handle *handle, 3720 int *indir_n, 3721 struct rte_flow_action **translated_actions, 3722 struct rte_flow_error *error) 3723 { 3724 struct mlx5_priv *priv = dev->data->dev_private; 3725 struct rte_flow_action *translated = NULL; 3726 size_t actions_size; 3727 int n; 3728 int copied_n = 0; 3729 struct mlx5_translated_action_handle *handle_end = NULL; 3730 3731 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3732 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3733 continue; 3734 if (copied_n == *indir_n) { 3735 return rte_flow_error_set 3736 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3737 NULL, "too many shared actions"); 3738 } 3739 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3740 sizeof(actions[n].conf)); 3741 handle[copied_n].index = n; 3742 copied_n++; 3743 } 3744 n++; 3745 *indir_n = copied_n; 3746 if (!copied_n) 3747 return 0; 3748 actions_size = sizeof(struct rte_flow_action) * n; 3749 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3750 if (!translated) { 3751 rte_errno = ENOMEM; 3752 return -ENOMEM; 3753 } 3754 memcpy(translated, actions, actions_size); 3755 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3756 struct mlx5_shared_action_rss *shared_rss; 3757 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3758 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3759 uint32_t idx = act_idx & 3760 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3761 3762 switch (type) { 3763 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3764 shared_rss = mlx5_ipool_get 3765 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3766 translated[handle->index].type = 3767 RTE_FLOW_ACTION_TYPE_RSS; 3768 translated[handle->index].conf = 3769 &shared_rss->origin; 3770 break; 3771 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3772 translated[handle->index].type = 3773 (enum rte_flow_action_type) 3774 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3775 translated[handle->index].conf = (void *)(uintptr_t)idx; 3776 break; 3777 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3778 if (priv->sh->flow_hit_aso_en) { 3779 translated[handle->index].type = 3780 (enum rte_flow_action_type) 3781 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3782 translated[handle->index].conf = 3783 (void *)(uintptr_t)idx; 3784 break; 3785 } 3786 /* Fall-through */ 3787 case MLX5_INDIRECT_ACTION_TYPE_CT: 3788 if (priv->sh->ct_aso_en) { 3789 translated[handle->index].type = 3790 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3791 translated[handle->index].conf = 3792 (void *)(uintptr_t)idx; 3793 break; 3794 } 3795 /* Fall-through */ 3796 default: 3797 mlx5_free(translated); 3798 return rte_flow_error_set 3799 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3800 NULL, "invalid indirect action type"); 3801 } 3802 } 3803 *translated_actions = translated; 3804 return 0; 3805 } 3806 3807 /** 3808 * Get Shared RSS action from the action list. 3809 * 3810 * @param[in] dev 3811 * Pointer to Ethernet device. 3812 * @param[in] shared 3813 * Pointer to the list of actions. 3814 * @param[in] shared_n 3815 * Actions list length. 3816 * 3817 * @return 3818 * The MLX5 RSS action ID if exists, otherwise return 0. 3819 */ 3820 static uint32_t 3821 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3822 struct mlx5_translated_action_handle *handle, 3823 int shared_n) 3824 { 3825 struct mlx5_translated_action_handle *handle_end; 3826 struct mlx5_priv *priv = dev->data->dev_private; 3827 struct mlx5_shared_action_rss *shared_rss; 3828 3829 3830 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3831 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3832 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3833 uint32_t idx = act_idx & 3834 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3835 switch (type) { 3836 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3837 shared_rss = mlx5_ipool_get 3838 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3839 idx); 3840 __atomic_add_fetch(&shared_rss->refcnt, 1, 3841 __ATOMIC_RELAXED); 3842 return idx; 3843 default: 3844 break; 3845 } 3846 } 3847 return 0; 3848 } 3849 3850 static unsigned int 3851 find_graph_root(uint32_t rss_level) 3852 { 3853 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3854 MLX5_EXPANSION_ROOT_OUTER; 3855 } 3856 3857 /** 3858 * Get layer flags from the prefix flow. 3859 * 3860 * Some flows may be split to several subflows, the prefix subflow gets the 3861 * match items and the suffix sub flow gets the actions. 3862 * Some actions need the user defined match item flags to get the detail for 3863 * the action. 3864 * This function helps the suffix flow to get the item layer flags from prefix 3865 * subflow. 3866 * 3867 * @param[in] dev_flow 3868 * Pointer the created preifx subflow. 3869 * 3870 * @return 3871 * The layers get from prefix subflow. 3872 */ 3873 static inline uint64_t 3874 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3875 { 3876 uint64_t layers = 0; 3877 3878 /* 3879 * Layers bits could be localization, but usually the compiler will 3880 * help to do the optimization work for source code. 3881 * If no decap actions, use the layers directly. 3882 */ 3883 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3884 return dev_flow->handle->layers; 3885 /* Convert L3 layers with decap action. */ 3886 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3887 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3888 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3889 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3890 /* Convert L4 layers with decap action. */ 3891 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3892 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3893 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3894 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3895 return layers; 3896 } 3897 3898 /** 3899 * Get metadata split action information. 3900 * 3901 * @param[in] actions 3902 * Pointer to the list of actions. 3903 * @param[out] qrss 3904 * Pointer to the return pointer. 3905 * @param[out] qrss_type 3906 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3907 * if no QUEUE/RSS is found. 3908 * @param[out] encap_idx 3909 * Pointer to the index of the encap action if exists, otherwise the last 3910 * action index. 3911 * 3912 * @return 3913 * Total number of actions. 3914 */ 3915 static int 3916 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3917 const struct rte_flow_action **qrss, 3918 int *encap_idx) 3919 { 3920 const struct rte_flow_action_raw_encap *raw_encap; 3921 int actions_n = 0; 3922 int raw_decap_idx = -1; 3923 3924 *encap_idx = -1; 3925 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3926 switch (actions->type) { 3927 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3928 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3929 *encap_idx = actions_n; 3930 break; 3931 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3932 raw_decap_idx = actions_n; 3933 break; 3934 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3935 raw_encap = actions->conf; 3936 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3937 *encap_idx = raw_decap_idx != -1 ? 3938 raw_decap_idx : actions_n; 3939 break; 3940 case RTE_FLOW_ACTION_TYPE_QUEUE: 3941 case RTE_FLOW_ACTION_TYPE_RSS: 3942 *qrss = actions; 3943 break; 3944 default: 3945 break; 3946 } 3947 actions_n++; 3948 } 3949 if (*encap_idx == -1) 3950 *encap_idx = actions_n; 3951 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3952 return actions_n + 1; 3953 } 3954 3955 /** 3956 * Check if the action will change packet. 3957 * 3958 * @param dev 3959 * Pointer to Ethernet device. 3960 * @param[in] type 3961 * action type. 3962 * 3963 * @return 3964 * true if action will change packet, false otherwise. 3965 */ 3966 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 3967 enum rte_flow_action_type type) 3968 { 3969 struct mlx5_priv *priv = dev->data->dev_private; 3970 3971 switch (type) { 3972 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3973 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3974 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3975 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3976 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3977 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3978 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3979 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3980 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3981 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3982 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3983 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3984 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3985 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3986 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 3987 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 3988 case RTE_FLOW_ACTION_TYPE_SET_META: 3989 case RTE_FLOW_ACTION_TYPE_SET_TAG: 3990 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 3991 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3992 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3993 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3994 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3995 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3996 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3997 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3998 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3999 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4000 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4001 return true; 4002 case RTE_FLOW_ACTION_TYPE_FLAG: 4003 case RTE_FLOW_ACTION_TYPE_MARK: 4004 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4005 return true; 4006 else 4007 return false; 4008 default: 4009 return false; 4010 } 4011 } 4012 4013 /** 4014 * Check meter action from the action list. 4015 * 4016 * @param dev 4017 * Pointer to Ethernet device. 4018 * @param[in] actions 4019 * Pointer to the list of actions. 4020 * @param[out] has_mtr 4021 * Pointer to the meter exist flag. 4022 * @param[out] has_modify 4023 * Pointer to the flag showing there's packet change action. 4024 * @param[out] meter_id 4025 * Pointer to the meter id. 4026 * 4027 * @return 4028 * Total number of actions. 4029 */ 4030 static int 4031 flow_check_meter_action(struct rte_eth_dev *dev, 4032 const struct rte_flow_action actions[], 4033 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4034 { 4035 const struct rte_flow_action_meter *mtr = NULL; 4036 int actions_n = 0; 4037 4038 MLX5_ASSERT(has_mtr); 4039 *has_mtr = false; 4040 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4041 switch (actions->type) { 4042 case RTE_FLOW_ACTION_TYPE_METER: 4043 mtr = actions->conf; 4044 *meter_id = mtr->mtr_id; 4045 *has_mtr = true; 4046 break; 4047 default: 4048 break; 4049 } 4050 if (!*has_mtr) 4051 *has_modify |= flow_check_modify_action_type(dev, 4052 actions->type); 4053 actions_n++; 4054 } 4055 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4056 return actions_n + 1; 4057 } 4058 4059 /** 4060 * Check if the flow should be split due to hairpin. 4061 * The reason for the split is that in current HW we can't 4062 * support encap and push-vlan on Rx, so if a flow contains 4063 * these actions we move it to Tx. 4064 * 4065 * @param dev 4066 * Pointer to Ethernet device. 4067 * @param[in] attr 4068 * Flow rule attributes. 4069 * @param[in] actions 4070 * Associated actions (list terminated by the END action). 4071 * 4072 * @return 4073 * > 0 the number of actions and the flow should be split, 4074 * 0 when no split required. 4075 */ 4076 static int 4077 flow_check_hairpin_split(struct rte_eth_dev *dev, 4078 const struct rte_flow_attr *attr, 4079 const struct rte_flow_action actions[]) 4080 { 4081 int queue_action = 0; 4082 int action_n = 0; 4083 int split = 0; 4084 const struct rte_flow_action_queue *queue; 4085 const struct rte_flow_action_rss *rss; 4086 const struct rte_flow_action_raw_encap *raw_encap; 4087 const struct rte_eth_hairpin_conf *conf; 4088 4089 if (!attr->ingress) 4090 return 0; 4091 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4092 switch (actions->type) { 4093 case RTE_FLOW_ACTION_TYPE_QUEUE: 4094 queue = actions->conf; 4095 if (queue == NULL) 4096 return 0; 4097 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4098 if (conf == NULL || conf->tx_explicit != 0) 4099 return 0; 4100 queue_action = 1; 4101 action_n++; 4102 break; 4103 case RTE_FLOW_ACTION_TYPE_RSS: 4104 rss = actions->conf; 4105 if (rss == NULL || rss->queue_num == 0) 4106 return 0; 4107 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4108 if (conf == NULL || conf->tx_explicit != 0) 4109 return 0; 4110 queue_action = 1; 4111 action_n++; 4112 break; 4113 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4114 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4115 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4116 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4117 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4118 split++; 4119 action_n++; 4120 break; 4121 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4122 raw_encap = actions->conf; 4123 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4124 split++; 4125 action_n++; 4126 break; 4127 default: 4128 action_n++; 4129 break; 4130 } 4131 } 4132 if (split && queue_action) 4133 return action_n; 4134 return 0; 4135 } 4136 4137 /* Declare flow create/destroy prototype in advance. */ 4138 static uint32_t 4139 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4140 const struct rte_flow_attr *attr, 4141 const struct rte_flow_item items[], 4142 const struct rte_flow_action actions[], 4143 bool external, struct rte_flow_error *error); 4144 4145 static void 4146 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4147 uint32_t flow_idx); 4148 4149 int 4150 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4151 struct mlx5_list_entry *entry, void *cb_ctx) 4152 { 4153 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4154 struct mlx5_flow_mreg_copy_resource *mcp_res = 4155 container_of(entry, typeof(*mcp_res), hlist_ent); 4156 4157 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4158 } 4159 4160 struct mlx5_list_entry * 4161 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4162 { 4163 struct rte_eth_dev *dev = tool_ctx; 4164 struct mlx5_priv *priv = dev->data->dev_private; 4165 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4166 struct mlx5_flow_mreg_copy_resource *mcp_res; 4167 struct rte_flow_error *error = ctx->error; 4168 uint32_t idx = 0; 4169 int ret; 4170 uint32_t mark_id = *(uint32_t *)(ctx->data); 4171 struct rte_flow_attr attr = { 4172 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4173 .ingress = 1, 4174 }; 4175 struct mlx5_rte_flow_item_tag tag_spec = { 4176 .data = mark_id, 4177 }; 4178 struct rte_flow_item items[] = { 4179 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4180 }; 4181 struct rte_flow_action_mark ftag = { 4182 .id = mark_id, 4183 }; 4184 struct mlx5_flow_action_copy_mreg cp_mreg = { 4185 .dst = REG_B, 4186 .src = REG_NON, 4187 }; 4188 struct rte_flow_action_jump jump = { 4189 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4190 }; 4191 struct rte_flow_action actions[] = { 4192 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4193 }; 4194 4195 /* Fill the register fileds in the flow. */ 4196 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4197 if (ret < 0) 4198 return NULL; 4199 tag_spec.id = ret; 4200 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4201 if (ret < 0) 4202 return NULL; 4203 cp_mreg.src = ret; 4204 /* Provide the full width of FLAG specific value. */ 4205 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4206 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4207 /* Build a new flow. */ 4208 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4209 items[0] = (struct rte_flow_item){ 4210 .type = (enum rte_flow_item_type) 4211 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4212 .spec = &tag_spec, 4213 }; 4214 items[1] = (struct rte_flow_item){ 4215 .type = RTE_FLOW_ITEM_TYPE_END, 4216 }; 4217 actions[0] = (struct rte_flow_action){ 4218 .type = (enum rte_flow_action_type) 4219 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4220 .conf = &ftag, 4221 }; 4222 actions[1] = (struct rte_flow_action){ 4223 .type = (enum rte_flow_action_type) 4224 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4225 .conf = &cp_mreg, 4226 }; 4227 actions[2] = (struct rte_flow_action){ 4228 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4229 .conf = &jump, 4230 }; 4231 actions[3] = (struct rte_flow_action){ 4232 .type = RTE_FLOW_ACTION_TYPE_END, 4233 }; 4234 } else { 4235 /* Default rule, wildcard match. */ 4236 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4237 items[0] = (struct rte_flow_item){ 4238 .type = RTE_FLOW_ITEM_TYPE_END, 4239 }; 4240 actions[0] = (struct rte_flow_action){ 4241 .type = (enum rte_flow_action_type) 4242 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4243 .conf = &cp_mreg, 4244 }; 4245 actions[1] = (struct rte_flow_action){ 4246 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4247 .conf = &jump, 4248 }; 4249 actions[2] = (struct rte_flow_action){ 4250 .type = RTE_FLOW_ACTION_TYPE_END, 4251 }; 4252 } 4253 /* Build a new entry. */ 4254 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4255 if (!mcp_res) { 4256 rte_errno = ENOMEM; 4257 return NULL; 4258 } 4259 mcp_res->idx = idx; 4260 mcp_res->mark_id = mark_id; 4261 /* 4262 * The copy Flows are not included in any list. There 4263 * ones are referenced from other Flows and can not 4264 * be applied, removed, deleted in ardbitrary order 4265 * by list traversing. 4266 */ 4267 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4268 &attr, items, actions, false, error); 4269 if (!mcp_res->rix_flow) { 4270 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4271 return NULL; 4272 } 4273 return &mcp_res->hlist_ent; 4274 } 4275 4276 struct mlx5_list_entry * 4277 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4278 void *cb_ctx __rte_unused) 4279 { 4280 struct rte_eth_dev *dev = tool_ctx; 4281 struct mlx5_priv *priv = dev->data->dev_private; 4282 struct mlx5_flow_mreg_copy_resource *mcp_res; 4283 uint32_t idx = 0; 4284 4285 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4286 if (!mcp_res) { 4287 rte_errno = ENOMEM; 4288 return NULL; 4289 } 4290 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4291 mcp_res->idx = idx; 4292 return &mcp_res->hlist_ent; 4293 } 4294 4295 void 4296 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4297 { 4298 struct mlx5_flow_mreg_copy_resource *mcp_res = 4299 container_of(entry, typeof(*mcp_res), hlist_ent); 4300 struct rte_eth_dev *dev = tool_ctx; 4301 struct mlx5_priv *priv = dev->data->dev_private; 4302 4303 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4304 } 4305 4306 /** 4307 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4308 * 4309 * As mark_id is unique, if there's already a registered flow for the mark_id, 4310 * return by increasing the reference counter of the resource. Otherwise, create 4311 * the resource (mcp_res) and flow. 4312 * 4313 * Flow looks like, 4314 * - If ingress port is ANY and reg_c[1] is mark_id, 4315 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4316 * 4317 * For default flow (zero mark_id), flow is like, 4318 * - If ingress port is ANY, 4319 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4320 * 4321 * @param dev 4322 * Pointer to Ethernet device. 4323 * @param mark_id 4324 * ID of MARK action, zero means default flow for META. 4325 * @param[out] error 4326 * Perform verbose error reporting if not NULL. 4327 * 4328 * @return 4329 * Associated resource on success, NULL otherwise and rte_errno is set. 4330 */ 4331 static struct mlx5_flow_mreg_copy_resource * 4332 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4333 struct rte_flow_error *error) 4334 { 4335 struct mlx5_priv *priv = dev->data->dev_private; 4336 struct mlx5_list_entry *entry; 4337 struct mlx5_flow_cb_ctx ctx = { 4338 .dev = dev, 4339 .error = error, 4340 .data = &mark_id, 4341 }; 4342 4343 /* Check if already registered. */ 4344 MLX5_ASSERT(priv->mreg_cp_tbl); 4345 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4346 if (!entry) 4347 return NULL; 4348 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4349 hlist_ent); 4350 } 4351 4352 void 4353 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4354 { 4355 struct mlx5_flow_mreg_copy_resource *mcp_res = 4356 container_of(entry, typeof(*mcp_res), hlist_ent); 4357 struct rte_eth_dev *dev = tool_ctx; 4358 struct mlx5_priv *priv = dev->data->dev_private; 4359 4360 MLX5_ASSERT(mcp_res->rix_flow); 4361 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4362 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4363 } 4364 4365 /** 4366 * Release flow in RX_CP_TBL. 4367 * 4368 * @param dev 4369 * Pointer to Ethernet device. 4370 * @flow 4371 * Parent flow for wich copying is provided. 4372 */ 4373 static void 4374 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4375 struct rte_flow *flow) 4376 { 4377 struct mlx5_flow_mreg_copy_resource *mcp_res; 4378 struct mlx5_priv *priv = dev->data->dev_private; 4379 4380 if (!flow->rix_mreg_copy) 4381 return; 4382 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4383 flow->rix_mreg_copy); 4384 if (!mcp_res || !priv->mreg_cp_tbl) 4385 return; 4386 MLX5_ASSERT(mcp_res->rix_flow); 4387 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4388 flow->rix_mreg_copy = 0; 4389 } 4390 4391 /** 4392 * Remove the default copy action from RX_CP_TBL. 4393 * 4394 * This functions is called in the mlx5_dev_start(). No thread safe 4395 * is guaranteed. 4396 * 4397 * @param dev 4398 * Pointer to Ethernet device. 4399 */ 4400 static void 4401 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4402 { 4403 struct mlx5_list_entry *entry; 4404 struct mlx5_priv *priv = dev->data->dev_private; 4405 struct mlx5_flow_cb_ctx ctx; 4406 uint32_t mark_id; 4407 4408 /* Check if default flow is registered. */ 4409 if (!priv->mreg_cp_tbl) 4410 return; 4411 mark_id = MLX5_DEFAULT_COPY_ID; 4412 ctx.data = &mark_id; 4413 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4414 if (!entry) 4415 return; 4416 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4417 } 4418 4419 /** 4420 * Add the default copy action in in RX_CP_TBL. 4421 * 4422 * This functions is called in the mlx5_dev_start(). No thread safe 4423 * is guaranteed. 4424 * 4425 * @param dev 4426 * Pointer to Ethernet device. 4427 * @param[out] error 4428 * Perform verbose error reporting if not NULL. 4429 * 4430 * @return 4431 * 0 for success, negative value otherwise and rte_errno is set. 4432 */ 4433 static int 4434 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4435 struct rte_flow_error *error) 4436 { 4437 struct mlx5_priv *priv = dev->data->dev_private; 4438 struct mlx5_flow_mreg_copy_resource *mcp_res; 4439 struct mlx5_flow_cb_ctx ctx; 4440 uint32_t mark_id; 4441 4442 /* Check whether extensive metadata feature is engaged. */ 4443 if (!priv->config.dv_flow_en || 4444 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4445 !mlx5_flow_ext_mreg_supported(dev) || 4446 !priv->sh->dv_regc0_mask) 4447 return 0; 4448 /* 4449 * Add default mreg copy flow may be called multiple time, but 4450 * only be called once in stop. Avoid register it twice. 4451 */ 4452 mark_id = MLX5_DEFAULT_COPY_ID; 4453 ctx.data = &mark_id; 4454 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4455 return 0; 4456 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4457 if (!mcp_res) 4458 return -rte_errno; 4459 return 0; 4460 } 4461 4462 /** 4463 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4464 * 4465 * All the flow having Q/RSS action should be split by 4466 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4467 * performs the following, 4468 * - CQE->flow_tag := reg_c[1] (MARK) 4469 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4470 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4471 * but there should be a flow per each MARK ID set by MARK action. 4472 * 4473 * For the aforementioned reason, if there's a MARK action in flow's action 4474 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4475 * the MARK ID to CQE's flow_tag like, 4476 * - If reg_c[1] is mark_id, 4477 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4478 * 4479 * For SET_META action which stores value in reg_c[0], as the destination is 4480 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4481 * MARK ID means the default flow. The default flow looks like, 4482 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4483 * 4484 * @param dev 4485 * Pointer to Ethernet device. 4486 * @param flow 4487 * Pointer to flow structure. 4488 * @param[in] actions 4489 * Pointer to the list of actions. 4490 * @param[out] error 4491 * Perform verbose error reporting if not NULL. 4492 * 4493 * @return 4494 * 0 on success, negative value otherwise and rte_errno is set. 4495 */ 4496 static int 4497 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4498 struct rte_flow *flow, 4499 const struct rte_flow_action *actions, 4500 struct rte_flow_error *error) 4501 { 4502 struct mlx5_priv *priv = dev->data->dev_private; 4503 struct mlx5_dev_config *config = &priv->config; 4504 struct mlx5_flow_mreg_copy_resource *mcp_res; 4505 const struct rte_flow_action_mark *mark; 4506 4507 /* Check whether extensive metadata feature is engaged. */ 4508 if (!config->dv_flow_en || 4509 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4510 !mlx5_flow_ext_mreg_supported(dev) || 4511 !priv->sh->dv_regc0_mask) 4512 return 0; 4513 /* Find MARK action. */ 4514 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4515 switch (actions->type) { 4516 case RTE_FLOW_ACTION_TYPE_FLAG: 4517 mcp_res = flow_mreg_add_copy_action 4518 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4519 if (!mcp_res) 4520 return -rte_errno; 4521 flow->rix_mreg_copy = mcp_res->idx; 4522 return 0; 4523 case RTE_FLOW_ACTION_TYPE_MARK: 4524 mark = (const struct rte_flow_action_mark *) 4525 actions->conf; 4526 mcp_res = 4527 flow_mreg_add_copy_action(dev, mark->id, error); 4528 if (!mcp_res) 4529 return -rte_errno; 4530 flow->rix_mreg_copy = mcp_res->idx; 4531 return 0; 4532 default: 4533 break; 4534 } 4535 } 4536 return 0; 4537 } 4538 4539 #define MLX5_MAX_SPLIT_ACTIONS 24 4540 #define MLX5_MAX_SPLIT_ITEMS 24 4541 4542 /** 4543 * Split the hairpin flow. 4544 * Since HW can't support encap and push-vlan on Rx, we move these 4545 * actions to Tx. 4546 * If the count action is after the encap then we also 4547 * move the count action. in this case the count will also measure 4548 * the outer bytes. 4549 * 4550 * @param dev 4551 * Pointer to Ethernet device. 4552 * @param[in] actions 4553 * Associated actions (list terminated by the END action). 4554 * @param[out] actions_rx 4555 * Rx flow actions. 4556 * @param[out] actions_tx 4557 * Tx flow actions.. 4558 * @param[out] pattern_tx 4559 * The pattern items for the Tx flow. 4560 * @param[out] flow_id 4561 * The flow ID connected to this flow. 4562 * 4563 * @return 4564 * 0 on success. 4565 */ 4566 static int 4567 flow_hairpin_split(struct rte_eth_dev *dev, 4568 const struct rte_flow_action actions[], 4569 struct rte_flow_action actions_rx[], 4570 struct rte_flow_action actions_tx[], 4571 struct rte_flow_item pattern_tx[], 4572 uint32_t flow_id) 4573 { 4574 const struct rte_flow_action_raw_encap *raw_encap; 4575 const struct rte_flow_action_raw_decap *raw_decap; 4576 struct mlx5_rte_flow_action_set_tag *set_tag; 4577 struct rte_flow_action *tag_action; 4578 struct mlx5_rte_flow_item_tag *tag_item; 4579 struct rte_flow_item *item; 4580 char *addr; 4581 int encap = 0; 4582 4583 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4584 switch (actions->type) { 4585 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4586 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4587 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4588 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4589 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4590 rte_memcpy(actions_tx, actions, 4591 sizeof(struct rte_flow_action)); 4592 actions_tx++; 4593 break; 4594 case RTE_FLOW_ACTION_TYPE_COUNT: 4595 if (encap) { 4596 rte_memcpy(actions_tx, actions, 4597 sizeof(struct rte_flow_action)); 4598 actions_tx++; 4599 } else { 4600 rte_memcpy(actions_rx, actions, 4601 sizeof(struct rte_flow_action)); 4602 actions_rx++; 4603 } 4604 break; 4605 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4606 raw_encap = actions->conf; 4607 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4608 memcpy(actions_tx, actions, 4609 sizeof(struct rte_flow_action)); 4610 actions_tx++; 4611 encap = 1; 4612 } else { 4613 rte_memcpy(actions_rx, actions, 4614 sizeof(struct rte_flow_action)); 4615 actions_rx++; 4616 } 4617 break; 4618 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4619 raw_decap = actions->conf; 4620 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4621 memcpy(actions_tx, actions, 4622 sizeof(struct rte_flow_action)); 4623 actions_tx++; 4624 } else { 4625 rte_memcpy(actions_rx, actions, 4626 sizeof(struct rte_flow_action)); 4627 actions_rx++; 4628 } 4629 break; 4630 default: 4631 rte_memcpy(actions_rx, actions, 4632 sizeof(struct rte_flow_action)); 4633 actions_rx++; 4634 break; 4635 } 4636 } 4637 /* Add set meta action and end action for the Rx flow. */ 4638 tag_action = actions_rx; 4639 tag_action->type = (enum rte_flow_action_type) 4640 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4641 actions_rx++; 4642 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4643 actions_rx++; 4644 set_tag = (void *)actions_rx; 4645 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4646 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4647 .data = flow_id, 4648 }; 4649 MLX5_ASSERT(set_tag->id > REG_NON); 4650 tag_action->conf = set_tag; 4651 /* Create Tx item list. */ 4652 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4653 addr = (void *)&pattern_tx[2]; 4654 item = pattern_tx; 4655 item->type = (enum rte_flow_item_type) 4656 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4657 tag_item = (void *)addr; 4658 tag_item->data = flow_id; 4659 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4660 MLX5_ASSERT(set_tag->id > REG_NON); 4661 item->spec = tag_item; 4662 addr += sizeof(struct mlx5_rte_flow_item_tag); 4663 tag_item = (void *)addr; 4664 tag_item->data = UINT32_MAX; 4665 tag_item->id = UINT16_MAX; 4666 item->mask = tag_item; 4667 item->last = NULL; 4668 item++; 4669 item->type = RTE_FLOW_ITEM_TYPE_END; 4670 return 0; 4671 } 4672 4673 /** 4674 * The last stage of splitting chain, just creates the subflow 4675 * without any modification. 4676 * 4677 * @param[in] dev 4678 * Pointer to Ethernet device. 4679 * @param[in] flow 4680 * Parent flow structure pointer. 4681 * @param[in, out] sub_flow 4682 * Pointer to return the created subflow, may be NULL. 4683 * @param[in] attr 4684 * Flow rule attributes. 4685 * @param[in] items 4686 * Pattern specification (list terminated by the END pattern item). 4687 * @param[in] actions 4688 * Associated actions (list terminated by the END action). 4689 * @param[in] flow_split_info 4690 * Pointer to flow split info structure. 4691 * @param[out] error 4692 * Perform verbose error reporting if not NULL. 4693 * @return 4694 * 0 on success, negative value otherwise 4695 */ 4696 static int 4697 flow_create_split_inner(struct rte_eth_dev *dev, 4698 struct rte_flow *flow, 4699 struct mlx5_flow **sub_flow, 4700 const struct rte_flow_attr *attr, 4701 const struct rte_flow_item items[], 4702 const struct rte_flow_action actions[], 4703 struct mlx5_flow_split_info *flow_split_info, 4704 struct rte_flow_error *error) 4705 { 4706 struct mlx5_flow *dev_flow; 4707 4708 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4709 flow_split_info->flow_idx, error); 4710 if (!dev_flow) 4711 return -rte_errno; 4712 dev_flow->flow = flow; 4713 dev_flow->external = flow_split_info->external; 4714 dev_flow->skip_scale = flow_split_info->skip_scale; 4715 /* Subflow object was created, we must include one in the list. */ 4716 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4717 dev_flow->handle, next); 4718 /* 4719 * If dev_flow is as one of the suffix flow, some actions in suffix 4720 * flow may need some user defined item layer flags, and pass the 4721 * Metadate rxq mark flag to suffix flow as well. 4722 */ 4723 if (flow_split_info->prefix_layers) 4724 dev_flow->handle->layers = flow_split_info->prefix_layers; 4725 if (flow_split_info->prefix_mark) 4726 dev_flow->handle->mark = 1; 4727 if (sub_flow) 4728 *sub_flow = dev_flow; 4729 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4730 dev_flow->dv.table_id = flow_split_info->table_id; 4731 #endif 4732 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4733 } 4734 4735 /** 4736 * Get the sub policy of a meter. 4737 * 4738 * @param[in] dev 4739 * Pointer to Ethernet device. 4740 * @param[in] flow 4741 * Parent flow structure pointer. 4742 * @param wks 4743 * Pointer to thread flow work space. 4744 * @param[in] attr 4745 * Flow rule attributes. 4746 * @param[in] items 4747 * Pattern specification (list terminated by the END pattern item). 4748 * @param[out] error 4749 * Perform verbose error reporting if not NULL. 4750 * 4751 * @return 4752 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4753 */ 4754 static struct mlx5_flow_meter_sub_policy * 4755 get_meter_sub_policy(struct rte_eth_dev *dev, 4756 struct rte_flow *flow, 4757 struct mlx5_flow_workspace *wks, 4758 const struct rte_flow_attr *attr, 4759 const struct rte_flow_item items[], 4760 struct rte_flow_error *error) 4761 { 4762 struct mlx5_flow_meter_policy *policy; 4763 struct mlx5_flow_meter_policy *final_policy; 4764 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4765 4766 policy = wks->policy; 4767 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4768 if (final_policy->is_rss || final_policy->is_queue) { 4769 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4770 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4771 uint32_t i; 4772 4773 /* 4774 * This is a tmp dev_flow, 4775 * no need to register any matcher for it in translate. 4776 */ 4777 wks->skip_matcher_reg = 1; 4778 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4779 struct mlx5_flow dev_flow = {0}; 4780 struct mlx5_flow_handle dev_handle = { {0} }; 4781 uint8_t fate = final_policy->act_cnt[i].fate_action; 4782 4783 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4784 const struct rte_flow_action_rss *rss_act = 4785 final_policy->act_cnt[i].rss->conf; 4786 struct rte_flow_action rss_actions[2] = { 4787 [0] = { 4788 .type = RTE_FLOW_ACTION_TYPE_RSS, 4789 .conf = rss_act, 4790 }, 4791 [1] = { 4792 .type = RTE_FLOW_ACTION_TYPE_END, 4793 .conf = NULL, 4794 } 4795 }; 4796 4797 dev_flow.handle = &dev_handle; 4798 dev_flow.ingress = attr->ingress; 4799 dev_flow.flow = flow; 4800 dev_flow.external = 0; 4801 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4802 dev_flow.dv.transfer = attr->transfer; 4803 #endif 4804 /** 4805 * Translate RSS action to get rss hash fields. 4806 */ 4807 if (flow_drv_translate(dev, &dev_flow, attr, 4808 items, rss_actions, error)) 4809 goto exit; 4810 rss_desc_v[i] = wks->rss_desc; 4811 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4812 rss_desc_v[i].hash_fields = 4813 dev_flow.hash_fields; 4814 rss_desc_v[i].queue_num = 4815 rss_desc_v[i].hash_fields ? 4816 rss_desc_v[i].queue_num : 1; 4817 rss_desc_v[i].tunnel = 4818 !!(dev_flow.handle->layers & 4819 MLX5_FLOW_LAYER_TUNNEL); 4820 /* Use the RSS queues in the containers. */ 4821 rss_desc_v[i].queue = 4822 (uint16_t *)(uintptr_t)rss_act->queue; 4823 rss_desc[i] = &rss_desc_v[i]; 4824 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4825 /* This is queue action. */ 4826 rss_desc_v[i] = wks->rss_desc; 4827 rss_desc_v[i].key_len = 0; 4828 rss_desc_v[i].hash_fields = 0; 4829 rss_desc_v[i].queue = 4830 &final_policy->act_cnt[i].queue; 4831 rss_desc_v[i].queue_num = 1; 4832 rss_desc[i] = &rss_desc_v[i]; 4833 } else { 4834 rss_desc[i] = NULL; 4835 } 4836 } 4837 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4838 flow, policy, rss_desc); 4839 } else { 4840 enum mlx5_meter_domain mtr_domain = 4841 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4842 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4843 MLX5_MTR_DOMAIN_INGRESS); 4844 sub_policy = policy->sub_policys[mtr_domain][0]; 4845 } 4846 if (!sub_policy) 4847 rte_flow_error_set(error, EINVAL, 4848 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4849 "Failed to get meter sub-policy."); 4850 exit: 4851 return sub_policy; 4852 } 4853 4854 /** 4855 * Split the meter flow. 4856 * 4857 * As meter flow will split to three sub flow, other than meter 4858 * action, the other actions make sense to only meter accepts 4859 * the packet. If it need to be dropped, no other additional 4860 * actions should be take. 4861 * 4862 * One kind of special action which decapsulates the L3 tunnel 4863 * header will be in the prefix sub flow, as not to take the 4864 * L3 tunnel header into account. 4865 * 4866 * @param[in] dev 4867 * Pointer to Ethernet device. 4868 * @param[in] flow 4869 * Parent flow structure pointer. 4870 * @param wks 4871 * Pointer to thread flow work space. 4872 * @param[in] attr 4873 * Flow rule attributes. 4874 * @param[in] items 4875 * Pattern specification (list terminated by the END pattern item). 4876 * @param[out] sfx_items 4877 * Suffix flow match items (list terminated by the END pattern item). 4878 * @param[in] actions 4879 * Associated actions (list terminated by the END action). 4880 * @param[out] actions_sfx 4881 * Suffix flow actions. 4882 * @param[out] actions_pre 4883 * Prefix flow actions. 4884 * @param[out] mtr_flow_id 4885 * Pointer to meter flow id. 4886 * @param[out] error 4887 * Perform verbose error reporting if not NULL. 4888 * 4889 * @return 4890 * 0 on success, a negative errno value otherwise and rte_errno is set. 4891 */ 4892 static int 4893 flow_meter_split_prep(struct rte_eth_dev *dev, 4894 struct rte_flow *flow, 4895 struct mlx5_flow_workspace *wks, 4896 const struct rte_flow_attr *attr, 4897 const struct rte_flow_item items[], 4898 struct rte_flow_item sfx_items[], 4899 const struct rte_flow_action actions[], 4900 struct rte_flow_action actions_sfx[], 4901 struct rte_flow_action actions_pre[], 4902 uint32_t *mtr_flow_id, 4903 struct rte_flow_error *error) 4904 { 4905 struct mlx5_priv *priv = dev->data->dev_private; 4906 struct mlx5_flow_meter_info *fm = wks->fm; 4907 struct rte_flow_action *tag_action = NULL; 4908 struct rte_flow_item *tag_item; 4909 struct mlx5_rte_flow_action_set_tag *set_tag; 4910 const struct rte_flow_action_raw_encap *raw_encap; 4911 const struct rte_flow_action_raw_decap *raw_decap; 4912 struct mlx5_rte_flow_item_tag *tag_item_spec; 4913 struct mlx5_rte_flow_item_tag *tag_item_mask; 4914 uint32_t tag_id = 0; 4915 struct rte_flow_item *vlan_item_dst = NULL; 4916 const struct rte_flow_item *vlan_item_src = NULL; 4917 struct rte_flow_action *hw_mtr_action; 4918 struct rte_flow_action *action_pre_head = NULL; 4919 int32_t flow_src_port = priv->representor_id; 4920 bool mtr_first; 4921 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 4922 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 4923 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 4924 uint32_t flow_id = 0; 4925 uint32_t flow_id_reversed = 0; 4926 uint8_t flow_id_bits = 0; 4927 int shift; 4928 4929 /* Prepare the suffix subflow items. */ 4930 tag_item = sfx_items++; 4931 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4932 struct mlx5_priv *port_priv; 4933 const struct rte_flow_item_port_id *pid_v; 4934 int item_type = items->type; 4935 4936 switch (item_type) { 4937 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4938 pid_v = items->spec; 4939 MLX5_ASSERT(pid_v); 4940 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 4941 if (!port_priv) 4942 return rte_flow_error_set(error, 4943 rte_errno, 4944 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 4945 pid_v, 4946 "Failed to get port info."); 4947 flow_src_port = port_priv->representor_id; 4948 if (!fm->def_policy && wks->policy->is_hierarchy && 4949 flow_src_port != priv->representor_id) { 4950 if (flow_drv_mtr_hierarchy_rule_create(dev, 4951 flow, fm, 4952 flow_src_port, 4953 items, 4954 error)) 4955 return -rte_errno; 4956 } 4957 memcpy(sfx_items, items, sizeof(*sfx_items)); 4958 sfx_items++; 4959 break; 4960 case RTE_FLOW_ITEM_TYPE_VLAN: 4961 /* Determine if copy vlan item below. */ 4962 vlan_item_src = items; 4963 vlan_item_dst = sfx_items++; 4964 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 4965 break; 4966 default: 4967 break; 4968 } 4969 } 4970 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4971 sfx_items++; 4972 mtr_first = priv->sh->meter_aso_en && 4973 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 4974 /* For ASO meter, meter must be before tag in TX direction. */ 4975 if (mtr_first) { 4976 action_pre_head = actions_pre++; 4977 /* Leave space for tag action. */ 4978 tag_action = actions_pre++; 4979 } 4980 /* Prepare the actions for prefix and suffix flow. */ 4981 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4982 struct rte_flow_action *action_cur = NULL; 4983 4984 switch (actions->type) { 4985 case RTE_FLOW_ACTION_TYPE_METER: 4986 if (mtr_first) { 4987 action_cur = action_pre_head; 4988 } else { 4989 /* Leave space for tag action. */ 4990 tag_action = actions_pre++; 4991 action_cur = actions_pre++; 4992 } 4993 break; 4994 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4995 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4996 action_cur = actions_pre++; 4997 break; 4998 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4999 raw_encap = actions->conf; 5000 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5001 action_cur = actions_pre++; 5002 break; 5003 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5004 raw_decap = actions->conf; 5005 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5006 action_cur = actions_pre++; 5007 break; 5008 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5009 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5010 if (vlan_item_dst && vlan_item_src) { 5011 memcpy(vlan_item_dst, vlan_item_src, 5012 sizeof(*vlan_item_dst)); 5013 /* 5014 * Convert to internal match item, it is used 5015 * for vlan push and set vid. 5016 */ 5017 vlan_item_dst->type = (enum rte_flow_item_type) 5018 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5019 } 5020 break; 5021 default: 5022 break; 5023 } 5024 if (!action_cur) 5025 action_cur = (fm->def_policy) ? 5026 actions_sfx++ : actions_pre++; 5027 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5028 } 5029 /* Add end action to the actions. */ 5030 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5031 if (priv->sh->meter_aso_en) { 5032 /** 5033 * For ASO meter, need to add an extra jump action explicitly, 5034 * to jump from meter to policer table. 5035 */ 5036 struct mlx5_flow_meter_sub_policy *sub_policy; 5037 struct mlx5_flow_tbl_data_entry *tbl_data; 5038 5039 if (!fm->def_policy) { 5040 sub_policy = get_meter_sub_policy(dev, flow, wks, 5041 attr, items, error); 5042 if (!sub_policy) 5043 return -rte_errno; 5044 } else { 5045 enum mlx5_meter_domain mtr_domain = 5046 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5047 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5048 MLX5_MTR_DOMAIN_INGRESS); 5049 5050 sub_policy = 5051 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5052 } 5053 tbl_data = container_of(sub_policy->tbl_rsc, 5054 struct mlx5_flow_tbl_data_entry, tbl); 5055 hw_mtr_action = actions_pre++; 5056 hw_mtr_action->type = (enum rte_flow_action_type) 5057 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5058 hw_mtr_action->conf = tbl_data->jump.action; 5059 } 5060 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5061 actions_pre++; 5062 if (!tag_action) 5063 return rte_flow_error_set(error, ENOMEM, 5064 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5065 NULL, "No tag action space."); 5066 if (!mtr_flow_id) { 5067 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5068 goto exit; 5069 } 5070 /* Only default-policy Meter creates mtr flow id. */ 5071 if (fm->def_policy) { 5072 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5073 if (!tag_id) 5074 return rte_flow_error_set(error, ENOMEM, 5075 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5076 "Failed to allocate meter flow id."); 5077 flow_id = tag_id - 1; 5078 flow_id_bits = (!flow_id) ? 1 : 5079 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5080 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5081 mtr_reg_bits) { 5082 mlx5_ipool_free(fm->flow_ipool, tag_id); 5083 return rte_flow_error_set(error, EINVAL, 5084 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5085 "Meter flow id exceeds max limit."); 5086 } 5087 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5088 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5089 } 5090 /* Build tag actions and items for meter_id/meter flow_id. */ 5091 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5092 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5093 tag_item_mask = tag_item_spec + 1; 5094 /* Both flow_id and meter_id share the same register. */ 5095 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5096 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5097 0, error), 5098 .offset = mtr_id_offset, 5099 .length = mtr_reg_bits, 5100 .data = flow->meter, 5101 }; 5102 /* 5103 * The color Reg bits used by flow_id are growing from 5104 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5105 */ 5106 for (shift = 0; shift < flow_id_bits; shift++) 5107 flow_id_reversed = (flow_id_reversed << 1) | 5108 ((flow_id >> shift) & 0x1); 5109 set_tag->data |= 5110 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5111 tag_item_spec->id = set_tag->id; 5112 tag_item_spec->data = set_tag->data << mtr_id_offset; 5113 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5114 tag_action->type = (enum rte_flow_action_type) 5115 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5116 tag_action->conf = set_tag; 5117 tag_item->type = (enum rte_flow_item_type) 5118 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5119 tag_item->spec = tag_item_spec; 5120 tag_item->last = NULL; 5121 tag_item->mask = tag_item_mask; 5122 exit: 5123 if (mtr_flow_id) 5124 *mtr_flow_id = tag_id; 5125 return 0; 5126 } 5127 5128 /** 5129 * Split action list having QUEUE/RSS for metadata register copy. 5130 * 5131 * Once Q/RSS action is detected in user's action list, the flow action 5132 * should be split in order to copy metadata registers, which will happen in 5133 * RX_CP_TBL like, 5134 * - CQE->flow_tag := reg_c[1] (MARK) 5135 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5136 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5137 * This is because the last action of each flow must be a terminal action 5138 * (QUEUE, RSS or DROP). 5139 * 5140 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5141 * stored and kept in the mlx5_flow structure per each sub_flow. 5142 * 5143 * The Q/RSS action is replaced with, 5144 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5145 * And the following JUMP action is added at the end, 5146 * - JUMP, to RX_CP_TBL. 5147 * 5148 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5149 * flow_create_split_metadata() routine. The flow will look like, 5150 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5151 * 5152 * @param dev 5153 * Pointer to Ethernet device. 5154 * @param[out] split_actions 5155 * Pointer to store split actions to jump to CP_TBL. 5156 * @param[in] actions 5157 * Pointer to the list of original flow actions. 5158 * @param[in] qrss 5159 * Pointer to the Q/RSS action. 5160 * @param[in] actions_n 5161 * Number of original actions. 5162 * @param[out] error 5163 * Perform verbose error reporting if not NULL. 5164 * 5165 * @return 5166 * non-zero unique flow_id on success, otherwise 0 and 5167 * error/rte_error are set. 5168 */ 5169 static uint32_t 5170 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5171 struct rte_flow_action *split_actions, 5172 const struct rte_flow_action *actions, 5173 const struct rte_flow_action *qrss, 5174 int actions_n, struct rte_flow_error *error) 5175 { 5176 struct mlx5_priv *priv = dev->data->dev_private; 5177 struct mlx5_rte_flow_action_set_tag *set_tag; 5178 struct rte_flow_action_jump *jump; 5179 const int qrss_idx = qrss - actions; 5180 uint32_t flow_id = 0; 5181 int ret = 0; 5182 5183 /* 5184 * Given actions will be split 5185 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5186 * - Add jump to mreg CP_TBL. 5187 * As a result, there will be one more action. 5188 */ 5189 ++actions_n; 5190 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5191 set_tag = (void *)(split_actions + actions_n); 5192 /* 5193 * If tag action is not set to void(it means we are not the meter 5194 * suffix flow), add the tag action. Since meter suffix flow already 5195 * has the tag added. 5196 */ 5197 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 5198 /* 5199 * Allocate the new subflow ID. This one is unique within 5200 * device and not shared with representors. Otherwise, 5201 * we would have to resolve multi-thread access synch 5202 * issue. Each flow on the shared device is appended 5203 * with source vport identifier, so the resulting 5204 * flows will be unique in the shared (by master and 5205 * representors) domain even if they have coinciding 5206 * IDs. 5207 */ 5208 mlx5_ipool_malloc(priv->sh->ipool 5209 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5210 if (!flow_id) 5211 return rte_flow_error_set(error, ENOMEM, 5212 RTE_FLOW_ERROR_TYPE_ACTION, 5213 NULL, "can't allocate id " 5214 "for split Q/RSS subflow"); 5215 /* Internal SET_TAG action to set flow ID. */ 5216 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5217 .data = flow_id, 5218 }; 5219 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5220 if (ret < 0) 5221 return ret; 5222 set_tag->id = ret; 5223 /* Construct new actions array. */ 5224 /* Replace QUEUE/RSS action. */ 5225 split_actions[qrss_idx] = (struct rte_flow_action){ 5226 .type = (enum rte_flow_action_type) 5227 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5228 .conf = set_tag, 5229 }; 5230 } 5231 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5232 jump = (void *)(set_tag + 1); 5233 *jump = (struct rte_flow_action_jump){ 5234 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5235 }; 5236 split_actions[actions_n - 2] = (struct rte_flow_action){ 5237 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5238 .conf = jump, 5239 }; 5240 split_actions[actions_n - 1] = (struct rte_flow_action){ 5241 .type = RTE_FLOW_ACTION_TYPE_END, 5242 }; 5243 return flow_id; 5244 } 5245 5246 /** 5247 * Extend the given action list for Tx metadata copy. 5248 * 5249 * Copy the given action list to the ext_actions and add flow metadata register 5250 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5251 * 5252 * @param[out] ext_actions 5253 * Pointer to the extended action list. 5254 * @param[in] actions 5255 * Pointer to the list of actions. 5256 * @param[in] actions_n 5257 * Number of actions in the list. 5258 * @param[out] error 5259 * Perform verbose error reporting if not NULL. 5260 * @param[in] encap_idx 5261 * The encap action inndex. 5262 * 5263 * @return 5264 * 0 on success, negative value otherwise 5265 */ 5266 static int 5267 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5268 struct rte_flow_action *ext_actions, 5269 const struct rte_flow_action *actions, 5270 int actions_n, struct rte_flow_error *error, 5271 int encap_idx) 5272 { 5273 struct mlx5_flow_action_copy_mreg *cp_mreg = 5274 (struct mlx5_flow_action_copy_mreg *) 5275 (ext_actions + actions_n + 1); 5276 int ret; 5277 5278 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5279 if (ret < 0) 5280 return ret; 5281 cp_mreg->dst = ret; 5282 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5283 if (ret < 0) 5284 return ret; 5285 cp_mreg->src = ret; 5286 if (encap_idx != 0) 5287 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5288 if (encap_idx == actions_n - 1) { 5289 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5290 .type = (enum rte_flow_action_type) 5291 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5292 .conf = cp_mreg, 5293 }; 5294 ext_actions[actions_n] = (struct rte_flow_action){ 5295 .type = RTE_FLOW_ACTION_TYPE_END, 5296 }; 5297 } else { 5298 ext_actions[encap_idx] = (struct rte_flow_action){ 5299 .type = (enum rte_flow_action_type) 5300 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5301 .conf = cp_mreg, 5302 }; 5303 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5304 sizeof(*ext_actions) * (actions_n - encap_idx)); 5305 } 5306 return 0; 5307 } 5308 5309 /** 5310 * Check the match action from the action list. 5311 * 5312 * @param[in] actions 5313 * Pointer to the list of actions. 5314 * @param[in] attr 5315 * Flow rule attributes. 5316 * @param[in] action 5317 * The action to be check if exist. 5318 * @param[out] match_action_pos 5319 * Pointer to the position of the matched action if exists, otherwise is -1. 5320 * @param[out] qrss_action_pos 5321 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5322 * @param[out] modify_after_mirror 5323 * Pointer to the flag of modify action after FDB mirroring. 5324 * 5325 * @return 5326 * > 0 the total number of actions. 5327 * 0 if not found match action in action list. 5328 */ 5329 static int 5330 flow_check_match_action(const struct rte_flow_action actions[], 5331 const struct rte_flow_attr *attr, 5332 enum rte_flow_action_type action, 5333 int *match_action_pos, int *qrss_action_pos, 5334 int *modify_after_mirror) 5335 { 5336 const struct rte_flow_action_sample *sample; 5337 const struct rte_flow_action_raw_decap *decap; 5338 int actions_n = 0; 5339 uint32_t ratio = 0; 5340 int sub_type = 0; 5341 int flag = 0; 5342 int fdb_mirror = 0; 5343 5344 *match_action_pos = -1; 5345 *qrss_action_pos = -1; 5346 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5347 if (actions->type == action) { 5348 flag = 1; 5349 *match_action_pos = actions_n; 5350 } 5351 switch (actions->type) { 5352 case RTE_FLOW_ACTION_TYPE_QUEUE: 5353 case RTE_FLOW_ACTION_TYPE_RSS: 5354 *qrss_action_pos = actions_n; 5355 break; 5356 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5357 sample = actions->conf; 5358 ratio = sample->ratio; 5359 sub_type = ((const struct rte_flow_action *) 5360 (sample->actions))->type; 5361 if (ratio == 1 && attr->transfer) 5362 fdb_mirror = 1; 5363 break; 5364 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5365 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5366 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5367 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5368 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5369 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5370 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5371 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5372 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5373 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5374 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5375 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5376 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5377 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5378 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5379 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5380 case RTE_FLOW_ACTION_TYPE_FLAG: 5381 case RTE_FLOW_ACTION_TYPE_MARK: 5382 case RTE_FLOW_ACTION_TYPE_SET_META: 5383 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5384 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5385 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5386 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5387 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5388 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5389 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5390 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5391 case RTE_FLOW_ACTION_TYPE_METER: 5392 if (fdb_mirror) 5393 *modify_after_mirror = 1; 5394 break; 5395 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5396 decap = actions->conf; 5397 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5398 ; 5399 actions_n++; 5400 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5401 const struct rte_flow_action_raw_encap *encap = 5402 actions->conf; 5403 if (decap->size <= 5404 MLX5_ENCAPSULATION_DECISION_SIZE && 5405 encap->size > 5406 MLX5_ENCAPSULATION_DECISION_SIZE) 5407 /* L3 encap. */ 5408 break; 5409 } 5410 if (fdb_mirror) 5411 *modify_after_mirror = 1; 5412 break; 5413 default: 5414 break; 5415 } 5416 actions_n++; 5417 } 5418 if (flag && fdb_mirror && !*modify_after_mirror) { 5419 /* FDB mirroring uses the destination array to implement 5420 * instead of FLOW_SAMPLER object. 5421 */ 5422 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5423 flag = 0; 5424 } 5425 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5426 return flag ? actions_n + 1 : 0; 5427 } 5428 5429 #define SAMPLE_SUFFIX_ITEM 2 5430 5431 /** 5432 * Split the sample flow. 5433 * 5434 * As sample flow will split to two sub flow, sample flow with 5435 * sample action, the other actions will move to new suffix flow. 5436 * 5437 * Also add unique tag id with tag action in the sample flow, 5438 * the same tag id will be as match in the suffix flow. 5439 * 5440 * @param dev 5441 * Pointer to Ethernet device. 5442 * @param[in] add_tag 5443 * Add extra tag action flag. 5444 * @param[out] sfx_items 5445 * Suffix flow match items (list terminated by the END pattern item). 5446 * @param[in] actions 5447 * Associated actions (list terminated by the END action). 5448 * @param[out] actions_sfx 5449 * Suffix flow actions. 5450 * @param[out] actions_pre 5451 * Prefix flow actions. 5452 * @param[in] actions_n 5453 * The total number of actions. 5454 * @param[in] sample_action_pos 5455 * The sample action position. 5456 * @param[in] qrss_action_pos 5457 * The Queue/RSS action position. 5458 * @param[in] jump_table 5459 * Add extra jump action flag. 5460 * @param[out] error 5461 * Perform verbose error reporting if not NULL. 5462 * 5463 * @return 5464 * 0 on success, or unique flow_id, a negative errno value 5465 * otherwise and rte_errno is set. 5466 */ 5467 static int 5468 flow_sample_split_prep(struct rte_eth_dev *dev, 5469 int add_tag, 5470 struct rte_flow_item sfx_items[], 5471 const struct rte_flow_action actions[], 5472 struct rte_flow_action actions_sfx[], 5473 struct rte_flow_action actions_pre[], 5474 int actions_n, 5475 int sample_action_pos, 5476 int qrss_action_pos, 5477 int jump_table, 5478 struct rte_flow_error *error) 5479 { 5480 struct mlx5_priv *priv = dev->data->dev_private; 5481 struct mlx5_rte_flow_action_set_tag *set_tag; 5482 struct mlx5_rte_flow_item_tag *tag_spec; 5483 struct mlx5_rte_flow_item_tag *tag_mask; 5484 struct rte_flow_action_jump *jump_action; 5485 uint32_t tag_id = 0; 5486 int index; 5487 int append_index = 0; 5488 int ret; 5489 5490 if (sample_action_pos < 0) 5491 return rte_flow_error_set(error, EINVAL, 5492 RTE_FLOW_ERROR_TYPE_ACTION, 5493 NULL, "invalid position of sample " 5494 "action in list"); 5495 /* Prepare the actions for prefix and suffix flow. */ 5496 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5497 index = qrss_action_pos; 5498 /* Put the preceding the Queue/RSS action into prefix flow. */ 5499 if (index != 0) 5500 memcpy(actions_pre, actions, 5501 sizeof(struct rte_flow_action) * index); 5502 /* Put others preceding the sample action into prefix flow. */ 5503 if (sample_action_pos > index + 1) 5504 memcpy(actions_pre + index, actions + index + 1, 5505 sizeof(struct rte_flow_action) * 5506 (sample_action_pos - index - 1)); 5507 index = sample_action_pos - 1; 5508 /* Put Queue/RSS action into Suffix flow. */ 5509 memcpy(actions_sfx, actions + qrss_action_pos, 5510 sizeof(struct rte_flow_action)); 5511 actions_sfx++; 5512 } else { 5513 index = sample_action_pos; 5514 if (index != 0) 5515 memcpy(actions_pre, actions, 5516 sizeof(struct rte_flow_action) * index); 5517 } 5518 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5519 * For CX6DX and above, metadata registers Cx preserve their value, 5520 * add an extra tag action for NIC-RX and E-Switch Domain. 5521 */ 5522 if (add_tag) { 5523 /* Prepare the prefix tag action. */ 5524 append_index++; 5525 set_tag = (void *)(actions_pre + actions_n + append_index); 5526 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5527 if (ret < 0) 5528 return ret; 5529 mlx5_ipool_malloc(priv->sh->ipool 5530 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5531 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5532 .id = ret, 5533 .data = tag_id, 5534 }; 5535 /* Prepare the suffix subflow items. */ 5536 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5537 tag_spec->data = tag_id; 5538 tag_spec->id = set_tag->id; 5539 tag_mask = tag_spec + 1; 5540 tag_mask->data = UINT32_MAX; 5541 sfx_items[0] = (struct rte_flow_item){ 5542 .type = (enum rte_flow_item_type) 5543 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5544 .spec = tag_spec, 5545 .last = NULL, 5546 .mask = tag_mask, 5547 }; 5548 sfx_items[1] = (struct rte_flow_item){ 5549 .type = (enum rte_flow_item_type) 5550 RTE_FLOW_ITEM_TYPE_END, 5551 }; 5552 /* Prepare the tag action in prefix subflow. */ 5553 actions_pre[index++] = 5554 (struct rte_flow_action){ 5555 .type = (enum rte_flow_action_type) 5556 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5557 .conf = set_tag, 5558 }; 5559 } 5560 memcpy(actions_pre + index, actions + sample_action_pos, 5561 sizeof(struct rte_flow_action)); 5562 index += 1; 5563 /* For the modify action after the sample action in E-Switch mirroring, 5564 * Add the extra jump action in prefix subflow and jump into the next 5565 * table, then do the modify action in the new table. 5566 */ 5567 if (jump_table) { 5568 /* Prepare the prefix jump action. */ 5569 append_index++; 5570 jump_action = (void *)(actions_pre + actions_n + append_index); 5571 jump_action->group = jump_table; 5572 actions_pre[index++] = 5573 (struct rte_flow_action){ 5574 .type = (enum rte_flow_action_type) 5575 RTE_FLOW_ACTION_TYPE_JUMP, 5576 .conf = jump_action, 5577 }; 5578 } 5579 actions_pre[index] = (struct rte_flow_action){ 5580 .type = (enum rte_flow_action_type) 5581 RTE_FLOW_ACTION_TYPE_END, 5582 }; 5583 /* Put the actions after sample into Suffix flow. */ 5584 memcpy(actions_sfx, actions + sample_action_pos + 1, 5585 sizeof(struct rte_flow_action) * 5586 (actions_n - sample_action_pos - 1)); 5587 return tag_id; 5588 } 5589 5590 /** 5591 * The splitting for metadata feature. 5592 * 5593 * - Q/RSS action on NIC Rx should be split in order to pass by 5594 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5595 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5596 * 5597 * - All the actions on NIC Tx should have a mreg copy action to 5598 * copy reg_a from WQE to reg_c[0]. 5599 * 5600 * @param dev 5601 * Pointer to Ethernet device. 5602 * @param[in] flow 5603 * Parent flow structure pointer. 5604 * @param[in] attr 5605 * Flow rule attributes. 5606 * @param[in] items 5607 * Pattern specification (list terminated by the END pattern item). 5608 * @param[in] actions 5609 * Associated actions (list terminated by the END action). 5610 * @param[in] flow_split_info 5611 * Pointer to flow split info structure. 5612 * @param[out] error 5613 * Perform verbose error reporting if not NULL. 5614 * @return 5615 * 0 on success, negative value otherwise 5616 */ 5617 static int 5618 flow_create_split_metadata(struct rte_eth_dev *dev, 5619 struct rte_flow *flow, 5620 const struct rte_flow_attr *attr, 5621 const struct rte_flow_item items[], 5622 const struct rte_flow_action actions[], 5623 struct mlx5_flow_split_info *flow_split_info, 5624 struct rte_flow_error *error) 5625 { 5626 struct mlx5_priv *priv = dev->data->dev_private; 5627 struct mlx5_dev_config *config = &priv->config; 5628 const struct rte_flow_action *qrss = NULL; 5629 struct rte_flow_action *ext_actions = NULL; 5630 struct mlx5_flow *dev_flow = NULL; 5631 uint32_t qrss_id = 0; 5632 int mtr_sfx = 0; 5633 size_t act_size; 5634 int actions_n; 5635 int encap_idx; 5636 int ret; 5637 5638 /* Check whether extensive metadata feature is engaged. */ 5639 if (!config->dv_flow_en || 5640 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5641 !mlx5_flow_ext_mreg_supported(dev)) 5642 return flow_create_split_inner(dev, flow, NULL, attr, items, 5643 actions, flow_split_info, error); 5644 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5645 &encap_idx); 5646 if (qrss) { 5647 /* Exclude hairpin flows from splitting. */ 5648 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5649 const struct rte_flow_action_queue *queue; 5650 5651 queue = qrss->conf; 5652 if (mlx5_rxq_get_type(dev, queue->index) == 5653 MLX5_RXQ_TYPE_HAIRPIN) 5654 qrss = NULL; 5655 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5656 const struct rte_flow_action_rss *rss; 5657 5658 rss = qrss->conf; 5659 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5660 MLX5_RXQ_TYPE_HAIRPIN) 5661 qrss = NULL; 5662 } 5663 } 5664 if (qrss) { 5665 /* Check if it is in meter suffix table. */ 5666 mtr_sfx = attr->group == (attr->transfer ? 5667 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5668 MLX5_FLOW_TABLE_LEVEL_METER); 5669 /* 5670 * Q/RSS action on NIC Rx should be split in order to pass by 5671 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5672 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5673 */ 5674 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5675 sizeof(struct rte_flow_action_set_tag) + 5676 sizeof(struct rte_flow_action_jump); 5677 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5678 SOCKET_ID_ANY); 5679 if (!ext_actions) 5680 return rte_flow_error_set(error, ENOMEM, 5681 RTE_FLOW_ERROR_TYPE_ACTION, 5682 NULL, "no memory to split " 5683 "metadata flow"); 5684 /* 5685 * If we are the suffix flow of meter, tag already exist. 5686 * Set the tag action to void. 5687 */ 5688 if (mtr_sfx) 5689 ext_actions[qrss - actions].type = 5690 RTE_FLOW_ACTION_TYPE_VOID; 5691 else 5692 ext_actions[qrss - actions].type = 5693 (enum rte_flow_action_type) 5694 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5695 /* 5696 * Create the new actions list with removed Q/RSS action 5697 * and appended set tag and jump to register copy table 5698 * (RX_CP_TBL). We should preallocate unique tag ID here 5699 * in advance, because it is needed for set tag action. 5700 */ 5701 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5702 qrss, actions_n, error); 5703 if (!mtr_sfx && !qrss_id) { 5704 ret = -rte_errno; 5705 goto exit; 5706 } 5707 } else if (attr->egress && !attr->transfer) { 5708 /* 5709 * All the actions on NIC Tx should have a metadata register 5710 * copy action to copy reg_a from WQE to reg_c[meta] 5711 */ 5712 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5713 sizeof(struct mlx5_flow_action_copy_mreg); 5714 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5715 SOCKET_ID_ANY); 5716 if (!ext_actions) 5717 return rte_flow_error_set(error, ENOMEM, 5718 RTE_FLOW_ERROR_TYPE_ACTION, 5719 NULL, "no memory to split " 5720 "metadata flow"); 5721 /* Create the action list appended with copy register. */ 5722 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5723 actions_n, error, encap_idx); 5724 if (ret < 0) 5725 goto exit; 5726 } 5727 /* Add the unmodified original or prefix subflow. */ 5728 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5729 items, ext_actions ? ext_actions : 5730 actions, flow_split_info, error); 5731 if (ret < 0) 5732 goto exit; 5733 MLX5_ASSERT(dev_flow); 5734 if (qrss) { 5735 const struct rte_flow_attr q_attr = { 5736 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5737 .ingress = 1, 5738 }; 5739 /* Internal PMD action to set register. */ 5740 struct mlx5_rte_flow_item_tag q_tag_spec = { 5741 .data = qrss_id, 5742 .id = REG_NON, 5743 }; 5744 struct rte_flow_item q_items[] = { 5745 { 5746 .type = (enum rte_flow_item_type) 5747 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5748 .spec = &q_tag_spec, 5749 .last = NULL, 5750 .mask = NULL, 5751 }, 5752 { 5753 .type = RTE_FLOW_ITEM_TYPE_END, 5754 }, 5755 }; 5756 struct rte_flow_action q_actions[] = { 5757 { 5758 .type = qrss->type, 5759 .conf = qrss->conf, 5760 }, 5761 { 5762 .type = RTE_FLOW_ACTION_TYPE_END, 5763 }, 5764 }; 5765 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5766 5767 /* 5768 * Configure the tag item only if there is no meter subflow. 5769 * Since tag is already marked in the meter suffix subflow 5770 * we can just use the meter suffix items as is. 5771 */ 5772 if (qrss_id) { 5773 /* Not meter subflow. */ 5774 MLX5_ASSERT(!mtr_sfx); 5775 /* 5776 * Put unique id in prefix flow due to it is destroyed 5777 * after suffix flow and id will be freed after there 5778 * is no actual flows with this id and identifier 5779 * reallocation becomes possible (for example, for 5780 * other flows in other threads). 5781 */ 5782 dev_flow->handle->split_flow_id = qrss_id; 5783 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5784 error); 5785 if (ret < 0) 5786 goto exit; 5787 q_tag_spec.id = ret; 5788 } 5789 dev_flow = NULL; 5790 /* Add suffix subflow to execute Q/RSS. */ 5791 flow_split_info->prefix_layers = layers; 5792 flow_split_info->prefix_mark = 0; 5793 ret = flow_create_split_inner(dev, flow, &dev_flow, 5794 &q_attr, mtr_sfx ? items : 5795 q_items, q_actions, 5796 flow_split_info, error); 5797 if (ret < 0) 5798 goto exit; 5799 /* qrss ID should be freed if failed. */ 5800 qrss_id = 0; 5801 MLX5_ASSERT(dev_flow); 5802 } 5803 5804 exit: 5805 /* 5806 * We do not destroy the partially created sub_flows in case of error. 5807 * These ones are included into parent flow list and will be destroyed 5808 * by flow_drv_destroy. 5809 */ 5810 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5811 qrss_id); 5812 mlx5_free(ext_actions); 5813 return ret; 5814 } 5815 5816 /** 5817 * Create meter internal drop flow with the original pattern. 5818 * 5819 * @param dev 5820 * Pointer to Ethernet device. 5821 * @param[in] flow 5822 * Parent flow structure pointer. 5823 * @param[in] attr 5824 * Flow rule attributes. 5825 * @param[in] items 5826 * Pattern specification (list terminated by the END pattern item). 5827 * @param[in] flow_split_info 5828 * Pointer to flow split info structure. 5829 * @param[in] fm 5830 * Pointer to flow meter structure. 5831 * @param[out] error 5832 * Perform verbose error reporting if not NULL. 5833 * @return 5834 * 0 on success, negative value otherwise 5835 */ 5836 static uint32_t 5837 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5838 struct rte_flow *flow, 5839 const struct rte_flow_attr *attr, 5840 const struct rte_flow_item items[], 5841 struct mlx5_flow_split_info *flow_split_info, 5842 struct mlx5_flow_meter_info *fm, 5843 struct rte_flow_error *error) 5844 { 5845 struct mlx5_flow *dev_flow = NULL; 5846 struct rte_flow_attr drop_attr = *attr; 5847 struct rte_flow_action drop_actions[3]; 5848 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5849 5850 MLX5_ASSERT(fm->drop_cnt); 5851 drop_actions[0].type = 5852 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5853 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5854 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5855 drop_actions[1].conf = NULL; 5856 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5857 drop_actions[2].conf = NULL; 5858 drop_split_info.external = false; 5859 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5860 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5861 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5862 return flow_create_split_inner(dev, flow, &dev_flow, 5863 &drop_attr, items, drop_actions, 5864 &drop_split_info, error); 5865 } 5866 5867 /** 5868 * The splitting for meter feature. 5869 * 5870 * - The meter flow will be split to two flows as prefix and 5871 * suffix flow. The packets make sense only it pass the prefix 5872 * meter action. 5873 * 5874 * - Reg_C_5 is used for the packet to match betweend prefix and 5875 * suffix flow. 5876 * 5877 * @param dev 5878 * Pointer to Ethernet device. 5879 * @param[in] flow 5880 * Parent flow structure pointer. 5881 * @param[in] attr 5882 * Flow rule attributes. 5883 * @param[in] items 5884 * Pattern specification (list terminated by the END pattern item). 5885 * @param[in] actions 5886 * Associated actions (list terminated by the END action). 5887 * @param[in] flow_split_info 5888 * Pointer to flow split info structure. 5889 * @param[out] error 5890 * Perform verbose error reporting if not NULL. 5891 * @return 5892 * 0 on success, negative value otherwise 5893 */ 5894 static int 5895 flow_create_split_meter(struct rte_eth_dev *dev, 5896 struct rte_flow *flow, 5897 const struct rte_flow_attr *attr, 5898 const struct rte_flow_item items[], 5899 const struct rte_flow_action actions[], 5900 struct mlx5_flow_split_info *flow_split_info, 5901 struct rte_flow_error *error) 5902 { 5903 struct mlx5_priv *priv = dev->data->dev_private; 5904 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5905 struct rte_flow_action *sfx_actions = NULL; 5906 struct rte_flow_action *pre_actions = NULL; 5907 struct rte_flow_item *sfx_items = NULL; 5908 struct mlx5_flow *dev_flow = NULL; 5909 struct rte_flow_attr sfx_attr = *attr; 5910 struct mlx5_flow_meter_info *fm = NULL; 5911 uint8_t skip_scale_restore; 5912 bool has_mtr = false; 5913 bool has_modify = false; 5914 bool set_mtr_reg = true; 5915 bool is_mtr_hierarchy = false; 5916 uint32_t meter_id = 0; 5917 uint32_t mtr_idx = 0; 5918 uint32_t mtr_flow_id = 0; 5919 size_t act_size; 5920 size_t item_size; 5921 int actions_n = 0; 5922 int ret = 0; 5923 5924 if (priv->mtr_en) 5925 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 5926 &has_modify, &meter_id); 5927 if (has_mtr) { 5928 if (flow->meter) { 5929 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 5930 if (!fm) 5931 return rte_flow_error_set(error, EINVAL, 5932 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5933 NULL, "Meter not found."); 5934 } else { 5935 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 5936 if (!fm) 5937 return rte_flow_error_set(error, EINVAL, 5938 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5939 NULL, "Meter not found."); 5940 ret = mlx5_flow_meter_attach(priv, fm, 5941 &sfx_attr, error); 5942 if (ret) 5943 return -rte_errno; 5944 flow->meter = mtr_idx; 5945 } 5946 MLX5_ASSERT(wks); 5947 wks->fm = fm; 5948 if (!fm->def_policy) { 5949 wks->policy = mlx5_flow_meter_policy_find(dev, 5950 fm->policy_id, 5951 NULL); 5952 MLX5_ASSERT(wks->policy); 5953 if (wks->policy->is_hierarchy) { 5954 wks->final_policy = 5955 mlx5_flow_meter_hierarchy_get_final_policy(dev, 5956 wks->policy); 5957 if (!wks->final_policy) 5958 return rte_flow_error_set(error, 5959 EINVAL, 5960 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5961 "Failed to find terminal policy of hierarchy."); 5962 is_mtr_hierarchy = true; 5963 } 5964 } 5965 /* 5966 * If it isn't default-policy Meter, and 5967 * 1. There's no action in flow to change 5968 * packet (modify/encap/decap etc.), OR 5969 * 2. No drop count needed for this meter. 5970 * 3. It's not meter hierarchy. 5971 * Then no need to use regC to save meter id anymore. 5972 */ 5973 if (!fm->def_policy && !is_mtr_hierarchy && 5974 (!has_modify || !fm->drop_cnt)) 5975 set_mtr_reg = false; 5976 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 5977 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 5978 sizeof(struct mlx5_rte_flow_action_set_tag); 5979 /* Suffix items: tag, vlan, port id, end. */ 5980 #define METER_SUFFIX_ITEM 4 5981 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5982 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5983 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5984 0, SOCKET_ID_ANY); 5985 if (!sfx_actions) 5986 return rte_flow_error_set(error, ENOMEM, 5987 RTE_FLOW_ERROR_TYPE_ACTION, 5988 NULL, "no memory to split " 5989 "meter flow"); 5990 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5991 act_size); 5992 /* There's no suffix flow for meter of non-default policy. */ 5993 if (!fm->def_policy) 5994 pre_actions = sfx_actions + 1; 5995 else 5996 pre_actions = sfx_actions + actions_n; 5997 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 5998 items, sfx_items, actions, 5999 sfx_actions, pre_actions, 6000 (set_mtr_reg ? &mtr_flow_id : NULL), 6001 error); 6002 if (ret) { 6003 ret = -rte_errno; 6004 goto exit; 6005 } 6006 /* Add the prefix subflow. */ 6007 flow_split_info->prefix_mark = 0; 6008 skip_scale_restore = flow_split_info->skip_scale; 6009 flow_split_info->skip_scale |= 6010 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6011 ret = flow_create_split_inner(dev, flow, &dev_flow, 6012 attr, items, pre_actions, 6013 flow_split_info, error); 6014 flow_split_info->skip_scale = skip_scale_restore; 6015 if (ret) { 6016 if (mtr_flow_id) 6017 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6018 ret = -rte_errno; 6019 goto exit; 6020 } 6021 if (mtr_flow_id) { 6022 dev_flow->handle->split_flow_id = mtr_flow_id; 6023 dev_flow->handle->is_meter_flow_id = 1; 6024 } 6025 if (!fm->def_policy) { 6026 if (!set_mtr_reg && fm->drop_cnt) 6027 ret = 6028 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6029 &sfx_attr, items, 6030 flow_split_info, 6031 fm, error); 6032 goto exit; 6033 } 6034 /* Setting the sfx group atrr. */ 6035 sfx_attr.group = sfx_attr.transfer ? 6036 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6037 MLX5_FLOW_TABLE_LEVEL_METER; 6038 flow_split_info->prefix_layers = 6039 flow_get_prefix_layer_flags(dev_flow); 6040 flow_split_info->prefix_mark = dev_flow->handle->mark; 6041 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6042 } 6043 /* Add the prefix subflow. */ 6044 ret = flow_create_split_metadata(dev, flow, 6045 &sfx_attr, sfx_items ? 6046 sfx_items : items, 6047 sfx_actions ? sfx_actions : actions, 6048 flow_split_info, error); 6049 exit: 6050 if (sfx_actions) 6051 mlx5_free(sfx_actions); 6052 return ret; 6053 } 6054 6055 /** 6056 * The splitting for sample feature. 6057 * 6058 * Once Sample action is detected in the action list, the flow actions should 6059 * be split into prefix sub flow and suffix sub flow. 6060 * 6061 * The original items remain in the prefix sub flow, all actions preceding the 6062 * sample action and the sample action itself will be copied to the prefix 6063 * sub flow, the actions following the sample action will be copied to the 6064 * suffix sub flow, Queue action always be located in the suffix sub flow. 6065 * 6066 * In order to make the packet from prefix sub flow matches with suffix sub 6067 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6068 * flow uses tag item with the unique flow id. 6069 * 6070 * @param dev 6071 * Pointer to Ethernet device. 6072 * @param[in] flow 6073 * Parent flow structure pointer. 6074 * @param[in] attr 6075 * Flow rule attributes. 6076 * @param[in] items 6077 * Pattern specification (list terminated by the END pattern item). 6078 * @param[in] actions 6079 * Associated actions (list terminated by the END action). 6080 * @param[in] flow_split_info 6081 * Pointer to flow split info structure. 6082 * @param[out] error 6083 * Perform verbose error reporting if not NULL. 6084 * @return 6085 * 0 on success, negative value otherwise 6086 */ 6087 static int 6088 flow_create_split_sample(struct rte_eth_dev *dev, 6089 struct rte_flow *flow, 6090 const struct rte_flow_attr *attr, 6091 const struct rte_flow_item items[], 6092 const struct rte_flow_action actions[], 6093 struct mlx5_flow_split_info *flow_split_info, 6094 struct rte_flow_error *error) 6095 { 6096 struct mlx5_priv *priv = dev->data->dev_private; 6097 struct rte_flow_action *sfx_actions = NULL; 6098 struct rte_flow_action *pre_actions = NULL; 6099 struct rte_flow_item *sfx_items = NULL; 6100 struct mlx5_flow *dev_flow = NULL; 6101 struct rte_flow_attr sfx_attr = *attr; 6102 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6103 struct mlx5_flow_dv_sample_resource *sample_res; 6104 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6105 struct mlx5_flow_tbl_resource *sfx_tbl; 6106 #endif 6107 size_t act_size; 6108 size_t item_size; 6109 uint32_t fdb_tx = 0; 6110 int32_t tag_id = 0; 6111 int actions_n = 0; 6112 int sample_action_pos; 6113 int qrss_action_pos; 6114 int add_tag = 0; 6115 int modify_after_mirror = 0; 6116 uint16_t jump_table = 0; 6117 const uint32_t next_ft_step = 1; 6118 int ret = 0; 6119 6120 if (priv->sampler_en) 6121 actions_n = flow_check_match_action(actions, attr, 6122 RTE_FLOW_ACTION_TYPE_SAMPLE, 6123 &sample_action_pos, &qrss_action_pos, 6124 &modify_after_mirror); 6125 if (actions_n) { 6126 /* The prefix actions must includes sample, tag, end. */ 6127 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6128 + sizeof(struct mlx5_rte_flow_action_set_tag); 6129 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6130 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6131 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6132 item_size), 0, SOCKET_ID_ANY); 6133 if (!sfx_actions) 6134 return rte_flow_error_set(error, ENOMEM, 6135 RTE_FLOW_ERROR_TYPE_ACTION, 6136 NULL, "no memory to split " 6137 "sample flow"); 6138 /* The representor_id is UINT16_MAX for uplink. */ 6139 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6140 /* 6141 * When reg_c_preserve is set, metadata registers Cx preserve 6142 * their value even through packet duplication. 6143 */ 6144 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 6145 if (add_tag) 6146 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6147 + act_size); 6148 if (modify_after_mirror) 6149 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6150 next_ft_step; 6151 pre_actions = sfx_actions + actions_n; 6152 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6153 actions, sfx_actions, 6154 pre_actions, actions_n, 6155 sample_action_pos, 6156 qrss_action_pos, jump_table, 6157 error); 6158 if (tag_id < 0 || (add_tag && !tag_id)) { 6159 ret = -rte_errno; 6160 goto exit; 6161 } 6162 if (modify_after_mirror) 6163 flow_split_info->skip_scale = 6164 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6165 /* Add the prefix subflow. */ 6166 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6167 items, pre_actions, 6168 flow_split_info, error); 6169 if (ret) { 6170 ret = -rte_errno; 6171 goto exit; 6172 } 6173 dev_flow->handle->split_flow_id = tag_id; 6174 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6175 if (!modify_after_mirror) { 6176 /* Set the sfx group attr. */ 6177 sample_res = (struct mlx5_flow_dv_sample_resource *) 6178 dev_flow->dv.sample_res; 6179 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6180 sample_res->normal_path_tbl; 6181 sfx_tbl_data = container_of(sfx_tbl, 6182 struct mlx5_flow_tbl_data_entry, 6183 tbl); 6184 sfx_attr.group = sfx_attr.transfer ? 6185 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6186 } else { 6187 MLX5_ASSERT(attr->transfer); 6188 sfx_attr.group = jump_table; 6189 } 6190 flow_split_info->prefix_layers = 6191 flow_get_prefix_layer_flags(dev_flow); 6192 flow_split_info->prefix_mark = dev_flow->handle->mark; 6193 /* Suffix group level already be scaled with factor, set 6194 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6195 * again in translation. 6196 */ 6197 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6198 #endif 6199 } 6200 /* Add the suffix subflow. */ 6201 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6202 sfx_items ? sfx_items : items, 6203 sfx_actions ? sfx_actions : actions, 6204 flow_split_info, error); 6205 exit: 6206 if (sfx_actions) 6207 mlx5_free(sfx_actions); 6208 return ret; 6209 } 6210 6211 /** 6212 * Split the flow to subflow set. The splitters might be linked 6213 * in the chain, like this: 6214 * flow_create_split_outer() calls: 6215 * flow_create_split_meter() calls: 6216 * flow_create_split_metadata(meter_subflow_0) calls: 6217 * flow_create_split_inner(metadata_subflow_0) 6218 * flow_create_split_inner(metadata_subflow_1) 6219 * flow_create_split_inner(metadata_subflow_2) 6220 * flow_create_split_metadata(meter_subflow_1) calls: 6221 * flow_create_split_inner(metadata_subflow_0) 6222 * flow_create_split_inner(metadata_subflow_1) 6223 * flow_create_split_inner(metadata_subflow_2) 6224 * 6225 * This provide flexible way to add new levels of flow splitting. 6226 * The all of successfully created subflows are included to the 6227 * parent flow dev_flow list. 6228 * 6229 * @param dev 6230 * Pointer to Ethernet device. 6231 * @param[in] flow 6232 * Parent flow structure pointer. 6233 * @param[in] attr 6234 * Flow rule attributes. 6235 * @param[in] items 6236 * Pattern specification (list terminated by the END pattern item). 6237 * @param[in] actions 6238 * Associated actions (list terminated by the END action). 6239 * @param[in] flow_split_info 6240 * Pointer to flow split info structure. 6241 * @param[out] error 6242 * Perform verbose error reporting if not NULL. 6243 * @return 6244 * 0 on success, negative value otherwise 6245 */ 6246 static int 6247 flow_create_split_outer(struct rte_eth_dev *dev, 6248 struct rte_flow *flow, 6249 const struct rte_flow_attr *attr, 6250 const struct rte_flow_item items[], 6251 const struct rte_flow_action actions[], 6252 struct mlx5_flow_split_info *flow_split_info, 6253 struct rte_flow_error *error) 6254 { 6255 int ret; 6256 6257 ret = flow_create_split_sample(dev, flow, attr, items, 6258 actions, flow_split_info, error); 6259 MLX5_ASSERT(ret <= 0); 6260 return ret; 6261 } 6262 6263 static inline struct mlx5_flow_tunnel * 6264 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6265 { 6266 struct mlx5_flow_tunnel *tunnel; 6267 6268 #pragma GCC diagnostic push 6269 #pragma GCC diagnostic ignored "-Wcast-qual" 6270 tunnel = (typeof(tunnel))flow->tunnel; 6271 #pragma GCC diagnostic pop 6272 6273 return tunnel; 6274 } 6275 6276 /** 6277 * Adjust flow RSS workspace if needed. 6278 * 6279 * @param wks 6280 * Pointer to thread flow work space. 6281 * @param rss_desc 6282 * Pointer to RSS descriptor. 6283 * @param[in] nrssq_num 6284 * New RSS queue number. 6285 * 6286 * @return 6287 * 0 on success, -1 otherwise and rte_errno is set. 6288 */ 6289 static int 6290 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6291 struct mlx5_flow_rss_desc *rss_desc, 6292 uint32_t nrssq_num) 6293 { 6294 if (likely(nrssq_num <= wks->rssq_num)) 6295 return 0; 6296 rss_desc->queue = realloc(rss_desc->queue, 6297 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6298 if (!rss_desc->queue) { 6299 rte_errno = ENOMEM; 6300 return -1; 6301 } 6302 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6303 return 0; 6304 } 6305 6306 /** 6307 * Create a flow and add it to @p list. 6308 * 6309 * @param dev 6310 * Pointer to Ethernet device. 6311 * @param list 6312 * Pointer to a TAILQ flow list. If this parameter NULL, 6313 * no list insertion occurred, flow is just created, 6314 * this is caller's responsibility to track the 6315 * created flow. 6316 * @param[in] attr 6317 * Flow rule attributes. 6318 * @param[in] items 6319 * Pattern specification (list terminated by the END pattern item). 6320 * @param[in] actions 6321 * Associated actions (list terminated by the END action). 6322 * @param[in] external 6323 * This flow rule is created by request external to PMD. 6324 * @param[out] error 6325 * Perform verbose error reporting if not NULL. 6326 * 6327 * @return 6328 * A flow index on success, 0 otherwise and rte_errno is set. 6329 */ 6330 static uint32_t 6331 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6332 const struct rte_flow_attr *attr, 6333 const struct rte_flow_item items[], 6334 const struct rte_flow_action original_actions[], 6335 bool external, struct rte_flow_error *error) 6336 { 6337 struct mlx5_priv *priv = dev->data->dev_private; 6338 struct rte_flow *flow = NULL; 6339 struct mlx5_flow *dev_flow; 6340 const struct rte_flow_action_rss *rss = NULL; 6341 struct mlx5_translated_action_handle 6342 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6343 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6344 union { 6345 struct mlx5_flow_expand_rss buf; 6346 uint8_t buffer[4096]; 6347 } expand_buffer; 6348 union { 6349 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6350 uint8_t buffer[2048]; 6351 } actions_rx; 6352 union { 6353 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6354 uint8_t buffer[2048]; 6355 } actions_hairpin_tx; 6356 union { 6357 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6358 uint8_t buffer[2048]; 6359 } items_tx; 6360 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6361 struct mlx5_flow_rss_desc *rss_desc; 6362 const struct rte_flow_action *p_actions_rx; 6363 uint32_t i; 6364 uint32_t idx = 0; 6365 int hairpin_flow; 6366 struct rte_flow_attr attr_tx = { .priority = 0 }; 6367 const struct rte_flow_action *actions; 6368 struct rte_flow_action *translated_actions = NULL; 6369 struct mlx5_flow_tunnel *tunnel; 6370 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6371 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6372 struct mlx5_flow_split_info flow_split_info = { 6373 .external = !!external, 6374 .skip_scale = 0, 6375 .flow_idx = 0, 6376 .prefix_mark = 0, 6377 .prefix_layers = 0, 6378 .table_id = 0 6379 }; 6380 int ret; 6381 6382 MLX5_ASSERT(wks); 6383 rss_desc = &wks->rss_desc; 6384 ret = flow_action_handles_translate(dev, original_actions, 6385 indir_actions, 6386 &indir_actions_n, 6387 &translated_actions, error); 6388 if (ret < 0) { 6389 MLX5_ASSERT(translated_actions == NULL); 6390 return 0; 6391 } 6392 actions = translated_actions ? translated_actions : original_actions; 6393 p_actions_rx = actions; 6394 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6395 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6396 external, hairpin_flow, error); 6397 if (ret < 0) 6398 goto error_before_hairpin_split; 6399 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6400 if (!flow) { 6401 rte_errno = ENOMEM; 6402 goto error_before_hairpin_split; 6403 } 6404 if (hairpin_flow > 0) { 6405 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6406 rte_errno = EINVAL; 6407 goto error_before_hairpin_split; 6408 } 6409 flow_hairpin_split(dev, actions, actions_rx.actions, 6410 actions_hairpin_tx.actions, items_tx.items, 6411 idx); 6412 p_actions_rx = actions_rx.actions; 6413 } 6414 flow_split_info.flow_idx = idx; 6415 flow->drv_type = flow_get_drv_type(dev, attr); 6416 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6417 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6418 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6419 /* RSS Action only works on NIC RX domain */ 6420 if (attr->ingress && !attr->transfer) 6421 rss = flow_get_rss_action(dev, p_actions_rx); 6422 if (rss) { 6423 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6424 return 0; 6425 /* 6426 * The following information is required by 6427 * mlx5_flow_hashfields_adjust() in advance. 6428 */ 6429 rss_desc->level = rss->level; 6430 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6431 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6432 } 6433 flow->dev_handles = 0; 6434 if (rss && rss->types) { 6435 unsigned int graph_root; 6436 6437 graph_root = find_graph_root(rss->level); 6438 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6439 items, rss->types, 6440 mlx5_support_expansion, graph_root); 6441 MLX5_ASSERT(ret > 0 && 6442 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6443 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6444 for (i = 0; i < buf->entries; ++i) 6445 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6446 } 6447 } else { 6448 buf->entries = 1; 6449 buf->entry[0].pattern = (void *)(uintptr_t)items; 6450 } 6451 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6452 indir_actions_n); 6453 for (i = 0; i < buf->entries; ++i) { 6454 /* Initialize flow split data. */ 6455 flow_split_info.prefix_layers = 0; 6456 flow_split_info.prefix_mark = 0; 6457 flow_split_info.skip_scale = 0; 6458 /* 6459 * The splitter may create multiple dev_flows, 6460 * depending on configuration. In the simplest 6461 * case it just creates unmodified original flow. 6462 */ 6463 ret = flow_create_split_outer(dev, flow, attr, 6464 buf->entry[i].pattern, 6465 p_actions_rx, &flow_split_info, 6466 error); 6467 if (ret < 0) 6468 goto error; 6469 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6470 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6471 p_actions_rx, 6472 idx, 6473 wks->flows[0].tunnel, 6474 &default_miss_ctx, 6475 error); 6476 if (ret < 0) { 6477 mlx5_free(default_miss_ctx.queue); 6478 goto error; 6479 } 6480 } 6481 } 6482 /* Create the tx flow. */ 6483 if (hairpin_flow) { 6484 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6485 attr_tx.ingress = 0; 6486 attr_tx.egress = 1; 6487 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6488 actions_hairpin_tx.actions, 6489 idx, error); 6490 if (!dev_flow) 6491 goto error; 6492 dev_flow->flow = flow; 6493 dev_flow->external = 0; 6494 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6495 dev_flow->handle, next); 6496 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6497 items_tx.items, 6498 actions_hairpin_tx.actions, error); 6499 if (ret < 0) 6500 goto error; 6501 } 6502 /* 6503 * Update the metadata register copy table. If extensive 6504 * metadata feature is enabled and registers are supported 6505 * we might create the extra rte_flow for each unique 6506 * MARK/FLAG action ID. 6507 * 6508 * The table is updated for ingress Flows only, because 6509 * the egress Flows belong to the different device and 6510 * copy table should be updated in peer NIC Rx domain. 6511 */ 6512 if (attr->ingress && 6513 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6514 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6515 if (ret) 6516 goto error; 6517 } 6518 /* 6519 * If the flow is external (from application) OR device is started, 6520 * OR mreg discover, then apply immediately. 6521 */ 6522 if (external || dev->data->dev_started || 6523 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6524 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6525 ret = flow_drv_apply(dev, flow, error); 6526 if (ret < 0) 6527 goto error; 6528 } 6529 flow->type = type; 6530 flow_rxq_flags_set(dev, flow); 6531 rte_free(translated_actions); 6532 tunnel = flow_tunnel_from_rule(wks->flows); 6533 if (tunnel) { 6534 flow->tunnel = 1; 6535 flow->tunnel_id = tunnel->tunnel_id; 6536 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6537 mlx5_free(default_miss_ctx.queue); 6538 } 6539 mlx5_flow_pop_thread_workspace(); 6540 return idx; 6541 error: 6542 MLX5_ASSERT(flow); 6543 ret = rte_errno; /* Save rte_errno before cleanup. */ 6544 flow_mreg_del_copy_action(dev, flow); 6545 flow_drv_destroy(dev, flow); 6546 if (rss_desc->shared_rss) 6547 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6548 mlx5_ipool_get 6549 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6550 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6551 mlx5_ipool_free(priv->flows[type], idx); 6552 rte_errno = ret; /* Restore rte_errno. */ 6553 ret = rte_errno; 6554 rte_errno = ret; 6555 mlx5_flow_pop_thread_workspace(); 6556 error_before_hairpin_split: 6557 rte_free(translated_actions); 6558 return 0; 6559 } 6560 6561 /** 6562 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6563 * incoming packets to table 1. 6564 * 6565 * Other flow rules, requested for group n, will be created in 6566 * e-switch table n+1. 6567 * Jump action to e-switch group n will be created to group n+1. 6568 * 6569 * Used when working in switchdev mode, to utilise advantages of table 1 6570 * and above. 6571 * 6572 * @param dev 6573 * Pointer to Ethernet device. 6574 * 6575 * @return 6576 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6577 */ 6578 struct rte_flow * 6579 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6580 { 6581 const struct rte_flow_attr attr = { 6582 .group = 0, 6583 .priority = 0, 6584 .ingress = 1, 6585 .egress = 0, 6586 .transfer = 1, 6587 }; 6588 const struct rte_flow_item pattern = { 6589 .type = RTE_FLOW_ITEM_TYPE_END, 6590 }; 6591 struct rte_flow_action_jump jump = { 6592 .group = 1, 6593 }; 6594 const struct rte_flow_action actions[] = { 6595 { 6596 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6597 .conf = &jump, 6598 }, 6599 { 6600 .type = RTE_FLOW_ACTION_TYPE_END, 6601 }, 6602 }; 6603 struct rte_flow_error error; 6604 6605 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6606 &attr, &pattern, 6607 actions, false, &error); 6608 } 6609 6610 /** 6611 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6612 * and sq number, directs all packets to peer vport. 6613 * 6614 * @param dev 6615 * Pointer to Ethernet device. 6616 * @param txq 6617 * Txq index. 6618 * 6619 * @return 6620 * Flow ID on success, 0 otherwise and rte_errno is set. 6621 */ 6622 uint32_t 6623 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6624 { 6625 struct rte_flow_attr attr = { 6626 .group = 0, 6627 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6628 .ingress = 1, 6629 .egress = 0, 6630 .transfer = 1, 6631 }; 6632 struct rte_flow_item_port_id port_spec = { 6633 .id = MLX5_PORT_ESW_MGR, 6634 }; 6635 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6636 .queue = txq, 6637 }; 6638 struct rte_flow_item pattern[] = { 6639 { 6640 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6641 .spec = &port_spec, 6642 }, 6643 { 6644 .type = (enum rte_flow_item_type) 6645 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6646 .spec = &txq_spec, 6647 }, 6648 { 6649 .type = RTE_FLOW_ITEM_TYPE_END, 6650 }, 6651 }; 6652 struct rte_flow_action_jump jump = { 6653 .group = 1, 6654 }; 6655 struct rte_flow_action_port_id port = { 6656 .id = dev->data->port_id, 6657 }; 6658 struct rte_flow_action actions[] = { 6659 { 6660 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6661 .conf = &jump, 6662 }, 6663 { 6664 .type = RTE_FLOW_ACTION_TYPE_END, 6665 }, 6666 }; 6667 struct rte_flow_error error; 6668 6669 /* 6670 * Creates group 0, highest priority jump flow. 6671 * Matches txq to bypass kernel packets. 6672 */ 6673 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6674 false, &error) == 0) 6675 return 0; 6676 /* Create group 1, lowest priority redirect flow for txq. */ 6677 attr.group = 1; 6678 actions[0].conf = &port; 6679 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6680 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6681 actions, false, &error); 6682 } 6683 6684 /** 6685 * Validate a flow supported by the NIC. 6686 * 6687 * @see rte_flow_validate() 6688 * @see rte_flow_ops 6689 */ 6690 int 6691 mlx5_flow_validate(struct rte_eth_dev *dev, 6692 const struct rte_flow_attr *attr, 6693 const struct rte_flow_item items[], 6694 const struct rte_flow_action original_actions[], 6695 struct rte_flow_error *error) 6696 { 6697 int hairpin_flow; 6698 struct mlx5_translated_action_handle 6699 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6700 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6701 const struct rte_flow_action *actions; 6702 struct rte_flow_action *translated_actions = NULL; 6703 int ret = flow_action_handles_translate(dev, original_actions, 6704 indir_actions, 6705 &indir_actions_n, 6706 &translated_actions, error); 6707 6708 if (ret) 6709 return ret; 6710 actions = translated_actions ? translated_actions : original_actions; 6711 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6712 ret = flow_drv_validate(dev, attr, items, actions, 6713 true, hairpin_flow, error); 6714 rte_free(translated_actions); 6715 return ret; 6716 } 6717 6718 /** 6719 * Create a flow. 6720 * 6721 * @see rte_flow_create() 6722 * @see rte_flow_ops 6723 */ 6724 struct rte_flow * 6725 mlx5_flow_create(struct rte_eth_dev *dev, 6726 const struct rte_flow_attr *attr, 6727 const struct rte_flow_item items[], 6728 const struct rte_flow_action actions[], 6729 struct rte_flow_error *error) 6730 { 6731 /* 6732 * If the device is not started yet, it is not allowed to created a 6733 * flow from application. PMD default flows and traffic control flows 6734 * are not affected. 6735 */ 6736 if (unlikely(!dev->data->dev_started)) { 6737 DRV_LOG(DEBUG, "port %u is not started when " 6738 "inserting a flow", dev->data->port_id); 6739 rte_flow_error_set(error, ENODEV, 6740 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6741 NULL, 6742 "port not started"); 6743 return NULL; 6744 } 6745 6746 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6747 attr, items, actions, 6748 true, error); 6749 } 6750 6751 /** 6752 * Destroy a flow in a list. 6753 * 6754 * @param dev 6755 * Pointer to Ethernet device. 6756 * @param[in] flow_idx 6757 * Index of flow to destroy. 6758 */ 6759 static void 6760 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6761 uint32_t flow_idx) 6762 { 6763 struct mlx5_priv *priv = dev->data->dev_private; 6764 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6765 6766 if (!flow) 6767 return; 6768 MLX5_ASSERT(flow->type == type); 6769 /* 6770 * Update RX queue flags only if port is started, otherwise it is 6771 * already clean. 6772 */ 6773 if (dev->data->dev_started) 6774 flow_rxq_flags_trim(dev, flow); 6775 flow_drv_destroy(dev, flow); 6776 if (flow->tunnel) { 6777 struct mlx5_flow_tunnel *tunnel; 6778 6779 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6780 RTE_VERIFY(tunnel); 6781 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6782 mlx5_flow_tunnel_free(dev, tunnel); 6783 } 6784 flow_mreg_del_copy_action(dev, flow); 6785 mlx5_ipool_free(priv->flows[type], flow_idx); 6786 } 6787 6788 /** 6789 * Destroy all flows. 6790 * 6791 * @param dev 6792 * Pointer to Ethernet device. 6793 * @param type 6794 * Flow type to be flushed. 6795 * @param active 6796 * If flushing is called avtively. 6797 */ 6798 void 6799 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6800 bool active) 6801 { 6802 struct mlx5_priv *priv = dev->data->dev_private; 6803 uint32_t num_flushed = 0, fidx = 1; 6804 struct rte_flow *flow; 6805 6806 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6807 flow_list_destroy(dev, type, fidx); 6808 num_flushed++; 6809 } 6810 if (active) { 6811 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6812 dev->data->port_id, num_flushed); 6813 } 6814 } 6815 6816 /** 6817 * Stop all default actions for flows. 6818 * 6819 * @param dev 6820 * Pointer to Ethernet device. 6821 */ 6822 void 6823 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6824 { 6825 flow_mreg_del_default_copy_action(dev); 6826 flow_rxq_flags_clear(dev); 6827 } 6828 6829 /** 6830 * Start all default actions for flows. 6831 * 6832 * @param dev 6833 * Pointer to Ethernet device. 6834 * @return 6835 * 0 on success, a negative errno value otherwise and rte_errno is set. 6836 */ 6837 int 6838 mlx5_flow_start_default(struct rte_eth_dev *dev) 6839 { 6840 struct rte_flow_error error; 6841 6842 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6843 return flow_mreg_add_default_copy_action(dev, &error); 6844 } 6845 6846 /** 6847 * Release key of thread specific flow workspace data. 6848 */ 6849 void 6850 flow_release_workspace(void *data) 6851 { 6852 struct mlx5_flow_workspace *wks = data; 6853 struct mlx5_flow_workspace *next; 6854 6855 while (wks) { 6856 next = wks->next; 6857 free(wks->rss_desc.queue); 6858 free(wks); 6859 wks = next; 6860 } 6861 } 6862 6863 /** 6864 * Get thread specific current flow workspace. 6865 * 6866 * @return pointer to thread specific flow workspace data, NULL on error. 6867 */ 6868 struct mlx5_flow_workspace* 6869 mlx5_flow_get_thread_workspace(void) 6870 { 6871 struct mlx5_flow_workspace *data; 6872 6873 data = mlx5_flow_os_get_specific_workspace(); 6874 MLX5_ASSERT(data && data->inuse); 6875 if (!data || !data->inuse) 6876 DRV_LOG(ERR, "flow workspace not initialized."); 6877 return data; 6878 } 6879 6880 /** 6881 * Allocate and init new flow workspace. 6882 * 6883 * @return pointer to flow workspace data, NULL on error. 6884 */ 6885 static struct mlx5_flow_workspace* 6886 flow_alloc_thread_workspace(void) 6887 { 6888 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6889 6890 if (!data) { 6891 DRV_LOG(ERR, "Failed to allocate flow workspace " 6892 "memory."); 6893 return NULL; 6894 } 6895 data->rss_desc.queue = calloc(1, 6896 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6897 if (!data->rss_desc.queue) 6898 goto err; 6899 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6900 return data; 6901 err: 6902 if (data->rss_desc.queue) 6903 free(data->rss_desc.queue); 6904 free(data); 6905 return NULL; 6906 } 6907 6908 /** 6909 * Get new thread specific flow workspace. 6910 * 6911 * If current workspace inuse, create new one and set as current. 6912 * 6913 * @return pointer to thread specific flow workspace data, NULL on error. 6914 */ 6915 static struct mlx5_flow_workspace* 6916 mlx5_flow_push_thread_workspace(void) 6917 { 6918 struct mlx5_flow_workspace *curr; 6919 struct mlx5_flow_workspace *data; 6920 6921 curr = mlx5_flow_os_get_specific_workspace(); 6922 if (!curr) { 6923 data = flow_alloc_thread_workspace(); 6924 if (!data) 6925 return NULL; 6926 } else if (!curr->inuse) { 6927 data = curr; 6928 } else if (curr->next) { 6929 data = curr->next; 6930 } else { 6931 data = flow_alloc_thread_workspace(); 6932 if (!data) 6933 return NULL; 6934 curr->next = data; 6935 data->prev = curr; 6936 } 6937 data->inuse = 1; 6938 data->flow_idx = 0; 6939 /* Set as current workspace */ 6940 if (mlx5_flow_os_set_specific_workspace(data)) 6941 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6942 return data; 6943 } 6944 6945 /** 6946 * Close current thread specific flow workspace. 6947 * 6948 * If previous workspace available, set it as current. 6949 * 6950 * @return pointer to thread specific flow workspace data, NULL on error. 6951 */ 6952 static void 6953 mlx5_flow_pop_thread_workspace(void) 6954 { 6955 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6956 6957 if (!data) 6958 return; 6959 if (!data->inuse) { 6960 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6961 return; 6962 } 6963 data->inuse = 0; 6964 if (!data->prev) 6965 return; 6966 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6967 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6968 } 6969 6970 /** 6971 * Verify the flow list is empty 6972 * 6973 * @param dev 6974 * Pointer to Ethernet device. 6975 * 6976 * @return the number of flows not released. 6977 */ 6978 int 6979 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 6980 { 6981 struct mlx5_priv *priv = dev->data->dev_private; 6982 struct rte_flow *flow; 6983 uint32_t idx = 0; 6984 int ret = 0, i; 6985 6986 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 6987 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 6988 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6989 dev->data->port_id, (void *)flow); 6990 ret++; 6991 } 6992 } 6993 return ret; 6994 } 6995 6996 /** 6997 * Enable default hairpin egress flow. 6998 * 6999 * @param dev 7000 * Pointer to Ethernet device. 7001 * @param queue 7002 * The queue index. 7003 * 7004 * @return 7005 * 0 on success, a negative errno value otherwise and rte_errno is set. 7006 */ 7007 int 7008 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7009 uint32_t queue) 7010 { 7011 const struct rte_flow_attr attr = { 7012 .egress = 1, 7013 .priority = 0, 7014 }; 7015 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7016 .queue = queue, 7017 }; 7018 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7019 .queue = UINT32_MAX, 7020 }; 7021 struct rte_flow_item items[] = { 7022 { 7023 .type = (enum rte_flow_item_type) 7024 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7025 .spec = &queue_spec, 7026 .last = NULL, 7027 .mask = &queue_mask, 7028 }, 7029 { 7030 .type = RTE_FLOW_ITEM_TYPE_END, 7031 }, 7032 }; 7033 struct rte_flow_action_jump jump = { 7034 .group = MLX5_HAIRPIN_TX_TABLE, 7035 }; 7036 struct rte_flow_action actions[2]; 7037 uint32_t flow_idx; 7038 struct rte_flow_error error; 7039 7040 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7041 actions[0].conf = &jump; 7042 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7043 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7044 &attr, items, actions, false, &error); 7045 if (!flow_idx) { 7046 DRV_LOG(DEBUG, 7047 "Failed to create ctrl flow: rte_errno(%d)," 7048 " type(%d), message(%s)", 7049 rte_errno, error.type, 7050 error.message ? error.message : " (no stated reason)"); 7051 return -rte_errno; 7052 } 7053 return 0; 7054 } 7055 7056 /** 7057 * Enable a control flow configured from the control plane. 7058 * 7059 * @param dev 7060 * Pointer to Ethernet device. 7061 * @param eth_spec 7062 * An Ethernet flow spec to apply. 7063 * @param eth_mask 7064 * An Ethernet flow mask to apply. 7065 * @param vlan_spec 7066 * A VLAN flow spec to apply. 7067 * @param vlan_mask 7068 * A VLAN flow mask to apply. 7069 * 7070 * @return 7071 * 0 on success, a negative errno value otherwise and rte_errno is set. 7072 */ 7073 int 7074 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7075 struct rte_flow_item_eth *eth_spec, 7076 struct rte_flow_item_eth *eth_mask, 7077 struct rte_flow_item_vlan *vlan_spec, 7078 struct rte_flow_item_vlan *vlan_mask) 7079 { 7080 struct mlx5_priv *priv = dev->data->dev_private; 7081 const struct rte_flow_attr attr = { 7082 .ingress = 1, 7083 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7084 }; 7085 struct rte_flow_item items[] = { 7086 { 7087 .type = RTE_FLOW_ITEM_TYPE_ETH, 7088 .spec = eth_spec, 7089 .last = NULL, 7090 .mask = eth_mask, 7091 }, 7092 { 7093 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7094 RTE_FLOW_ITEM_TYPE_END, 7095 .spec = vlan_spec, 7096 .last = NULL, 7097 .mask = vlan_mask, 7098 }, 7099 { 7100 .type = RTE_FLOW_ITEM_TYPE_END, 7101 }, 7102 }; 7103 uint16_t queue[priv->reta_idx_n]; 7104 struct rte_flow_action_rss action_rss = { 7105 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7106 .level = 0, 7107 .types = priv->rss_conf.rss_hf, 7108 .key_len = priv->rss_conf.rss_key_len, 7109 .queue_num = priv->reta_idx_n, 7110 .key = priv->rss_conf.rss_key, 7111 .queue = queue, 7112 }; 7113 struct rte_flow_action actions[] = { 7114 { 7115 .type = RTE_FLOW_ACTION_TYPE_RSS, 7116 .conf = &action_rss, 7117 }, 7118 { 7119 .type = RTE_FLOW_ACTION_TYPE_END, 7120 }, 7121 }; 7122 uint32_t flow_idx; 7123 struct rte_flow_error error; 7124 unsigned int i; 7125 7126 if (!priv->reta_idx_n || !priv->rxqs_n) { 7127 return 0; 7128 } 7129 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7130 action_rss.types = 0; 7131 for (i = 0; i != priv->reta_idx_n; ++i) 7132 queue[i] = (*priv->reta_idx)[i]; 7133 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7134 &attr, items, actions, false, &error); 7135 if (!flow_idx) 7136 return -rte_errno; 7137 return 0; 7138 } 7139 7140 /** 7141 * Enable a flow control configured from the control plane. 7142 * 7143 * @param dev 7144 * Pointer to Ethernet device. 7145 * @param eth_spec 7146 * An Ethernet flow spec to apply. 7147 * @param eth_mask 7148 * An Ethernet flow mask to apply. 7149 * 7150 * @return 7151 * 0 on success, a negative errno value otherwise and rte_errno is set. 7152 */ 7153 int 7154 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7155 struct rte_flow_item_eth *eth_spec, 7156 struct rte_flow_item_eth *eth_mask) 7157 { 7158 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7159 } 7160 7161 /** 7162 * Create default miss flow rule matching lacp traffic 7163 * 7164 * @param dev 7165 * Pointer to Ethernet device. 7166 * @param eth_spec 7167 * An Ethernet flow spec to apply. 7168 * 7169 * @return 7170 * 0 on success, a negative errno value otherwise and rte_errno is set. 7171 */ 7172 int 7173 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7174 { 7175 /* 7176 * The LACP matching is done by only using ether type since using 7177 * a multicast dst mac causes kernel to give low priority to this flow. 7178 */ 7179 static const struct rte_flow_item_eth lacp_spec = { 7180 .type = RTE_BE16(0x8809), 7181 }; 7182 static const struct rte_flow_item_eth lacp_mask = { 7183 .type = 0xffff, 7184 }; 7185 const struct rte_flow_attr attr = { 7186 .ingress = 1, 7187 }; 7188 struct rte_flow_item items[] = { 7189 { 7190 .type = RTE_FLOW_ITEM_TYPE_ETH, 7191 .spec = &lacp_spec, 7192 .mask = &lacp_mask, 7193 }, 7194 { 7195 .type = RTE_FLOW_ITEM_TYPE_END, 7196 }, 7197 }; 7198 struct rte_flow_action actions[] = { 7199 { 7200 .type = (enum rte_flow_action_type) 7201 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7202 }, 7203 { 7204 .type = RTE_FLOW_ACTION_TYPE_END, 7205 }, 7206 }; 7207 struct rte_flow_error error; 7208 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7209 &attr, items, actions, 7210 false, &error); 7211 7212 if (!flow_idx) 7213 return -rte_errno; 7214 return 0; 7215 } 7216 7217 /** 7218 * Destroy a flow. 7219 * 7220 * @see rte_flow_destroy() 7221 * @see rte_flow_ops 7222 */ 7223 int 7224 mlx5_flow_destroy(struct rte_eth_dev *dev, 7225 struct rte_flow *flow, 7226 struct rte_flow_error *error __rte_unused) 7227 { 7228 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7229 (uintptr_t)(void *)flow); 7230 return 0; 7231 } 7232 7233 /** 7234 * Destroy all flows. 7235 * 7236 * @see rte_flow_flush() 7237 * @see rte_flow_ops 7238 */ 7239 int 7240 mlx5_flow_flush(struct rte_eth_dev *dev, 7241 struct rte_flow_error *error __rte_unused) 7242 { 7243 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7244 return 0; 7245 } 7246 7247 /** 7248 * Isolated mode. 7249 * 7250 * @see rte_flow_isolate() 7251 * @see rte_flow_ops 7252 */ 7253 int 7254 mlx5_flow_isolate(struct rte_eth_dev *dev, 7255 int enable, 7256 struct rte_flow_error *error) 7257 { 7258 struct mlx5_priv *priv = dev->data->dev_private; 7259 7260 if (dev->data->dev_started) { 7261 rte_flow_error_set(error, EBUSY, 7262 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7263 NULL, 7264 "port must be stopped first"); 7265 return -rte_errno; 7266 } 7267 priv->isolated = !!enable; 7268 if (enable) 7269 dev->dev_ops = &mlx5_dev_ops_isolate; 7270 else 7271 dev->dev_ops = &mlx5_dev_ops; 7272 7273 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7274 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7275 7276 return 0; 7277 } 7278 7279 /** 7280 * Query a flow. 7281 * 7282 * @see rte_flow_query() 7283 * @see rte_flow_ops 7284 */ 7285 static int 7286 flow_drv_query(struct rte_eth_dev *dev, 7287 uint32_t flow_idx, 7288 const struct rte_flow_action *actions, 7289 void *data, 7290 struct rte_flow_error *error) 7291 { 7292 struct mlx5_priv *priv = dev->data->dev_private; 7293 const struct mlx5_flow_driver_ops *fops; 7294 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7295 flow_idx); 7296 enum mlx5_flow_drv_type ftype; 7297 7298 if (!flow) { 7299 return rte_flow_error_set(error, ENOENT, 7300 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7301 NULL, 7302 "invalid flow handle"); 7303 } 7304 ftype = flow->drv_type; 7305 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7306 fops = flow_get_drv_ops(ftype); 7307 7308 return fops->query(dev, flow, actions, data, error); 7309 } 7310 7311 /** 7312 * Query a flow. 7313 * 7314 * @see rte_flow_query() 7315 * @see rte_flow_ops 7316 */ 7317 int 7318 mlx5_flow_query(struct rte_eth_dev *dev, 7319 struct rte_flow *flow, 7320 const struct rte_flow_action *actions, 7321 void *data, 7322 struct rte_flow_error *error) 7323 { 7324 int ret; 7325 7326 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7327 error); 7328 if (ret < 0) 7329 return ret; 7330 return 0; 7331 } 7332 7333 /** 7334 * Get rte_flow callbacks. 7335 * 7336 * @param dev 7337 * Pointer to Ethernet device structure. 7338 * @param ops 7339 * Pointer to operation-specific structure. 7340 * 7341 * @return 0 7342 */ 7343 int 7344 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7345 const struct rte_flow_ops **ops) 7346 { 7347 *ops = &mlx5_flow_ops; 7348 return 0; 7349 } 7350 7351 /** 7352 * Validate meter policy actions. 7353 * Dispatcher for action type specific validation. 7354 * 7355 * @param[in] dev 7356 * Pointer to the Ethernet device structure. 7357 * @param[in] action 7358 * The meter policy action object to validate. 7359 * @param[in] attr 7360 * Attributes of flow to determine steering domain. 7361 * @param[out] is_rss 7362 * Is RSS or not. 7363 * @param[out] domain_bitmap 7364 * Domain bitmap. 7365 * @param[out] is_def_policy 7366 * Is default policy or not. 7367 * @param[out] error 7368 * Perform verbose error reporting if not NULL. Initialized in case of 7369 * error only. 7370 * 7371 * @return 7372 * 0 on success, otherwise negative errno value. 7373 */ 7374 int 7375 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7376 const struct rte_flow_action *actions[RTE_COLORS], 7377 struct rte_flow_attr *attr, 7378 bool *is_rss, 7379 uint8_t *domain_bitmap, 7380 uint8_t *policy_mode, 7381 struct rte_mtr_error *error) 7382 { 7383 const struct mlx5_flow_driver_ops *fops; 7384 7385 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7386 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7387 domain_bitmap, policy_mode, error); 7388 } 7389 7390 /** 7391 * Destroy the meter table set. 7392 * 7393 * @param[in] dev 7394 * Pointer to Ethernet device. 7395 * @param[in] mtr_policy 7396 * Meter policy struct. 7397 */ 7398 void 7399 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7400 struct mlx5_flow_meter_policy *mtr_policy) 7401 { 7402 const struct mlx5_flow_driver_ops *fops; 7403 7404 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7405 fops->destroy_mtr_acts(dev, mtr_policy); 7406 } 7407 7408 /** 7409 * Create policy action, lock free, 7410 * (mutex should be acquired by caller). 7411 * Dispatcher for action type specific call. 7412 * 7413 * @param[in] dev 7414 * Pointer to the Ethernet device structure. 7415 * @param[in] mtr_policy 7416 * Meter policy struct. 7417 * @param[in] action 7418 * Action specification used to create meter actions. 7419 * @param[out] error 7420 * Perform verbose error reporting if not NULL. Initialized in case of 7421 * error only. 7422 * 7423 * @return 7424 * 0 on success, otherwise negative errno value. 7425 */ 7426 int 7427 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7428 struct mlx5_flow_meter_policy *mtr_policy, 7429 const struct rte_flow_action *actions[RTE_COLORS], 7430 struct rte_mtr_error *error) 7431 { 7432 const struct mlx5_flow_driver_ops *fops; 7433 7434 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7435 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7436 } 7437 7438 /** 7439 * Create policy rules, lock free, 7440 * (mutex should be acquired by caller). 7441 * Dispatcher for action type specific call. 7442 * 7443 * @param[in] dev 7444 * Pointer to the Ethernet device structure. 7445 * @param[in] mtr_policy 7446 * Meter policy struct. 7447 * 7448 * @return 7449 * 0 on success, -1 otherwise. 7450 */ 7451 int 7452 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7453 struct mlx5_flow_meter_policy *mtr_policy) 7454 { 7455 const struct mlx5_flow_driver_ops *fops; 7456 7457 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7458 return fops->create_policy_rules(dev, mtr_policy); 7459 } 7460 7461 /** 7462 * Destroy policy rules, lock free, 7463 * (mutex should be acquired by caller). 7464 * Dispatcher for action type specific call. 7465 * 7466 * @param[in] dev 7467 * Pointer to the Ethernet device structure. 7468 * @param[in] mtr_policy 7469 * Meter policy struct. 7470 */ 7471 void 7472 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7473 struct mlx5_flow_meter_policy *mtr_policy) 7474 { 7475 const struct mlx5_flow_driver_ops *fops; 7476 7477 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7478 fops->destroy_policy_rules(dev, mtr_policy); 7479 } 7480 7481 /** 7482 * Destroy the default policy table set. 7483 * 7484 * @param[in] dev 7485 * Pointer to Ethernet device. 7486 */ 7487 void 7488 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7489 { 7490 const struct mlx5_flow_driver_ops *fops; 7491 7492 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7493 fops->destroy_def_policy(dev); 7494 } 7495 7496 /** 7497 * Destroy the default policy table set. 7498 * 7499 * @param[in] dev 7500 * Pointer to Ethernet device. 7501 * 7502 * @return 7503 * 0 on success, -1 otherwise. 7504 */ 7505 int 7506 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7507 { 7508 const struct mlx5_flow_driver_ops *fops; 7509 7510 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7511 return fops->create_def_policy(dev); 7512 } 7513 7514 /** 7515 * Create the needed meter and suffix tables. 7516 * 7517 * @param[in] dev 7518 * Pointer to Ethernet device. 7519 * 7520 * @return 7521 * 0 on success, -1 otherwise. 7522 */ 7523 int 7524 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7525 struct mlx5_flow_meter_info *fm, 7526 uint32_t mtr_idx, 7527 uint8_t domain_bitmap) 7528 { 7529 const struct mlx5_flow_driver_ops *fops; 7530 7531 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7532 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7533 } 7534 7535 /** 7536 * Destroy the meter table set. 7537 * 7538 * @param[in] dev 7539 * Pointer to Ethernet device. 7540 * @param[in] tbl 7541 * Pointer to the meter table set. 7542 */ 7543 void 7544 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7545 struct mlx5_flow_meter_info *fm) 7546 { 7547 const struct mlx5_flow_driver_ops *fops; 7548 7549 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7550 fops->destroy_mtr_tbls(dev, fm); 7551 } 7552 7553 /** 7554 * Destroy the global meter drop table. 7555 * 7556 * @param[in] dev 7557 * Pointer to Ethernet device. 7558 */ 7559 void 7560 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7561 { 7562 const struct mlx5_flow_driver_ops *fops; 7563 7564 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7565 fops->destroy_mtr_drop_tbls(dev); 7566 } 7567 7568 /** 7569 * Destroy the sub policy table with RX queue. 7570 * 7571 * @param[in] dev 7572 * Pointer to Ethernet device. 7573 * @param[in] mtr_policy 7574 * Pointer to meter policy table. 7575 */ 7576 void 7577 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7578 struct mlx5_flow_meter_policy *mtr_policy) 7579 { 7580 const struct mlx5_flow_driver_ops *fops; 7581 7582 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7583 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7584 } 7585 7586 /** 7587 * Allocate the needed aso flow meter id. 7588 * 7589 * @param[in] dev 7590 * Pointer to Ethernet device. 7591 * 7592 * @return 7593 * Index to aso flow meter on success, NULL otherwise. 7594 */ 7595 uint32_t 7596 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7597 { 7598 const struct mlx5_flow_driver_ops *fops; 7599 7600 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7601 return fops->create_meter(dev); 7602 } 7603 7604 /** 7605 * Free the aso flow meter id. 7606 * 7607 * @param[in] dev 7608 * Pointer to Ethernet device. 7609 * @param[in] mtr_idx 7610 * Index to aso flow meter to be free. 7611 * 7612 * @return 7613 * 0 on success. 7614 */ 7615 void 7616 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7617 { 7618 const struct mlx5_flow_driver_ops *fops; 7619 7620 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7621 fops->free_meter(dev, mtr_idx); 7622 } 7623 7624 /** 7625 * Allocate a counter. 7626 * 7627 * @param[in] dev 7628 * Pointer to Ethernet device structure. 7629 * 7630 * @return 7631 * Index to allocated counter on success, 0 otherwise. 7632 */ 7633 uint32_t 7634 mlx5_counter_alloc(struct rte_eth_dev *dev) 7635 { 7636 const struct mlx5_flow_driver_ops *fops; 7637 struct rte_flow_attr attr = { .transfer = 0 }; 7638 7639 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7640 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7641 return fops->counter_alloc(dev); 7642 } 7643 DRV_LOG(ERR, 7644 "port %u counter allocate is not supported.", 7645 dev->data->port_id); 7646 return 0; 7647 } 7648 7649 /** 7650 * Free a counter. 7651 * 7652 * @param[in] dev 7653 * Pointer to Ethernet device structure. 7654 * @param[in] cnt 7655 * Index to counter to be free. 7656 */ 7657 void 7658 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7659 { 7660 const struct mlx5_flow_driver_ops *fops; 7661 struct rte_flow_attr attr = { .transfer = 0 }; 7662 7663 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7664 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7665 fops->counter_free(dev, cnt); 7666 return; 7667 } 7668 DRV_LOG(ERR, 7669 "port %u counter free is not supported.", 7670 dev->data->port_id); 7671 } 7672 7673 /** 7674 * Query counter statistics. 7675 * 7676 * @param[in] dev 7677 * Pointer to Ethernet device structure. 7678 * @param[in] cnt 7679 * Index to counter to query. 7680 * @param[in] clear 7681 * Set to clear counter statistics. 7682 * @param[out] pkts 7683 * The counter hits packets number to save. 7684 * @param[out] bytes 7685 * The counter hits bytes number to save. 7686 * 7687 * @return 7688 * 0 on success, a negative errno value otherwise. 7689 */ 7690 int 7691 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7692 bool clear, uint64_t *pkts, uint64_t *bytes) 7693 { 7694 const struct mlx5_flow_driver_ops *fops; 7695 struct rte_flow_attr attr = { .transfer = 0 }; 7696 7697 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7698 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7699 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7700 } 7701 DRV_LOG(ERR, 7702 "port %u counter query is not supported.", 7703 dev->data->port_id); 7704 return -ENOTSUP; 7705 } 7706 7707 /** 7708 * Allocate a new memory for the counter values wrapped by all the needed 7709 * management. 7710 * 7711 * @param[in] sh 7712 * Pointer to mlx5_dev_ctx_shared object. 7713 * 7714 * @return 7715 * 0 on success, a negative errno value otherwise. 7716 */ 7717 static int 7718 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7719 { 7720 struct mlx5_devx_mkey_attr mkey_attr; 7721 struct mlx5_counter_stats_mem_mng *mem_mng; 7722 volatile struct flow_counter_stats *raw_data; 7723 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7724 int size = (sizeof(struct flow_counter_stats) * 7725 MLX5_COUNTERS_PER_POOL + 7726 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7727 sizeof(struct mlx5_counter_stats_mem_mng); 7728 size_t pgsize = rte_mem_page_size(); 7729 uint8_t *mem; 7730 int i; 7731 7732 if (pgsize == (size_t)-1) { 7733 DRV_LOG(ERR, "Failed to get mem page size"); 7734 rte_errno = ENOMEM; 7735 return -ENOMEM; 7736 } 7737 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7738 if (!mem) { 7739 rte_errno = ENOMEM; 7740 return -ENOMEM; 7741 } 7742 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7743 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7744 mem_mng->umem = mlx5_os_umem_reg(sh->cdev->ctx, mem, size, 7745 IBV_ACCESS_LOCAL_WRITE); 7746 if (!mem_mng->umem) { 7747 rte_errno = errno; 7748 mlx5_free(mem); 7749 return -rte_errno; 7750 } 7751 memset(&mkey_attr, 0, sizeof(mkey_attr)); 7752 mkey_attr.addr = (uintptr_t)mem; 7753 mkey_attr.size = size; 7754 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7755 mkey_attr.pd = sh->cdev->pdn; 7756 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 7757 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 7758 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->cdev->ctx, &mkey_attr); 7759 if (!mem_mng->dm) { 7760 mlx5_os_umem_dereg(mem_mng->umem); 7761 rte_errno = errno; 7762 mlx5_free(mem); 7763 return -rte_errno; 7764 } 7765 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7766 raw_data = (volatile struct flow_counter_stats *)mem; 7767 for (i = 0; i < raws_n; ++i) { 7768 mem_mng->raws[i].mem_mng = mem_mng; 7769 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7770 } 7771 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7772 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7773 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7774 next); 7775 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7776 sh->cmng.mem_mng = mem_mng; 7777 return 0; 7778 } 7779 7780 /** 7781 * Set the statistic memory to the new counter pool. 7782 * 7783 * @param[in] sh 7784 * Pointer to mlx5_dev_ctx_shared object. 7785 * @param[in] pool 7786 * Pointer to the pool to set the statistic memory. 7787 * 7788 * @return 7789 * 0 on success, a negative errno value otherwise. 7790 */ 7791 static int 7792 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7793 struct mlx5_flow_counter_pool *pool) 7794 { 7795 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7796 /* Resize statistic memory once used out. */ 7797 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7798 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7799 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7800 return -1; 7801 } 7802 rte_spinlock_lock(&pool->sl); 7803 pool->raw = cmng->mem_mng->raws + pool->index % 7804 MLX5_CNT_CONTAINER_RESIZE; 7805 rte_spinlock_unlock(&pool->sl); 7806 pool->raw_hw = NULL; 7807 return 0; 7808 } 7809 7810 #define MLX5_POOL_QUERY_FREQ_US 1000000 7811 7812 /** 7813 * Set the periodic procedure for triggering asynchronous batch queries for all 7814 * the counter pools. 7815 * 7816 * @param[in] sh 7817 * Pointer to mlx5_dev_ctx_shared object. 7818 */ 7819 void 7820 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7821 { 7822 uint32_t pools_n, us; 7823 7824 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7825 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7826 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7827 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7828 sh->cmng.query_thread_on = 0; 7829 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7830 } else { 7831 sh->cmng.query_thread_on = 1; 7832 } 7833 } 7834 7835 /** 7836 * The periodic procedure for triggering asynchronous batch queries for all the 7837 * counter pools. This function is probably called by the host thread. 7838 * 7839 * @param[in] arg 7840 * The parameter for the alarm process. 7841 */ 7842 void 7843 mlx5_flow_query_alarm(void *arg) 7844 { 7845 struct mlx5_dev_ctx_shared *sh = arg; 7846 int ret; 7847 uint16_t pool_index = sh->cmng.pool_index; 7848 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7849 struct mlx5_flow_counter_pool *pool; 7850 uint16_t n_valid; 7851 7852 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7853 goto set_alarm; 7854 rte_spinlock_lock(&cmng->pool_update_sl); 7855 pool = cmng->pools[pool_index]; 7856 n_valid = cmng->n_valid; 7857 rte_spinlock_unlock(&cmng->pool_update_sl); 7858 /* Set the statistic memory to the new created pool. */ 7859 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7860 goto set_alarm; 7861 if (pool->raw_hw) 7862 /* There is a pool query in progress. */ 7863 goto set_alarm; 7864 pool->raw_hw = 7865 LIST_FIRST(&sh->cmng.free_stat_raws); 7866 if (!pool->raw_hw) 7867 /* No free counter statistics raw memory. */ 7868 goto set_alarm; 7869 /* 7870 * Identify the counters released between query trigger and query 7871 * handle more efficiently. The counter released in this gap period 7872 * should wait for a new round of query as the new arrived packets 7873 * will not be taken into account. 7874 */ 7875 pool->query_gen++; 7876 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7877 MLX5_COUNTERS_PER_POOL, 7878 NULL, NULL, 7879 pool->raw_hw->mem_mng->dm->id, 7880 (void *)(uintptr_t) 7881 pool->raw_hw->data, 7882 sh->devx_comp, 7883 (uint64_t)(uintptr_t)pool); 7884 if (ret) { 7885 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7886 " %d", pool->min_dcs->id); 7887 pool->raw_hw = NULL; 7888 goto set_alarm; 7889 } 7890 LIST_REMOVE(pool->raw_hw, next); 7891 sh->cmng.pending_queries++; 7892 pool_index++; 7893 if (pool_index >= n_valid) 7894 pool_index = 0; 7895 set_alarm: 7896 sh->cmng.pool_index = pool_index; 7897 mlx5_set_query_alarm(sh); 7898 } 7899 7900 /** 7901 * Check and callback event for new aged flow in the counter pool 7902 * 7903 * @param[in] sh 7904 * Pointer to mlx5_dev_ctx_shared object. 7905 * @param[in] pool 7906 * Pointer to Current counter pool. 7907 */ 7908 static void 7909 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7910 struct mlx5_flow_counter_pool *pool) 7911 { 7912 struct mlx5_priv *priv; 7913 struct mlx5_flow_counter *cnt; 7914 struct mlx5_age_info *age_info; 7915 struct mlx5_age_param *age_param; 7916 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7917 struct mlx5_counter_stats_raw *prev = pool->raw; 7918 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7919 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7920 uint16_t expected = AGE_CANDIDATE; 7921 uint32_t i; 7922 7923 pool->time_of_last_age_check = curr_time; 7924 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7925 cnt = MLX5_POOL_GET_CNT(pool, i); 7926 age_param = MLX5_CNT_TO_AGE(cnt); 7927 if (__atomic_load_n(&age_param->state, 7928 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7929 continue; 7930 if (cur->data[i].hits != prev->data[i].hits) { 7931 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7932 __ATOMIC_RELAXED); 7933 continue; 7934 } 7935 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7936 time_delta, 7937 __ATOMIC_RELAXED) <= age_param->timeout) 7938 continue; 7939 /** 7940 * Hold the lock first, or if between the 7941 * state AGE_TMOUT and tailq operation the 7942 * release happened, the release procedure 7943 * may delete a non-existent tailq node. 7944 */ 7945 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7946 age_info = GET_PORT_AGE_INFO(priv); 7947 rte_spinlock_lock(&age_info->aged_sl); 7948 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7949 AGE_TMOUT, false, 7950 __ATOMIC_RELAXED, 7951 __ATOMIC_RELAXED)) { 7952 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7953 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7954 } 7955 rte_spinlock_unlock(&age_info->aged_sl); 7956 } 7957 mlx5_age_event_prepare(sh); 7958 } 7959 7960 /** 7961 * Handler for the HW respond about ready values from an asynchronous batch 7962 * query. This function is probably called by the host thread. 7963 * 7964 * @param[in] sh 7965 * The pointer to the shared device context. 7966 * @param[in] async_id 7967 * The Devx async ID. 7968 * @param[in] status 7969 * The status of the completion. 7970 */ 7971 void 7972 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7973 uint64_t async_id, int status) 7974 { 7975 struct mlx5_flow_counter_pool *pool = 7976 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7977 struct mlx5_counter_stats_raw *raw_to_free; 7978 uint8_t query_gen = pool->query_gen ^ 1; 7979 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7980 enum mlx5_counter_type cnt_type = 7981 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7982 MLX5_COUNTER_TYPE_ORIGIN; 7983 7984 if (unlikely(status)) { 7985 raw_to_free = pool->raw_hw; 7986 } else { 7987 raw_to_free = pool->raw; 7988 if (pool->is_aged) 7989 mlx5_flow_aging_check(sh, pool); 7990 rte_spinlock_lock(&pool->sl); 7991 pool->raw = pool->raw_hw; 7992 rte_spinlock_unlock(&pool->sl); 7993 /* Be sure the new raw counters data is updated in memory. */ 7994 rte_io_wmb(); 7995 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7996 rte_spinlock_lock(&cmng->csl[cnt_type]); 7997 TAILQ_CONCAT(&cmng->counters[cnt_type], 7998 &pool->counters[query_gen], next); 7999 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8000 } 8001 } 8002 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8003 pool->raw_hw = NULL; 8004 sh->cmng.pending_queries--; 8005 } 8006 8007 static int 8008 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8009 const struct flow_grp_info *grp_info, 8010 struct rte_flow_error *error) 8011 { 8012 if (grp_info->transfer && grp_info->external && 8013 grp_info->fdb_def_rule) { 8014 if (group == UINT32_MAX) 8015 return rte_flow_error_set 8016 (error, EINVAL, 8017 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8018 NULL, 8019 "group index not supported"); 8020 *table = group + 1; 8021 } else { 8022 *table = group; 8023 } 8024 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8025 return 0; 8026 } 8027 8028 /** 8029 * Translate the rte_flow group index to HW table value. 8030 * 8031 * If tunnel offload is disabled, all group ids converted to flow table 8032 * id using the standard method. 8033 * If tunnel offload is enabled, group id can be converted using the 8034 * standard or tunnel conversion method. Group conversion method 8035 * selection depends on flags in `grp_info` parameter: 8036 * - Internal (grp_info.external == 0) groups conversion uses the 8037 * standard method. 8038 * - Group ids in JUMP action converted with the tunnel conversion. 8039 * - Group id in rule attribute conversion depends on a rule type and 8040 * group id value: 8041 * ** non zero group attributes converted with the tunnel method 8042 * ** zero group attribute in non-tunnel rule is converted using the 8043 * standard method - there's only one root table 8044 * ** zero group attribute in steer tunnel rule is converted with the 8045 * standard method - single root table 8046 * ** zero group attribute in match tunnel rule is a special OvS 8047 * case: that value is used for portability reasons. That group 8048 * id is converted with the tunnel conversion method. 8049 * 8050 * @param[in] dev 8051 * Port device 8052 * @param[in] tunnel 8053 * PMD tunnel offload object 8054 * @param[in] group 8055 * rte_flow group index value. 8056 * @param[out] table 8057 * HW table value. 8058 * @param[in] grp_info 8059 * flags used for conversion 8060 * @param[out] error 8061 * Pointer to error structure. 8062 * 8063 * @return 8064 * 0 on success, a negative errno value otherwise and rte_errno is set. 8065 */ 8066 int 8067 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8068 const struct mlx5_flow_tunnel *tunnel, 8069 uint32_t group, uint32_t *table, 8070 const struct flow_grp_info *grp_info, 8071 struct rte_flow_error *error) 8072 { 8073 int ret; 8074 bool standard_translation; 8075 8076 if (!grp_info->skip_scale && grp_info->external && 8077 group < MLX5_MAX_TABLES_EXTERNAL) 8078 group *= MLX5_FLOW_TABLE_FACTOR; 8079 if (is_tunnel_offload_active(dev)) { 8080 standard_translation = !grp_info->external || 8081 grp_info->std_tbl_fix; 8082 } else { 8083 standard_translation = true; 8084 } 8085 DRV_LOG(DEBUG, 8086 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8087 dev->data->port_id, group, grp_info->transfer, 8088 grp_info->external, grp_info->fdb_def_rule, 8089 standard_translation ? "STANDARD" : "TUNNEL"); 8090 if (standard_translation) 8091 ret = flow_group_to_table(dev->data->port_id, group, table, 8092 grp_info, error); 8093 else 8094 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8095 table, error); 8096 8097 return ret; 8098 } 8099 8100 /** 8101 * Discover availability of metadata reg_c's. 8102 * 8103 * Iteratively use test flows to check availability. 8104 * 8105 * @param[in] dev 8106 * Pointer to the Ethernet device structure. 8107 * 8108 * @return 8109 * 0 on success, a negative errno value otherwise and rte_errno is set. 8110 */ 8111 int 8112 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8113 { 8114 struct mlx5_priv *priv = dev->data->dev_private; 8115 struct mlx5_dev_config *config = &priv->config; 8116 enum modify_reg idx; 8117 int n = 0; 8118 8119 /* reg_c[0] and reg_c[1] are reserved. */ 8120 config->flow_mreg_c[n++] = REG_C_0; 8121 config->flow_mreg_c[n++] = REG_C_1; 8122 /* Discover availability of other reg_c's. */ 8123 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8124 struct rte_flow_attr attr = { 8125 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8126 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8127 .ingress = 1, 8128 }; 8129 struct rte_flow_item items[] = { 8130 [0] = { 8131 .type = RTE_FLOW_ITEM_TYPE_END, 8132 }, 8133 }; 8134 struct rte_flow_action actions[] = { 8135 [0] = { 8136 .type = (enum rte_flow_action_type) 8137 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8138 .conf = &(struct mlx5_flow_action_copy_mreg){ 8139 .src = REG_C_1, 8140 .dst = idx, 8141 }, 8142 }, 8143 [1] = { 8144 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8145 .conf = &(struct rte_flow_action_jump){ 8146 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8147 }, 8148 }, 8149 [2] = { 8150 .type = RTE_FLOW_ACTION_TYPE_END, 8151 }, 8152 }; 8153 uint32_t flow_idx; 8154 struct rte_flow *flow; 8155 struct rte_flow_error error; 8156 8157 if (!config->dv_flow_en) 8158 break; 8159 /* Create internal flow, validation skips copy action. */ 8160 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8161 items, actions, false, &error); 8162 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8163 flow_idx); 8164 if (!flow) 8165 continue; 8166 config->flow_mreg_c[n++] = idx; 8167 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8168 } 8169 for (; n < MLX5_MREG_C_NUM; ++n) 8170 config->flow_mreg_c[n] = REG_NON; 8171 return 0; 8172 } 8173 8174 int 8175 save_dump_file(const uint8_t *data, uint32_t size, 8176 uint32_t type, uint32_t id, void *arg, FILE *file) 8177 { 8178 char line[BUF_SIZE]; 8179 uint32_t out = 0; 8180 uint32_t k; 8181 uint32_t actions_num; 8182 struct rte_flow_query_count *count; 8183 8184 memset(line, 0, BUF_SIZE); 8185 switch (type) { 8186 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8187 actions_num = *(uint32_t *)(arg); 8188 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,", 8189 type, id, actions_num); 8190 break; 8191 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8192 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,", 8193 type, id); 8194 break; 8195 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8196 count = (struct rte_flow_query_count *)arg; 8197 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type, 8198 id, count->hits, count->bytes); 8199 return 0; 8200 default: 8201 return -1; 8202 } 8203 8204 for (k = 0; k < size; k++) { 8205 /* Make sure we do not overrun the line buffer length. */ 8206 if (out >= BUF_SIZE - 4) { 8207 line[out] = '\0'; 8208 break; 8209 } 8210 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8211 (data[k]) & 0xff); 8212 } 8213 fprintf(file, "%s\n", line); 8214 return 0; 8215 } 8216 8217 int 8218 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8219 struct rte_flow_query_count *count, struct rte_flow_error *error) 8220 { 8221 struct rte_flow_action action[2]; 8222 enum mlx5_flow_drv_type ftype; 8223 const struct mlx5_flow_driver_ops *fops; 8224 8225 if (!flow) { 8226 return rte_flow_error_set(error, ENOENT, 8227 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8228 NULL, 8229 "invalid flow handle"); 8230 } 8231 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8232 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8233 if (flow->counter) { 8234 memset(count, 0, sizeof(struct rte_flow_query_count)); 8235 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8236 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8237 ftype < MLX5_FLOW_TYPE_MAX); 8238 fops = flow_get_drv_ops(ftype); 8239 return fops->query(dev, flow, action, count, error); 8240 } 8241 return -1; 8242 } 8243 8244 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8245 /** 8246 * Dump flow ipool data to file 8247 * 8248 * @param[in] dev 8249 * The pointer to Ethernet device. 8250 * @param[in] file 8251 * A pointer to a file for output. 8252 * @param[out] error 8253 * Perform verbose error reporting if not NULL. PMDs initialize this 8254 * structure in case of error only. 8255 * @return 8256 * 0 on success, a negative value otherwise. 8257 */ 8258 int 8259 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8260 struct rte_flow *flow, FILE *file, 8261 struct rte_flow_error *error) 8262 { 8263 struct mlx5_priv *priv = dev->data->dev_private; 8264 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8265 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8266 uint32_t handle_idx; 8267 struct mlx5_flow_handle *dh; 8268 struct rte_flow_query_count count; 8269 uint32_t actions_num; 8270 const uint8_t *data; 8271 size_t size; 8272 uint32_t id; 8273 uint32_t type; 8274 8275 if (!flow) { 8276 return rte_flow_error_set(error, ENOENT, 8277 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8278 NULL, 8279 "invalid flow handle"); 8280 } 8281 handle_idx = flow->dev_handles; 8282 while (handle_idx) { 8283 dh = mlx5_ipool_get(priv->sh->ipool 8284 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8285 if (!dh) 8286 continue; 8287 handle_idx = dh->next.next; 8288 id = (uint32_t)(uintptr_t)dh->drv_flow; 8289 8290 /* query counter */ 8291 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8292 if (!mlx5_flow_query_counter(dev, flow, &count, error)) 8293 save_dump_file(NULL, 0, type, 8294 id, (void *)&count, file); 8295 8296 /* Get modify_hdr and encap_decap buf from ipools. */ 8297 encap_decap = NULL; 8298 modify_hdr = dh->dvh.modify_hdr; 8299 8300 if (dh->dvh.rix_encap_decap) { 8301 encap_decap = mlx5_ipool_get(priv->sh->ipool 8302 [MLX5_IPOOL_DECAP_ENCAP], 8303 dh->dvh.rix_encap_decap); 8304 } 8305 if (modify_hdr) { 8306 data = (const uint8_t *)modify_hdr->actions; 8307 size = (size_t)(modify_hdr->actions_num) * 8; 8308 actions_num = modify_hdr->actions_num; 8309 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8310 save_dump_file(data, size, type, id, 8311 (void *)(&actions_num), file); 8312 } 8313 if (encap_decap) { 8314 data = encap_decap->buf; 8315 size = encap_decap->size; 8316 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8317 save_dump_file(data, size, type, 8318 id, NULL, file); 8319 } 8320 } 8321 return 0; 8322 } 8323 #endif 8324 8325 /** 8326 * Dump flow raw hw data to file 8327 * 8328 * @param[in] dev 8329 * The pointer to Ethernet device. 8330 * @param[in] file 8331 * A pointer to a file for output. 8332 * @param[out] error 8333 * Perform verbose error reporting if not NULL. PMDs initialize this 8334 * structure in case of error only. 8335 * @return 8336 * 0 on success, a nagative value otherwise. 8337 */ 8338 int 8339 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8340 FILE *file, 8341 struct rte_flow_error *error __rte_unused) 8342 { 8343 struct mlx5_priv *priv = dev->data->dev_private; 8344 struct mlx5_dev_ctx_shared *sh = priv->sh; 8345 uint32_t handle_idx; 8346 int ret; 8347 struct mlx5_flow_handle *dh; 8348 struct rte_flow *flow; 8349 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8350 uint32_t idx; 8351 #endif 8352 8353 if (!priv->config.dv_flow_en) { 8354 if (fputs("device dv flow disabled\n", file) <= 0) 8355 return -errno; 8356 return -ENOTSUP; 8357 } 8358 8359 /* dump all */ 8360 if (!flow_idx) { 8361 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8362 MLX5_IPOOL_FOREACH(priv->flows[MLX5_FLOW_TYPE_GEN], idx, flow) 8363 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8364 #endif 8365 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8366 sh->rx_domain, 8367 sh->tx_domain, file); 8368 } 8369 /* dump one */ 8370 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8371 (uintptr_t)(void *)flow_idx); 8372 if (!flow) 8373 return -ENOENT; 8374 8375 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8376 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8377 #endif 8378 handle_idx = flow->dev_handles; 8379 while (handle_idx) { 8380 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8381 handle_idx); 8382 if (!dh) 8383 return -ENOENT; 8384 if (dh->drv_flow) { 8385 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8386 file); 8387 if (ret) 8388 return -ENOENT; 8389 } 8390 handle_idx = dh->next.next; 8391 } 8392 return 0; 8393 } 8394 8395 /** 8396 * Get aged-out flows. 8397 * 8398 * @param[in] dev 8399 * Pointer to the Ethernet device structure. 8400 * @param[in] context 8401 * The address of an array of pointers to the aged-out flows contexts. 8402 * @param[in] nb_countexts 8403 * The length of context array pointers. 8404 * @param[out] error 8405 * Perform verbose error reporting if not NULL. Initialized in case of 8406 * error only. 8407 * 8408 * @return 8409 * how many contexts get in success, otherwise negative errno value. 8410 * if nb_contexts is 0, return the amount of all aged contexts. 8411 * if nb_contexts is not 0 , return the amount of aged flows reported 8412 * in the context array. 8413 */ 8414 int 8415 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8416 uint32_t nb_contexts, struct rte_flow_error *error) 8417 { 8418 const struct mlx5_flow_driver_ops *fops; 8419 struct rte_flow_attr attr = { .transfer = 0 }; 8420 8421 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8422 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8423 return fops->get_aged_flows(dev, contexts, nb_contexts, 8424 error); 8425 } 8426 DRV_LOG(ERR, 8427 "port %u get aged flows is not supported.", 8428 dev->data->port_id); 8429 return -ENOTSUP; 8430 } 8431 8432 /* Wrapper for driver action_validate op callback */ 8433 static int 8434 flow_drv_action_validate(struct rte_eth_dev *dev, 8435 const struct rte_flow_indir_action_conf *conf, 8436 const struct rte_flow_action *action, 8437 const struct mlx5_flow_driver_ops *fops, 8438 struct rte_flow_error *error) 8439 { 8440 static const char err_msg[] = "indirect action validation unsupported"; 8441 8442 if (!fops->action_validate) { 8443 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8444 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8445 NULL, err_msg); 8446 return -rte_errno; 8447 } 8448 return fops->action_validate(dev, conf, action, error); 8449 } 8450 8451 /** 8452 * Destroys the shared action by handle. 8453 * 8454 * @param dev 8455 * Pointer to Ethernet device structure. 8456 * @param[in] handle 8457 * Handle for the indirect action object to be destroyed. 8458 * @param[out] error 8459 * Perform verbose error reporting if not NULL. PMDs initialize this 8460 * structure in case of error only. 8461 * 8462 * @return 8463 * 0 on success, a negative errno value otherwise and rte_errno is set. 8464 * 8465 * @note: wrapper for driver action_create op callback. 8466 */ 8467 static int 8468 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8469 struct rte_flow_action_handle *handle, 8470 struct rte_flow_error *error) 8471 { 8472 static const char err_msg[] = "indirect action destruction unsupported"; 8473 struct rte_flow_attr attr = { .transfer = 0 }; 8474 const struct mlx5_flow_driver_ops *fops = 8475 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8476 8477 if (!fops->action_destroy) { 8478 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8479 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8480 NULL, err_msg); 8481 return -rte_errno; 8482 } 8483 return fops->action_destroy(dev, handle, error); 8484 } 8485 8486 /* Wrapper for driver action_destroy op callback */ 8487 static int 8488 flow_drv_action_update(struct rte_eth_dev *dev, 8489 struct rte_flow_action_handle *handle, 8490 const void *update, 8491 const struct mlx5_flow_driver_ops *fops, 8492 struct rte_flow_error *error) 8493 { 8494 static const char err_msg[] = "indirect action update unsupported"; 8495 8496 if (!fops->action_update) { 8497 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8498 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8499 NULL, err_msg); 8500 return -rte_errno; 8501 } 8502 return fops->action_update(dev, handle, update, error); 8503 } 8504 8505 /* Wrapper for driver action_destroy op callback */ 8506 static int 8507 flow_drv_action_query(struct rte_eth_dev *dev, 8508 const struct rte_flow_action_handle *handle, 8509 void *data, 8510 const struct mlx5_flow_driver_ops *fops, 8511 struct rte_flow_error *error) 8512 { 8513 static const char err_msg[] = "indirect action query unsupported"; 8514 8515 if (!fops->action_query) { 8516 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8517 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8518 NULL, err_msg); 8519 return -rte_errno; 8520 } 8521 return fops->action_query(dev, handle, data, error); 8522 } 8523 8524 /** 8525 * Create indirect action for reuse in multiple flow rules. 8526 * 8527 * @param dev 8528 * Pointer to Ethernet device structure. 8529 * @param conf 8530 * Pointer to indirect action object configuration. 8531 * @param[in] action 8532 * Action configuration for indirect action object creation. 8533 * @param[out] error 8534 * Perform verbose error reporting if not NULL. PMDs initialize this 8535 * structure in case of error only. 8536 * @return 8537 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8538 */ 8539 static struct rte_flow_action_handle * 8540 mlx5_action_handle_create(struct rte_eth_dev *dev, 8541 const struct rte_flow_indir_action_conf *conf, 8542 const struct rte_flow_action *action, 8543 struct rte_flow_error *error) 8544 { 8545 static const char err_msg[] = "indirect action creation unsupported"; 8546 struct rte_flow_attr attr = { .transfer = 0 }; 8547 const struct mlx5_flow_driver_ops *fops = 8548 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8549 8550 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8551 return NULL; 8552 if (!fops->action_create) { 8553 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8554 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8555 NULL, err_msg); 8556 return NULL; 8557 } 8558 return fops->action_create(dev, conf, action, error); 8559 } 8560 8561 /** 8562 * Updates inplace the indirect action configuration pointed by *handle* 8563 * with the configuration provided as *update* argument. 8564 * The update of the indirect action configuration effects all flow rules 8565 * reusing the action via handle. 8566 * 8567 * @param dev 8568 * Pointer to Ethernet device structure. 8569 * @param[in] handle 8570 * Handle for the indirect action to be updated. 8571 * @param[in] update 8572 * Action specification used to modify the action pointed by handle. 8573 * *update* could be of same type with the action pointed by the *handle* 8574 * handle argument, or some other structures like a wrapper, depending on 8575 * the indirect action type. 8576 * @param[out] error 8577 * Perform verbose error reporting if not NULL. PMDs initialize this 8578 * structure in case of error only. 8579 * 8580 * @return 8581 * 0 on success, a negative errno value otherwise and rte_errno is set. 8582 */ 8583 static int 8584 mlx5_action_handle_update(struct rte_eth_dev *dev, 8585 struct rte_flow_action_handle *handle, 8586 const void *update, 8587 struct rte_flow_error *error) 8588 { 8589 struct rte_flow_attr attr = { .transfer = 0 }; 8590 const struct mlx5_flow_driver_ops *fops = 8591 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8592 int ret; 8593 8594 ret = flow_drv_action_validate(dev, NULL, 8595 (const struct rte_flow_action *)update, fops, error); 8596 if (ret) 8597 return ret; 8598 return flow_drv_action_update(dev, handle, update, fops, 8599 error); 8600 } 8601 8602 /** 8603 * Query the indirect action by handle. 8604 * 8605 * This function allows retrieving action-specific data such as counters. 8606 * Data is gathered by special action which may be present/referenced in 8607 * more than one flow rule definition. 8608 * 8609 * see @RTE_FLOW_ACTION_TYPE_COUNT 8610 * 8611 * @param dev 8612 * Pointer to Ethernet device structure. 8613 * @param[in] handle 8614 * Handle for the indirect action to query. 8615 * @param[in, out] data 8616 * Pointer to storage for the associated query data type. 8617 * @param[out] error 8618 * Perform verbose error reporting if not NULL. PMDs initialize this 8619 * structure in case of error only. 8620 * 8621 * @return 8622 * 0 on success, a negative errno value otherwise and rte_errno is set. 8623 */ 8624 static int 8625 mlx5_action_handle_query(struct rte_eth_dev *dev, 8626 const struct rte_flow_action_handle *handle, 8627 void *data, 8628 struct rte_flow_error *error) 8629 { 8630 struct rte_flow_attr attr = { .transfer = 0 }; 8631 const struct mlx5_flow_driver_ops *fops = 8632 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8633 8634 return flow_drv_action_query(dev, handle, data, fops, error); 8635 } 8636 8637 /** 8638 * Destroy all indirect actions (shared RSS). 8639 * 8640 * @param dev 8641 * Pointer to Ethernet device. 8642 * 8643 * @return 8644 * 0 on success, a negative errno value otherwise and rte_errno is set. 8645 */ 8646 int 8647 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8648 { 8649 struct rte_flow_error error; 8650 struct mlx5_priv *priv = dev->data->dev_private; 8651 struct mlx5_shared_action_rss *shared_rss; 8652 int ret = 0; 8653 uint32_t idx; 8654 8655 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8656 priv->rss_shared_actions, idx, shared_rss, next) { 8657 ret |= mlx5_action_handle_destroy(dev, 8658 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8659 } 8660 return ret; 8661 } 8662 8663 #ifndef HAVE_MLX5DV_DR 8664 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8665 #else 8666 #define MLX5_DOMAIN_SYNC_FLOW \ 8667 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8668 #endif 8669 8670 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8671 { 8672 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8673 const struct mlx5_flow_driver_ops *fops; 8674 int ret; 8675 struct rte_flow_attr attr = { .transfer = 0 }; 8676 8677 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8678 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8679 if (ret > 0) 8680 ret = -ret; 8681 return ret; 8682 } 8683 8684 const struct mlx5_flow_tunnel * 8685 mlx5_get_tof(const struct rte_flow_item *item, 8686 const struct rte_flow_action *action, 8687 enum mlx5_tof_rule_type *rule_type) 8688 { 8689 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 8690 if (item->type == (typeof(item->type)) 8691 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 8692 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 8693 return flow_items_to_tunnel(item); 8694 } 8695 } 8696 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 8697 if (action->type == (typeof(action->type)) 8698 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 8699 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 8700 return flow_actions_to_tunnel(action); 8701 } 8702 } 8703 return NULL; 8704 } 8705 8706 /** 8707 * tunnel offload functionalilty is defined for DV environment only 8708 */ 8709 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8710 __extension__ 8711 union tunnel_offload_mark { 8712 uint32_t val; 8713 struct { 8714 uint32_t app_reserve:8; 8715 uint32_t table_id:15; 8716 uint32_t transfer:1; 8717 uint32_t _unused_:8; 8718 }; 8719 }; 8720 8721 static bool 8722 mlx5_access_tunnel_offload_db 8723 (struct rte_eth_dev *dev, 8724 bool (*match)(struct rte_eth_dev *, 8725 struct mlx5_flow_tunnel *, const void *), 8726 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8727 void (*miss)(struct rte_eth_dev *, void *), 8728 void *ctx, bool lock_op); 8729 8730 static int 8731 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 8732 struct rte_flow *flow, 8733 const struct rte_flow_attr *attr, 8734 const struct rte_flow_action *app_actions, 8735 uint32_t flow_idx, 8736 const struct mlx5_flow_tunnel *tunnel, 8737 struct tunnel_default_miss_ctx *ctx, 8738 struct rte_flow_error *error) 8739 { 8740 struct mlx5_priv *priv = dev->data->dev_private; 8741 struct mlx5_flow *dev_flow; 8742 struct rte_flow_attr miss_attr = *attr; 8743 const struct rte_flow_item miss_items[2] = { 8744 { 8745 .type = RTE_FLOW_ITEM_TYPE_ETH, 8746 .spec = NULL, 8747 .last = NULL, 8748 .mask = NULL 8749 }, 8750 { 8751 .type = RTE_FLOW_ITEM_TYPE_END, 8752 .spec = NULL, 8753 .last = NULL, 8754 .mask = NULL 8755 } 8756 }; 8757 union tunnel_offload_mark mark_id; 8758 struct rte_flow_action_mark miss_mark; 8759 struct rte_flow_action miss_actions[3] = { 8760 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 8761 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 8762 }; 8763 const struct rte_flow_action_jump *jump_data; 8764 uint32_t i, flow_table = 0; /* prevent compilation warning */ 8765 struct flow_grp_info grp_info = { 8766 .external = 1, 8767 .transfer = attr->transfer, 8768 .fdb_def_rule = !!priv->fdb_def_rule, 8769 .std_tbl_fix = 0, 8770 }; 8771 int ret; 8772 8773 if (!attr->transfer) { 8774 uint32_t q_size; 8775 8776 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 8777 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 8778 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 8779 0, SOCKET_ID_ANY); 8780 if (!ctx->queue) 8781 return rte_flow_error_set 8782 (error, ENOMEM, 8783 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8784 NULL, "invalid default miss RSS"); 8785 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8786 ctx->action_rss.level = 0, 8787 ctx->action_rss.types = priv->rss_conf.rss_hf, 8788 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 8789 ctx->action_rss.queue_num = priv->reta_idx_n, 8790 ctx->action_rss.key = priv->rss_conf.rss_key, 8791 ctx->action_rss.queue = ctx->queue; 8792 if (!priv->reta_idx_n || !priv->rxqs_n) 8793 return rte_flow_error_set 8794 (error, EINVAL, 8795 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8796 NULL, "invalid port configuration"); 8797 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8798 ctx->action_rss.types = 0; 8799 for (i = 0; i != priv->reta_idx_n; ++i) 8800 ctx->queue[i] = (*priv->reta_idx)[i]; 8801 } else { 8802 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 8803 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 8804 } 8805 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 8806 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 8807 jump_data = app_actions->conf; 8808 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 8809 miss_attr.group = jump_data->group; 8810 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 8811 &flow_table, &grp_info, error); 8812 if (ret) 8813 return rte_flow_error_set(error, EINVAL, 8814 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8815 NULL, "invalid tunnel id"); 8816 mark_id.app_reserve = 0; 8817 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 8818 mark_id.transfer = !!attr->transfer; 8819 mark_id._unused_ = 0; 8820 miss_mark.id = mark_id.val; 8821 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 8822 miss_items, miss_actions, flow_idx, error); 8823 if (!dev_flow) 8824 return -rte_errno; 8825 dev_flow->flow = flow; 8826 dev_flow->external = true; 8827 dev_flow->tunnel = tunnel; 8828 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 8829 /* Subflow object was created, we must include one in the list. */ 8830 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 8831 dev_flow->handle, next); 8832 DRV_LOG(DEBUG, 8833 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 8834 dev->data->port_id, tunnel->app_tunnel.type, 8835 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 8836 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 8837 miss_actions, error); 8838 if (!ret) 8839 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 8840 error); 8841 8842 return ret; 8843 } 8844 8845 static const struct mlx5_flow_tbl_data_entry * 8846 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 8847 { 8848 struct mlx5_priv *priv = dev->data->dev_private; 8849 struct mlx5_dev_ctx_shared *sh = priv->sh; 8850 struct mlx5_list_entry *he; 8851 union tunnel_offload_mark mbits = { .val = mark }; 8852 union mlx5_flow_tbl_key table_key = { 8853 { 8854 .level = tunnel_id_to_flow_tbl(mbits.table_id), 8855 .id = 0, 8856 .reserved = 0, 8857 .dummy = 0, 8858 .is_fdb = !!mbits.transfer, 8859 .is_egress = 0, 8860 } 8861 }; 8862 struct mlx5_flow_cb_ctx ctx = { 8863 .data = &table_key.v64, 8864 }; 8865 8866 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 8867 return he ? 8868 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 8869 } 8870 8871 static void 8872 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 8873 struct mlx5_list_entry *entry) 8874 { 8875 struct mlx5_dev_ctx_shared *sh = tool_ctx; 8876 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8877 8878 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8879 tunnel_flow_tbl_to_id(tte->flow_table)); 8880 mlx5_free(tte); 8881 } 8882 8883 static int 8884 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 8885 struct mlx5_list_entry *entry, void *cb_ctx) 8886 { 8887 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 8888 union tunnel_tbl_key tbl = { 8889 .val = *(uint64_t *)(ctx->data), 8890 }; 8891 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8892 8893 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 8894 } 8895 8896 static struct mlx5_list_entry * 8897 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 8898 { 8899 struct mlx5_dev_ctx_shared *sh = tool_ctx; 8900 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 8901 struct tunnel_tbl_entry *tte; 8902 union tunnel_tbl_key tbl = { 8903 .val = *(uint64_t *)(ctx->data), 8904 }; 8905 8906 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 8907 sizeof(*tte), 0, 8908 SOCKET_ID_ANY); 8909 if (!tte) 8910 goto err; 8911 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8912 &tte->flow_table); 8913 if (tte->flow_table >= MLX5_MAX_TABLES) { 8914 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 8915 tte->flow_table); 8916 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8917 tte->flow_table); 8918 goto err; 8919 } else if (!tte->flow_table) { 8920 goto err; 8921 } 8922 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 8923 tte->tunnel_id = tbl.tunnel_id; 8924 tte->group = tbl.group; 8925 return &tte->hash; 8926 err: 8927 if (tte) 8928 mlx5_free(tte); 8929 return NULL; 8930 } 8931 8932 static struct mlx5_list_entry * 8933 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 8934 struct mlx5_list_entry *oentry, 8935 void *cb_ctx __rte_unused) 8936 { 8937 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 8938 0, SOCKET_ID_ANY); 8939 8940 if (!tte) 8941 return NULL; 8942 memcpy(tte, oentry, sizeof(*tte)); 8943 return &tte->hash; 8944 } 8945 8946 static void 8947 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 8948 struct mlx5_list_entry *entry) 8949 { 8950 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8951 8952 mlx5_free(tte); 8953 } 8954 8955 static uint32_t 8956 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 8957 const struct mlx5_flow_tunnel *tunnel, 8958 uint32_t group, uint32_t *table, 8959 struct rte_flow_error *error) 8960 { 8961 struct mlx5_list_entry *he; 8962 struct tunnel_tbl_entry *tte; 8963 union tunnel_tbl_key key = { 8964 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 8965 .group = group 8966 }; 8967 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8968 struct mlx5_hlist *group_hash; 8969 struct mlx5_flow_cb_ctx ctx = { 8970 .data = &key.val, 8971 }; 8972 8973 group_hash = tunnel ? tunnel->groups : thub->groups; 8974 he = mlx5_hlist_register(group_hash, key.val, &ctx); 8975 if (!he) 8976 return rte_flow_error_set(error, EINVAL, 8977 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8978 NULL, 8979 "tunnel group index not supported"); 8980 tte = container_of(he, typeof(*tte), hash); 8981 *table = tte->flow_table; 8982 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 8983 dev->data->port_id, key.tunnel_id, group, *table); 8984 return 0; 8985 } 8986 8987 static void 8988 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8989 struct mlx5_flow_tunnel *tunnel) 8990 { 8991 struct mlx5_priv *priv = dev->data->dev_private; 8992 struct mlx5_indexed_pool *ipool; 8993 8994 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8995 dev->data->port_id, tunnel->tunnel_id); 8996 LIST_REMOVE(tunnel, chain); 8997 mlx5_hlist_destroy(tunnel->groups); 8998 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8999 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9000 } 9001 9002 static bool 9003 mlx5_access_tunnel_offload_db 9004 (struct rte_eth_dev *dev, 9005 bool (*match)(struct rte_eth_dev *, 9006 struct mlx5_flow_tunnel *, const void *), 9007 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9008 void (*miss)(struct rte_eth_dev *, void *), 9009 void *ctx, bool lock_op) 9010 { 9011 bool verdict = false; 9012 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9013 struct mlx5_flow_tunnel *tunnel; 9014 9015 rte_spinlock_lock(&thub->sl); 9016 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 9017 verdict = match(dev, tunnel, (const void *)ctx); 9018 if (verdict) 9019 break; 9020 } 9021 if (!lock_op) 9022 rte_spinlock_unlock(&thub->sl); 9023 if (verdict && hit) 9024 hit(dev, tunnel, ctx); 9025 if (!verdict && miss) 9026 miss(dev, ctx); 9027 if (lock_op) 9028 rte_spinlock_unlock(&thub->sl); 9029 9030 return verdict; 9031 } 9032 9033 struct tunnel_db_find_tunnel_id_ctx { 9034 uint32_t tunnel_id; 9035 struct mlx5_flow_tunnel *tunnel; 9036 }; 9037 9038 static bool 9039 find_tunnel_id_match(struct rte_eth_dev *dev, 9040 struct mlx5_flow_tunnel *tunnel, const void *x) 9041 { 9042 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9043 9044 RTE_SET_USED(dev); 9045 return tunnel->tunnel_id == ctx->tunnel_id; 9046 } 9047 9048 static void 9049 find_tunnel_id_hit(struct rte_eth_dev *dev, 9050 struct mlx5_flow_tunnel *tunnel, void *x) 9051 { 9052 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9053 RTE_SET_USED(dev); 9054 ctx->tunnel = tunnel; 9055 } 9056 9057 static struct mlx5_flow_tunnel * 9058 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 9059 { 9060 struct tunnel_db_find_tunnel_id_ctx ctx = { 9061 .tunnel_id = id, 9062 }; 9063 9064 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 9065 find_tunnel_id_hit, NULL, &ctx, true); 9066 9067 return ctx.tunnel; 9068 } 9069 9070 static struct mlx5_flow_tunnel * 9071 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 9072 const struct rte_flow_tunnel *app_tunnel) 9073 { 9074 struct mlx5_priv *priv = dev->data->dev_private; 9075 struct mlx5_indexed_pool *ipool; 9076 struct mlx5_flow_tunnel *tunnel; 9077 uint32_t id; 9078 9079 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9080 tunnel = mlx5_ipool_zmalloc(ipool, &id); 9081 if (!tunnel) 9082 return NULL; 9083 if (id >= MLX5_MAX_TUNNELS) { 9084 mlx5_ipool_free(ipool, id); 9085 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 9086 return NULL; 9087 } 9088 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 9089 priv->sh, 9090 mlx5_flow_tunnel_grp2tbl_create_cb, 9091 mlx5_flow_tunnel_grp2tbl_match_cb, 9092 mlx5_flow_tunnel_grp2tbl_remove_cb, 9093 mlx5_flow_tunnel_grp2tbl_clone_cb, 9094 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9095 if (!tunnel->groups) { 9096 mlx5_ipool_free(ipool, id); 9097 return NULL; 9098 } 9099 /* initiate new PMD tunnel */ 9100 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 9101 tunnel->tunnel_id = id; 9102 tunnel->action.type = (typeof(tunnel->action.type)) 9103 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 9104 tunnel->action.conf = tunnel; 9105 tunnel->item.type = (typeof(tunnel->item.type)) 9106 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 9107 tunnel->item.spec = tunnel; 9108 tunnel->item.last = NULL; 9109 tunnel->item.mask = NULL; 9110 9111 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 9112 dev->data->port_id, tunnel->tunnel_id); 9113 9114 return tunnel; 9115 } 9116 9117 struct tunnel_db_get_tunnel_ctx { 9118 const struct rte_flow_tunnel *app_tunnel; 9119 struct mlx5_flow_tunnel *tunnel; 9120 }; 9121 9122 static bool get_tunnel_match(struct rte_eth_dev *dev, 9123 struct mlx5_flow_tunnel *tunnel, const void *x) 9124 { 9125 const struct tunnel_db_get_tunnel_ctx *ctx = x; 9126 9127 RTE_SET_USED(dev); 9128 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 9129 sizeof(*ctx->app_tunnel)); 9130 } 9131 9132 static void get_tunnel_hit(struct rte_eth_dev *dev, 9133 struct mlx5_flow_tunnel *tunnel, void *x) 9134 { 9135 /* called under tunnel spinlock protection */ 9136 struct tunnel_db_get_tunnel_ctx *ctx = x; 9137 9138 RTE_SET_USED(dev); 9139 tunnel->refctn++; 9140 ctx->tunnel = tunnel; 9141 } 9142 9143 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 9144 { 9145 /* called under tunnel spinlock protection */ 9146 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9147 struct tunnel_db_get_tunnel_ctx *ctx = x; 9148 9149 rte_spinlock_unlock(&thub->sl); 9150 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 9151 rte_spinlock_lock(&thub->sl); 9152 if (ctx->tunnel) { 9153 ctx->tunnel->refctn = 1; 9154 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 9155 } 9156 } 9157 9158 9159 static int 9160 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 9161 const struct rte_flow_tunnel *app_tunnel, 9162 struct mlx5_flow_tunnel **tunnel) 9163 { 9164 struct tunnel_db_get_tunnel_ctx ctx = { 9165 .app_tunnel = app_tunnel, 9166 }; 9167 9168 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 9169 get_tunnel_miss, &ctx, true); 9170 *tunnel = ctx.tunnel; 9171 return ctx.tunnel ? 0 : -ENOMEM; 9172 } 9173 9174 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9175 { 9176 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9177 9178 if (!thub) 9179 return; 9180 if (!LIST_EMPTY(&thub->tunnels)) 9181 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9182 mlx5_hlist_destroy(thub->groups); 9183 mlx5_free(thub); 9184 } 9185 9186 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9187 { 9188 int err; 9189 struct mlx5_flow_tunnel_hub *thub; 9190 9191 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9192 0, SOCKET_ID_ANY); 9193 if (!thub) 9194 return -ENOMEM; 9195 LIST_INIT(&thub->tunnels); 9196 rte_spinlock_init(&thub->sl); 9197 thub->groups = mlx5_hlist_create("flow groups", 64, 9198 false, true, sh, 9199 mlx5_flow_tunnel_grp2tbl_create_cb, 9200 mlx5_flow_tunnel_grp2tbl_match_cb, 9201 mlx5_flow_tunnel_grp2tbl_remove_cb, 9202 mlx5_flow_tunnel_grp2tbl_clone_cb, 9203 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9204 if (!thub->groups) { 9205 err = -rte_errno; 9206 goto err; 9207 } 9208 sh->tunnel_hub = thub; 9209 9210 return 0; 9211 9212 err: 9213 if (thub->groups) 9214 mlx5_hlist_destroy(thub->groups); 9215 if (thub) 9216 mlx5_free(thub); 9217 return err; 9218 } 9219 9220 static inline bool 9221 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9222 struct rte_flow_tunnel *tunnel, 9223 const char *err_msg) 9224 { 9225 err_msg = NULL; 9226 if (!is_tunnel_offload_active(dev)) { 9227 err_msg = "tunnel offload was not activated"; 9228 goto out; 9229 } else if (!tunnel) { 9230 err_msg = "no application tunnel"; 9231 goto out; 9232 } 9233 9234 switch (tunnel->type) { 9235 default: 9236 err_msg = "unsupported tunnel type"; 9237 goto out; 9238 case RTE_FLOW_ITEM_TYPE_VXLAN: 9239 case RTE_FLOW_ITEM_TYPE_GRE: 9240 case RTE_FLOW_ITEM_TYPE_NVGRE: 9241 case RTE_FLOW_ITEM_TYPE_GENEVE: 9242 break; 9243 } 9244 9245 out: 9246 return !err_msg; 9247 } 9248 9249 static int 9250 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9251 struct rte_flow_tunnel *app_tunnel, 9252 struct rte_flow_action **actions, 9253 uint32_t *num_of_actions, 9254 struct rte_flow_error *error) 9255 { 9256 int ret; 9257 struct mlx5_flow_tunnel *tunnel; 9258 const char *err_msg = NULL; 9259 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 9260 9261 if (!verdict) 9262 return rte_flow_error_set(error, EINVAL, 9263 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9264 err_msg); 9265 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9266 if (ret < 0) { 9267 return rte_flow_error_set(error, ret, 9268 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9269 "failed to initialize pmd tunnel"); 9270 } 9271 *actions = &tunnel->action; 9272 *num_of_actions = 1; 9273 return 0; 9274 } 9275 9276 static int 9277 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9278 struct rte_flow_tunnel *app_tunnel, 9279 struct rte_flow_item **items, 9280 uint32_t *num_of_items, 9281 struct rte_flow_error *error) 9282 { 9283 int ret; 9284 struct mlx5_flow_tunnel *tunnel; 9285 const char *err_msg = NULL; 9286 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 9287 9288 if (!verdict) 9289 return rte_flow_error_set(error, EINVAL, 9290 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9291 err_msg); 9292 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9293 if (ret < 0) { 9294 return rte_flow_error_set(error, ret, 9295 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9296 "failed to initialize pmd tunnel"); 9297 } 9298 *items = &tunnel->item; 9299 *num_of_items = 1; 9300 return 0; 9301 } 9302 9303 struct tunnel_db_element_release_ctx { 9304 struct rte_flow_item *items; 9305 struct rte_flow_action *actions; 9306 uint32_t num_elements; 9307 struct rte_flow_error *error; 9308 int ret; 9309 }; 9310 9311 static bool 9312 tunnel_element_release_match(struct rte_eth_dev *dev, 9313 struct mlx5_flow_tunnel *tunnel, const void *x) 9314 { 9315 const struct tunnel_db_element_release_ctx *ctx = x; 9316 9317 RTE_SET_USED(dev); 9318 if (ctx->num_elements != 1) 9319 return false; 9320 else if (ctx->items) 9321 return ctx->items == &tunnel->item; 9322 else if (ctx->actions) 9323 return ctx->actions == &tunnel->action; 9324 9325 return false; 9326 } 9327 9328 static void 9329 tunnel_element_release_hit(struct rte_eth_dev *dev, 9330 struct mlx5_flow_tunnel *tunnel, void *x) 9331 { 9332 struct tunnel_db_element_release_ctx *ctx = x; 9333 ctx->ret = 0; 9334 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9335 mlx5_flow_tunnel_free(dev, tunnel); 9336 } 9337 9338 static void 9339 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9340 { 9341 struct tunnel_db_element_release_ctx *ctx = x; 9342 RTE_SET_USED(dev); 9343 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9344 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9345 "invalid argument"); 9346 } 9347 9348 static int 9349 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9350 struct rte_flow_item *pmd_items, 9351 uint32_t num_items, struct rte_flow_error *err) 9352 { 9353 struct tunnel_db_element_release_ctx ctx = { 9354 .items = pmd_items, 9355 .actions = NULL, 9356 .num_elements = num_items, 9357 .error = err, 9358 }; 9359 9360 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9361 tunnel_element_release_hit, 9362 tunnel_element_release_miss, &ctx, false); 9363 9364 return ctx.ret; 9365 } 9366 9367 static int 9368 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9369 struct rte_flow_action *pmd_actions, 9370 uint32_t num_actions, struct rte_flow_error *err) 9371 { 9372 struct tunnel_db_element_release_ctx ctx = { 9373 .items = NULL, 9374 .actions = pmd_actions, 9375 .num_elements = num_actions, 9376 .error = err, 9377 }; 9378 9379 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9380 tunnel_element_release_hit, 9381 tunnel_element_release_miss, &ctx, false); 9382 9383 return ctx.ret; 9384 } 9385 9386 static int 9387 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9388 struct rte_mbuf *m, 9389 struct rte_flow_restore_info *info, 9390 struct rte_flow_error *err) 9391 { 9392 uint64_t ol_flags = m->ol_flags; 9393 const struct mlx5_flow_tbl_data_entry *tble; 9394 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 9395 9396 if (!is_tunnel_offload_active(dev)) { 9397 info->flags = 0; 9398 return 0; 9399 } 9400 9401 if ((ol_flags & mask) != mask) 9402 goto err; 9403 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9404 if (!tble) { 9405 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9406 dev->data->port_id, m->hash.fdir.hi); 9407 goto err; 9408 } 9409 MLX5_ASSERT(tble->tunnel); 9410 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9411 info->group_id = tble->group_id; 9412 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9413 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9414 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9415 9416 return 0; 9417 9418 err: 9419 return rte_flow_error_set(err, EINVAL, 9420 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9421 "failed to get restore info"); 9422 } 9423 9424 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9425 static int 9426 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9427 __rte_unused struct rte_flow_tunnel *app_tunnel, 9428 __rte_unused struct rte_flow_action **actions, 9429 __rte_unused uint32_t *num_of_actions, 9430 __rte_unused struct rte_flow_error *error) 9431 { 9432 return -ENOTSUP; 9433 } 9434 9435 static int 9436 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9437 __rte_unused struct rte_flow_tunnel *app_tunnel, 9438 __rte_unused struct rte_flow_item **items, 9439 __rte_unused uint32_t *num_of_items, 9440 __rte_unused struct rte_flow_error *error) 9441 { 9442 return -ENOTSUP; 9443 } 9444 9445 static int 9446 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9447 __rte_unused struct rte_flow_item *pmd_items, 9448 __rte_unused uint32_t num_items, 9449 __rte_unused struct rte_flow_error *err) 9450 { 9451 return -ENOTSUP; 9452 } 9453 9454 static int 9455 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9456 __rte_unused struct rte_flow_action *pmd_action, 9457 __rte_unused uint32_t num_actions, 9458 __rte_unused struct rte_flow_error *err) 9459 { 9460 return -ENOTSUP; 9461 } 9462 9463 static int 9464 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9465 __rte_unused struct rte_mbuf *m, 9466 __rte_unused struct rte_flow_restore_info *i, 9467 __rte_unused struct rte_flow_error *err) 9468 { 9469 return -ENOTSUP; 9470 } 9471 9472 static int 9473 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9474 __rte_unused struct rte_flow *flow, 9475 __rte_unused const struct rte_flow_attr *attr, 9476 __rte_unused const struct rte_flow_action *actions, 9477 __rte_unused uint32_t flow_idx, 9478 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9479 __rte_unused struct tunnel_default_miss_ctx *ctx, 9480 __rte_unused struct rte_flow_error *error) 9481 { 9482 return -ENOTSUP; 9483 } 9484 9485 static struct mlx5_flow_tunnel * 9486 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9487 __rte_unused uint32_t id) 9488 { 9489 return NULL; 9490 } 9491 9492 static void 9493 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9494 __rte_unused struct mlx5_flow_tunnel *tunnel) 9495 { 9496 } 9497 9498 static uint32_t 9499 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9500 __rte_unused const struct mlx5_flow_tunnel *t, 9501 __rte_unused uint32_t group, 9502 __rte_unused uint32_t *table, 9503 struct rte_flow_error *error) 9504 { 9505 return rte_flow_error_set(error, ENOTSUP, 9506 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9507 "tunnel offload requires DV support"); 9508 } 9509 9510 void 9511 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9512 __rte_unused uint16_t port_id) 9513 { 9514 } 9515 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9516 9517 static void 9518 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9519 { 9520 int ret; 9521 struct rte_flow_error error; 9522 9523 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9524 char *item_name; 9525 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9526 sizeof(item_name), 9527 (void *)(uintptr_t)item->type, &error); 9528 if (ret > 0) 9529 printf("%s ", item_name); 9530 else 9531 printf("%d\n", (int)item->type); 9532 } 9533 printf("END\n"); 9534 } 9535 9536 static int 9537 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 9538 { 9539 const struct rte_flow_item_udp *spec = udp_item->spec; 9540 const struct rte_flow_item_udp *mask = udp_item->mask; 9541 uint16_t udp_dport = 0; 9542 9543 if (spec != NULL) { 9544 if (!mask) 9545 mask = &rte_flow_item_udp_mask; 9546 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 9547 mask->hdr.dst_port); 9548 } 9549 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 9550 } 9551 9552 static const struct mlx5_flow_expand_node * 9553 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 9554 unsigned int item_idx, 9555 const struct mlx5_flow_expand_node graph[], 9556 const struct mlx5_flow_expand_node *node) 9557 { 9558 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 9559 switch (item->type) { 9560 case RTE_FLOW_ITEM_TYPE_VXLAN: 9561 MLX5_ASSERT(item_idx > 0); 9562 prev_item = pattern + item_idx - 1; 9563 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 9564 if (mlx5_flow_is_std_vxlan_port(prev_item)) 9565 return &graph[MLX5_EXPANSION_STD_VXLAN]; 9566 else 9567 return &graph[MLX5_EXPANSION_L3_VXLAN]; 9568 break; 9569 default: 9570 return node; 9571 } 9572 } 9573