1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rxtx.h" 32 #include "mlx5_common_os.h" 33 #include "rte_pmd_mlx5.h" 34 35 struct tunnel_default_miss_ctx { 36 uint16_t *queue; 37 __extension__ 38 union { 39 struct rte_flow_action_rss action_rss; 40 struct rte_flow_action_queue miss_queue; 41 struct rte_flow_action_jump miss_jump; 42 uint8_t raw[0]; 43 }; 44 }; 45 46 static int 47 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 48 struct rte_flow *flow, 49 const struct rte_flow_attr *attr, 50 const struct rte_flow_action *app_actions, 51 uint32_t flow_idx, 52 struct tunnel_default_miss_ctx *ctx, 53 struct rte_flow_error *error); 54 static struct mlx5_flow_tunnel * 55 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 56 static void 57 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 58 static uint32_t 59 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 60 const struct mlx5_flow_tunnel *tunnel, 61 uint32_t group, uint32_t *table, 62 struct rte_flow_error *error); 63 64 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 65 static void mlx5_flow_pop_thread_workspace(void); 66 67 68 /** Device flow drivers. */ 69 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 70 71 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 72 73 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 74 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 75 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 76 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 77 #endif 78 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 79 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 80 }; 81 82 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 83 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 84 (const int []){ \ 85 __VA_ARGS__, 0, \ 86 } 87 88 /** Node object of input graph for mlx5_flow_expand_rss(). */ 89 struct mlx5_flow_expand_node { 90 const int *const next; 91 /**< 92 * List of next node indexes. Index 0 is interpreted as a terminator. 93 */ 94 const enum rte_flow_item_type type; 95 /**< Pattern item type of current node. */ 96 uint64_t rss_types; 97 /**< 98 * RSS types bit-field associated with this node 99 * (see ETH_RSS_* definitions). 100 */ 101 }; 102 103 /** Object returned by mlx5_flow_expand_rss(). */ 104 struct mlx5_flow_expand_rss { 105 uint32_t entries; 106 /**< Number of entries @p patterns and @p priorities. */ 107 struct { 108 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 109 uint32_t priority; /**< Priority offset for each expansion. */ 110 } entry[]; 111 }; 112 113 static enum rte_flow_item_type 114 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 115 { 116 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 117 uint16_t ether_type = 0; 118 uint16_t ether_type_m; 119 uint8_t ip_next_proto = 0; 120 uint8_t ip_next_proto_m; 121 122 if (item == NULL || item->spec == NULL) 123 return ret; 124 switch (item->type) { 125 case RTE_FLOW_ITEM_TYPE_ETH: 126 if (item->mask) 127 ether_type_m = ((const struct rte_flow_item_eth *) 128 (item->mask))->type; 129 else 130 ether_type_m = rte_flow_item_eth_mask.type; 131 if (ether_type_m != RTE_BE16(0xFFFF)) 132 break; 133 ether_type = ((const struct rte_flow_item_eth *) 134 (item->spec))->type; 135 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 136 ret = RTE_FLOW_ITEM_TYPE_IPV4; 137 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 138 ret = RTE_FLOW_ITEM_TYPE_IPV6; 139 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 140 ret = RTE_FLOW_ITEM_TYPE_VLAN; 141 else 142 ret = RTE_FLOW_ITEM_TYPE_END; 143 break; 144 case RTE_FLOW_ITEM_TYPE_VLAN: 145 if (item->mask) 146 ether_type_m = ((const struct rte_flow_item_vlan *) 147 (item->mask))->inner_type; 148 else 149 ether_type_m = rte_flow_item_vlan_mask.inner_type; 150 if (ether_type_m != RTE_BE16(0xFFFF)) 151 break; 152 ether_type = ((const struct rte_flow_item_vlan *) 153 (item->spec))->inner_type; 154 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 155 ret = RTE_FLOW_ITEM_TYPE_IPV4; 156 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 157 ret = RTE_FLOW_ITEM_TYPE_IPV6; 158 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 159 ret = RTE_FLOW_ITEM_TYPE_VLAN; 160 else 161 ret = RTE_FLOW_ITEM_TYPE_END; 162 break; 163 case RTE_FLOW_ITEM_TYPE_IPV4: 164 if (item->mask) 165 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 166 (item->mask))->hdr.next_proto_id; 167 else 168 ip_next_proto_m = 169 rte_flow_item_ipv4_mask.hdr.next_proto_id; 170 if (ip_next_proto_m != 0xFF) 171 break; 172 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 173 (item->spec))->hdr.next_proto_id; 174 if (ip_next_proto == IPPROTO_UDP) 175 ret = RTE_FLOW_ITEM_TYPE_UDP; 176 else if (ip_next_proto == IPPROTO_TCP) 177 ret = RTE_FLOW_ITEM_TYPE_TCP; 178 else if (ip_next_proto == IPPROTO_IP) 179 ret = RTE_FLOW_ITEM_TYPE_IPV4; 180 else if (ip_next_proto == IPPROTO_IPV6) 181 ret = RTE_FLOW_ITEM_TYPE_IPV6; 182 else 183 ret = RTE_FLOW_ITEM_TYPE_END; 184 break; 185 case RTE_FLOW_ITEM_TYPE_IPV6: 186 if (item->mask) 187 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 188 (item->mask))->hdr.proto; 189 else 190 ip_next_proto_m = 191 rte_flow_item_ipv6_mask.hdr.proto; 192 if (ip_next_proto_m != 0xFF) 193 break; 194 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 195 (item->spec))->hdr.proto; 196 if (ip_next_proto == IPPROTO_UDP) 197 ret = RTE_FLOW_ITEM_TYPE_UDP; 198 else if (ip_next_proto == IPPROTO_TCP) 199 ret = RTE_FLOW_ITEM_TYPE_TCP; 200 else if (ip_next_proto == IPPROTO_IP) 201 ret = RTE_FLOW_ITEM_TYPE_IPV4; 202 else if (ip_next_proto == IPPROTO_IPV6) 203 ret = RTE_FLOW_ITEM_TYPE_IPV6; 204 else 205 ret = RTE_FLOW_ITEM_TYPE_END; 206 break; 207 default: 208 ret = RTE_FLOW_ITEM_TYPE_VOID; 209 break; 210 } 211 return ret; 212 } 213 214 #define MLX5_RSS_EXP_ELT_N 8 215 216 /** 217 * Expand RSS flows into several possible flows according to the RSS hash 218 * fields requested and the driver capabilities. 219 * 220 * @param[out] buf 221 * Buffer to store the result expansion. 222 * @param[in] size 223 * Buffer size in bytes. If 0, @p buf can be NULL. 224 * @param[in] pattern 225 * User flow pattern. 226 * @param[in] types 227 * RSS types to expand (see ETH_RSS_* definitions). 228 * @param[in] graph 229 * Input graph to expand @p pattern according to @p types. 230 * @param[in] graph_root_index 231 * Index of root node in @p graph, typically 0. 232 * 233 * @return 234 * A positive value representing the size of @p buf in bytes regardless of 235 * @p size on success, a negative errno value otherwise and rte_errno is 236 * set, the following errors are defined: 237 * 238 * -E2BIG: graph-depth @p graph is too deep. 239 */ 240 static int 241 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 242 const struct rte_flow_item *pattern, uint64_t types, 243 const struct mlx5_flow_expand_node graph[], 244 int graph_root_index) 245 { 246 const struct rte_flow_item *item; 247 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 248 const int *next_node; 249 const int *stack[MLX5_RSS_EXP_ELT_N]; 250 int stack_pos = 0; 251 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 252 unsigned int i; 253 size_t lsize; 254 size_t user_pattern_size = 0; 255 void *addr = NULL; 256 const struct mlx5_flow_expand_node *next = NULL; 257 struct rte_flow_item missed_item; 258 int missed = 0; 259 int elt = 0; 260 const struct rte_flow_item *last_item = NULL; 261 262 memset(&missed_item, 0, sizeof(missed_item)); 263 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 264 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 265 if (lsize <= size) { 266 buf->entry[0].priority = 0; 267 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 268 buf->entries = 0; 269 addr = buf->entry[0].pattern; 270 } 271 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 272 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 273 last_item = item; 274 for (i = 0; node->next && node->next[i]; ++i) { 275 next = &graph[node->next[i]]; 276 if (next->type == item->type) 277 break; 278 } 279 if (next) 280 node = next; 281 user_pattern_size += sizeof(*item); 282 } 283 user_pattern_size += sizeof(*item); /* Handle END item. */ 284 lsize += user_pattern_size; 285 /* Copy the user pattern in the first entry of the buffer. */ 286 if (lsize <= size) { 287 rte_memcpy(addr, pattern, user_pattern_size); 288 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 289 buf->entries = 1; 290 } 291 /* Start expanding. */ 292 memset(flow_items, 0, sizeof(flow_items)); 293 user_pattern_size -= sizeof(*item); 294 /* 295 * Check if the last valid item has spec set, need complete pattern, 296 * and the pattern can be used for expansion. 297 */ 298 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 299 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 300 /* Item type END indicates expansion is not required. */ 301 return lsize; 302 } 303 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 304 next = NULL; 305 missed = 1; 306 for (i = 0; node->next && node->next[i]; ++i) { 307 next = &graph[node->next[i]]; 308 if (next->type == missed_item.type) { 309 flow_items[0].type = missed_item.type; 310 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 311 break; 312 } 313 next = NULL; 314 } 315 } 316 if (next && missed) { 317 elt = 2; /* missed item + item end. */ 318 node = next; 319 lsize += elt * sizeof(*item) + user_pattern_size; 320 if ((node->rss_types & types) && lsize <= size) { 321 buf->entry[buf->entries].priority = 1; 322 buf->entry[buf->entries].pattern = addr; 323 buf->entries++; 324 rte_memcpy(addr, buf->entry[0].pattern, 325 user_pattern_size); 326 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 327 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 328 addr = (void *)(((uintptr_t)addr) + 329 elt * sizeof(*item)); 330 } 331 } 332 memset(flow_items, 0, sizeof(flow_items)); 333 next_node = node->next; 334 stack[stack_pos] = next_node; 335 node = next_node ? &graph[*next_node] : NULL; 336 while (node) { 337 flow_items[stack_pos].type = node->type; 338 if (node->rss_types & types) { 339 /* 340 * compute the number of items to copy from the 341 * expansion and copy it. 342 * When the stack_pos is 0, there are 1 element in it, 343 * plus the addition END item. 344 */ 345 elt = stack_pos + 2; 346 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 347 lsize += elt * sizeof(*item) + user_pattern_size; 348 if (lsize <= size) { 349 size_t n = elt * sizeof(*item); 350 351 buf->entry[buf->entries].priority = 352 stack_pos + 1 + missed; 353 buf->entry[buf->entries].pattern = addr; 354 buf->entries++; 355 rte_memcpy(addr, buf->entry[0].pattern, 356 user_pattern_size); 357 addr = (void *)(((uintptr_t)addr) + 358 user_pattern_size); 359 rte_memcpy(addr, &missed_item, 360 missed * sizeof(*item)); 361 addr = (void *)(((uintptr_t)addr) + 362 missed * sizeof(*item)); 363 rte_memcpy(addr, flow_items, n); 364 addr = (void *)(((uintptr_t)addr) + n); 365 } 366 } 367 /* Go deeper. */ 368 if (node->next) { 369 next_node = node->next; 370 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 371 rte_errno = E2BIG; 372 return -rte_errno; 373 } 374 stack[stack_pos] = next_node; 375 } else if (*(next_node + 1)) { 376 /* Follow up with the next possibility. */ 377 ++next_node; 378 } else { 379 /* Move to the next path. */ 380 if (stack_pos) 381 next_node = stack[--stack_pos]; 382 next_node++; 383 stack[stack_pos] = next_node; 384 } 385 node = *next_node ? &graph[*next_node] : NULL; 386 }; 387 return lsize; 388 } 389 390 enum mlx5_expansion { 391 MLX5_EXPANSION_ROOT, 392 MLX5_EXPANSION_ROOT_OUTER, 393 MLX5_EXPANSION_ROOT_ETH_VLAN, 394 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 395 MLX5_EXPANSION_OUTER_ETH, 396 MLX5_EXPANSION_OUTER_ETH_VLAN, 397 MLX5_EXPANSION_OUTER_VLAN, 398 MLX5_EXPANSION_OUTER_IPV4, 399 MLX5_EXPANSION_OUTER_IPV4_UDP, 400 MLX5_EXPANSION_OUTER_IPV4_TCP, 401 MLX5_EXPANSION_OUTER_IPV6, 402 MLX5_EXPANSION_OUTER_IPV6_UDP, 403 MLX5_EXPANSION_OUTER_IPV6_TCP, 404 MLX5_EXPANSION_VXLAN, 405 MLX5_EXPANSION_VXLAN_GPE, 406 MLX5_EXPANSION_GRE, 407 MLX5_EXPANSION_MPLS, 408 MLX5_EXPANSION_ETH, 409 MLX5_EXPANSION_ETH_VLAN, 410 MLX5_EXPANSION_VLAN, 411 MLX5_EXPANSION_IPV4, 412 MLX5_EXPANSION_IPV4_UDP, 413 MLX5_EXPANSION_IPV4_TCP, 414 MLX5_EXPANSION_IPV6, 415 MLX5_EXPANSION_IPV6_UDP, 416 MLX5_EXPANSION_IPV6_TCP, 417 }; 418 419 /** Supported expansion of items. */ 420 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 421 [MLX5_EXPANSION_ROOT] = { 422 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 423 MLX5_EXPANSION_IPV4, 424 MLX5_EXPANSION_IPV6), 425 .type = RTE_FLOW_ITEM_TYPE_END, 426 }, 427 [MLX5_EXPANSION_ROOT_OUTER] = { 428 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 429 MLX5_EXPANSION_OUTER_IPV4, 430 MLX5_EXPANSION_OUTER_IPV6), 431 .type = RTE_FLOW_ITEM_TYPE_END, 432 }, 433 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 434 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 435 .type = RTE_FLOW_ITEM_TYPE_END, 436 }, 437 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 438 .next = MLX5_FLOW_EXPAND_RSS_NEXT 439 (MLX5_EXPANSION_OUTER_ETH_VLAN), 440 .type = RTE_FLOW_ITEM_TYPE_END, 441 }, 442 [MLX5_EXPANSION_OUTER_ETH] = { 443 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 444 MLX5_EXPANSION_OUTER_IPV6, 445 MLX5_EXPANSION_MPLS), 446 .type = RTE_FLOW_ITEM_TYPE_ETH, 447 .rss_types = 0, 448 }, 449 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 450 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 451 .type = RTE_FLOW_ITEM_TYPE_ETH, 452 .rss_types = 0, 453 }, 454 [MLX5_EXPANSION_OUTER_VLAN] = { 455 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 456 MLX5_EXPANSION_OUTER_IPV6), 457 .type = RTE_FLOW_ITEM_TYPE_VLAN, 458 }, 459 [MLX5_EXPANSION_OUTER_IPV4] = { 460 .next = MLX5_FLOW_EXPAND_RSS_NEXT 461 (MLX5_EXPANSION_OUTER_IPV4_UDP, 462 MLX5_EXPANSION_OUTER_IPV4_TCP, 463 MLX5_EXPANSION_GRE, 464 MLX5_EXPANSION_IPV4, 465 MLX5_EXPANSION_IPV6), 466 .type = RTE_FLOW_ITEM_TYPE_IPV4, 467 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 468 ETH_RSS_NONFRAG_IPV4_OTHER, 469 }, 470 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 471 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 472 MLX5_EXPANSION_VXLAN_GPE), 473 .type = RTE_FLOW_ITEM_TYPE_UDP, 474 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 475 }, 476 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 477 .type = RTE_FLOW_ITEM_TYPE_TCP, 478 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 479 }, 480 [MLX5_EXPANSION_OUTER_IPV6] = { 481 .next = MLX5_FLOW_EXPAND_RSS_NEXT 482 (MLX5_EXPANSION_OUTER_IPV6_UDP, 483 MLX5_EXPANSION_OUTER_IPV6_TCP, 484 MLX5_EXPANSION_IPV4, 485 MLX5_EXPANSION_IPV6, 486 MLX5_EXPANSION_GRE), 487 .type = RTE_FLOW_ITEM_TYPE_IPV6, 488 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 489 ETH_RSS_NONFRAG_IPV6_OTHER, 490 }, 491 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 492 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 493 MLX5_EXPANSION_VXLAN_GPE), 494 .type = RTE_FLOW_ITEM_TYPE_UDP, 495 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 496 }, 497 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 498 .type = RTE_FLOW_ITEM_TYPE_TCP, 499 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 500 }, 501 [MLX5_EXPANSION_VXLAN] = { 502 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 503 MLX5_EXPANSION_IPV4, 504 MLX5_EXPANSION_IPV6), 505 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 506 }, 507 [MLX5_EXPANSION_VXLAN_GPE] = { 508 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 509 MLX5_EXPANSION_IPV4, 510 MLX5_EXPANSION_IPV6), 511 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 512 }, 513 [MLX5_EXPANSION_GRE] = { 514 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 515 MLX5_EXPANSION_IPV6), 516 .type = RTE_FLOW_ITEM_TYPE_GRE, 517 }, 518 [MLX5_EXPANSION_MPLS] = { 519 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 520 MLX5_EXPANSION_IPV6), 521 .type = RTE_FLOW_ITEM_TYPE_MPLS, 522 }, 523 [MLX5_EXPANSION_ETH] = { 524 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 525 MLX5_EXPANSION_IPV6), 526 .type = RTE_FLOW_ITEM_TYPE_ETH, 527 }, 528 [MLX5_EXPANSION_ETH_VLAN] = { 529 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 530 .type = RTE_FLOW_ITEM_TYPE_ETH, 531 }, 532 [MLX5_EXPANSION_VLAN] = { 533 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 534 MLX5_EXPANSION_IPV6), 535 .type = RTE_FLOW_ITEM_TYPE_VLAN, 536 }, 537 [MLX5_EXPANSION_IPV4] = { 538 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 539 MLX5_EXPANSION_IPV4_TCP), 540 .type = RTE_FLOW_ITEM_TYPE_IPV4, 541 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 542 ETH_RSS_NONFRAG_IPV4_OTHER, 543 }, 544 [MLX5_EXPANSION_IPV4_UDP] = { 545 .type = RTE_FLOW_ITEM_TYPE_UDP, 546 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 547 }, 548 [MLX5_EXPANSION_IPV4_TCP] = { 549 .type = RTE_FLOW_ITEM_TYPE_TCP, 550 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 551 }, 552 [MLX5_EXPANSION_IPV6] = { 553 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 554 MLX5_EXPANSION_IPV6_TCP), 555 .type = RTE_FLOW_ITEM_TYPE_IPV6, 556 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 557 ETH_RSS_NONFRAG_IPV6_OTHER, 558 }, 559 [MLX5_EXPANSION_IPV6_UDP] = { 560 .type = RTE_FLOW_ITEM_TYPE_UDP, 561 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 562 }, 563 [MLX5_EXPANSION_IPV6_TCP] = { 564 .type = RTE_FLOW_ITEM_TYPE_TCP, 565 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 566 }, 567 }; 568 569 static struct rte_flow_shared_action * 570 mlx5_shared_action_create(struct rte_eth_dev *dev, 571 const struct rte_flow_shared_action_conf *conf, 572 const struct rte_flow_action *action, 573 struct rte_flow_error *error); 574 static int mlx5_shared_action_destroy 575 (struct rte_eth_dev *dev, 576 struct rte_flow_shared_action *shared_action, 577 struct rte_flow_error *error); 578 static int mlx5_shared_action_update 579 (struct rte_eth_dev *dev, 580 struct rte_flow_shared_action *shared_action, 581 const struct rte_flow_action *action, 582 struct rte_flow_error *error); 583 static int mlx5_shared_action_query 584 (struct rte_eth_dev *dev, 585 const struct rte_flow_shared_action *action, 586 void *data, 587 struct rte_flow_error *error); 588 static int 589 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 590 struct rte_flow_tunnel *app_tunnel, 591 struct rte_flow_action **actions, 592 uint32_t *num_of_actions, 593 struct rte_flow_error *error); 594 static int 595 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 596 struct rte_flow_tunnel *app_tunnel, 597 struct rte_flow_item **items, 598 uint32_t *num_of_items, 599 struct rte_flow_error *error); 600 static int 601 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 602 struct rte_flow_item *pmd_items, 603 uint32_t num_items, struct rte_flow_error *err); 604 static int 605 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 606 struct rte_flow_action *pmd_actions, 607 uint32_t num_actions, 608 struct rte_flow_error *err); 609 static int 610 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 611 struct rte_mbuf *m, 612 struct rte_flow_restore_info *info, 613 struct rte_flow_error *err); 614 615 static const struct rte_flow_ops mlx5_flow_ops = { 616 .validate = mlx5_flow_validate, 617 .create = mlx5_flow_create, 618 .destroy = mlx5_flow_destroy, 619 .flush = mlx5_flow_flush, 620 .isolate = mlx5_flow_isolate, 621 .query = mlx5_flow_query, 622 .dev_dump = mlx5_flow_dev_dump, 623 .get_aged_flows = mlx5_flow_get_aged_flows, 624 .shared_action_create = mlx5_shared_action_create, 625 .shared_action_destroy = mlx5_shared_action_destroy, 626 .shared_action_update = mlx5_shared_action_update, 627 .shared_action_query = mlx5_shared_action_query, 628 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 629 .tunnel_match = mlx5_flow_tunnel_match, 630 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 631 .tunnel_item_release = mlx5_flow_tunnel_item_release, 632 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 633 }; 634 635 /* Tunnel information. */ 636 struct mlx5_flow_tunnel_info { 637 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 638 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 639 }; 640 641 static struct mlx5_flow_tunnel_info tunnels_info[] = { 642 { 643 .tunnel = MLX5_FLOW_LAYER_VXLAN, 644 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 645 }, 646 { 647 .tunnel = MLX5_FLOW_LAYER_GENEVE, 648 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 649 }, 650 { 651 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 652 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 653 }, 654 { 655 .tunnel = MLX5_FLOW_LAYER_GRE, 656 .ptype = RTE_PTYPE_TUNNEL_GRE, 657 }, 658 { 659 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 660 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 661 }, 662 { 663 .tunnel = MLX5_FLOW_LAYER_MPLS, 664 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 665 }, 666 { 667 .tunnel = MLX5_FLOW_LAYER_NVGRE, 668 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 669 }, 670 { 671 .tunnel = MLX5_FLOW_LAYER_IPIP, 672 .ptype = RTE_PTYPE_TUNNEL_IP, 673 }, 674 { 675 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 676 .ptype = RTE_PTYPE_TUNNEL_IP, 677 }, 678 { 679 .tunnel = MLX5_FLOW_LAYER_GTP, 680 .ptype = RTE_PTYPE_TUNNEL_GTPU, 681 }, 682 }; 683 684 685 686 /** 687 * Translate tag ID to register. 688 * 689 * @param[in] dev 690 * Pointer to the Ethernet device structure. 691 * @param[in] feature 692 * The feature that request the register. 693 * @param[in] id 694 * The request register ID. 695 * @param[out] error 696 * Error description in case of any. 697 * 698 * @return 699 * The request register on success, a negative errno 700 * value otherwise and rte_errno is set. 701 */ 702 int 703 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 704 enum mlx5_feature_name feature, 705 uint32_t id, 706 struct rte_flow_error *error) 707 { 708 struct mlx5_priv *priv = dev->data->dev_private; 709 struct mlx5_dev_config *config = &priv->config; 710 enum modify_reg start_reg; 711 bool skip_mtr_reg = false; 712 713 switch (feature) { 714 case MLX5_HAIRPIN_RX: 715 return REG_B; 716 case MLX5_HAIRPIN_TX: 717 return REG_A; 718 case MLX5_METADATA_RX: 719 switch (config->dv_xmeta_en) { 720 case MLX5_XMETA_MODE_LEGACY: 721 return REG_B; 722 case MLX5_XMETA_MODE_META16: 723 return REG_C_0; 724 case MLX5_XMETA_MODE_META32: 725 return REG_C_1; 726 } 727 break; 728 case MLX5_METADATA_TX: 729 return REG_A; 730 case MLX5_METADATA_FDB: 731 switch (config->dv_xmeta_en) { 732 case MLX5_XMETA_MODE_LEGACY: 733 return REG_NON; 734 case MLX5_XMETA_MODE_META16: 735 return REG_C_0; 736 case MLX5_XMETA_MODE_META32: 737 return REG_C_1; 738 } 739 break; 740 case MLX5_FLOW_MARK: 741 switch (config->dv_xmeta_en) { 742 case MLX5_XMETA_MODE_LEGACY: 743 return REG_NON; 744 case MLX5_XMETA_MODE_META16: 745 return REG_C_1; 746 case MLX5_XMETA_MODE_META32: 747 return REG_C_0; 748 } 749 break; 750 case MLX5_MTR_SFX: 751 /* 752 * If meter color and flow match share one register, flow match 753 * should use the meter color register for match. 754 */ 755 if (priv->mtr_reg_share) 756 return priv->mtr_color_reg; 757 else 758 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 759 REG_C_3; 760 case MLX5_MTR_COLOR: 761 case MLX5_ASO_FLOW_HIT: /* Both features use the same REG_C. */ 762 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 763 return priv->mtr_color_reg; 764 case MLX5_COPY_MARK: 765 /* 766 * Metadata COPY_MARK register using is in meter suffix sub 767 * flow while with meter. It's safe to share the same register. 768 */ 769 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 770 case MLX5_APP_TAG: 771 /* 772 * If meter is enable, it will engage the register for color 773 * match and flow match. If meter color match is not using the 774 * REG_C_2, need to skip the REG_C_x be used by meter color 775 * match. 776 * If meter is disable, free to use all available registers. 777 */ 778 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 779 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 780 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 781 if (id > (uint32_t)(REG_C_7 - start_reg)) 782 return rte_flow_error_set(error, EINVAL, 783 RTE_FLOW_ERROR_TYPE_ITEM, 784 NULL, "invalid tag id"); 785 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 786 return rte_flow_error_set(error, ENOTSUP, 787 RTE_FLOW_ERROR_TYPE_ITEM, 788 NULL, "unsupported tag id"); 789 /* 790 * This case means meter is using the REG_C_x great than 2. 791 * Take care not to conflict with meter color REG_C_x. 792 * If the available index REG_C_y >= REG_C_x, skip the 793 * color register. 794 */ 795 if (skip_mtr_reg && config->flow_mreg_c 796 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 797 if (id >= (uint32_t)(REG_C_7 - start_reg)) 798 return rte_flow_error_set(error, EINVAL, 799 RTE_FLOW_ERROR_TYPE_ITEM, 800 NULL, "invalid tag id"); 801 if (config->flow_mreg_c 802 [id + 1 + start_reg - REG_C_0] != REG_NON) 803 return config->flow_mreg_c 804 [id + 1 + start_reg - REG_C_0]; 805 return rte_flow_error_set(error, ENOTSUP, 806 RTE_FLOW_ERROR_TYPE_ITEM, 807 NULL, "unsupported tag id"); 808 } 809 return config->flow_mreg_c[id + start_reg - REG_C_0]; 810 } 811 MLX5_ASSERT(false); 812 return rte_flow_error_set(error, EINVAL, 813 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 814 NULL, "invalid feature name"); 815 } 816 817 /** 818 * Check extensive flow metadata register support. 819 * 820 * @param dev 821 * Pointer to rte_eth_dev structure. 822 * 823 * @return 824 * True if device supports extensive flow metadata register, otherwise false. 825 */ 826 bool 827 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 828 { 829 struct mlx5_priv *priv = dev->data->dev_private; 830 struct mlx5_dev_config *config = &priv->config; 831 832 /* 833 * Having available reg_c can be regarded inclusively as supporting 834 * extensive flow metadata register, which could mean, 835 * - metadata register copy action by modify header. 836 * - 16 modify header actions is supported. 837 * - reg_c's are preserved across different domain (FDB and NIC) on 838 * packet loopback by flow lookup miss. 839 */ 840 return config->flow_mreg_c[2] != REG_NON; 841 } 842 843 /** 844 * Get the lowest priority. 845 * 846 * @param[in] dev 847 * Pointer to the Ethernet device structure. 848 * @param[in] attributes 849 * Pointer to device flow rule attributes. 850 * 851 * @return 852 * The value of lowest priority of flow. 853 */ 854 uint32_t 855 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 856 const struct rte_flow_attr *attr) 857 { 858 struct mlx5_priv *priv = dev->data->dev_private; 859 860 if (!attr->group && !attr->transfer) 861 return priv->config.flow_prio - 2; 862 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 863 } 864 865 /** 866 * Calculate matcher priority of the flow. 867 * 868 * @param[in] dev 869 * Pointer to the Ethernet device structure. 870 * @param[in] attr 871 * Pointer to device flow rule attributes. 872 * @param[in] subpriority 873 * The priority based on the items. 874 * @return 875 * The matcher priority of the flow. 876 */ 877 uint16_t 878 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 879 const struct rte_flow_attr *attr, 880 uint32_t subpriority) 881 { 882 uint16_t priority = (uint16_t)attr->priority; 883 struct mlx5_priv *priv = dev->data->dev_private; 884 885 if (!attr->group && !attr->transfer) { 886 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 887 priority = priv->config.flow_prio - 1; 888 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 889 } 890 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 891 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 892 return priority * 3 + subpriority; 893 } 894 895 /** 896 * Verify the @p item specifications (spec, last, mask) are compatible with the 897 * NIC capabilities. 898 * 899 * @param[in] item 900 * Item specification. 901 * @param[in] mask 902 * @p item->mask or flow default bit-masks. 903 * @param[in] nic_mask 904 * Bit-masks covering supported fields by the NIC to compare with user mask. 905 * @param[in] size 906 * Bit-masks size in bytes. 907 * @param[in] range_accepted 908 * True if range of values is accepted for specific fields, false otherwise. 909 * @param[out] error 910 * Pointer to error structure. 911 * 912 * @return 913 * 0 on success, a negative errno value otherwise and rte_errno is set. 914 */ 915 int 916 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 917 const uint8_t *mask, 918 const uint8_t *nic_mask, 919 unsigned int size, 920 bool range_accepted, 921 struct rte_flow_error *error) 922 { 923 unsigned int i; 924 925 MLX5_ASSERT(nic_mask); 926 for (i = 0; i < size; ++i) 927 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 928 return rte_flow_error_set(error, ENOTSUP, 929 RTE_FLOW_ERROR_TYPE_ITEM, 930 item, 931 "mask enables non supported" 932 " bits"); 933 if (!item->spec && (item->mask || item->last)) 934 return rte_flow_error_set(error, EINVAL, 935 RTE_FLOW_ERROR_TYPE_ITEM, item, 936 "mask/last without a spec is not" 937 " supported"); 938 if (item->spec && item->last && !range_accepted) { 939 uint8_t spec[size]; 940 uint8_t last[size]; 941 unsigned int i; 942 int ret; 943 944 for (i = 0; i < size; ++i) { 945 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 946 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 947 } 948 ret = memcmp(spec, last, size); 949 if (ret != 0) 950 return rte_flow_error_set(error, EINVAL, 951 RTE_FLOW_ERROR_TYPE_ITEM, 952 item, 953 "range is not valid"); 954 } 955 return 0; 956 } 957 958 /** 959 * Adjust the hash fields according to the @p flow information. 960 * 961 * @param[in] dev_flow. 962 * Pointer to the mlx5_flow. 963 * @param[in] tunnel 964 * 1 when the hash field is for a tunnel item. 965 * @param[in] layer_types 966 * ETH_RSS_* types. 967 * @param[in] hash_fields 968 * Item hash fields. 969 * 970 * @return 971 * The hash fields that should be used. 972 */ 973 uint64_t 974 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 975 int tunnel __rte_unused, uint64_t layer_types, 976 uint64_t hash_fields) 977 { 978 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 979 int rss_request_inner = rss_desc->level >= 2; 980 981 /* Check RSS hash level for tunnel. */ 982 if (tunnel && rss_request_inner) 983 hash_fields |= IBV_RX_HASH_INNER; 984 else if (tunnel || rss_request_inner) 985 return 0; 986 #endif 987 /* Check if requested layer matches RSS hash fields. */ 988 if (!(rss_desc->types & layer_types)) 989 return 0; 990 return hash_fields; 991 } 992 993 /** 994 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 995 * if several tunnel rules are used on this queue, the tunnel ptype will be 996 * cleared. 997 * 998 * @param rxq_ctrl 999 * Rx queue to update. 1000 */ 1001 static void 1002 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1003 { 1004 unsigned int i; 1005 uint32_t tunnel_ptype = 0; 1006 1007 /* Look up for the ptype to use. */ 1008 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1009 if (!rxq_ctrl->flow_tunnels_n[i]) 1010 continue; 1011 if (!tunnel_ptype) { 1012 tunnel_ptype = tunnels_info[i].ptype; 1013 } else { 1014 tunnel_ptype = 0; 1015 break; 1016 } 1017 } 1018 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1019 } 1020 1021 /** 1022 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1023 * flow. 1024 * 1025 * @param[in] dev 1026 * Pointer to the Ethernet device structure. 1027 * @param[in] dev_handle 1028 * Pointer to device flow handle structure. 1029 */ 1030 static void 1031 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1032 struct mlx5_flow_handle *dev_handle) 1033 { 1034 struct mlx5_priv *priv = dev->data->dev_private; 1035 const int mark = dev_handle->mark; 1036 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1037 struct mlx5_ind_table_obj *ind_tbl = NULL; 1038 unsigned int i; 1039 1040 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1041 struct mlx5_hrxq *hrxq; 1042 1043 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1044 dev_handle->rix_hrxq); 1045 if (hrxq) 1046 ind_tbl = hrxq->ind_table; 1047 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1048 struct mlx5_shared_action_rss *shared_rss; 1049 1050 shared_rss = mlx5_ipool_get 1051 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1052 dev_handle->rix_srss); 1053 if (shared_rss) 1054 ind_tbl = shared_rss->ind_tbl; 1055 } 1056 if (!ind_tbl) 1057 return; 1058 for (i = 0; i != ind_tbl->queues_n; ++i) { 1059 int idx = ind_tbl->queues[i]; 1060 struct mlx5_rxq_ctrl *rxq_ctrl = 1061 container_of((*priv->rxqs)[idx], 1062 struct mlx5_rxq_ctrl, rxq); 1063 1064 /* 1065 * To support metadata register copy on Tx loopback, 1066 * this must be always enabled (metadata may arive 1067 * from other port - not from local flows only. 1068 */ 1069 if (priv->config.dv_flow_en && 1070 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1071 mlx5_flow_ext_mreg_supported(dev)) { 1072 rxq_ctrl->rxq.mark = 1; 1073 rxq_ctrl->flow_mark_n = 1; 1074 } else if (mark) { 1075 rxq_ctrl->rxq.mark = 1; 1076 rxq_ctrl->flow_mark_n++; 1077 } 1078 if (tunnel) { 1079 unsigned int j; 1080 1081 /* Increase the counter matching the flow. */ 1082 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1083 if ((tunnels_info[j].tunnel & 1084 dev_handle->layers) == 1085 tunnels_info[j].tunnel) { 1086 rxq_ctrl->flow_tunnels_n[j]++; 1087 break; 1088 } 1089 } 1090 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1091 } 1092 } 1093 } 1094 1095 /** 1096 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1097 * 1098 * @param[in] dev 1099 * Pointer to the Ethernet device structure. 1100 * @param[in] flow 1101 * Pointer to flow structure. 1102 */ 1103 static void 1104 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1105 { 1106 struct mlx5_priv *priv = dev->data->dev_private; 1107 uint32_t handle_idx; 1108 struct mlx5_flow_handle *dev_handle; 1109 1110 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1111 handle_idx, dev_handle, next) 1112 flow_drv_rxq_flags_set(dev, dev_handle); 1113 } 1114 1115 /** 1116 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1117 * device flow if no other flow uses it with the same kind of request. 1118 * 1119 * @param dev 1120 * Pointer to Ethernet device. 1121 * @param[in] dev_handle 1122 * Pointer to the device flow handle structure. 1123 */ 1124 static void 1125 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1126 struct mlx5_flow_handle *dev_handle) 1127 { 1128 struct mlx5_priv *priv = dev->data->dev_private; 1129 const int mark = dev_handle->mark; 1130 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1131 struct mlx5_ind_table_obj *ind_tbl = NULL; 1132 unsigned int i; 1133 1134 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1135 struct mlx5_hrxq *hrxq; 1136 1137 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1138 dev_handle->rix_hrxq); 1139 if (hrxq) 1140 ind_tbl = hrxq->ind_table; 1141 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1142 struct mlx5_shared_action_rss *shared_rss; 1143 1144 shared_rss = mlx5_ipool_get 1145 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1146 dev_handle->rix_srss); 1147 if (shared_rss) 1148 ind_tbl = shared_rss->ind_tbl; 1149 } 1150 if (!ind_tbl) 1151 return; 1152 MLX5_ASSERT(dev->data->dev_started); 1153 for (i = 0; i != ind_tbl->queues_n; ++i) { 1154 int idx = ind_tbl->queues[i]; 1155 struct mlx5_rxq_ctrl *rxq_ctrl = 1156 container_of((*priv->rxqs)[idx], 1157 struct mlx5_rxq_ctrl, rxq); 1158 1159 if (priv->config.dv_flow_en && 1160 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1161 mlx5_flow_ext_mreg_supported(dev)) { 1162 rxq_ctrl->rxq.mark = 1; 1163 rxq_ctrl->flow_mark_n = 1; 1164 } else if (mark) { 1165 rxq_ctrl->flow_mark_n--; 1166 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1167 } 1168 if (tunnel) { 1169 unsigned int j; 1170 1171 /* Decrease the counter matching the flow. */ 1172 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1173 if ((tunnels_info[j].tunnel & 1174 dev_handle->layers) == 1175 tunnels_info[j].tunnel) { 1176 rxq_ctrl->flow_tunnels_n[j]--; 1177 break; 1178 } 1179 } 1180 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1181 } 1182 } 1183 } 1184 1185 /** 1186 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1187 * @p flow if no other flow uses it with the same kind of request. 1188 * 1189 * @param dev 1190 * Pointer to Ethernet device. 1191 * @param[in] flow 1192 * Pointer to the flow. 1193 */ 1194 static void 1195 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1196 { 1197 struct mlx5_priv *priv = dev->data->dev_private; 1198 uint32_t handle_idx; 1199 struct mlx5_flow_handle *dev_handle; 1200 1201 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1202 handle_idx, dev_handle, next) 1203 flow_drv_rxq_flags_trim(dev, dev_handle); 1204 } 1205 1206 /** 1207 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1208 * 1209 * @param dev 1210 * Pointer to Ethernet device. 1211 */ 1212 static void 1213 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1214 { 1215 struct mlx5_priv *priv = dev->data->dev_private; 1216 unsigned int i; 1217 1218 for (i = 0; i != priv->rxqs_n; ++i) { 1219 struct mlx5_rxq_ctrl *rxq_ctrl; 1220 unsigned int j; 1221 1222 if (!(*priv->rxqs)[i]) 1223 continue; 1224 rxq_ctrl = container_of((*priv->rxqs)[i], 1225 struct mlx5_rxq_ctrl, rxq); 1226 rxq_ctrl->flow_mark_n = 0; 1227 rxq_ctrl->rxq.mark = 0; 1228 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1229 rxq_ctrl->flow_tunnels_n[j] = 0; 1230 rxq_ctrl->rxq.tunnel = 0; 1231 } 1232 } 1233 1234 /** 1235 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1236 * 1237 * @param[in] dev 1238 * Pointer to the Ethernet device structure. 1239 */ 1240 void 1241 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1242 { 1243 struct mlx5_priv *priv = dev->data->dev_private; 1244 struct mlx5_rxq_data *data; 1245 unsigned int i; 1246 1247 for (i = 0; i != priv->rxqs_n; ++i) { 1248 if (!(*priv->rxqs)[i]) 1249 continue; 1250 data = (*priv->rxqs)[i]; 1251 if (!rte_flow_dynf_metadata_avail()) { 1252 data->dynf_meta = 0; 1253 data->flow_meta_mask = 0; 1254 data->flow_meta_offset = -1; 1255 data->flow_meta_port_mask = 0; 1256 } else { 1257 data->dynf_meta = 1; 1258 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1259 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1260 data->flow_meta_port_mask = (uint32_t)~0; 1261 if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16) 1262 data->flow_meta_port_mask >>= 16; 1263 } 1264 } 1265 } 1266 1267 /* 1268 * return a pointer to the desired action in the list of actions. 1269 * 1270 * @param[in] actions 1271 * The list of actions to search the action in. 1272 * @param[in] action 1273 * The action to find. 1274 * 1275 * @return 1276 * Pointer to the action in the list, if found. NULL otherwise. 1277 */ 1278 const struct rte_flow_action * 1279 mlx5_flow_find_action(const struct rte_flow_action *actions, 1280 enum rte_flow_action_type action) 1281 { 1282 if (actions == NULL) 1283 return NULL; 1284 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1285 if (actions->type == action) 1286 return actions; 1287 return NULL; 1288 } 1289 1290 /* 1291 * Validate the flag action. 1292 * 1293 * @param[in] action_flags 1294 * Bit-fields that holds the actions detected until now. 1295 * @param[in] attr 1296 * Attributes of flow that includes this action. 1297 * @param[out] error 1298 * Pointer to error structure. 1299 * 1300 * @return 1301 * 0 on success, a negative errno value otherwise and rte_errno is set. 1302 */ 1303 int 1304 mlx5_flow_validate_action_flag(uint64_t action_flags, 1305 const struct rte_flow_attr *attr, 1306 struct rte_flow_error *error) 1307 { 1308 if (action_flags & MLX5_FLOW_ACTION_MARK) 1309 return rte_flow_error_set(error, EINVAL, 1310 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1311 "can't mark and flag in same flow"); 1312 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1313 return rte_flow_error_set(error, EINVAL, 1314 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1315 "can't have 2 flag" 1316 " actions in same flow"); 1317 if (attr->egress) 1318 return rte_flow_error_set(error, ENOTSUP, 1319 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1320 "flag action not supported for " 1321 "egress"); 1322 return 0; 1323 } 1324 1325 /* 1326 * Validate the mark action. 1327 * 1328 * @param[in] action 1329 * Pointer to the queue action. 1330 * @param[in] action_flags 1331 * Bit-fields that holds the actions detected until now. 1332 * @param[in] attr 1333 * Attributes of flow that includes this action. 1334 * @param[out] error 1335 * Pointer to error structure. 1336 * 1337 * @return 1338 * 0 on success, a negative errno value otherwise and rte_errno is set. 1339 */ 1340 int 1341 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1342 uint64_t action_flags, 1343 const struct rte_flow_attr *attr, 1344 struct rte_flow_error *error) 1345 { 1346 const struct rte_flow_action_mark *mark = action->conf; 1347 1348 if (!mark) 1349 return rte_flow_error_set(error, EINVAL, 1350 RTE_FLOW_ERROR_TYPE_ACTION, 1351 action, 1352 "configuration cannot be null"); 1353 if (mark->id >= MLX5_FLOW_MARK_MAX) 1354 return rte_flow_error_set(error, EINVAL, 1355 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1356 &mark->id, 1357 "mark id must in 0 <= id < " 1358 RTE_STR(MLX5_FLOW_MARK_MAX)); 1359 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1360 return rte_flow_error_set(error, EINVAL, 1361 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1362 "can't flag and mark in same flow"); 1363 if (action_flags & MLX5_FLOW_ACTION_MARK) 1364 return rte_flow_error_set(error, EINVAL, 1365 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1366 "can't have 2 mark actions in same" 1367 " flow"); 1368 if (attr->egress) 1369 return rte_flow_error_set(error, ENOTSUP, 1370 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1371 "mark action not supported for " 1372 "egress"); 1373 return 0; 1374 } 1375 1376 /* 1377 * Validate the drop action. 1378 * 1379 * @param[in] action_flags 1380 * Bit-fields that holds the actions detected until now. 1381 * @param[in] attr 1382 * Attributes of flow that includes this action. 1383 * @param[out] error 1384 * Pointer to error structure. 1385 * 1386 * @return 1387 * 0 on success, a negative errno value otherwise and rte_errno is set. 1388 */ 1389 int 1390 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1391 const struct rte_flow_attr *attr, 1392 struct rte_flow_error *error) 1393 { 1394 if (attr->egress) 1395 return rte_flow_error_set(error, ENOTSUP, 1396 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1397 "drop action not supported for " 1398 "egress"); 1399 return 0; 1400 } 1401 1402 /* 1403 * Validate the queue action. 1404 * 1405 * @param[in] action 1406 * Pointer to the queue action. 1407 * @param[in] action_flags 1408 * Bit-fields that holds the actions detected until now. 1409 * @param[in] dev 1410 * Pointer to the Ethernet device structure. 1411 * @param[in] attr 1412 * Attributes of flow that includes this action. 1413 * @param[out] error 1414 * Pointer to error structure. 1415 * 1416 * @return 1417 * 0 on success, a negative errno value otherwise and rte_errno is set. 1418 */ 1419 int 1420 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1421 uint64_t action_flags, 1422 struct rte_eth_dev *dev, 1423 const struct rte_flow_attr *attr, 1424 struct rte_flow_error *error) 1425 { 1426 struct mlx5_priv *priv = dev->data->dev_private; 1427 const struct rte_flow_action_queue *queue = action->conf; 1428 1429 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1430 return rte_flow_error_set(error, EINVAL, 1431 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1432 "can't have 2 fate actions in" 1433 " same flow"); 1434 if (!priv->rxqs_n) 1435 return rte_flow_error_set(error, EINVAL, 1436 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1437 NULL, "No Rx queues configured"); 1438 if (queue->index >= priv->rxqs_n) 1439 return rte_flow_error_set(error, EINVAL, 1440 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1441 &queue->index, 1442 "queue index out of range"); 1443 if (!(*priv->rxqs)[queue->index]) 1444 return rte_flow_error_set(error, EINVAL, 1445 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1446 &queue->index, 1447 "queue is not configured"); 1448 if (attr->egress) 1449 return rte_flow_error_set(error, ENOTSUP, 1450 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1451 "queue action not supported for " 1452 "egress"); 1453 return 0; 1454 } 1455 1456 /* 1457 * Validate the rss action. 1458 * 1459 * @param[in] dev 1460 * Pointer to the Ethernet device structure. 1461 * @param[in] action 1462 * Pointer to the queue action. 1463 * @param[out] error 1464 * Pointer to error structure. 1465 * 1466 * @return 1467 * 0 on success, a negative errno value otherwise and rte_errno is set. 1468 */ 1469 int 1470 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1471 const struct rte_flow_action *action, 1472 struct rte_flow_error *error) 1473 { 1474 struct mlx5_priv *priv = dev->data->dev_private; 1475 const struct rte_flow_action_rss *rss = action->conf; 1476 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1477 unsigned int i; 1478 1479 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1480 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1481 return rte_flow_error_set(error, ENOTSUP, 1482 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1483 &rss->func, 1484 "RSS hash function not supported"); 1485 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1486 if (rss->level > 2) 1487 #else 1488 if (rss->level > 1) 1489 #endif 1490 return rte_flow_error_set(error, ENOTSUP, 1491 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1492 &rss->level, 1493 "tunnel RSS is not supported"); 1494 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1495 if (rss->key_len == 0 && rss->key != NULL) 1496 return rte_flow_error_set(error, ENOTSUP, 1497 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1498 &rss->key_len, 1499 "RSS hash key length 0"); 1500 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1501 return rte_flow_error_set(error, ENOTSUP, 1502 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1503 &rss->key_len, 1504 "RSS hash key too small"); 1505 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1506 return rte_flow_error_set(error, ENOTSUP, 1507 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1508 &rss->key_len, 1509 "RSS hash key too large"); 1510 if (rss->queue_num > priv->config.ind_table_max_size) 1511 return rte_flow_error_set(error, ENOTSUP, 1512 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1513 &rss->queue_num, 1514 "number of queues too large"); 1515 if (rss->types & MLX5_RSS_HF_MASK) 1516 return rte_flow_error_set(error, ENOTSUP, 1517 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1518 &rss->types, 1519 "some RSS protocols are not" 1520 " supported"); 1521 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1522 !(rss->types & ETH_RSS_IP)) 1523 return rte_flow_error_set(error, EINVAL, 1524 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1525 "L3 partial RSS requested but L3 RSS" 1526 " type not specified"); 1527 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1528 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1529 return rte_flow_error_set(error, EINVAL, 1530 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1531 "L4 partial RSS requested but L4 RSS" 1532 " type not specified"); 1533 if (!priv->rxqs_n) 1534 return rte_flow_error_set(error, EINVAL, 1535 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1536 NULL, "No Rx queues configured"); 1537 if (!rss->queue_num) 1538 return rte_flow_error_set(error, EINVAL, 1539 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1540 NULL, "No queues configured"); 1541 for (i = 0; i != rss->queue_num; ++i) { 1542 struct mlx5_rxq_ctrl *rxq_ctrl; 1543 1544 if (rss->queue[i] >= priv->rxqs_n) 1545 return rte_flow_error_set 1546 (error, EINVAL, 1547 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1548 &rss->queue[i], "queue index out of range"); 1549 if (!(*priv->rxqs)[rss->queue[i]]) 1550 return rte_flow_error_set 1551 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1552 &rss->queue[i], "queue is not configured"); 1553 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1554 struct mlx5_rxq_ctrl, rxq); 1555 if (i == 0) 1556 rxq_type = rxq_ctrl->type; 1557 if (rxq_type != rxq_ctrl->type) 1558 return rte_flow_error_set 1559 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1560 &rss->queue[i], 1561 "combining hairpin and regular RSS queues is not supported"); 1562 } 1563 return 0; 1564 } 1565 1566 /* 1567 * Validate the rss action. 1568 * 1569 * @param[in] action 1570 * Pointer to the queue action. 1571 * @param[in] action_flags 1572 * Bit-fields that holds the actions detected until now. 1573 * @param[in] dev 1574 * Pointer to the Ethernet device structure. 1575 * @param[in] attr 1576 * Attributes of flow that includes this action. 1577 * @param[in] item_flags 1578 * Items that were detected. 1579 * @param[out] error 1580 * Pointer to error structure. 1581 * 1582 * @return 1583 * 0 on success, a negative errno value otherwise and rte_errno is set. 1584 */ 1585 int 1586 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1587 uint64_t action_flags, 1588 struct rte_eth_dev *dev, 1589 const struct rte_flow_attr *attr, 1590 uint64_t item_flags, 1591 struct rte_flow_error *error) 1592 { 1593 const struct rte_flow_action_rss *rss = action->conf; 1594 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1595 int ret; 1596 1597 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1598 return rte_flow_error_set(error, EINVAL, 1599 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1600 "can't have 2 fate actions" 1601 " in same flow"); 1602 ret = mlx5_validate_action_rss(dev, action, error); 1603 if (ret) 1604 return ret; 1605 if (attr->egress) 1606 return rte_flow_error_set(error, ENOTSUP, 1607 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1608 "rss action not supported for " 1609 "egress"); 1610 if (rss->level > 1 && !tunnel) 1611 return rte_flow_error_set(error, EINVAL, 1612 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1613 "inner RSS is not supported for " 1614 "non-tunnel flows"); 1615 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1616 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1617 return rte_flow_error_set(error, EINVAL, 1618 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1619 "RSS on eCPRI is not supported now"); 1620 } 1621 return 0; 1622 } 1623 1624 /* 1625 * Validate the default miss action. 1626 * 1627 * @param[in] action_flags 1628 * Bit-fields that holds the actions detected until now. 1629 * @param[out] error 1630 * Pointer to error structure. 1631 * 1632 * @return 1633 * 0 on success, a negative errno value otherwise and rte_errno is set. 1634 */ 1635 int 1636 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1637 const struct rte_flow_attr *attr, 1638 struct rte_flow_error *error) 1639 { 1640 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1641 return rte_flow_error_set(error, EINVAL, 1642 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1643 "can't have 2 fate actions in" 1644 " same flow"); 1645 if (attr->egress) 1646 return rte_flow_error_set(error, ENOTSUP, 1647 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1648 "default miss action not supported " 1649 "for egress"); 1650 if (attr->group) 1651 return rte_flow_error_set(error, ENOTSUP, 1652 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1653 "only group 0 is supported"); 1654 if (attr->transfer) 1655 return rte_flow_error_set(error, ENOTSUP, 1656 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1657 NULL, "transfer is not supported"); 1658 return 0; 1659 } 1660 1661 /* 1662 * Validate the count action. 1663 * 1664 * @param[in] dev 1665 * Pointer to the Ethernet device structure. 1666 * @param[in] attr 1667 * Attributes of flow that includes this action. 1668 * @param[out] error 1669 * Pointer to error structure. 1670 * 1671 * @return 1672 * 0 on success, a negative errno value otherwise and rte_errno is set. 1673 */ 1674 int 1675 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1676 const struct rte_flow_attr *attr, 1677 struct rte_flow_error *error) 1678 { 1679 if (attr->egress) 1680 return rte_flow_error_set(error, ENOTSUP, 1681 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1682 "count action not supported for " 1683 "egress"); 1684 return 0; 1685 } 1686 1687 /** 1688 * Verify the @p attributes will be correctly understood by the NIC and store 1689 * them in the @p flow if everything is correct. 1690 * 1691 * @param[in] dev 1692 * Pointer to the Ethernet device structure. 1693 * @param[in] attributes 1694 * Pointer to flow attributes 1695 * @param[out] error 1696 * Pointer to error structure. 1697 * 1698 * @return 1699 * 0 on success, a negative errno value otherwise and rte_errno is set. 1700 */ 1701 int 1702 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1703 const struct rte_flow_attr *attributes, 1704 struct rte_flow_error *error) 1705 { 1706 struct mlx5_priv *priv = dev->data->dev_private; 1707 uint32_t priority_max = priv->config.flow_prio - 1; 1708 1709 if (attributes->group) 1710 return rte_flow_error_set(error, ENOTSUP, 1711 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1712 NULL, "groups is not supported"); 1713 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1714 attributes->priority >= priority_max) 1715 return rte_flow_error_set(error, ENOTSUP, 1716 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1717 NULL, "priority out of range"); 1718 if (attributes->egress) 1719 return rte_flow_error_set(error, ENOTSUP, 1720 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1721 "egress is not supported"); 1722 if (attributes->transfer && !priv->config.dv_esw_en) 1723 return rte_flow_error_set(error, ENOTSUP, 1724 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1725 NULL, "transfer is not supported"); 1726 if (!attributes->ingress) 1727 return rte_flow_error_set(error, EINVAL, 1728 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1729 NULL, 1730 "ingress attribute is mandatory"); 1731 return 0; 1732 } 1733 1734 /** 1735 * Validate ICMP6 item. 1736 * 1737 * @param[in] item 1738 * Item specification. 1739 * @param[in] item_flags 1740 * Bit-fields that holds the items detected until now. 1741 * @param[in] ext_vlan_sup 1742 * Whether extended VLAN features are supported or not. 1743 * @param[out] error 1744 * Pointer to error structure. 1745 * 1746 * @return 1747 * 0 on success, a negative errno value otherwise and rte_errno is set. 1748 */ 1749 int 1750 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1751 uint64_t item_flags, 1752 uint8_t target_protocol, 1753 struct rte_flow_error *error) 1754 { 1755 const struct rte_flow_item_icmp6 *mask = item->mask; 1756 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1757 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1758 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1759 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1760 MLX5_FLOW_LAYER_OUTER_L4; 1761 int ret; 1762 1763 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1764 return rte_flow_error_set(error, EINVAL, 1765 RTE_FLOW_ERROR_TYPE_ITEM, item, 1766 "protocol filtering not compatible" 1767 " with ICMP6 layer"); 1768 if (!(item_flags & l3m)) 1769 return rte_flow_error_set(error, EINVAL, 1770 RTE_FLOW_ERROR_TYPE_ITEM, item, 1771 "IPv6 is mandatory to filter on" 1772 " ICMP6"); 1773 if (item_flags & l4m) 1774 return rte_flow_error_set(error, EINVAL, 1775 RTE_FLOW_ERROR_TYPE_ITEM, item, 1776 "multiple L4 layers not supported"); 1777 if (!mask) 1778 mask = &rte_flow_item_icmp6_mask; 1779 ret = mlx5_flow_item_acceptable 1780 (item, (const uint8_t *)mask, 1781 (const uint8_t *)&rte_flow_item_icmp6_mask, 1782 sizeof(struct rte_flow_item_icmp6), 1783 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1784 if (ret < 0) 1785 return ret; 1786 return 0; 1787 } 1788 1789 /** 1790 * Validate ICMP item. 1791 * 1792 * @param[in] item 1793 * Item specification. 1794 * @param[in] item_flags 1795 * Bit-fields that holds the items detected until now. 1796 * @param[out] error 1797 * Pointer to error structure. 1798 * 1799 * @return 1800 * 0 on success, a negative errno value otherwise and rte_errno is set. 1801 */ 1802 int 1803 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1804 uint64_t item_flags, 1805 uint8_t target_protocol, 1806 struct rte_flow_error *error) 1807 { 1808 const struct rte_flow_item_icmp *mask = item->mask; 1809 const struct rte_flow_item_icmp nic_mask = { 1810 .hdr.icmp_type = 0xff, 1811 .hdr.icmp_code = 0xff, 1812 .hdr.icmp_ident = RTE_BE16(0xffff), 1813 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1814 }; 1815 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1816 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1817 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1818 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1819 MLX5_FLOW_LAYER_OUTER_L4; 1820 int ret; 1821 1822 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1823 return rte_flow_error_set(error, EINVAL, 1824 RTE_FLOW_ERROR_TYPE_ITEM, item, 1825 "protocol filtering not compatible" 1826 " with ICMP layer"); 1827 if (!(item_flags & l3m)) 1828 return rte_flow_error_set(error, EINVAL, 1829 RTE_FLOW_ERROR_TYPE_ITEM, item, 1830 "IPv4 is mandatory to filter" 1831 " on ICMP"); 1832 if (item_flags & l4m) 1833 return rte_flow_error_set(error, EINVAL, 1834 RTE_FLOW_ERROR_TYPE_ITEM, item, 1835 "multiple L4 layers not supported"); 1836 if (!mask) 1837 mask = &nic_mask; 1838 ret = mlx5_flow_item_acceptable 1839 (item, (const uint8_t *)mask, 1840 (const uint8_t *)&nic_mask, 1841 sizeof(struct rte_flow_item_icmp), 1842 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1843 if (ret < 0) 1844 return ret; 1845 return 0; 1846 } 1847 1848 /** 1849 * Validate Ethernet item. 1850 * 1851 * @param[in] item 1852 * Item specification. 1853 * @param[in] item_flags 1854 * Bit-fields that holds the items detected until now. 1855 * @param[out] error 1856 * Pointer to error structure. 1857 * 1858 * @return 1859 * 0 on success, a negative errno value otherwise and rte_errno is set. 1860 */ 1861 int 1862 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1863 uint64_t item_flags, bool ext_vlan_sup, 1864 struct rte_flow_error *error) 1865 { 1866 const struct rte_flow_item_eth *mask = item->mask; 1867 const struct rte_flow_item_eth nic_mask = { 1868 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1869 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1870 .type = RTE_BE16(0xffff), 1871 .has_vlan = ext_vlan_sup ? 1 : 0, 1872 }; 1873 int ret; 1874 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1875 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1876 MLX5_FLOW_LAYER_OUTER_L2; 1877 1878 if (item_flags & ethm) 1879 return rte_flow_error_set(error, ENOTSUP, 1880 RTE_FLOW_ERROR_TYPE_ITEM, item, 1881 "multiple L2 layers not supported"); 1882 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1883 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1884 return rte_flow_error_set(error, EINVAL, 1885 RTE_FLOW_ERROR_TYPE_ITEM, item, 1886 "L2 layer should not follow " 1887 "L3 layers"); 1888 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1889 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1890 return rte_flow_error_set(error, EINVAL, 1891 RTE_FLOW_ERROR_TYPE_ITEM, item, 1892 "L2 layer should not follow VLAN"); 1893 if (!mask) 1894 mask = &rte_flow_item_eth_mask; 1895 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1896 (const uint8_t *)&nic_mask, 1897 sizeof(struct rte_flow_item_eth), 1898 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1899 return ret; 1900 } 1901 1902 /** 1903 * Validate VLAN item. 1904 * 1905 * @param[in] item 1906 * Item specification. 1907 * @param[in] item_flags 1908 * Bit-fields that holds the items detected until now. 1909 * @param[in] dev 1910 * Ethernet device flow is being created on. 1911 * @param[out] error 1912 * Pointer to error structure. 1913 * 1914 * @return 1915 * 0 on success, a negative errno value otherwise and rte_errno is set. 1916 */ 1917 int 1918 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1919 uint64_t item_flags, 1920 struct rte_eth_dev *dev, 1921 struct rte_flow_error *error) 1922 { 1923 const struct rte_flow_item_vlan *spec = item->spec; 1924 const struct rte_flow_item_vlan *mask = item->mask; 1925 const struct rte_flow_item_vlan nic_mask = { 1926 .tci = RTE_BE16(UINT16_MAX), 1927 .inner_type = RTE_BE16(UINT16_MAX), 1928 }; 1929 uint16_t vlan_tag = 0; 1930 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1931 int ret; 1932 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1933 MLX5_FLOW_LAYER_INNER_L4) : 1934 (MLX5_FLOW_LAYER_OUTER_L3 | 1935 MLX5_FLOW_LAYER_OUTER_L4); 1936 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1937 MLX5_FLOW_LAYER_OUTER_VLAN; 1938 1939 if (item_flags & vlanm) 1940 return rte_flow_error_set(error, EINVAL, 1941 RTE_FLOW_ERROR_TYPE_ITEM, item, 1942 "multiple VLAN layers not supported"); 1943 else if ((item_flags & l34m) != 0) 1944 return rte_flow_error_set(error, EINVAL, 1945 RTE_FLOW_ERROR_TYPE_ITEM, item, 1946 "VLAN cannot follow L3/L4 layer"); 1947 if (!mask) 1948 mask = &rte_flow_item_vlan_mask; 1949 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1950 (const uint8_t *)&nic_mask, 1951 sizeof(struct rte_flow_item_vlan), 1952 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1953 if (ret) 1954 return ret; 1955 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1956 struct mlx5_priv *priv = dev->data->dev_private; 1957 1958 if (priv->vmwa_context) { 1959 /* 1960 * Non-NULL context means we have a virtual machine 1961 * and SR-IOV enabled, we have to create VLAN interface 1962 * to make hypervisor to setup E-Switch vport 1963 * context correctly. We avoid creating the multiple 1964 * VLAN interfaces, so we cannot support VLAN tag mask. 1965 */ 1966 return rte_flow_error_set(error, EINVAL, 1967 RTE_FLOW_ERROR_TYPE_ITEM, 1968 item, 1969 "VLAN tag mask is not" 1970 " supported in virtual" 1971 " environment"); 1972 } 1973 } 1974 if (spec) { 1975 vlan_tag = spec->tci; 1976 vlan_tag &= mask->tci; 1977 } 1978 /* 1979 * From verbs perspective an empty VLAN is equivalent 1980 * to a packet without VLAN layer. 1981 */ 1982 if (!vlan_tag) 1983 return rte_flow_error_set(error, EINVAL, 1984 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1985 item->spec, 1986 "VLAN cannot be empty"); 1987 return 0; 1988 } 1989 1990 /** 1991 * Validate IPV4 item. 1992 * 1993 * @param[in] item 1994 * Item specification. 1995 * @param[in] item_flags 1996 * Bit-fields that holds the items detected until now. 1997 * @param[in] last_item 1998 * Previous validated item in the pattern items. 1999 * @param[in] ether_type 2000 * Type in the ethernet layer header (including dot1q). 2001 * @param[in] acc_mask 2002 * Acceptable mask, if NULL default internal default mask 2003 * will be used to check whether item fields are supported. 2004 * @param[in] range_accepted 2005 * True if range of values is accepted for specific fields, false otherwise. 2006 * @param[out] error 2007 * Pointer to error structure. 2008 * 2009 * @return 2010 * 0 on success, a negative errno value otherwise and rte_errno is set. 2011 */ 2012 int 2013 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2014 uint64_t item_flags, 2015 uint64_t last_item, 2016 uint16_t ether_type, 2017 const struct rte_flow_item_ipv4 *acc_mask, 2018 bool range_accepted, 2019 struct rte_flow_error *error) 2020 { 2021 const struct rte_flow_item_ipv4 *mask = item->mask; 2022 const struct rte_flow_item_ipv4 *spec = item->spec; 2023 const struct rte_flow_item_ipv4 nic_mask = { 2024 .hdr = { 2025 .src_addr = RTE_BE32(0xffffffff), 2026 .dst_addr = RTE_BE32(0xffffffff), 2027 .type_of_service = 0xff, 2028 .next_proto_id = 0xff, 2029 }, 2030 }; 2031 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2032 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2033 MLX5_FLOW_LAYER_OUTER_L3; 2034 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2035 MLX5_FLOW_LAYER_OUTER_L4; 2036 int ret; 2037 uint8_t next_proto = 0xFF; 2038 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2039 MLX5_FLOW_LAYER_OUTER_VLAN | 2040 MLX5_FLOW_LAYER_INNER_VLAN); 2041 2042 if ((last_item & l2_vlan) && ether_type && 2043 ether_type != RTE_ETHER_TYPE_IPV4) 2044 return rte_flow_error_set(error, EINVAL, 2045 RTE_FLOW_ERROR_TYPE_ITEM, item, 2046 "IPv4 cannot follow L2/VLAN layer " 2047 "which ether type is not IPv4"); 2048 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2049 if (mask && spec) 2050 next_proto = mask->hdr.next_proto_id & 2051 spec->hdr.next_proto_id; 2052 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2053 return rte_flow_error_set(error, EINVAL, 2054 RTE_FLOW_ERROR_TYPE_ITEM, 2055 item, 2056 "multiple tunnel " 2057 "not supported"); 2058 } 2059 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2060 return rte_flow_error_set(error, EINVAL, 2061 RTE_FLOW_ERROR_TYPE_ITEM, item, 2062 "wrong tunnel type - IPv6 specified " 2063 "but IPv4 item provided"); 2064 if (item_flags & l3m) 2065 return rte_flow_error_set(error, ENOTSUP, 2066 RTE_FLOW_ERROR_TYPE_ITEM, item, 2067 "multiple L3 layers not supported"); 2068 else if (item_flags & l4m) 2069 return rte_flow_error_set(error, EINVAL, 2070 RTE_FLOW_ERROR_TYPE_ITEM, item, 2071 "L3 cannot follow an L4 layer."); 2072 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2073 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2074 return rte_flow_error_set(error, EINVAL, 2075 RTE_FLOW_ERROR_TYPE_ITEM, item, 2076 "L3 cannot follow an NVGRE layer."); 2077 if (!mask) 2078 mask = &rte_flow_item_ipv4_mask; 2079 else if (mask->hdr.next_proto_id != 0 && 2080 mask->hdr.next_proto_id != 0xff) 2081 return rte_flow_error_set(error, EINVAL, 2082 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2083 "partial mask is not supported" 2084 " for protocol"); 2085 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2086 acc_mask ? (const uint8_t *)acc_mask 2087 : (const uint8_t *)&nic_mask, 2088 sizeof(struct rte_flow_item_ipv4), 2089 range_accepted, error); 2090 if (ret < 0) 2091 return ret; 2092 return 0; 2093 } 2094 2095 /** 2096 * Validate IPV6 item. 2097 * 2098 * @param[in] item 2099 * Item specification. 2100 * @param[in] item_flags 2101 * Bit-fields that holds the items detected until now. 2102 * @param[in] last_item 2103 * Previous validated item in the pattern items. 2104 * @param[in] ether_type 2105 * Type in the ethernet layer header (including dot1q). 2106 * @param[in] acc_mask 2107 * Acceptable mask, if NULL default internal default mask 2108 * will be used to check whether item fields are supported. 2109 * @param[out] error 2110 * Pointer to error structure. 2111 * 2112 * @return 2113 * 0 on success, a negative errno value otherwise and rte_errno is set. 2114 */ 2115 int 2116 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2117 uint64_t item_flags, 2118 uint64_t last_item, 2119 uint16_t ether_type, 2120 const struct rte_flow_item_ipv6 *acc_mask, 2121 struct rte_flow_error *error) 2122 { 2123 const struct rte_flow_item_ipv6 *mask = item->mask; 2124 const struct rte_flow_item_ipv6 *spec = item->spec; 2125 const struct rte_flow_item_ipv6 nic_mask = { 2126 .hdr = { 2127 .src_addr = 2128 "\xff\xff\xff\xff\xff\xff\xff\xff" 2129 "\xff\xff\xff\xff\xff\xff\xff\xff", 2130 .dst_addr = 2131 "\xff\xff\xff\xff\xff\xff\xff\xff" 2132 "\xff\xff\xff\xff\xff\xff\xff\xff", 2133 .vtc_flow = RTE_BE32(0xffffffff), 2134 .proto = 0xff, 2135 }, 2136 }; 2137 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2138 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2139 MLX5_FLOW_LAYER_OUTER_L3; 2140 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2141 MLX5_FLOW_LAYER_OUTER_L4; 2142 int ret; 2143 uint8_t next_proto = 0xFF; 2144 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2145 MLX5_FLOW_LAYER_OUTER_VLAN | 2146 MLX5_FLOW_LAYER_INNER_VLAN); 2147 2148 if ((last_item & l2_vlan) && ether_type && 2149 ether_type != RTE_ETHER_TYPE_IPV6) 2150 return rte_flow_error_set(error, EINVAL, 2151 RTE_FLOW_ERROR_TYPE_ITEM, item, 2152 "IPv6 cannot follow L2/VLAN layer " 2153 "which ether type is not IPv6"); 2154 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2155 next_proto = spec->hdr.proto; 2156 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 2157 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2158 return rte_flow_error_set(error, EINVAL, 2159 RTE_FLOW_ERROR_TYPE_ITEM, 2160 item, 2161 "multiple tunnel " 2162 "not supported"); 2163 } 2164 if (next_proto == IPPROTO_HOPOPTS || 2165 next_proto == IPPROTO_ROUTING || 2166 next_proto == IPPROTO_FRAGMENT || 2167 next_proto == IPPROTO_ESP || 2168 next_proto == IPPROTO_AH || 2169 next_proto == IPPROTO_DSTOPTS) 2170 return rte_flow_error_set(error, EINVAL, 2171 RTE_FLOW_ERROR_TYPE_ITEM, item, 2172 "IPv6 proto (next header) should " 2173 "not be set as extension header"); 2174 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2175 return rte_flow_error_set(error, EINVAL, 2176 RTE_FLOW_ERROR_TYPE_ITEM, item, 2177 "wrong tunnel type - IPv4 specified " 2178 "but IPv6 item provided"); 2179 if (item_flags & l3m) 2180 return rte_flow_error_set(error, ENOTSUP, 2181 RTE_FLOW_ERROR_TYPE_ITEM, item, 2182 "multiple L3 layers not supported"); 2183 else if (item_flags & l4m) 2184 return rte_flow_error_set(error, EINVAL, 2185 RTE_FLOW_ERROR_TYPE_ITEM, item, 2186 "L3 cannot follow an L4 layer."); 2187 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2188 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2189 return rte_flow_error_set(error, EINVAL, 2190 RTE_FLOW_ERROR_TYPE_ITEM, item, 2191 "L3 cannot follow an NVGRE layer."); 2192 if (!mask) 2193 mask = &rte_flow_item_ipv6_mask; 2194 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2195 acc_mask ? (const uint8_t *)acc_mask 2196 : (const uint8_t *)&nic_mask, 2197 sizeof(struct rte_flow_item_ipv6), 2198 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2199 if (ret < 0) 2200 return ret; 2201 return 0; 2202 } 2203 2204 /** 2205 * Validate UDP item. 2206 * 2207 * @param[in] item 2208 * Item specification. 2209 * @param[in] item_flags 2210 * Bit-fields that holds the items detected until now. 2211 * @param[in] target_protocol 2212 * The next protocol in the previous item. 2213 * @param[in] flow_mask 2214 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2215 * @param[out] error 2216 * Pointer to error structure. 2217 * 2218 * @return 2219 * 0 on success, a negative errno value otherwise and rte_errno is set. 2220 */ 2221 int 2222 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2223 uint64_t item_flags, 2224 uint8_t target_protocol, 2225 struct rte_flow_error *error) 2226 { 2227 const struct rte_flow_item_udp *mask = item->mask; 2228 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2229 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2230 MLX5_FLOW_LAYER_OUTER_L3; 2231 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2232 MLX5_FLOW_LAYER_OUTER_L4; 2233 int ret; 2234 2235 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2236 return rte_flow_error_set(error, EINVAL, 2237 RTE_FLOW_ERROR_TYPE_ITEM, item, 2238 "protocol filtering not compatible" 2239 " with UDP layer"); 2240 if (!(item_flags & l3m)) 2241 return rte_flow_error_set(error, EINVAL, 2242 RTE_FLOW_ERROR_TYPE_ITEM, item, 2243 "L3 is mandatory to filter on L4"); 2244 if (item_flags & l4m) 2245 return rte_flow_error_set(error, EINVAL, 2246 RTE_FLOW_ERROR_TYPE_ITEM, item, 2247 "multiple L4 layers not supported"); 2248 if (!mask) 2249 mask = &rte_flow_item_udp_mask; 2250 ret = mlx5_flow_item_acceptable 2251 (item, (const uint8_t *)mask, 2252 (const uint8_t *)&rte_flow_item_udp_mask, 2253 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2254 error); 2255 if (ret < 0) 2256 return ret; 2257 return 0; 2258 } 2259 2260 /** 2261 * Validate TCP item. 2262 * 2263 * @param[in] item 2264 * Item specification. 2265 * @param[in] item_flags 2266 * Bit-fields that holds the items detected until now. 2267 * @param[in] target_protocol 2268 * The next protocol in the previous item. 2269 * @param[out] error 2270 * Pointer to error structure. 2271 * 2272 * @return 2273 * 0 on success, a negative errno value otherwise and rte_errno is set. 2274 */ 2275 int 2276 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2277 uint64_t item_flags, 2278 uint8_t target_protocol, 2279 const struct rte_flow_item_tcp *flow_mask, 2280 struct rte_flow_error *error) 2281 { 2282 const struct rte_flow_item_tcp *mask = item->mask; 2283 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2284 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2285 MLX5_FLOW_LAYER_OUTER_L3; 2286 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2287 MLX5_FLOW_LAYER_OUTER_L4; 2288 int ret; 2289 2290 MLX5_ASSERT(flow_mask); 2291 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2292 return rte_flow_error_set(error, EINVAL, 2293 RTE_FLOW_ERROR_TYPE_ITEM, item, 2294 "protocol filtering not compatible" 2295 " with TCP layer"); 2296 if (!(item_flags & l3m)) 2297 return rte_flow_error_set(error, EINVAL, 2298 RTE_FLOW_ERROR_TYPE_ITEM, item, 2299 "L3 is mandatory to filter on L4"); 2300 if (item_flags & l4m) 2301 return rte_flow_error_set(error, EINVAL, 2302 RTE_FLOW_ERROR_TYPE_ITEM, item, 2303 "multiple L4 layers not supported"); 2304 if (!mask) 2305 mask = &rte_flow_item_tcp_mask; 2306 ret = mlx5_flow_item_acceptable 2307 (item, (const uint8_t *)mask, 2308 (const uint8_t *)flow_mask, 2309 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2310 error); 2311 if (ret < 0) 2312 return ret; 2313 return 0; 2314 } 2315 2316 /** 2317 * Validate VXLAN item. 2318 * 2319 * @param[in] item 2320 * Item specification. 2321 * @param[in] item_flags 2322 * Bit-fields that holds the items detected until now. 2323 * @param[in] target_protocol 2324 * The next protocol in the previous item. 2325 * @param[out] error 2326 * Pointer to error structure. 2327 * 2328 * @return 2329 * 0 on success, a negative errno value otherwise and rte_errno is set. 2330 */ 2331 int 2332 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2333 uint64_t item_flags, 2334 struct rte_flow_error *error) 2335 { 2336 const struct rte_flow_item_vxlan *spec = item->spec; 2337 const struct rte_flow_item_vxlan *mask = item->mask; 2338 int ret; 2339 union vni { 2340 uint32_t vlan_id; 2341 uint8_t vni[4]; 2342 } id = { .vlan_id = 0, }; 2343 2344 2345 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2346 return rte_flow_error_set(error, ENOTSUP, 2347 RTE_FLOW_ERROR_TYPE_ITEM, item, 2348 "multiple tunnel layers not" 2349 " supported"); 2350 /* 2351 * Verify only UDPv4 is present as defined in 2352 * https://tools.ietf.org/html/rfc7348 2353 */ 2354 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2355 return rte_flow_error_set(error, EINVAL, 2356 RTE_FLOW_ERROR_TYPE_ITEM, item, 2357 "no outer UDP layer found"); 2358 if (!mask) 2359 mask = &rte_flow_item_vxlan_mask; 2360 ret = mlx5_flow_item_acceptable 2361 (item, (const uint8_t *)mask, 2362 (const uint8_t *)&rte_flow_item_vxlan_mask, 2363 sizeof(struct rte_flow_item_vxlan), 2364 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2365 if (ret < 0) 2366 return ret; 2367 if (spec) { 2368 memcpy(&id.vni[1], spec->vni, 3); 2369 memcpy(&id.vni[1], mask->vni, 3); 2370 } 2371 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2372 return rte_flow_error_set(error, ENOTSUP, 2373 RTE_FLOW_ERROR_TYPE_ITEM, item, 2374 "VXLAN tunnel must be fully defined"); 2375 return 0; 2376 } 2377 2378 /** 2379 * Validate VXLAN_GPE item. 2380 * 2381 * @param[in] item 2382 * Item specification. 2383 * @param[in] item_flags 2384 * Bit-fields that holds the items detected until now. 2385 * @param[in] priv 2386 * Pointer to the private data structure. 2387 * @param[in] target_protocol 2388 * The next protocol in the previous item. 2389 * @param[out] error 2390 * Pointer to error structure. 2391 * 2392 * @return 2393 * 0 on success, a negative errno value otherwise and rte_errno is set. 2394 */ 2395 int 2396 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2397 uint64_t item_flags, 2398 struct rte_eth_dev *dev, 2399 struct rte_flow_error *error) 2400 { 2401 struct mlx5_priv *priv = dev->data->dev_private; 2402 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2403 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2404 int ret; 2405 union vni { 2406 uint32_t vlan_id; 2407 uint8_t vni[4]; 2408 } id = { .vlan_id = 0, }; 2409 2410 if (!priv->config.l3_vxlan_en) 2411 return rte_flow_error_set(error, ENOTSUP, 2412 RTE_FLOW_ERROR_TYPE_ITEM, item, 2413 "L3 VXLAN is not enabled by device" 2414 " parameter and/or not configured in" 2415 " firmware"); 2416 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2417 return rte_flow_error_set(error, ENOTSUP, 2418 RTE_FLOW_ERROR_TYPE_ITEM, item, 2419 "multiple tunnel layers not" 2420 " supported"); 2421 /* 2422 * Verify only UDPv4 is present as defined in 2423 * https://tools.ietf.org/html/rfc7348 2424 */ 2425 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2426 return rte_flow_error_set(error, EINVAL, 2427 RTE_FLOW_ERROR_TYPE_ITEM, item, 2428 "no outer UDP layer found"); 2429 if (!mask) 2430 mask = &rte_flow_item_vxlan_gpe_mask; 2431 ret = mlx5_flow_item_acceptable 2432 (item, (const uint8_t *)mask, 2433 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2434 sizeof(struct rte_flow_item_vxlan_gpe), 2435 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2436 if (ret < 0) 2437 return ret; 2438 if (spec) { 2439 if (spec->protocol) 2440 return rte_flow_error_set(error, ENOTSUP, 2441 RTE_FLOW_ERROR_TYPE_ITEM, 2442 item, 2443 "VxLAN-GPE protocol" 2444 " not supported"); 2445 memcpy(&id.vni[1], spec->vni, 3); 2446 memcpy(&id.vni[1], mask->vni, 3); 2447 } 2448 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2449 return rte_flow_error_set(error, ENOTSUP, 2450 RTE_FLOW_ERROR_TYPE_ITEM, item, 2451 "VXLAN-GPE tunnel must be fully" 2452 " defined"); 2453 return 0; 2454 } 2455 /** 2456 * Validate GRE Key item. 2457 * 2458 * @param[in] item 2459 * Item specification. 2460 * @param[in] item_flags 2461 * Bit flags to mark detected items. 2462 * @param[in] gre_item 2463 * Pointer to gre_item 2464 * @param[out] error 2465 * Pointer to error structure. 2466 * 2467 * @return 2468 * 0 on success, a negative errno value otherwise and rte_errno is set. 2469 */ 2470 int 2471 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2472 uint64_t item_flags, 2473 const struct rte_flow_item *gre_item, 2474 struct rte_flow_error *error) 2475 { 2476 const rte_be32_t *mask = item->mask; 2477 int ret = 0; 2478 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2479 const struct rte_flow_item_gre *gre_spec; 2480 const struct rte_flow_item_gre *gre_mask; 2481 2482 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2483 return rte_flow_error_set(error, ENOTSUP, 2484 RTE_FLOW_ERROR_TYPE_ITEM, item, 2485 "Multiple GRE key not support"); 2486 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2487 return rte_flow_error_set(error, ENOTSUP, 2488 RTE_FLOW_ERROR_TYPE_ITEM, item, 2489 "No preceding GRE header"); 2490 if (item_flags & MLX5_FLOW_LAYER_INNER) 2491 return rte_flow_error_set(error, ENOTSUP, 2492 RTE_FLOW_ERROR_TYPE_ITEM, item, 2493 "GRE key following a wrong item"); 2494 gre_mask = gre_item->mask; 2495 if (!gre_mask) 2496 gre_mask = &rte_flow_item_gre_mask; 2497 gre_spec = gre_item->spec; 2498 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2499 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2500 return rte_flow_error_set(error, EINVAL, 2501 RTE_FLOW_ERROR_TYPE_ITEM, item, 2502 "Key bit must be on"); 2503 2504 if (!mask) 2505 mask = &gre_key_default_mask; 2506 ret = mlx5_flow_item_acceptable 2507 (item, (const uint8_t *)mask, 2508 (const uint8_t *)&gre_key_default_mask, 2509 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2510 return ret; 2511 } 2512 2513 /** 2514 * Validate GRE item. 2515 * 2516 * @param[in] item 2517 * Item specification. 2518 * @param[in] item_flags 2519 * Bit flags to mark detected items. 2520 * @param[in] target_protocol 2521 * The next protocol in the previous item. 2522 * @param[out] error 2523 * Pointer to error structure. 2524 * 2525 * @return 2526 * 0 on success, a negative errno value otherwise and rte_errno is set. 2527 */ 2528 int 2529 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2530 uint64_t item_flags, 2531 uint8_t target_protocol, 2532 struct rte_flow_error *error) 2533 { 2534 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2535 const struct rte_flow_item_gre *mask = item->mask; 2536 int ret; 2537 const struct rte_flow_item_gre nic_mask = { 2538 .c_rsvd0_ver = RTE_BE16(0xB000), 2539 .protocol = RTE_BE16(UINT16_MAX), 2540 }; 2541 2542 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2543 return rte_flow_error_set(error, EINVAL, 2544 RTE_FLOW_ERROR_TYPE_ITEM, item, 2545 "protocol filtering not compatible" 2546 " with this GRE layer"); 2547 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2548 return rte_flow_error_set(error, ENOTSUP, 2549 RTE_FLOW_ERROR_TYPE_ITEM, item, 2550 "multiple tunnel layers not" 2551 " supported"); 2552 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2553 return rte_flow_error_set(error, ENOTSUP, 2554 RTE_FLOW_ERROR_TYPE_ITEM, item, 2555 "L3 Layer is missing"); 2556 if (!mask) 2557 mask = &rte_flow_item_gre_mask; 2558 ret = mlx5_flow_item_acceptable 2559 (item, (const uint8_t *)mask, 2560 (const uint8_t *)&nic_mask, 2561 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2562 error); 2563 if (ret < 0) 2564 return ret; 2565 #ifndef HAVE_MLX5DV_DR 2566 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2567 if (spec && (spec->protocol & mask->protocol)) 2568 return rte_flow_error_set(error, ENOTSUP, 2569 RTE_FLOW_ERROR_TYPE_ITEM, item, 2570 "without MPLS support the" 2571 " specification cannot be used for" 2572 " filtering"); 2573 #endif 2574 #endif 2575 return 0; 2576 } 2577 2578 /** 2579 * Validate Geneve item. 2580 * 2581 * @param[in] item 2582 * Item specification. 2583 * @param[in] itemFlags 2584 * Bit-fields that holds the items detected until now. 2585 * @param[in] enPriv 2586 * Pointer to the private data structure. 2587 * @param[out] error 2588 * Pointer to error structure. 2589 * 2590 * @return 2591 * 0 on success, a negative errno value otherwise and rte_errno is set. 2592 */ 2593 2594 int 2595 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2596 uint64_t item_flags, 2597 struct rte_eth_dev *dev, 2598 struct rte_flow_error *error) 2599 { 2600 struct mlx5_priv *priv = dev->data->dev_private; 2601 const struct rte_flow_item_geneve *spec = item->spec; 2602 const struct rte_flow_item_geneve *mask = item->mask; 2603 int ret; 2604 uint16_t gbhdr; 2605 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2606 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2607 const struct rte_flow_item_geneve nic_mask = { 2608 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2609 .vni = "\xff\xff\xff", 2610 .protocol = RTE_BE16(UINT16_MAX), 2611 }; 2612 2613 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2614 return rte_flow_error_set(error, ENOTSUP, 2615 RTE_FLOW_ERROR_TYPE_ITEM, item, 2616 "L3 Geneve is not enabled by device" 2617 " parameter and/or not configured in" 2618 " firmware"); 2619 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2620 return rte_flow_error_set(error, ENOTSUP, 2621 RTE_FLOW_ERROR_TYPE_ITEM, item, 2622 "multiple tunnel layers not" 2623 " supported"); 2624 /* 2625 * Verify only UDPv4 is present as defined in 2626 * https://tools.ietf.org/html/rfc7348 2627 */ 2628 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2629 return rte_flow_error_set(error, EINVAL, 2630 RTE_FLOW_ERROR_TYPE_ITEM, item, 2631 "no outer UDP layer found"); 2632 if (!mask) 2633 mask = &rte_flow_item_geneve_mask; 2634 ret = mlx5_flow_item_acceptable 2635 (item, (const uint8_t *)mask, 2636 (const uint8_t *)&nic_mask, 2637 sizeof(struct rte_flow_item_geneve), 2638 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2639 if (ret) 2640 return ret; 2641 if (spec) { 2642 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2643 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2644 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2645 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2646 return rte_flow_error_set(error, ENOTSUP, 2647 RTE_FLOW_ERROR_TYPE_ITEM, 2648 item, 2649 "Geneve protocol unsupported" 2650 " fields are being used"); 2651 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2652 return rte_flow_error_set 2653 (error, ENOTSUP, 2654 RTE_FLOW_ERROR_TYPE_ITEM, 2655 item, 2656 "Unsupported Geneve options length"); 2657 } 2658 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2659 return rte_flow_error_set 2660 (error, ENOTSUP, 2661 RTE_FLOW_ERROR_TYPE_ITEM, item, 2662 "Geneve tunnel must be fully defined"); 2663 return 0; 2664 } 2665 2666 /** 2667 * Validate Geneve TLV option item. 2668 * 2669 * @param[in] item 2670 * Item specification. 2671 * @param[in] last_item 2672 * Previous validated item in the pattern items. 2673 * @param[in] geneve_item 2674 * Previous GENEVE item specification. 2675 * @param[in] dev 2676 * Pointer to the rte_eth_dev structure. 2677 * @param[out] error 2678 * Pointer to error structure. 2679 * 2680 * @return 2681 * 0 on success, a negative errno value otherwise and rte_errno is set. 2682 */ 2683 int 2684 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2685 uint64_t last_item, 2686 const struct rte_flow_item *geneve_item, 2687 struct rte_eth_dev *dev, 2688 struct rte_flow_error *error) 2689 { 2690 struct mlx5_priv *priv = dev->data->dev_private; 2691 struct mlx5_dev_ctx_shared *sh = priv->sh; 2692 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2693 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2694 uint8_t data_max_supported = 2695 hca_attr->max_geneve_tlv_option_data_len * 4; 2696 struct mlx5_dev_config *config = &priv->config; 2697 const struct rte_flow_item_geneve *geneve_spec; 2698 const struct rte_flow_item_geneve *geneve_mask; 2699 const struct rte_flow_item_geneve_opt *spec = item->spec; 2700 const struct rte_flow_item_geneve_opt *mask = item->mask; 2701 unsigned int i; 2702 unsigned int data_len; 2703 uint8_t tlv_option_len; 2704 uint16_t optlen_m, optlen_v; 2705 const struct rte_flow_item_geneve_opt full_mask = { 2706 .option_class = RTE_BE16(0xffff), 2707 .option_type = 0xff, 2708 .option_len = 0x1f, 2709 }; 2710 2711 if (!mask) 2712 mask = &rte_flow_item_geneve_opt_mask; 2713 if (!spec) 2714 return rte_flow_error_set 2715 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2716 "Geneve TLV opt class/type/length must be specified"); 2717 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2718 return rte_flow_error_set 2719 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2720 "Geneve TLV opt length exceeeds the limit (31)"); 2721 /* Check if class type and length masks are full. */ 2722 if (full_mask.option_class != mask->option_class || 2723 full_mask.option_type != mask->option_type || 2724 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2725 return rte_flow_error_set 2726 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2727 "Geneve TLV opt class/type/length masks must be full"); 2728 /* Check if length is supported */ 2729 if ((uint32_t)spec->option_len > 2730 config->hca_attr.max_geneve_tlv_option_data_len) 2731 return rte_flow_error_set 2732 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2733 "Geneve TLV opt length not supported"); 2734 if (config->hca_attr.max_geneve_tlv_options > 1) 2735 DRV_LOG(DEBUG, 2736 "max_geneve_tlv_options supports more than 1 option"); 2737 /* Check GENEVE item preceding. */ 2738 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2739 return rte_flow_error_set 2740 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2741 "Geneve opt item must be preceded with Geneve item"); 2742 geneve_spec = geneve_item->spec; 2743 geneve_mask = geneve_item->mask ? geneve_item->mask : 2744 &rte_flow_item_geneve_mask; 2745 /* Check if GENEVE TLV option size doesn't exceed option length */ 2746 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2747 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2748 tlv_option_len = spec->option_len & mask->option_len; 2749 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2750 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2751 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2752 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2753 if ((optlen_v & optlen_m) <= tlv_option_len) 2754 return rte_flow_error_set 2755 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2756 "GENEVE TLV option length exceeds optlen"); 2757 } 2758 /* Check if length is 0 or data is 0. */ 2759 if (spec->data == NULL || spec->option_len == 0) 2760 return rte_flow_error_set 2761 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2762 "Geneve TLV opt with zero data/length not supported"); 2763 /* Check not all data & mask are 0. */ 2764 data_len = spec->option_len * 4; 2765 if (mask->data == NULL) { 2766 for (i = 0; i < data_len; i++) 2767 if (spec->data[i]) 2768 break; 2769 if (i == data_len) 2770 return rte_flow_error_set(error, ENOTSUP, 2771 RTE_FLOW_ERROR_TYPE_ITEM, item, 2772 "Can't match on Geneve option data 0"); 2773 } else { 2774 for (i = 0; i < data_len; i++) 2775 if (spec->data[i] & mask->data[i]) 2776 break; 2777 if (i == data_len) 2778 return rte_flow_error_set(error, ENOTSUP, 2779 RTE_FLOW_ERROR_TYPE_ITEM, item, 2780 "Can't match on Geneve option data and mask 0"); 2781 /* Check data mask supported. */ 2782 for (i = data_max_supported; i < data_len ; i++) 2783 if (mask->data[i]) 2784 return rte_flow_error_set(error, ENOTSUP, 2785 RTE_FLOW_ERROR_TYPE_ITEM, item, 2786 "Data mask is of unsupported size"); 2787 } 2788 /* Check GENEVE option is supported in NIC. */ 2789 if (!config->hca_attr.geneve_tlv_opt) 2790 return rte_flow_error_set 2791 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2792 "Geneve TLV opt not supported"); 2793 /* Check if we already have geneve option with different type/class. */ 2794 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2795 geneve_opt_resource = sh->geneve_tlv_option_resource; 2796 if (geneve_opt_resource != NULL) 2797 if (geneve_opt_resource->option_class != spec->option_class || 2798 geneve_opt_resource->option_type != spec->option_type || 2799 geneve_opt_resource->length != spec->option_len) { 2800 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2801 return rte_flow_error_set(error, ENOTSUP, 2802 RTE_FLOW_ERROR_TYPE_ITEM, item, 2803 "Only one Geneve TLV option supported"); 2804 } 2805 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2806 return 0; 2807 } 2808 2809 /** 2810 * Validate MPLS item. 2811 * 2812 * @param[in] dev 2813 * Pointer to the rte_eth_dev structure. 2814 * @param[in] item 2815 * Item specification. 2816 * @param[in] item_flags 2817 * Bit-fields that holds the items detected until now. 2818 * @param[in] prev_layer 2819 * The protocol layer indicated in previous item. 2820 * @param[out] error 2821 * Pointer to error structure. 2822 * 2823 * @return 2824 * 0 on success, a negative errno value otherwise and rte_errno is set. 2825 */ 2826 int 2827 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2828 const struct rte_flow_item *item __rte_unused, 2829 uint64_t item_flags __rte_unused, 2830 uint64_t prev_layer __rte_unused, 2831 struct rte_flow_error *error) 2832 { 2833 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2834 const struct rte_flow_item_mpls *mask = item->mask; 2835 struct mlx5_priv *priv = dev->data->dev_private; 2836 int ret; 2837 2838 if (!priv->config.mpls_en) 2839 return rte_flow_error_set(error, ENOTSUP, 2840 RTE_FLOW_ERROR_TYPE_ITEM, item, 2841 "MPLS not supported or" 2842 " disabled in firmware" 2843 " configuration."); 2844 /* MPLS over IP, UDP, GRE is allowed */ 2845 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2846 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2847 MLX5_FLOW_LAYER_GRE | 2848 MLX5_FLOW_LAYER_GRE_KEY))) 2849 return rte_flow_error_set(error, EINVAL, 2850 RTE_FLOW_ERROR_TYPE_ITEM, item, 2851 "protocol filtering not compatible" 2852 " with MPLS layer"); 2853 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2854 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2855 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2856 return rte_flow_error_set(error, ENOTSUP, 2857 RTE_FLOW_ERROR_TYPE_ITEM, item, 2858 "multiple tunnel layers not" 2859 " supported"); 2860 if (!mask) 2861 mask = &rte_flow_item_mpls_mask; 2862 ret = mlx5_flow_item_acceptable 2863 (item, (const uint8_t *)mask, 2864 (const uint8_t *)&rte_flow_item_mpls_mask, 2865 sizeof(struct rte_flow_item_mpls), 2866 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2867 if (ret < 0) 2868 return ret; 2869 return 0; 2870 #else 2871 return rte_flow_error_set(error, ENOTSUP, 2872 RTE_FLOW_ERROR_TYPE_ITEM, item, 2873 "MPLS is not supported by Verbs, please" 2874 " update."); 2875 #endif 2876 } 2877 2878 /** 2879 * Validate NVGRE item. 2880 * 2881 * @param[in] item 2882 * Item specification. 2883 * @param[in] item_flags 2884 * Bit flags to mark detected items. 2885 * @param[in] target_protocol 2886 * The next protocol in the previous item. 2887 * @param[out] error 2888 * Pointer to error structure. 2889 * 2890 * @return 2891 * 0 on success, a negative errno value otherwise and rte_errno is set. 2892 */ 2893 int 2894 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2895 uint64_t item_flags, 2896 uint8_t target_protocol, 2897 struct rte_flow_error *error) 2898 { 2899 const struct rte_flow_item_nvgre *mask = item->mask; 2900 int ret; 2901 2902 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2903 return rte_flow_error_set(error, EINVAL, 2904 RTE_FLOW_ERROR_TYPE_ITEM, item, 2905 "protocol filtering not compatible" 2906 " with this GRE layer"); 2907 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2908 return rte_flow_error_set(error, ENOTSUP, 2909 RTE_FLOW_ERROR_TYPE_ITEM, item, 2910 "multiple tunnel layers not" 2911 " supported"); 2912 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2913 return rte_flow_error_set(error, ENOTSUP, 2914 RTE_FLOW_ERROR_TYPE_ITEM, item, 2915 "L3 Layer is missing"); 2916 if (!mask) 2917 mask = &rte_flow_item_nvgre_mask; 2918 ret = mlx5_flow_item_acceptable 2919 (item, (const uint8_t *)mask, 2920 (const uint8_t *)&rte_flow_item_nvgre_mask, 2921 sizeof(struct rte_flow_item_nvgre), 2922 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2923 if (ret < 0) 2924 return ret; 2925 return 0; 2926 } 2927 2928 /** 2929 * Validate eCPRI item. 2930 * 2931 * @param[in] item 2932 * Item specification. 2933 * @param[in] item_flags 2934 * Bit-fields that holds the items detected until now. 2935 * @param[in] last_item 2936 * Previous validated item in the pattern items. 2937 * @param[in] ether_type 2938 * Type in the ethernet layer header (including dot1q). 2939 * @param[in] acc_mask 2940 * Acceptable mask, if NULL default internal default mask 2941 * will be used to check whether item fields are supported. 2942 * @param[out] error 2943 * Pointer to error structure. 2944 * 2945 * @return 2946 * 0 on success, a negative errno value otherwise and rte_errno is set. 2947 */ 2948 int 2949 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 2950 uint64_t item_flags, 2951 uint64_t last_item, 2952 uint16_t ether_type, 2953 const struct rte_flow_item_ecpri *acc_mask, 2954 struct rte_flow_error *error) 2955 { 2956 const struct rte_flow_item_ecpri *mask = item->mask; 2957 const struct rte_flow_item_ecpri nic_mask = { 2958 .hdr = { 2959 .common = { 2960 .u32 = 2961 RTE_BE32(((const struct rte_ecpri_common_hdr) { 2962 .type = 0xFF, 2963 }).u32), 2964 }, 2965 .dummy[0] = 0xFFFFFFFF, 2966 }, 2967 }; 2968 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 2969 MLX5_FLOW_LAYER_OUTER_VLAN); 2970 struct rte_flow_item_ecpri mask_lo; 2971 2972 if (!(last_item & outer_l2_vlan) && 2973 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 2974 return rte_flow_error_set(error, EINVAL, 2975 RTE_FLOW_ERROR_TYPE_ITEM, item, 2976 "eCPRI can only follow L2/VLAN layer or UDP layer"); 2977 if ((last_item & outer_l2_vlan) && ether_type && 2978 ether_type != RTE_ETHER_TYPE_ECPRI) 2979 return rte_flow_error_set(error, EINVAL, 2980 RTE_FLOW_ERROR_TYPE_ITEM, item, 2981 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 2982 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2983 return rte_flow_error_set(error, EINVAL, 2984 RTE_FLOW_ERROR_TYPE_ITEM, item, 2985 "eCPRI with tunnel is not supported right now"); 2986 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 2987 return rte_flow_error_set(error, ENOTSUP, 2988 RTE_FLOW_ERROR_TYPE_ITEM, item, 2989 "multiple L3 layers not supported"); 2990 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 2991 return rte_flow_error_set(error, EINVAL, 2992 RTE_FLOW_ERROR_TYPE_ITEM, item, 2993 "eCPRI cannot coexist with a TCP layer"); 2994 /* In specification, eCPRI could be over UDP layer. */ 2995 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 2996 return rte_flow_error_set(error, EINVAL, 2997 RTE_FLOW_ERROR_TYPE_ITEM, item, 2998 "eCPRI over UDP layer is not yet supported right now"); 2999 /* Mask for type field in common header could be zero. */ 3000 if (!mask) 3001 mask = &rte_flow_item_ecpri_mask; 3002 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3003 /* Input mask is in big-endian format. */ 3004 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3005 return rte_flow_error_set(error, EINVAL, 3006 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3007 "partial mask is not supported for protocol"); 3008 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3009 return rte_flow_error_set(error, EINVAL, 3010 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3011 "message header mask must be after a type mask"); 3012 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3013 acc_mask ? (const uint8_t *)acc_mask 3014 : (const uint8_t *)&nic_mask, 3015 sizeof(struct rte_flow_item_ecpri), 3016 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3017 } 3018 3019 /** 3020 * Release resource related QUEUE/RSS action split. 3021 * 3022 * @param dev 3023 * Pointer to Ethernet device. 3024 * @param flow 3025 * Flow to release id's from. 3026 */ 3027 static void 3028 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 3029 struct rte_flow *flow) 3030 { 3031 struct mlx5_priv *priv = dev->data->dev_private; 3032 uint32_t handle_idx; 3033 struct mlx5_flow_handle *dev_handle; 3034 3035 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 3036 handle_idx, dev_handle, next) 3037 if (dev_handle->split_flow_id) 3038 mlx5_ipool_free(priv->sh->ipool 3039 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 3040 dev_handle->split_flow_id); 3041 } 3042 3043 static int 3044 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3045 const struct rte_flow_attr *attr __rte_unused, 3046 const struct rte_flow_item items[] __rte_unused, 3047 const struct rte_flow_action actions[] __rte_unused, 3048 bool external __rte_unused, 3049 int hairpin __rte_unused, 3050 struct rte_flow_error *error) 3051 { 3052 return rte_flow_error_set(error, ENOTSUP, 3053 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3054 } 3055 3056 static struct mlx5_flow * 3057 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3058 const struct rte_flow_attr *attr __rte_unused, 3059 const struct rte_flow_item items[] __rte_unused, 3060 const struct rte_flow_action actions[] __rte_unused, 3061 struct rte_flow_error *error) 3062 { 3063 rte_flow_error_set(error, ENOTSUP, 3064 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3065 return NULL; 3066 } 3067 3068 static int 3069 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3070 struct mlx5_flow *dev_flow __rte_unused, 3071 const struct rte_flow_attr *attr __rte_unused, 3072 const struct rte_flow_item items[] __rte_unused, 3073 const struct rte_flow_action actions[] __rte_unused, 3074 struct rte_flow_error *error) 3075 { 3076 return rte_flow_error_set(error, ENOTSUP, 3077 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3078 } 3079 3080 static int 3081 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3082 struct rte_flow *flow __rte_unused, 3083 struct rte_flow_error *error) 3084 { 3085 return rte_flow_error_set(error, ENOTSUP, 3086 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3087 } 3088 3089 static void 3090 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3091 struct rte_flow *flow __rte_unused) 3092 { 3093 } 3094 3095 static void 3096 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3097 struct rte_flow *flow __rte_unused) 3098 { 3099 } 3100 3101 static int 3102 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3103 struct rte_flow *flow __rte_unused, 3104 const struct rte_flow_action *actions __rte_unused, 3105 void *data __rte_unused, 3106 struct rte_flow_error *error) 3107 { 3108 return rte_flow_error_set(error, ENOTSUP, 3109 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3110 } 3111 3112 static int 3113 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3114 uint32_t domains __rte_unused, 3115 uint32_t flags __rte_unused) 3116 { 3117 return 0; 3118 } 3119 3120 /* Void driver to protect from null pointer reference. */ 3121 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3122 .validate = flow_null_validate, 3123 .prepare = flow_null_prepare, 3124 .translate = flow_null_translate, 3125 .apply = flow_null_apply, 3126 .remove = flow_null_remove, 3127 .destroy = flow_null_destroy, 3128 .query = flow_null_query, 3129 .sync_domain = flow_null_sync_domain, 3130 }; 3131 3132 /** 3133 * Select flow driver type according to flow attributes and device 3134 * configuration. 3135 * 3136 * @param[in] dev 3137 * Pointer to the dev structure. 3138 * @param[in] attr 3139 * Pointer to the flow attributes. 3140 * 3141 * @return 3142 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3143 */ 3144 static enum mlx5_flow_drv_type 3145 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3146 { 3147 struct mlx5_priv *priv = dev->data->dev_private; 3148 /* The OS can determine first a specific flow type (DV, VERBS) */ 3149 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3150 3151 if (type != MLX5_FLOW_TYPE_MAX) 3152 return type; 3153 /* If no OS specific type - continue with DV/VERBS selection */ 3154 if (attr->transfer && priv->config.dv_esw_en) 3155 type = MLX5_FLOW_TYPE_DV; 3156 if (!attr->transfer) 3157 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3158 MLX5_FLOW_TYPE_VERBS; 3159 return type; 3160 } 3161 3162 #define flow_get_drv_ops(type) flow_drv_ops[type] 3163 3164 /** 3165 * Flow driver validation API. This abstracts calling driver specific functions. 3166 * The type of flow driver is determined according to flow attributes. 3167 * 3168 * @param[in] dev 3169 * Pointer to the dev structure. 3170 * @param[in] attr 3171 * Pointer to the flow attributes. 3172 * @param[in] items 3173 * Pointer to the list of items. 3174 * @param[in] actions 3175 * Pointer to the list of actions. 3176 * @param[in] external 3177 * This flow rule is created by request external to PMD. 3178 * @param[in] hairpin 3179 * Number of hairpin TX actions, 0 means classic flow. 3180 * @param[out] error 3181 * Pointer to the error structure. 3182 * 3183 * @return 3184 * 0 on success, a negative errno value otherwise and rte_errno is set. 3185 */ 3186 static inline int 3187 flow_drv_validate(struct rte_eth_dev *dev, 3188 const struct rte_flow_attr *attr, 3189 const struct rte_flow_item items[], 3190 const struct rte_flow_action actions[], 3191 bool external, int hairpin, struct rte_flow_error *error) 3192 { 3193 const struct mlx5_flow_driver_ops *fops; 3194 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3195 3196 fops = flow_get_drv_ops(type); 3197 return fops->validate(dev, attr, items, actions, external, 3198 hairpin, error); 3199 } 3200 3201 /** 3202 * Flow driver preparation API. This abstracts calling driver specific 3203 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3204 * calculates the size of memory required for device flow, allocates the memory, 3205 * initializes the device flow and returns the pointer. 3206 * 3207 * @note 3208 * This function initializes device flow structure such as dv or verbs in 3209 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3210 * rest. For example, adding returning device flow to flow->dev_flow list and 3211 * setting backward reference to the flow should be done out of this function. 3212 * layers field is not filled either. 3213 * 3214 * @param[in] dev 3215 * Pointer to the dev structure. 3216 * @param[in] attr 3217 * Pointer to the flow attributes. 3218 * @param[in] items 3219 * Pointer to the list of items. 3220 * @param[in] actions 3221 * Pointer to the list of actions. 3222 * @param[in] flow_idx 3223 * This memory pool index to the flow. 3224 * @param[out] error 3225 * Pointer to the error structure. 3226 * 3227 * @return 3228 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3229 */ 3230 static inline struct mlx5_flow * 3231 flow_drv_prepare(struct rte_eth_dev *dev, 3232 const struct rte_flow *flow, 3233 const struct rte_flow_attr *attr, 3234 const struct rte_flow_item items[], 3235 const struct rte_flow_action actions[], 3236 uint32_t flow_idx, 3237 struct rte_flow_error *error) 3238 { 3239 const struct mlx5_flow_driver_ops *fops; 3240 enum mlx5_flow_drv_type type = flow->drv_type; 3241 struct mlx5_flow *mlx5_flow = NULL; 3242 3243 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3244 fops = flow_get_drv_ops(type); 3245 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3246 if (mlx5_flow) 3247 mlx5_flow->flow_idx = flow_idx; 3248 return mlx5_flow; 3249 } 3250 3251 /** 3252 * Flow driver translation API. This abstracts calling driver specific 3253 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3254 * translates a generic flow into a driver flow. flow_drv_prepare() must 3255 * precede. 3256 * 3257 * @note 3258 * dev_flow->layers could be filled as a result of parsing during translation 3259 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3260 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3261 * flow->actions could be overwritten even though all the expanded dev_flows 3262 * have the same actions. 3263 * 3264 * @param[in] dev 3265 * Pointer to the rte dev structure. 3266 * @param[in, out] dev_flow 3267 * Pointer to the mlx5 flow. 3268 * @param[in] attr 3269 * Pointer to the flow attributes. 3270 * @param[in] items 3271 * Pointer to the list of items. 3272 * @param[in] actions 3273 * Pointer to the list of actions. 3274 * @param[out] error 3275 * Pointer to the error structure. 3276 * 3277 * @return 3278 * 0 on success, a negative errno value otherwise and rte_errno is set. 3279 */ 3280 static inline int 3281 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3282 const struct rte_flow_attr *attr, 3283 const struct rte_flow_item items[], 3284 const struct rte_flow_action actions[], 3285 struct rte_flow_error *error) 3286 { 3287 const struct mlx5_flow_driver_ops *fops; 3288 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3289 3290 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3291 fops = flow_get_drv_ops(type); 3292 return fops->translate(dev, dev_flow, attr, items, actions, error); 3293 } 3294 3295 /** 3296 * Flow driver apply API. This abstracts calling driver specific functions. 3297 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3298 * translated driver flows on to device. flow_drv_translate() must precede. 3299 * 3300 * @param[in] dev 3301 * Pointer to Ethernet device structure. 3302 * @param[in, out] flow 3303 * Pointer to flow structure. 3304 * @param[out] error 3305 * Pointer to error structure. 3306 * 3307 * @return 3308 * 0 on success, a negative errno value otherwise and rte_errno is set. 3309 */ 3310 static inline int 3311 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3312 struct rte_flow_error *error) 3313 { 3314 const struct mlx5_flow_driver_ops *fops; 3315 enum mlx5_flow_drv_type type = flow->drv_type; 3316 3317 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3318 fops = flow_get_drv_ops(type); 3319 return fops->apply(dev, flow, error); 3320 } 3321 3322 /** 3323 * Flow driver destroy API. This abstracts calling driver specific functions. 3324 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3325 * on device and releases resources of the flow. 3326 * 3327 * @param[in] dev 3328 * Pointer to Ethernet device. 3329 * @param[in, out] flow 3330 * Pointer to flow structure. 3331 */ 3332 static inline void 3333 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3334 { 3335 const struct mlx5_flow_driver_ops *fops; 3336 enum mlx5_flow_drv_type type = flow->drv_type; 3337 3338 flow_mreg_split_qrss_release(dev, flow); 3339 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3340 fops = flow_get_drv_ops(type); 3341 fops->destroy(dev, flow); 3342 } 3343 3344 /** 3345 * Get RSS action from the action list. 3346 * 3347 * @param[in] actions 3348 * Pointer to the list of actions. 3349 * 3350 * @return 3351 * Pointer to the RSS action if exist, else return NULL. 3352 */ 3353 static const struct rte_flow_action_rss* 3354 flow_get_rss_action(const struct rte_flow_action actions[]) 3355 { 3356 const struct rte_flow_action_rss *rss = NULL; 3357 3358 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3359 switch (actions->type) { 3360 case RTE_FLOW_ACTION_TYPE_RSS: 3361 rss = actions->conf; 3362 break; 3363 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3364 { 3365 const struct rte_flow_action_sample *sample = 3366 actions->conf; 3367 const struct rte_flow_action *act = sample->actions; 3368 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3369 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3370 rss = act->conf; 3371 break; 3372 } 3373 default: 3374 break; 3375 } 3376 } 3377 return rss; 3378 } 3379 3380 /** 3381 * Get ASO age action by index. 3382 * 3383 * @param[in] dev 3384 * Pointer to the Ethernet device structure. 3385 * @param[in] age_idx 3386 * Index to the ASO age action. 3387 * 3388 * @return 3389 * The specified ASO age action. 3390 */ 3391 struct mlx5_aso_age_action* 3392 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3393 { 3394 uint16_t pool_idx = age_idx & UINT16_MAX; 3395 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3396 struct mlx5_priv *priv = dev->data->dev_private; 3397 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3398 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3399 3400 return &pool->actions[offset - 1]; 3401 } 3402 3403 /* maps shared action to translated non shared in some actions array */ 3404 struct mlx5_translated_shared_action { 3405 struct rte_flow_shared_action *action; /**< Shared action */ 3406 int index; /**< Index in related array of rte_flow_action */ 3407 }; 3408 3409 /** 3410 * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related 3411 * non shared action if translation possible. 3412 * This functionality used to run same execution path for both shared & non 3413 * shared actions on flow create. All necessary preparations for shared 3414 * action handling should be preformed on *shared* actions list returned 3415 * from this call. 3416 * 3417 * @param[in] dev 3418 * Pointer to Ethernet device. 3419 * @param[in] actions 3420 * List of actions to translate. 3421 * @param[out] shared 3422 * List to store translated shared actions. 3423 * @param[in, out] shared_n 3424 * Size of *shared* array. On return should be updated with number of shared 3425 * actions retrieved from the *actions* list. 3426 * @param[out] translated_actions 3427 * List of actions where all shared actions were translated to non shared 3428 * if possible. NULL if no translation took place. 3429 * @param[out] error 3430 * Pointer to the error structure. 3431 * 3432 * @return 3433 * 0 on success, a negative errno value otherwise and rte_errno is set. 3434 */ 3435 static int 3436 flow_shared_actions_translate(struct rte_eth_dev *dev, 3437 const struct rte_flow_action actions[], 3438 struct mlx5_translated_shared_action *shared, 3439 int *shared_n, 3440 struct rte_flow_action **translated_actions, 3441 struct rte_flow_error *error) 3442 { 3443 struct mlx5_priv *priv = dev->data->dev_private; 3444 struct rte_flow_action *translated = NULL; 3445 size_t actions_size; 3446 int n; 3447 int copied_n = 0; 3448 struct mlx5_translated_shared_action *shared_end = NULL; 3449 3450 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3451 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED) 3452 continue; 3453 if (copied_n == *shared_n) { 3454 return rte_flow_error_set 3455 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3456 NULL, "too many shared actions"); 3457 } 3458 rte_memcpy(&shared[copied_n].action, &actions[n].conf, 3459 sizeof(actions[n].conf)); 3460 shared[copied_n].index = n; 3461 copied_n++; 3462 } 3463 n++; 3464 *shared_n = copied_n; 3465 if (!copied_n) 3466 return 0; 3467 actions_size = sizeof(struct rte_flow_action) * n; 3468 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3469 if (!translated) { 3470 rte_errno = ENOMEM; 3471 return -ENOMEM; 3472 } 3473 memcpy(translated, actions, actions_size); 3474 for (shared_end = shared + copied_n; shared < shared_end; shared++) { 3475 struct mlx5_shared_action_rss *shared_rss; 3476 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; 3477 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; 3478 uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) 3479 - 1); 3480 3481 switch (type) { 3482 case MLX5_SHARED_ACTION_TYPE_RSS: 3483 shared_rss = mlx5_ipool_get 3484 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3485 translated[shared->index].type = 3486 RTE_FLOW_ACTION_TYPE_RSS; 3487 translated[shared->index].conf = 3488 &shared_rss->origin; 3489 break; 3490 case MLX5_SHARED_ACTION_TYPE_AGE: 3491 if (priv->sh->flow_hit_aso_en) { 3492 translated[shared->index].type = 3493 (enum rte_flow_action_type) 3494 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3495 translated[shared->index].conf = 3496 (void *)(uintptr_t)idx; 3497 break; 3498 } 3499 /* Fall-through */ 3500 default: 3501 mlx5_free(translated); 3502 return rte_flow_error_set 3503 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3504 NULL, "invalid shared action type"); 3505 } 3506 } 3507 *translated_actions = translated; 3508 return 0; 3509 } 3510 3511 /** 3512 * Get Shared RSS action from the action list. 3513 * 3514 * @param[in] dev 3515 * Pointer to Ethernet device. 3516 * @param[in] shared 3517 * Pointer to the list of actions. 3518 * @param[in] shared_n 3519 * Actions list length. 3520 * 3521 * @return 3522 * The MLX5 RSS action ID if exists, otherwise return 0. 3523 */ 3524 static uint32_t 3525 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3526 struct mlx5_translated_shared_action *shared, 3527 int shared_n) 3528 { 3529 struct mlx5_translated_shared_action *shared_end; 3530 struct mlx5_priv *priv = dev->data->dev_private; 3531 struct mlx5_shared_action_rss *shared_rss; 3532 3533 3534 for (shared_end = shared + shared_n; shared < shared_end; shared++) { 3535 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; 3536 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; 3537 uint32_t idx = act_idx & 3538 ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); 3539 switch (type) { 3540 case MLX5_SHARED_ACTION_TYPE_RSS: 3541 shared_rss = mlx5_ipool_get 3542 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3543 idx); 3544 __atomic_add_fetch(&shared_rss->refcnt, 1, 3545 __ATOMIC_RELAXED); 3546 return idx; 3547 default: 3548 break; 3549 } 3550 } 3551 return 0; 3552 } 3553 3554 static unsigned int 3555 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3556 { 3557 const struct rte_flow_item *item; 3558 unsigned int has_vlan = 0; 3559 3560 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3561 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3562 has_vlan = 1; 3563 break; 3564 } 3565 } 3566 if (has_vlan) 3567 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3568 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3569 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3570 MLX5_EXPANSION_ROOT_OUTER; 3571 } 3572 3573 /** 3574 * Get layer flags from the prefix flow. 3575 * 3576 * Some flows may be split to several subflows, the prefix subflow gets the 3577 * match items and the suffix sub flow gets the actions. 3578 * Some actions need the user defined match item flags to get the detail for 3579 * the action. 3580 * This function helps the suffix flow to get the item layer flags from prefix 3581 * subflow. 3582 * 3583 * @param[in] dev_flow 3584 * Pointer the created preifx subflow. 3585 * 3586 * @return 3587 * The layers get from prefix subflow. 3588 */ 3589 static inline uint64_t 3590 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3591 { 3592 uint64_t layers = 0; 3593 3594 /* 3595 * Layers bits could be localization, but usually the compiler will 3596 * help to do the optimization work for source code. 3597 * If no decap actions, use the layers directly. 3598 */ 3599 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3600 return dev_flow->handle->layers; 3601 /* Convert L3 layers with decap action. */ 3602 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3603 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3604 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3605 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3606 /* Convert L4 layers with decap action. */ 3607 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3608 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3609 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3610 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3611 return layers; 3612 } 3613 3614 /** 3615 * Get metadata split action information. 3616 * 3617 * @param[in] actions 3618 * Pointer to the list of actions. 3619 * @param[out] qrss 3620 * Pointer to the return pointer. 3621 * @param[out] qrss_type 3622 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3623 * if no QUEUE/RSS is found. 3624 * @param[out] encap_idx 3625 * Pointer to the index of the encap action if exists, otherwise the last 3626 * action index. 3627 * 3628 * @return 3629 * Total number of actions. 3630 */ 3631 static int 3632 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3633 const struct rte_flow_action **qrss, 3634 int *encap_idx) 3635 { 3636 const struct rte_flow_action_raw_encap *raw_encap; 3637 int actions_n = 0; 3638 int raw_decap_idx = -1; 3639 3640 *encap_idx = -1; 3641 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3642 switch (actions->type) { 3643 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3644 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3645 *encap_idx = actions_n; 3646 break; 3647 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3648 raw_decap_idx = actions_n; 3649 break; 3650 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3651 raw_encap = actions->conf; 3652 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3653 *encap_idx = raw_decap_idx != -1 ? 3654 raw_decap_idx : actions_n; 3655 break; 3656 case RTE_FLOW_ACTION_TYPE_QUEUE: 3657 case RTE_FLOW_ACTION_TYPE_RSS: 3658 *qrss = actions; 3659 break; 3660 default: 3661 break; 3662 } 3663 actions_n++; 3664 } 3665 if (*encap_idx == -1) 3666 *encap_idx = actions_n; 3667 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3668 return actions_n + 1; 3669 } 3670 3671 /** 3672 * Check meter action from the action list. 3673 * 3674 * @param[in] actions 3675 * Pointer to the list of actions. 3676 * @param[out] mtr 3677 * Pointer to the meter exist flag. 3678 * 3679 * @return 3680 * Total number of actions. 3681 */ 3682 static int 3683 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 3684 { 3685 int actions_n = 0; 3686 3687 MLX5_ASSERT(mtr); 3688 *mtr = 0; 3689 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3690 switch (actions->type) { 3691 case RTE_FLOW_ACTION_TYPE_METER: 3692 *mtr = 1; 3693 break; 3694 default: 3695 break; 3696 } 3697 actions_n++; 3698 } 3699 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3700 return actions_n + 1; 3701 } 3702 3703 /** 3704 * Check if the flow should be split due to hairpin. 3705 * The reason for the split is that in current HW we can't 3706 * support encap and push-vlan on Rx, so if a flow contains 3707 * these actions we move it to Tx. 3708 * 3709 * @param dev 3710 * Pointer to Ethernet device. 3711 * @param[in] attr 3712 * Flow rule attributes. 3713 * @param[in] actions 3714 * Associated actions (list terminated by the END action). 3715 * 3716 * @return 3717 * > 0 the number of actions and the flow should be split, 3718 * 0 when no split required. 3719 */ 3720 static int 3721 flow_check_hairpin_split(struct rte_eth_dev *dev, 3722 const struct rte_flow_attr *attr, 3723 const struct rte_flow_action actions[]) 3724 { 3725 int queue_action = 0; 3726 int action_n = 0; 3727 int split = 0; 3728 const struct rte_flow_action_queue *queue; 3729 const struct rte_flow_action_rss *rss; 3730 const struct rte_flow_action_raw_encap *raw_encap; 3731 const struct rte_eth_hairpin_conf *conf; 3732 3733 if (!attr->ingress) 3734 return 0; 3735 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3736 switch (actions->type) { 3737 case RTE_FLOW_ACTION_TYPE_QUEUE: 3738 queue = actions->conf; 3739 if (queue == NULL) 3740 return 0; 3741 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3742 if (conf == NULL || conf->tx_explicit != 0) 3743 return 0; 3744 queue_action = 1; 3745 action_n++; 3746 break; 3747 case RTE_FLOW_ACTION_TYPE_RSS: 3748 rss = actions->conf; 3749 if (rss == NULL || rss->queue_num == 0) 3750 return 0; 3751 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3752 if (conf == NULL || conf->tx_explicit != 0) 3753 return 0; 3754 queue_action = 1; 3755 action_n++; 3756 break; 3757 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3758 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3759 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3760 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3761 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3762 split++; 3763 action_n++; 3764 break; 3765 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3766 raw_encap = actions->conf; 3767 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3768 split++; 3769 action_n++; 3770 break; 3771 default: 3772 action_n++; 3773 break; 3774 } 3775 } 3776 if (split && queue_action) 3777 return action_n; 3778 return 0; 3779 } 3780 3781 /* Declare flow create/destroy prototype in advance. */ 3782 static uint32_t 3783 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3784 const struct rte_flow_attr *attr, 3785 const struct rte_flow_item items[], 3786 const struct rte_flow_action actions[], 3787 bool external, struct rte_flow_error *error); 3788 3789 static void 3790 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3791 uint32_t flow_idx); 3792 3793 int 3794 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused, 3795 struct mlx5_hlist_entry *entry, 3796 uint64_t key, void *cb_ctx __rte_unused) 3797 { 3798 struct mlx5_flow_mreg_copy_resource *mcp_res = 3799 container_of(entry, typeof(*mcp_res), hlist_ent); 3800 3801 return mcp_res->mark_id != key; 3802 } 3803 3804 struct mlx5_hlist_entry * 3805 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 3806 void *cb_ctx) 3807 { 3808 struct rte_eth_dev *dev = list->ctx; 3809 struct mlx5_priv *priv = dev->data->dev_private; 3810 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3811 struct mlx5_flow_mreg_copy_resource *mcp_res; 3812 struct rte_flow_error *error = ctx->error; 3813 uint32_t idx = 0; 3814 int ret; 3815 uint32_t mark_id = key; 3816 struct rte_flow_attr attr = { 3817 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3818 .ingress = 1, 3819 }; 3820 struct mlx5_rte_flow_item_tag tag_spec = { 3821 .data = mark_id, 3822 }; 3823 struct rte_flow_item items[] = { 3824 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 3825 }; 3826 struct rte_flow_action_mark ftag = { 3827 .id = mark_id, 3828 }; 3829 struct mlx5_flow_action_copy_mreg cp_mreg = { 3830 .dst = REG_B, 3831 .src = REG_NON, 3832 }; 3833 struct rte_flow_action_jump jump = { 3834 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3835 }; 3836 struct rte_flow_action actions[] = { 3837 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 3838 }; 3839 3840 /* Fill the register fileds in the flow. */ 3841 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3842 if (ret < 0) 3843 return NULL; 3844 tag_spec.id = ret; 3845 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3846 if (ret < 0) 3847 return NULL; 3848 cp_mreg.src = ret; 3849 /* Provide the full width of FLAG specific value. */ 3850 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3851 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3852 /* Build a new flow. */ 3853 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3854 items[0] = (struct rte_flow_item){ 3855 .type = (enum rte_flow_item_type) 3856 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3857 .spec = &tag_spec, 3858 }; 3859 items[1] = (struct rte_flow_item){ 3860 .type = RTE_FLOW_ITEM_TYPE_END, 3861 }; 3862 actions[0] = (struct rte_flow_action){ 3863 .type = (enum rte_flow_action_type) 3864 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3865 .conf = &ftag, 3866 }; 3867 actions[1] = (struct rte_flow_action){ 3868 .type = (enum rte_flow_action_type) 3869 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3870 .conf = &cp_mreg, 3871 }; 3872 actions[2] = (struct rte_flow_action){ 3873 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3874 .conf = &jump, 3875 }; 3876 actions[3] = (struct rte_flow_action){ 3877 .type = RTE_FLOW_ACTION_TYPE_END, 3878 }; 3879 } else { 3880 /* Default rule, wildcard match. */ 3881 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 3882 items[0] = (struct rte_flow_item){ 3883 .type = RTE_FLOW_ITEM_TYPE_END, 3884 }; 3885 actions[0] = (struct rte_flow_action){ 3886 .type = (enum rte_flow_action_type) 3887 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3888 .conf = &cp_mreg, 3889 }; 3890 actions[1] = (struct rte_flow_action){ 3891 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3892 .conf = &jump, 3893 }; 3894 actions[2] = (struct rte_flow_action){ 3895 .type = RTE_FLOW_ACTION_TYPE_END, 3896 }; 3897 } 3898 /* Build a new entry. */ 3899 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3900 if (!mcp_res) { 3901 rte_errno = ENOMEM; 3902 return NULL; 3903 } 3904 mcp_res->idx = idx; 3905 mcp_res->mark_id = mark_id; 3906 /* 3907 * The copy Flows are not included in any list. There 3908 * ones are referenced from other Flows and can not 3909 * be applied, removed, deleted in ardbitrary order 3910 * by list traversing. 3911 */ 3912 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3913 actions, false, error); 3914 if (!mcp_res->rix_flow) { 3915 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 3916 return NULL; 3917 } 3918 return &mcp_res->hlist_ent; 3919 } 3920 3921 /** 3922 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3923 * 3924 * As mark_id is unique, if there's already a registered flow for the mark_id, 3925 * return by increasing the reference counter of the resource. Otherwise, create 3926 * the resource (mcp_res) and flow. 3927 * 3928 * Flow looks like, 3929 * - If ingress port is ANY and reg_c[1] is mark_id, 3930 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3931 * 3932 * For default flow (zero mark_id), flow is like, 3933 * - If ingress port is ANY, 3934 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 3935 * 3936 * @param dev 3937 * Pointer to Ethernet device. 3938 * @param mark_id 3939 * ID of MARK action, zero means default flow for META. 3940 * @param[out] error 3941 * Perform verbose error reporting if not NULL. 3942 * 3943 * @return 3944 * Associated resource on success, NULL otherwise and rte_errno is set. 3945 */ 3946 static struct mlx5_flow_mreg_copy_resource * 3947 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 3948 struct rte_flow_error *error) 3949 { 3950 struct mlx5_priv *priv = dev->data->dev_private; 3951 struct mlx5_hlist_entry *entry; 3952 struct mlx5_flow_cb_ctx ctx = { 3953 .dev = dev, 3954 .error = error, 3955 }; 3956 3957 /* Check if already registered. */ 3958 MLX5_ASSERT(priv->mreg_cp_tbl); 3959 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 3960 if (!entry) 3961 return NULL; 3962 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 3963 hlist_ent); 3964 } 3965 3966 void 3967 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 3968 { 3969 struct mlx5_flow_mreg_copy_resource *mcp_res = 3970 container_of(entry, typeof(*mcp_res), hlist_ent); 3971 struct rte_eth_dev *dev = list->ctx; 3972 struct mlx5_priv *priv = dev->data->dev_private; 3973 3974 MLX5_ASSERT(mcp_res->rix_flow); 3975 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3976 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3977 } 3978 3979 /** 3980 * Release flow in RX_CP_TBL. 3981 * 3982 * @param dev 3983 * Pointer to Ethernet device. 3984 * @flow 3985 * Parent flow for wich copying is provided. 3986 */ 3987 static void 3988 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3989 struct rte_flow *flow) 3990 { 3991 struct mlx5_flow_mreg_copy_resource *mcp_res; 3992 struct mlx5_priv *priv = dev->data->dev_private; 3993 3994 if (!flow->rix_mreg_copy) 3995 return; 3996 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3997 flow->rix_mreg_copy); 3998 if (!mcp_res || !priv->mreg_cp_tbl) 3999 return; 4000 MLX5_ASSERT(mcp_res->rix_flow); 4001 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4002 flow->rix_mreg_copy = 0; 4003 } 4004 4005 /** 4006 * Remove the default copy action from RX_CP_TBL. 4007 * 4008 * This functions is called in the mlx5_dev_start(). No thread safe 4009 * is guaranteed. 4010 * 4011 * @param dev 4012 * Pointer to Ethernet device. 4013 */ 4014 static void 4015 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4016 { 4017 struct mlx5_hlist_entry *entry; 4018 struct mlx5_priv *priv = dev->data->dev_private; 4019 4020 /* Check if default flow is registered. */ 4021 if (!priv->mreg_cp_tbl) 4022 return; 4023 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 4024 MLX5_DEFAULT_COPY_ID, NULL); 4025 if (!entry) 4026 return; 4027 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4028 } 4029 4030 /** 4031 * Add the default copy action in in RX_CP_TBL. 4032 * 4033 * This functions is called in the mlx5_dev_start(). No thread safe 4034 * is guaranteed. 4035 * 4036 * @param dev 4037 * Pointer to Ethernet device. 4038 * @param[out] error 4039 * Perform verbose error reporting if not NULL. 4040 * 4041 * @return 4042 * 0 for success, negative value otherwise and rte_errno is set. 4043 */ 4044 static int 4045 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4046 struct rte_flow_error *error) 4047 { 4048 struct mlx5_priv *priv = dev->data->dev_private; 4049 struct mlx5_flow_mreg_copy_resource *mcp_res; 4050 4051 /* Check whether extensive metadata feature is engaged. */ 4052 if (!priv->config.dv_flow_en || 4053 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4054 !mlx5_flow_ext_mreg_supported(dev) || 4055 !priv->sh->dv_regc0_mask) 4056 return 0; 4057 /* 4058 * Add default mreg copy flow may be called multiple time, but 4059 * only be called once in stop. Avoid register it twice. 4060 */ 4061 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 4062 return 0; 4063 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 4064 if (!mcp_res) 4065 return -rte_errno; 4066 return 0; 4067 } 4068 4069 /** 4070 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4071 * 4072 * All the flow having Q/RSS action should be split by 4073 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4074 * performs the following, 4075 * - CQE->flow_tag := reg_c[1] (MARK) 4076 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4077 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4078 * but there should be a flow per each MARK ID set by MARK action. 4079 * 4080 * For the aforementioned reason, if there's a MARK action in flow's action 4081 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4082 * the MARK ID to CQE's flow_tag like, 4083 * - If reg_c[1] is mark_id, 4084 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4085 * 4086 * For SET_META action which stores value in reg_c[0], as the destination is 4087 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4088 * MARK ID means the default flow. The default flow looks like, 4089 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4090 * 4091 * @param dev 4092 * Pointer to Ethernet device. 4093 * @param flow 4094 * Pointer to flow structure. 4095 * @param[in] actions 4096 * Pointer to the list of actions. 4097 * @param[out] error 4098 * Perform verbose error reporting if not NULL. 4099 * 4100 * @return 4101 * 0 on success, negative value otherwise and rte_errno is set. 4102 */ 4103 static int 4104 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4105 struct rte_flow *flow, 4106 const struct rte_flow_action *actions, 4107 struct rte_flow_error *error) 4108 { 4109 struct mlx5_priv *priv = dev->data->dev_private; 4110 struct mlx5_dev_config *config = &priv->config; 4111 struct mlx5_flow_mreg_copy_resource *mcp_res; 4112 const struct rte_flow_action_mark *mark; 4113 4114 /* Check whether extensive metadata feature is engaged. */ 4115 if (!config->dv_flow_en || 4116 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4117 !mlx5_flow_ext_mreg_supported(dev) || 4118 !priv->sh->dv_regc0_mask) 4119 return 0; 4120 /* Find MARK action. */ 4121 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4122 switch (actions->type) { 4123 case RTE_FLOW_ACTION_TYPE_FLAG: 4124 mcp_res = flow_mreg_add_copy_action 4125 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4126 if (!mcp_res) 4127 return -rte_errno; 4128 flow->rix_mreg_copy = mcp_res->idx; 4129 return 0; 4130 case RTE_FLOW_ACTION_TYPE_MARK: 4131 mark = (const struct rte_flow_action_mark *) 4132 actions->conf; 4133 mcp_res = 4134 flow_mreg_add_copy_action(dev, mark->id, error); 4135 if (!mcp_res) 4136 return -rte_errno; 4137 flow->rix_mreg_copy = mcp_res->idx; 4138 return 0; 4139 default: 4140 break; 4141 } 4142 } 4143 return 0; 4144 } 4145 4146 #define MLX5_MAX_SPLIT_ACTIONS 24 4147 #define MLX5_MAX_SPLIT_ITEMS 24 4148 4149 /** 4150 * Split the hairpin flow. 4151 * Since HW can't support encap and push-vlan on Rx, we move these 4152 * actions to Tx. 4153 * If the count action is after the encap then we also 4154 * move the count action. in this case the count will also measure 4155 * the outer bytes. 4156 * 4157 * @param dev 4158 * Pointer to Ethernet device. 4159 * @param[in] actions 4160 * Associated actions (list terminated by the END action). 4161 * @param[out] actions_rx 4162 * Rx flow actions. 4163 * @param[out] actions_tx 4164 * Tx flow actions.. 4165 * @param[out] pattern_tx 4166 * The pattern items for the Tx flow. 4167 * @param[out] flow_id 4168 * The flow ID connected to this flow. 4169 * 4170 * @return 4171 * 0 on success. 4172 */ 4173 static int 4174 flow_hairpin_split(struct rte_eth_dev *dev, 4175 const struct rte_flow_action actions[], 4176 struct rte_flow_action actions_rx[], 4177 struct rte_flow_action actions_tx[], 4178 struct rte_flow_item pattern_tx[], 4179 uint32_t flow_id) 4180 { 4181 const struct rte_flow_action_raw_encap *raw_encap; 4182 const struct rte_flow_action_raw_decap *raw_decap; 4183 struct mlx5_rte_flow_action_set_tag *set_tag; 4184 struct rte_flow_action *tag_action; 4185 struct mlx5_rte_flow_item_tag *tag_item; 4186 struct rte_flow_item *item; 4187 char *addr; 4188 int encap = 0; 4189 4190 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4191 switch (actions->type) { 4192 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4193 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4194 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4195 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4196 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4197 rte_memcpy(actions_tx, actions, 4198 sizeof(struct rte_flow_action)); 4199 actions_tx++; 4200 break; 4201 case RTE_FLOW_ACTION_TYPE_COUNT: 4202 if (encap) { 4203 rte_memcpy(actions_tx, actions, 4204 sizeof(struct rte_flow_action)); 4205 actions_tx++; 4206 } else { 4207 rte_memcpy(actions_rx, actions, 4208 sizeof(struct rte_flow_action)); 4209 actions_rx++; 4210 } 4211 break; 4212 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4213 raw_encap = actions->conf; 4214 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4215 memcpy(actions_tx, actions, 4216 sizeof(struct rte_flow_action)); 4217 actions_tx++; 4218 encap = 1; 4219 } else { 4220 rte_memcpy(actions_rx, actions, 4221 sizeof(struct rte_flow_action)); 4222 actions_rx++; 4223 } 4224 break; 4225 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4226 raw_decap = actions->conf; 4227 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4228 memcpy(actions_tx, actions, 4229 sizeof(struct rte_flow_action)); 4230 actions_tx++; 4231 } else { 4232 rte_memcpy(actions_rx, actions, 4233 sizeof(struct rte_flow_action)); 4234 actions_rx++; 4235 } 4236 break; 4237 default: 4238 rte_memcpy(actions_rx, actions, 4239 sizeof(struct rte_flow_action)); 4240 actions_rx++; 4241 break; 4242 } 4243 } 4244 /* Add set meta action and end action for the Rx flow. */ 4245 tag_action = actions_rx; 4246 tag_action->type = (enum rte_flow_action_type) 4247 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4248 actions_rx++; 4249 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4250 actions_rx++; 4251 set_tag = (void *)actions_rx; 4252 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 4253 MLX5_ASSERT(set_tag->id > REG_NON); 4254 set_tag->data = flow_id; 4255 tag_action->conf = set_tag; 4256 /* Create Tx item list. */ 4257 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4258 addr = (void *)&pattern_tx[2]; 4259 item = pattern_tx; 4260 item->type = (enum rte_flow_item_type) 4261 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4262 tag_item = (void *)addr; 4263 tag_item->data = flow_id; 4264 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4265 MLX5_ASSERT(set_tag->id > REG_NON); 4266 item->spec = tag_item; 4267 addr += sizeof(struct mlx5_rte_flow_item_tag); 4268 tag_item = (void *)addr; 4269 tag_item->data = UINT32_MAX; 4270 tag_item->id = UINT16_MAX; 4271 item->mask = tag_item; 4272 item->last = NULL; 4273 item++; 4274 item->type = RTE_FLOW_ITEM_TYPE_END; 4275 return 0; 4276 } 4277 4278 /** 4279 * The last stage of splitting chain, just creates the subflow 4280 * without any modification. 4281 * 4282 * @param[in] dev 4283 * Pointer to Ethernet device. 4284 * @param[in] flow 4285 * Parent flow structure pointer. 4286 * @param[in, out] sub_flow 4287 * Pointer to return the created subflow, may be NULL. 4288 * @param[in] attr 4289 * Flow rule attributes. 4290 * @param[in] items 4291 * Pattern specification (list terminated by the END pattern item). 4292 * @param[in] actions 4293 * Associated actions (list terminated by the END action). 4294 * @param[in] flow_split_info 4295 * Pointer to flow split info structure. 4296 * @param[out] error 4297 * Perform verbose error reporting if not NULL. 4298 * @return 4299 * 0 on success, negative value otherwise 4300 */ 4301 static int 4302 flow_create_split_inner(struct rte_eth_dev *dev, 4303 struct rte_flow *flow, 4304 struct mlx5_flow **sub_flow, 4305 const struct rte_flow_attr *attr, 4306 const struct rte_flow_item items[], 4307 const struct rte_flow_action actions[], 4308 struct mlx5_flow_split_info *flow_split_info, 4309 struct rte_flow_error *error) 4310 { 4311 struct mlx5_flow *dev_flow; 4312 4313 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4314 flow_split_info->flow_idx, error); 4315 if (!dev_flow) 4316 return -rte_errno; 4317 dev_flow->flow = flow; 4318 dev_flow->external = flow_split_info->external; 4319 dev_flow->skip_scale = flow_split_info->skip_scale; 4320 /* Subflow object was created, we must include one in the list. */ 4321 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4322 dev_flow->handle, next); 4323 /* 4324 * If dev_flow is as one of the suffix flow, some actions in suffix 4325 * flow may need some user defined item layer flags, and pass the 4326 * Metadate rxq mark flag to suffix flow as well. 4327 */ 4328 if (flow_split_info->prefix_layers) 4329 dev_flow->handle->layers = flow_split_info->prefix_layers; 4330 if (flow_split_info->prefix_mark) 4331 dev_flow->handle->mark = 1; 4332 if (sub_flow) 4333 *sub_flow = dev_flow; 4334 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4335 } 4336 4337 /** 4338 * Split the meter flow. 4339 * 4340 * As meter flow will split to three sub flow, other than meter 4341 * action, the other actions make sense to only meter accepts 4342 * the packet. If it need to be dropped, no other additional 4343 * actions should be take. 4344 * 4345 * One kind of special action which decapsulates the L3 tunnel 4346 * header will be in the prefix sub flow, as not to take the 4347 * L3 tunnel header into account. 4348 * 4349 * @param dev 4350 * Pointer to Ethernet device. 4351 * @param[in] items 4352 * Pattern specification (list terminated by the END pattern item). 4353 * @param[out] sfx_items 4354 * Suffix flow match items (list terminated by the END pattern item). 4355 * @param[in] actions 4356 * Associated actions (list terminated by the END action). 4357 * @param[out] actions_sfx 4358 * Suffix flow actions. 4359 * @param[out] actions_pre 4360 * Prefix flow actions. 4361 * @param[out] pattern_sfx 4362 * The pattern items for the suffix flow. 4363 * @param[out] tag_sfx 4364 * Pointer to suffix flow tag. 4365 * 4366 * @return 4367 * 0 on success. 4368 */ 4369 static int 4370 flow_meter_split_prep(struct rte_eth_dev *dev, 4371 const struct rte_flow_item items[], 4372 struct rte_flow_item sfx_items[], 4373 const struct rte_flow_action actions[], 4374 struct rte_flow_action actions_sfx[], 4375 struct rte_flow_action actions_pre[]) 4376 { 4377 struct mlx5_priv *priv = dev->data->dev_private; 4378 struct rte_flow_action *tag_action = NULL; 4379 struct rte_flow_item *tag_item; 4380 struct mlx5_rte_flow_action_set_tag *set_tag; 4381 struct rte_flow_error error; 4382 const struct rte_flow_action_raw_encap *raw_encap; 4383 const struct rte_flow_action_raw_decap *raw_decap; 4384 struct mlx5_rte_flow_item_tag *tag_spec; 4385 struct mlx5_rte_flow_item_tag *tag_mask; 4386 uint32_t tag_id = 0; 4387 bool copy_vlan = false; 4388 4389 /* Prepare the actions for prefix and suffix flow. */ 4390 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4391 struct rte_flow_action **action_cur = NULL; 4392 4393 switch (actions->type) { 4394 case RTE_FLOW_ACTION_TYPE_METER: 4395 /* Add the extra tag action first. */ 4396 tag_action = actions_pre; 4397 tag_action->type = (enum rte_flow_action_type) 4398 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4399 actions_pre++; 4400 action_cur = &actions_pre; 4401 break; 4402 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4403 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4404 action_cur = &actions_pre; 4405 break; 4406 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4407 raw_encap = actions->conf; 4408 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4409 action_cur = &actions_pre; 4410 break; 4411 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4412 raw_decap = actions->conf; 4413 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4414 action_cur = &actions_pre; 4415 break; 4416 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4417 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4418 copy_vlan = true; 4419 break; 4420 default: 4421 break; 4422 } 4423 if (!action_cur) 4424 action_cur = &actions_sfx; 4425 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 4426 (*action_cur)++; 4427 } 4428 /* Add end action to the actions. */ 4429 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4430 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4431 actions_pre++; 4432 /* Set the tag. */ 4433 set_tag = (void *)actions_pre; 4434 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4435 mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 4436 &tag_id); 4437 if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { 4438 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", 4439 dev->data->port_id); 4440 mlx5_ipool_free(priv->sh->ipool 4441 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); 4442 return 0; 4443 } else if (!tag_id) { 4444 return 0; 4445 } 4446 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 4447 assert(tag_action); 4448 tag_action->conf = set_tag; 4449 /* Prepare the suffix subflow items. */ 4450 tag_item = sfx_items++; 4451 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4452 int item_type = items->type; 4453 4454 switch (item_type) { 4455 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4456 memcpy(sfx_items, items, sizeof(*sfx_items)); 4457 sfx_items++; 4458 break; 4459 case RTE_FLOW_ITEM_TYPE_VLAN: 4460 if (copy_vlan) { 4461 memcpy(sfx_items, items, sizeof(*sfx_items)); 4462 /* 4463 * Convert to internal match item, it is used 4464 * for vlan push and set vid. 4465 */ 4466 sfx_items->type = (enum rte_flow_item_type) 4467 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4468 sfx_items++; 4469 } 4470 break; 4471 default: 4472 break; 4473 } 4474 } 4475 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4476 sfx_items++; 4477 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4478 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 4479 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4480 tag_mask = tag_spec + 1; 4481 tag_mask->data = 0xffffff00; 4482 tag_item->type = (enum rte_flow_item_type) 4483 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4484 tag_item->spec = tag_spec; 4485 tag_item->last = NULL; 4486 tag_item->mask = tag_mask; 4487 return tag_id; 4488 } 4489 4490 /** 4491 * Split action list having QUEUE/RSS for metadata register copy. 4492 * 4493 * Once Q/RSS action is detected in user's action list, the flow action 4494 * should be split in order to copy metadata registers, which will happen in 4495 * RX_CP_TBL like, 4496 * - CQE->flow_tag := reg_c[1] (MARK) 4497 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4498 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4499 * This is because the last action of each flow must be a terminal action 4500 * (QUEUE, RSS or DROP). 4501 * 4502 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4503 * stored and kept in the mlx5_flow structure per each sub_flow. 4504 * 4505 * The Q/RSS action is replaced with, 4506 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4507 * And the following JUMP action is added at the end, 4508 * - JUMP, to RX_CP_TBL. 4509 * 4510 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4511 * flow_create_split_metadata() routine. The flow will look like, 4512 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4513 * 4514 * @param dev 4515 * Pointer to Ethernet device. 4516 * @param[out] split_actions 4517 * Pointer to store split actions to jump to CP_TBL. 4518 * @param[in] actions 4519 * Pointer to the list of original flow actions. 4520 * @param[in] qrss 4521 * Pointer to the Q/RSS action. 4522 * @param[in] actions_n 4523 * Number of original actions. 4524 * @param[out] error 4525 * Perform verbose error reporting if not NULL. 4526 * 4527 * @return 4528 * non-zero unique flow_id on success, otherwise 0 and 4529 * error/rte_error are set. 4530 */ 4531 static uint32_t 4532 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4533 struct rte_flow_action *split_actions, 4534 const struct rte_flow_action *actions, 4535 const struct rte_flow_action *qrss, 4536 int actions_n, struct rte_flow_error *error) 4537 { 4538 struct mlx5_priv *priv = dev->data->dev_private; 4539 struct mlx5_rte_flow_action_set_tag *set_tag; 4540 struct rte_flow_action_jump *jump; 4541 const int qrss_idx = qrss - actions; 4542 uint32_t flow_id = 0; 4543 int ret = 0; 4544 4545 /* 4546 * Given actions will be split 4547 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4548 * - Add jump to mreg CP_TBL. 4549 * As a result, there will be one more action. 4550 */ 4551 ++actions_n; 4552 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4553 set_tag = (void *)(split_actions + actions_n); 4554 /* 4555 * If tag action is not set to void(it means we are not the meter 4556 * suffix flow), add the tag action. Since meter suffix flow already 4557 * has the tag added. 4558 */ 4559 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4560 /* 4561 * Allocate the new subflow ID. This one is unique within 4562 * device and not shared with representors. Otherwise, 4563 * we would have to resolve multi-thread access synch 4564 * issue. Each flow on the shared device is appended 4565 * with source vport identifier, so the resulting 4566 * flows will be unique in the shared (by master and 4567 * representors) domain even if they have coinciding 4568 * IDs. 4569 */ 4570 mlx5_ipool_malloc(priv->sh->ipool 4571 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 4572 if (!flow_id) 4573 return rte_flow_error_set(error, ENOMEM, 4574 RTE_FLOW_ERROR_TYPE_ACTION, 4575 NULL, "can't allocate id " 4576 "for split Q/RSS subflow"); 4577 /* Internal SET_TAG action to set flow ID. */ 4578 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4579 .data = flow_id, 4580 }; 4581 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4582 if (ret < 0) 4583 return ret; 4584 set_tag->id = ret; 4585 /* Construct new actions array. */ 4586 /* Replace QUEUE/RSS action. */ 4587 split_actions[qrss_idx] = (struct rte_flow_action){ 4588 .type = (enum rte_flow_action_type) 4589 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4590 .conf = set_tag, 4591 }; 4592 } 4593 /* JUMP action to jump to mreg copy table (CP_TBL). */ 4594 jump = (void *)(set_tag + 1); 4595 *jump = (struct rte_flow_action_jump){ 4596 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4597 }; 4598 split_actions[actions_n - 2] = (struct rte_flow_action){ 4599 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4600 .conf = jump, 4601 }; 4602 split_actions[actions_n - 1] = (struct rte_flow_action){ 4603 .type = RTE_FLOW_ACTION_TYPE_END, 4604 }; 4605 return flow_id; 4606 } 4607 4608 /** 4609 * Extend the given action list for Tx metadata copy. 4610 * 4611 * Copy the given action list to the ext_actions and add flow metadata register 4612 * copy action in order to copy reg_a set by WQE to reg_c[0]. 4613 * 4614 * @param[out] ext_actions 4615 * Pointer to the extended action list. 4616 * @param[in] actions 4617 * Pointer to the list of actions. 4618 * @param[in] actions_n 4619 * Number of actions in the list. 4620 * @param[out] error 4621 * Perform verbose error reporting if not NULL. 4622 * @param[in] encap_idx 4623 * The encap action inndex. 4624 * 4625 * @return 4626 * 0 on success, negative value otherwise 4627 */ 4628 static int 4629 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 4630 struct rte_flow_action *ext_actions, 4631 const struct rte_flow_action *actions, 4632 int actions_n, struct rte_flow_error *error, 4633 int encap_idx) 4634 { 4635 struct mlx5_flow_action_copy_mreg *cp_mreg = 4636 (struct mlx5_flow_action_copy_mreg *) 4637 (ext_actions + actions_n + 1); 4638 int ret; 4639 4640 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4641 if (ret < 0) 4642 return ret; 4643 cp_mreg->dst = ret; 4644 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 4645 if (ret < 0) 4646 return ret; 4647 cp_mreg->src = ret; 4648 if (encap_idx != 0) 4649 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 4650 if (encap_idx == actions_n - 1) { 4651 ext_actions[actions_n - 1] = (struct rte_flow_action){ 4652 .type = (enum rte_flow_action_type) 4653 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4654 .conf = cp_mreg, 4655 }; 4656 ext_actions[actions_n] = (struct rte_flow_action){ 4657 .type = RTE_FLOW_ACTION_TYPE_END, 4658 }; 4659 } else { 4660 ext_actions[encap_idx] = (struct rte_flow_action){ 4661 .type = (enum rte_flow_action_type) 4662 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4663 .conf = cp_mreg, 4664 }; 4665 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 4666 sizeof(*ext_actions) * (actions_n - encap_idx)); 4667 } 4668 return 0; 4669 } 4670 4671 /** 4672 * Check the match action from the action list. 4673 * 4674 * @param[in] actions 4675 * Pointer to the list of actions. 4676 * @param[in] attr 4677 * Flow rule attributes. 4678 * @param[in] action 4679 * The action to be check if exist. 4680 * @param[out] match_action_pos 4681 * Pointer to the position of the matched action if exists, otherwise is -1. 4682 * @param[out] qrss_action_pos 4683 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 4684 * @param[out] modify_after_mirror 4685 * Pointer to the flag of modify action after FDB mirroring. 4686 * 4687 * @return 4688 * > 0 the total number of actions. 4689 * 0 if not found match action in action list. 4690 */ 4691 static int 4692 flow_check_match_action(const struct rte_flow_action actions[], 4693 const struct rte_flow_attr *attr, 4694 enum rte_flow_action_type action, 4695 int *match_action_pos, int *qrss_action_pos, 4696 int *modify_after_mirror) 4697 { 4698 const struct rte_flow_action_sample *sample; 4699 int actions_n = 0; 4700 uint32_t ratio = 0; 4701 int sub_type = 0; 4702 int flag = 0; 4703 int fdb_mirror = 0; 4704 4705 *match_action_pos = -1; 4706 *qrss_action_pos = -1; 4707 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4708 if (actions->type == action) { 4709 flag = 1; 4710 *match_action_pos = actions_n; 4711 } 4712 switch (actions->type) { 4713 case RTE_FLOW_ACTION_TYPE_QUEUE: 4714 case RTE_FLOW_ACTION_TYPE_RSS: 4715 *qrss_action_pos = actions_n; 4716 break; 4717 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4718 sample = actions->conf; 4719 ratio = sample->ratio; 4720 sub_type = ((const struct rte_flow_action *) 4721 (sample->actions))->type; 4722 if (ratio == 1 && attr->transfer) 4723 fdb_mirror = 1; 4724 break; 4725 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4726 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4727 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4728 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4729 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4730 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4731 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4732 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4733 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4734 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4735 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4736 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4737 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4738 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4739 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4740 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4741 case RTE_FLOW_ACTION_TYPE_FLAG: 4742 case RTE_FLOW_ACTION_TYPE_MARK: 4743 case RTE_FLOW_ACTION_TYPE_SET_META: 4744 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4745 if (fdb_mirror) 4746 *modify_after_mirror = 1; 4747 break; 4748 default: 4749 break; 4750 } 4751 actions_n++; 4752 } 4753 if (flag && fdb_mirror && !*modify_after_mirror) { 4754 /* FDB mirroring uses the destination array to implement 4755 * instead of FLOW_SAMPLER object. 4756 */ 4757 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 4758 flag = 0; 4759 } 4760 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4761 return flag ? actions_n + 1 : 0; 4762 } 4763 4764 #define SAMPLE_SUFFIX_ITEM 2 4765 4766 /** 4767 * Split the sample flow. 4768 * 4769 * As sample flow will split to two sub flow, sample flow with 4770 * sample action, the other actions will move to new suffix flow. 4771 * 4772 * Also add unique tag id with tag action in the sample flow, 4773 * the same tag id will be as match in the suffix flow. 4774 * 4775 * @param dev 4776 * Pointer to Ethernet device. 4777 * @param[in] add_tag 4778 * Add extra tag action flag. 4779 * @param[out] sfx_items 4780 * Suffix flow match items (list terminated by the END pattern item). 4781 * @param[in] actions 4782 * Associated actions (list terminated by the END action). 4783 * @param[out] actions_sfx 4784 * Suffix flow actions. 4785 * @param[out] actions_pre 4786 * Prefix flow actions. 4787 * @param[in] actions_n 4788 * The total number of actions. 4789 * @param[in] sample_action_pos 4790 * The sample action position. 4791 * @param[in] qrss_action_pos 4792 * The Queue/RSS action position. 4793 * @param[in] jump_table 4794 * Add extra jump action flag. 4795 * @param[out] error 4796 * Perform verbose error reporting if not NULL. 4797 * 4798 * @return 4799 * 0 on success, or unique flow_id, a negative errno value 4800 * otherwise and rte_errno is set. 4801 */ 4802 static int 4803 flow_sample_split_prep(struct rte_eth_dev *dev, 4804 int add_tag, 4805 struct rte_flow_item sfx_items[], 4806 const struct rte_flow_action actions[], 4807 struct rte_flow_action actions_sfx[], 4808 struct rte_flow_action actions_pre[], 4809 int actions_n, 4810 int sample_action_pos, 4811 int qrss_action_pos, 4812 int jump_table, 4813 struct rte_flow_error *error) 4814 { 4815 struct mlx5_priv *priv = dev->data->dev_private; 4816 struct mlx5_rte_flow_action_set_tag *set_tag; 4817 struct mlx5_rte_flow_item_tag *tag_spec; 4818 struct mlx5_rte_flow_item_tag *tag_mask; 4819 struct rte_flow_action_jump *jump_action; 4820 uint32_t tag_id = 0; 4821 int index; 4822 int append_index = 0; 4823 int ret; 4824 4825 if (sample_action_pos < 0) 4826 return rte_flow_error_set(error, EINVAL, 4827 RTE_FLOW_ERROR_TYPE_ACTION, 4828 NULL, "invalid position of sample " 4829 "action in list"); 4830 /* Prepare the actions for prefix and suffix flow. */ 4831 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 4832 index = qrss_action_pos; 4833 /* Put the preceding the Queue/RSS action into prefix flow. */ 4834 if (index != 0) 4835 memcpy(actions_pre, actions, 4836 sizeof(struct rte_flow_action) * index); 4837 /* Put others preceding the sample action into prefix flow. */ 4838 if (sample_action_pos > index + 1) 4839 memcpy(actions_pre + index, actions + index + 1, 4840 sizeof(struct rte_flow_action) * 4841 (sample_action_pos - index - 1)); 4842 index = sample_action_pos - 1; 4843 /* Put Queue/RSS action into Suffix flow. */ 4844 memcpy(actions_sfx, actions + qrss_action_pos, 4845 sizeof(struct rte_flow_action)); 4846 actions_sfx++; 4847 } else { 4848 index = sample_action_pos; 4849 if (index != 0) 4850 memcpy(actions_pre, actions, 4851 sizeof(struct rte_flow_action) * index); 4852 } 4853 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 4854 * For CX6DX and above, metadata registers Cx preserve their value, 4855 * add an extra tag action for NIC-RX and E-Switch Domain. 4856 */ 4857 if (add_tag) { 4858 /* Prepare the prefix tag action. */ 4859 append_index++; 4860 set_tag = (void *)(actions_pre + actions_n + append_index); 4861 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 4862 if (ret < 0) 4863 return ret; 4864 set_tag->id = ret; 4865 mlx5_ipool_malloc(priv->sh->ipool 4866 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 4867 set_tag->data = tag_id; 4868 /* Prepare the suffix subflow items. */ 4869 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 4870 tag_spec->data = tag_id; 4871 tag_spec->id = set_tag->id; 4872 tag_mask = tag_spec + 1; 4873 tag_mask->data = UINT32_MAX; 4874 sfx_items[0] = (struct rte_flow_item){ 4875 .type = (enum rte_flow_item_type) 4876 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4877 .spec = tag_spec, 4878 .last = NULL, 4879 .mask = tag_mask, 4880 }; 4881 sfx_items[1] = (struct rte_flow_item){ 4882 .type = (enum rte_flow_item_type) 4883 RTE_FLOW_ITEM_TYPE_END, 4884 }; 4885 /* Prepare the tag action in prefix subflow. */ 4886 actions_pre[index++] = 4887 (struct rte_flow_action){ 4888 .type = (enum rte_flow_action_type) 4889 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4890 .conf = set_tag, 4891 }; 4892 } 4893 memcpy(actions_pre + index, actions + sample_action_pos, 4894 sizeof(struct rte_flow_action)); 4895 index += 1; 4896 /* For the modify action after the sample action in E-Switch mirroring, 4897 * Add the extra jump action in prefix subflow and jump into the next 4898 * table, then do the modify action in the new table. 4899 */ 4900 if (jump_table) { 4901 /* Prepare the prefix jump action. */ 4902 append_index++; 4903 jump_action = (void *)(actions_pre + actions_n + append_index); 4904 jump_action->group = jump_table; 4905 actions_pre[index++] = 4906 (struct rte_flow_action){ 4907 .type = (enum rte_flow_action_type) 4908 RTE_FLOW_ACTION_TYPE_JUMP, 4909 .conf = jump_action, 4910 }; 4911 } 4912 actions_pre[index] = (struct rte_flow_action){ 4913 .type = (enum rte_flow_action_type) 4914 RTE_FLOW_ACTION_TYPE_END, 4915 }; 4916 /* Put the actions after sample into Suffix flow. */ 4917 memcpy(actions_sfx, actions + sample_action_pos + 1, 4918 sizeof(struct rte_flow_action) * 4919 (actions_n - sample_action_pos - 1)); 4920 return tag_id; 4921 } 4922 4923 /** 4924 * The splitting for metadata feature. 4925 * 4926 * - Q/RSS action on NIC Rx should be split in order to pass by 4927 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4928 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4929 * 4930 * - All the actions on NIC Tx should have a mreg copy action to 4931 * copy reg_a from WQE to reg_c[0]. 4932 * 4933 * @param dev 4934 * Pointer to Ethernet device. 4935 * @param[in] flow 4936 * Parent flow structure pointer. 4937 * @param[in] attr 4938 * Flow rule attributes. 4939 * @param[in] items 4940 * Pattern specification (list terminated by the END pattern item). 4941 * @param[in] actions 4942 * Associated actions (list terminated by the END action). 4943 * @param[in] flow_split_info 4944 * Pointer to flow split info structure. 4945 * @param[out] error 4946 * Perform verbose error reporting if not NULL. 4947 * @return 4948 * 0 on success, negative value otherwise 4949 */ 4950 static int 4951 flow_create_split_metadata(struct rte_eth_dev *dev, 4952 struct rte_flow *flow, 4953 const struct rte_flow_attr *attr, 4954 const struct rte_flow_item items[], 4955 const struct rte_flow_action actions[], 4956 struct mlx5_flow_split_info *flow_split_info, 4957 struct rte_flow_error *error) 4958 { 4959 struct mlx5_priv *priv = dev->data->dev_private; 4960 struct mlx5_dev_config *config = &priv->config; 4961 const struct rte_flow_action *qrss = NULL; 4962 struct rte_flow_action *ext_actions = NULL; 4963 struct mlx5_flow *dev_flow = NULL; 4964 uint32_t qrss_id = 0; 4965 int mtr_sfx = 0; 4966 size_t act_size; 4967 int actions_n; 4968 int encap_idx; 4969 int ret; 4970 4971 /* Check whether extensive metadata feature is engaged. */ 4972 if (!config->dv_flow_en || 4973 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4974 !mlx5_flow_ext_mreg_supported(dev)) 4975 return flow_create_split_inner(dev, flow, NULL, attr, items, 4976 actions, flow_split_info, error); 4977 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 4978 &encap_idx); 4979 if (qrss) { 4980 /* Exclude hairpin flows from splitting. */ 4981 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 4982 const struct rte_flow_action_queue *queue; 4983 4984 queue = qrss->conf; 4985 if (mlx5_rxq_get_type(dev, queue->index) == 4986 MLX5_RXQ_TYPE_HAIRPIN) 4987 qrss = NULL; 4988 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 4989 const struct rte_flow_action_rss *rss; 4990 4991 rss = qrss->conf; 4992 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 4993 MLX5_RXQ_TYPE_HAIRPIN) 4994 qrss = NULL; 4995 } 4996 } 4997 if (qrss) { 4998 /* Check if it is in meter suffix table. */ 4999 mtr_sfx = attr->group == (attr->transfer ? 5000 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 5001 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 5002 /* 5003 * Q/RSS action on NIC Rx should be split in order to pass by 5004 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5005 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5006 */ 5007 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5008 sizeof(struct rte_flow_action_set_tag) + 5009 sizeof(struct rte_flow_action_jump); 5010 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5011 SOCKET_ID_ANY); 5012 if (!ext_actions) 5013 return rte_flow_error_set(error, ENOMEM, 5014 RTE_FLOW_ERROR_TYPE_ACTION, 5015 NULL, "no memory to split " 5016 "metadata flow"); 5017 /* 5018 * If we are the suffix flow of meter, tag already exist. 5019 * Set the tag action to void. 5020 */ 5021 if (mtr_sfx) 5022 ext_actions[qrss - actions].type = 5023 RTE_FLOW_ACTION_TYPE_VOID; 5024 else 5025 ext_actions[qrss - actions].type = 5026 (enum rte_flow_action_type) 5027 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5028 /* 5029 * Create the new actions list with removed Q/RSS action 5030 * and appended set tag and jump to register copy table 5031 * (RX_CP_TBL). We should preallocate unique tag ID here 5032 * in advance, because it is needed for set tag action. 5033 */ 5034 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5035 qrss, actions_n, error); 5036 if (!mtr_sfx && !qrss_id) { 5037 ret = -rte_errno; 5038 goto exit; 5039 } 5040 } else if (attr->egress && !attr->transfer) { 5041 /* 5042 * All the actions on NIC Tx should have a metadata register 5043 * copy action to copy reg_a from WQE to reg_c[meta] 5044 */ 5045 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5046 sizeof(struct mlx5_flow_action_copy_mreg); 5047 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5048 SOCKET_ID_ANY); 5049 if (!ext_actions) 5050 return rte_flow_error_set(error, ENOMEM, 5051 RTE_FLOW_ERROR_TYPE_ACTION, 5052 NULL, "no memory to split " 5053 "metadata flow"); 5054 /* Create the action list appended with copy register. */ 5055 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5056 actions_n, error, encap_idx); 5057 if (ret < 0) 5058 goto exit; 5059 } 5060 /* Add the unmodified original or prefix subflow. */ 5061 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5062 items, ext_actions ? ext_actions : 5063 actions, flow_split_info, error); 5064 if (ret < 0) 5065 goto exit; 5066 MLX5_ASSERT(dev_flow); 5067 if (qrss) { 5068 const struct rte_flow_attr q_attr = { 5069 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5070 .ingress = 1, 5071 }; 5072 /* Internal PMD action to set register. */ 5073 struct mlx5_rte_flow_item_tag q_tag_spec = { 5074 .data = qrss_id, 5075 .id = REG_NON, 5076 }; 5077 struct rte_flow_item q_items[] = { 5078 { 5079 .type = (enum rte_flow_item_type) 5080 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5081 .spec = &q_tag_spec, 5082 .last = NULL, 5083 .mask = NULL, 5084 }, 5085 { 5086 .type = RTE_FLOW_ITEM_TYPE_END, 5087 }, 5088 }; 5089 struct rte_flow_action q_actions[] = { 5090 { 5091 .type = qrss->type, 5092 .conf = qrss->conf, 5093 }, 5094 { 5095 .type = RTE_FLOW_ACTION_TYPE_END, 5096 }, 5097 }; 5098 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5099 5100 /* 5101 * Configure the tag item only if there is no meter subflow. 5102 * Since tag is already marked in the meter suffix subflow 5103 * we can just use the meter suffix items as is. 5104 */ 5105 if (qrss_id) { 5106 /* Not meter subflow. */ 5107 MLX5_ASSERT(!mtr_sfx); 5108 /* 5109 * Put unique id in prefix flow due to it is destroyed 5110 * after suffix flow and id will be freed after there 5111 * is no actual flows with this id and identifier 5112 * reallocation becomes possible (for example, for 5113 * other flows in other threads). 5114 */ 5115 dev_flow->handle->split_flow_id = qrss_id; 5116 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5117 error); 5118 if (ret < 0) 5119 goto exit; 5120 q_tag_spec.id = ret; 5121 } 5122 dev_flow = NULL; 5123 /* Add suffix subflow to execute Q/RSS. */ 5124 flow_split_info->prefix_layers = layers; 5125 flow_split_info->prefix_mark = 0; 5126 ret = flow_create_split_inner(dev, flow, &dev_flow, 5127 &q_attr, mtr_sfx ? items : 5128 q_items, q_actions, 5129 flow_split_info, error); 5130 if (ret < 0) 5131 goto exit; 5132 /* qrss ID should be freed if failed. */ 5133 qrss_id = 0; 5134 MLX5_ASSERT(dev_flow); 5135 } 5136 5137 exit: 5138 /* 5139 * We do not destroy the partially created sub_flows in case of error. 5140 * These ones are included into parent flow list and will be destroyed 5141 * by flow_drv_destroy. 5142 */ 5143 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5144 qrss_id); 5145 mlx5_free(ext_actions); 5146 return ret; 5147 } 5148 5149 /** 5150 * The splitting for meter feature. 5151 * 5152 * - The meter flow will be split to two flows as prefix and 5153 * suffix flow. The packets make sense only it pass the prefix 5154 * meter action. 5155 * 5156 * - Reg_C_5 is used for the packet to match betweend prefix and 5157 * suffix flow. 5158 * 5159 * @param dev 5160 * Pointer to Ethernet device. 5161 * @param[in] flow 5162 * Parent flow structure pointer. 5163 * @param[in] attr 5164 * Flow rule attributes. 5165 * @param[in] items 5166 * Pattern specification (list terminated by the END pattern item). 5167 * @param[in] actions 5168 * Associated actions (list terminated by the END action). 5169 * @param[in] flow_split_info 5170 * Pointer to flow split info structure. 5171 * @param[out] error 5172 * Perform verbose error reporting if not NULL. 5173 * @return 5174 * 0 on success, negative value otherwise 5175 */ 5176 static int 5177 flow_create_split_meter(struct rte_eth_dev *dev, 5178 struct rte_flow *flow, 5179 const struct rte_flow_attr *attr, 5180 const struct rte_flow_item items[], 5181 const struct rte_flow_action actions[], 5182 struct mlx5_flow_split_info *flow_split_info, 5183 struct rte_flow_error *error) 5184 { 5185 struct mlx5_priv *priv = dev->data->dev_private; 5186 struct rte_flow_action *sfx_actions = NULL; 5187 struct rte_flow_action *pre_actions = NULL; 5188 struct rte_flow_item *sfx_items = NULL; 5189 struct mlx5_flow *dev_flow = NULL; 5190 struct rte_flow_attr sfx_attr = *attr; 5191 uint32_t mtr = 0; 5192 uint32_t mtr_tag_id = 0; 5193 size_t act_size; 5194 size_t item_size; 5195 int actions_n = 0; 5196 int ret; 5197 5198 if (priv->mtr_en) 5199 actions_n = flow_check_meter_action(actions, &mtr); 5200 if (mtr) { 5201 /* The five prefix actions: meter, decap, encap, tag, end. */ 5202 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 5203 sizeof(struct mlx5_rte_flow_action_set_tag); 5204 /* tag, vlan, port id, end. */ 5205 #define METER_SUFFIX_ITEM 4 5206 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5207 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5208 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5209 0, SOCKET_ID_ANY); 5210 if (!sfx_actions) 5211 return rte_flow_error_set(error, ENOMEM, 5212 RTE_FLOW_ERROR_TYPE_ACTION, 5213 NULL, "no memory to split " 5214 "meter flow"); 5215 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5216 act_size); 5217 pre_actions = sfx_actions + actions_n; 5218 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 5219 actions, sfx_actions, 5220 pre_actions); 5221 if (!mtr_tag_id) { 5222 ret = -rte_errno; 5223 goto exit; 5224 } 5225 /* Add the prefix subflow. */ 5226 flow_split_info->prefix_mark = 0; 5227 ret = flow_create_split_inner(dev, flow, &dev_flow, 5228 attr, items, pre_actions, 5229 flow_split_info, error); 5230 if (ret) { 5231 ret = -rte_errno; 5232 goto exit; 5233 } 5234 dev_flow->handle->split_flow_id = mtr_tag_id; 5235 /* Setting the sfx group atrr. */ 5236 sfx_attr.group = sfx_attr.transfer ? 5237 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 5238 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 5239 flow_split_info->prefix_layers = 5240 flow_get_prefix_layer_flags(dev_flow); 5241 flow_split_info->prefix_mark = dev_flow->handle->mark; 5242 } 5243 /* Add the prefix subflow. */ 5244 ret = flow_create_split_metadata(dev, flow, 5245 &sfx_attr, sfx_items ? 5246 sfx_items : items, 5247 sfx_actions ? sfx_actions : actions, 5248 flow_split_info, error); 5249 exit: 5250 if (sfx_actions) 5251 mlx5_free(sfx_actions); 5252 return ret; 5253 } 5254 5255 /** 5256 * The splitting for sample feature. 5257 * 5258 * Once Sample action is detected in the action list, the flow actions should 5259 * be split into prefix sub flow and suffix sub flow. 5260 * 5261 * The original items remain in the prefix sub flow, all actions preceding the 5262 * sample action and the sample action itself will be copied to the prefix 5263 * sub flow, the actions following the sample action will be copied to the 5264 * suffix sub flow, Queue action always be located in the suffix sub flow. 5265 * 5266 * In order to make the packet from prefix sub flow matches with suffix sub 5267 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5268 * flow uses tag item with the unique flow id. 5269 * 5270 * @param dev 5271 * Pointer to Ethernet device. 5272 * @param[in] flow 5273 * Parent flow structure pointer. 5274 * @param[in] attr 5275 * Flow rule attributes. 5276 * @param[in] items 5277 * Pattern specification (list terminated by the END pattern item). 5278 * @param[in] actions 5279 * Associated actions (list terminated by the END action). 5280 * @param[in] flow_split_info 5281 * Pointer to flow split info structure. 5282 * @param[out] error 5283 * Perform verbose error reporting if not NULL. 5284 * @return 5285 * 0 on success, negative value otherwise 5286 */ 5287 static int 5288 flow_create_split_sample(struct rte_eth_dev *dev, 5289 struct rte_flow *flow, 5290 const struct rte_flow_attr *attr, 5291 const struct rte_flow_item items[], 5292 const struct rte_flow_action actions[], 5293 struct mlx5_flow_split_info *flow_split_info, 5294 struct rte_flow_error *error) 5295 { 5296 struct mlx5_priv *priv = dev->data->dev_private; 5297 struct rte_flow_action *sfx_actions = NULL; 5298 struct rte_flow_action *pre_actions = NULL; 5299 struct rte_flow_item *sfx_items = NULL; 5300 struct mlx5_flow *dev_flow = NULL; 5301 struct rte_flow_attr sfx_attr = *attr; 5302 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5303 struct mlx5_flow_dv_sample_resource *sample_res; 5304 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5305 struct mlx5_flow_tbl_resource *sfx_tbl; 5306 #endif 5307 size_t act_size; 5308 size_t item_size; 5309 uint32_t fdb_tx = 0; 5310 int32_t tag_id = 0; 5311 int actions_n = 0; 5312 int sample_action_pos; 5313 int qrss_action_pos; 5314 int add_tag = 0; 5315 int modify_after_mirror = 0; 5316 uint16_t jump_table = 0; 5317 const uint32_t next_ft_step = 1; 5318 int ret = 0; 5319 5320 if (priv->sampler_en) 5321 actions_n = flow_check_match_action(actions, attr, 5322 RTE_FLOW_ACTION_TYPE_SAMPLE, 5323 &sample_action_pos, &qrss_action_pos, 5324 &modify_after_mirror); 5325 if (actions_n) { 5326 /* The prefix actions must includes sample, tag, end. */ 5327 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5328 + sizeof(struct mlx5_rte_flow_action_set_tag); 5329 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5330 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5331 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5332 item_size), 0, SOCKET_ID_ANY); 5333 if (!sfx_actions) 5334 return rte_flow_error_set(error, ENOMEM, 5335 RTE_FLOW_ERROR_TYPE_ACTION, 5336 NULL, "no memory to split " 5337 "sample flow"); 5338 /* The representor_id is -1 for uplink. */ 5339 fdb_tx = (attr->transfer && priv->representor_id != -1); 5340 /* 5341 * When reg_c_preserve is set, metadata registers Cx preserve 5342 * their value even through packet duplication. 5343 */ 5344 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 5345 if (add_tag) 5346 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5347 + act_size); 5348 if (modify_after_mirror) 5349 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 5350 next_ft_step; 5351 pre_actions = sfx_actions + actions_n; 5352 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 5353 actions, sfx_actions, 5354 pre_actions, actions_n, 5355 sample_action_pos, 5356 qrss_action_pos, jump_table, 5357 error); 5358 if (tag_id < 0 || (add_tag && !tag_id)) { 5359 ret = -rte_errno; 5360 goto exit; 5361 } 5362 if (modify_after_mirror) 5363 flow_split_info->skip_scale = 5364 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5365 /* Add the prefix subflow. */ 5366 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5367 items, pre_actions, 5368 flow_split_info, error); 5369 if (ret) { 5370 ret = -rte_errno; 5371 goto exit; 5372 } 5373 dev_flow->handle->split_flow_id = tag_id; 5374 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5375 if (!modify_after_mirror) { 5376 /* Set the sfx group attr. */ 5377 sample_res = (struct mlx5_flow_dv_sample_resource *) 5378 dev_flow->dv.sample_res; 5379 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5380 sample_res->normal_path_tbl; 5381 sfx_tbl_data = container_of(sfx_tbl, 5382 struct mlx5_flow_tbl_data_entry, 5383 tbl); 5384 sfx_attr.group = sfx_attr.transfer ? 5385 (sfx_tbl_data->table_id - 1) : 5386 sfx_tbl_data->table_id; 5387 } else { 5388 MLX5_ASSERT(attr->transfer); 5389 sfx_attr.group = jump_table; 5390 } 5391 flow_split_info->prefix_layers = 5392 flow_get_prefix_layer_flags(dev_flow); 5393 flow_split_info->prefix_mark = dev_flow->handle->mark; 5394 /* Suffix group level already be scaled with factor, set 5395 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 5396 * again in translation. 5397 */ 5398 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5399 #endif 5400 } 5401 /* Add the suffix subflow. */ 5402 ret = flow_create_split_meter(dev, flow, &sfx_attr, 5403 sfx_items ? sfx_items : items, 5404 sfx_actions ? sfx_actions : actions, 5405 flow_split_info, error); 5406 exit: 5407 if (sfx_actions) 5408 mlx5_free(sfx_actions); 5409 return ret; 5410 } 5411 5412 /** 5413 * Split the flow to subflow set. The splitters might be linked 5414 * in the chain, like this: 5415 * flow_create_split_outer() calls: 5416 * flow_create_split_meter() calls: 5417 * flow_create_split_metadata(meter_subflow_0) calls: 5418 * flow_create_split_inner(metadata_subflow_0) 5419 * flow_create_split_inner(metadata_subflow_1) 5420 * flow_create_split_inner(metadata_subflow_2) 5421 * flow_create_split_metadata(meter_subflow_1) calls: 5422 * flow_create_split_inner(metadata_subflow_0) 5423 * flow_create_split_inner(metadata_subflow_1) 5424 * flow_create_split_inner(metadata_subflow_2) 5425 * 5426 * This provide flexible way to add new levels of flow splitting. 5427 * The all of successfully created subflows are included to the 5428 * parent flow dev_flow list. 5429 * 5430 * @param dev 5431 * Pointer to Ethernet device. 5432 * @param[in] flow 5433 * Parent flow structure pointer. 5434 * @param[in] attr 5435 * Flow rule attributes. 5436 * @param[in] items 5437 * Pattern specification (list terminated by the END pattern item). 5438 * @param[in] actions 5439 * Associated actions (list terminated by the END action). 5440 * @param[in] flow_split_info 5441 * Pointer to flow split info structure. 5442 * @param[out] error 5443 * Perform verbose error reporting if not NULL. 5444 * @return 5445 * 0 on success, negative value otherwise 5446 */ 5447 static int 5448 flow_create_split_outer(struct rte_eth_dev *dev, 5449 struct rte_flow *flow, 5450 const struct rte_flow_attr *attr, 5451 const struct rte_flow_item items[], 5452 const struct rte_flow_action actions[], 5453 struct mlx5_flow_split_info *flow_split_info, 5454 struct rte_flow_error *error) 5455 { 5456 int ret; 5457 5458 ret = flow_create_split_sample(dev, flow, attr, items, 5459 actions, flow_split_info, error); 5460 MLX5_ASSERT(ret <= 0); 5461 return ret; 5462 } 5463 5464 static struct mlx5_flow_tunnel * 5465 flow_tunnel_from_rule(struct rte_eth_dev *dev, 5466 const struct rte_flow_attr *attr, 5467 const struct rte_flow_item items[], 5468 const struct rte_flow_action actions[]) 5469 { 5470 struct mlx5_flow_tunnel *tunnel; 5471 5472 #pragma GCC diagnostic push 5473 #pragma GCC diagnostic ignored "-Wcast-qual" 5474 if (is_flow_tunnel_match_rule(dev, attr, items, actions)) 5475 tunnel = (struct mlx5_flow_tunnel *)items[0].spec; 5476 else if (is_flow_tunnel_steer_rule(dev, attr, items, actions)) 5477 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf; 5478 else 5479 tunnel = NULL; 5480 #pragma GCC diagnostic pop 5481 5482 return tunnel; 5483 } 5484 5485 /** 5486 * Adjust flow RSS workspace if needed. 5487 * 5488 * @param wks 5489 * Pointer to thread flow work space. 5490 * @param rss_desc 5491 * Pointer to RSS descriptor. 5492 * @param[in] nrssq_num 5493 * New RSS queue number. 5494 * 5495 * @return 5496 * 0 on success, -1 otherwise and rte_errno is set. 5497 */ 5498 static int 5499 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 5500 struct mlx5_flow_rss_desc *rss_desc, 5501 uint32_t nrssq_num) 5502 { 5503 if (likely(nrssq_num <= wks->rssq_num)) 5504 return 0; 5505 rss_desc->queue = realloc(rss_desc->queue, 5506 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 5507 if (!rss_desc->queue) { 5508 rte_errno = ENOMEM; 5509 return -1; 5510 } 5511 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 5512 return 0; 5513 } 5514 5515 /** 5516 * Create a flow and add it to @p list. 5517 * 5518 * @param dev 5519 * Pointer to Ethernet device. 5520 * @param list 5521 * Pointer to a TAILQ flow list. If this parameter NULL, 5522 * no list insertion occurred, flow is just created, 5523 * this is caller's responsibility to track the 5524 * created flow. 5525 * @param[in] attr 5526 * Flow rule attributes. 5527 * @param[in] items 5528 * Pattern specification (list terminated by the END pattern item). 5529 * @param[in] actions 5530 * Associated actions (list terminated by the END action). 5531 * @param[in] external 5532 * This flow rule is created by request external to PMD. 5533 * @param[out] error 5534 * Perform verbose error reporting if not NULL. 5535 * 5536 * @return 5537 * A flow index on success, 0 otherwise and rte_errno is set. 5538 */ 5539 static uint32_t 5540 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 5541 const struct rte_flow_attr *attr, 5542 const struct rte_flow_item items[], 5543 const struct rte_flow_action original_actions[], 5544 bool external, struct rte_flow_error *error) 5545 { 5546 struct mlx5_priv *priv = dev->data->dev_private; 5547 struct rte_flow *flow = NULL; 5548 struct mlx5_flow *dev_flow; 5549 const struct rte_flow_action_rss *rss = NULL; 5550 struct mlx5_translated_shared_action 5551 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5552 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5553 union { 5554 struct mlx5_flow_expand_rss buf; 5555 uint8_t buffer[2048]; 5556 } expand_buffer; 5557 union { 5558 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5559 uint8_t buffer[2048]; 5560 } actions_rx; 5561 union { 5562 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5563 uint8_t buffer[2048]; 5564 } actions_hairpin_tx; 5565 union { 5566 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 5567 uint8_t buffer[2048]; 5568 } items_tx; 5569 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 5570 struct mlx5_flow_rss_desc *rss_desc; 5571 const struct rte_flow_action *p_actions_rx; 5572 uint32_t i; 5573 uint32_t idx = 0; 5574 int hairpin_flow; 5575 struct rte_flow_attr attr_tx = { .priority = 0 }; 5576 const struct rte_flow_action *actions; 5577 struct rte_flow_action *translated_actions = NULL; 5578 struct mlx5_flow_tunnel *tunnel; 5579 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 5580 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 5581 struct mlx5_flow_split_info flow_split_info = { 5582 .external = !!external, 5583 .skip_scale = 0, 5584 .flow_idx = 0, 5585 .prefix_mark = 0, 5586 .prefix_layers = 0 5587 }; 5588 int ret; 5589 5590 MLX5_ASSERT(wks); 5591 rss_desc = &wks->rss_desc; 5592 ret = flow_shared_actions_translate(dev, original_actions, 5593 shared_actions, 5594 &shared_actions_n, 5595 &translated_actions, error); 5596 if (ret < 0) { 5597 MLX5_ASSERT(translated_actions == NULL); 5598 return 0; 5599 } 5600 actions = translated_actions ? translated_actions : original_actions; 5601 p_actions_rx = actions; 5602 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5603 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 5604 external, hairpin_flow, error); 5605 if (ret < 0) 5606 goto error_before_hairpin_split; 5607 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 5608 if (!flow) { 5609 rte_errno = ENOMEM; 5610 goto error_before_hairpin_split; 5611 } 5612 if (hairpin_flow > 0) { 5613 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 5614 rte_errno = EINVAL; 5615 goto error_before_hairpin_split; 5616 } 5617 flow_hairpin_split(dev, actions, actions_rx.actions, 5618 actions_hairpin_tx.actions, items_tx.items, 5619 idx); 5620 p_actions_rx = actions_rx.actions; 5621 } 5622 flow_split_info.flow_idx = idx; 5623 flow->drv_type = flow_get_drv_type(dev, attr); 5624 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 5625 flow->drv_type < MLX5_FLOW_TYPE_MAX); 5626 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 5627 /* RSS Action only works on NIC RX domain */ 5628 if (attr->ingress && !attr->transfer) 5629 rss = flow_get_rss_action(p_actions_rx); 5630 if (rss) { 5631 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 5632 return 0; 5633 /* 5634 * The following information is required by 5635 * mlx5_flow_hashfields_adjust() in advance. 5636 */ 5637 rss_desc->level = rss->level; 5638 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 5639 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 5640 } 5641 flow->dev_handles = 0; 5642 if (rss && rss->types) { 5643 unsigned int graph_root; 5644 5645 graph_root = find_graph_root(items, rss->level); 5646 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 5647 items, rss->types, 5648 mlx5_support_expansion, graph_root); 5649 MLX5_ASSERT(ret > 0 && 5650 (unsigned int)ret < sizeof(expand_buffer.buffer)); 5651 } else { 5652 buf->entries = 1; 5653 buf->entry[0].pattern = (void *)(uintptr_t)items; 5654 } 5655 rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions, 5656 shared_actions_n); 5657 for (i = 0; i < buf->entries; ++i) { 5658 /* Initialize flow split data. */ 5659 flow_split_info.prefix_layers = 0; 5660 flow_split_info.prefix_mark = 0; 5661 flow_split_info.skip_scale = 0; 5662 /* 5663 * The splitter may create multiple dev_flows, 5664 * depending on configuration. In the simplest 5665 * case it just creates unmodified original flow. 5666 */ 5667 ret = flow_create_split_outer(dev, flow, attr, 5668 buf->entry[i].pattern, 5669 p_actions_rx, &flow_split_info, 5670 error); 5671 if (ret < 0) 5672 goto error; 5673 if (is_flow_tunnel_steer_rule(dev, attr, 5674 buf->entry[i].pattern, 5675 p_actions_rx)) { 5676 ret = flow_tunnel_add_default_miss(dev, flow, attr, 5677 p_actions_rx, 5678 idx, 5679 &default_miss_ctx, 5680 error); 5681 if (ret < 0) { 5682 mlx5_free(default_miss_ctx.queue); 5683 goto error; 5684 } 5685 } 5686 } 5687 /* Create the tx flow. */ 5688 if (hairpin_flow) { 5689 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 5690 attr_tx.ingress = 0; 5691 attr_tx.egress = 1; 5692 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 5693 actions_hairpin_tx.actions, 5694 idx, error); 5695 if (!dev_flow) 5696 goto error; 5697 dev_flow->flow = flow; 5698 dev_flow->external = 0; 5699 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5700 dev_flow->handle, next); 5701 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 5702 items_tx.items, 5703 actions_hairpin_tx.actions, error); 5704 if (ret < 0) 5705 goto error; 5706 } 5707 /* 5708 * Update the metadata register copy table. If extensive 5709 * metadata feature is enabled and registers are supported 5710 * we might create the extra rte_flow for each unique 5711 * MARK/FLAG action ID. 5712 * 5713 * The table is updated for ingress Flows only, because 5714 * the egress Flows belong to the different device and 5715 * copy table should be updated in peer NIC Rx domain. 5716 */ 5717 if (attr->ingress && 5718 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 5719 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 5720 if (ret) 5721 goto error; 5722 } 5723 /* 5724 * If the flow is external (from application) OR device is started, 5725 * OR mreg discover, then apply immediately. 5726 */ 5727 if (external || dev->data->dev_started || 5728 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 5729 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 5730 ret = flow_drv_apply(dev, flow, error); 5731 if (ret < 0) 5732 goto error; 5733 } 5734 if (list) { 5735 rte_spinlock_lock(&priv->flow_list_lock); 5736 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 5737 flow, next); 5738 rte_spinlock_unlock(&priv->flow_list_lock); 5739 } 5740 flow_rxq_flags_set(dev, flow); 5741 rte_free(translated_actions); 5742 tunnel = flow_tunnel_from_rule(dev, attr, items, actions); 5743 if (tunnel) { 5744 flow->tunnel = 1; 5745 flow->tunnel_id = tunnel->tunnel_id; 5746 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 5747 mlx5_free(default_miss_ctx.queue); 5748 } 5749 mlx5_flow_pop_thread_workspace(); 5750 return idx; 5751 error: 5752 MLX5_ASSERT(flow); 5753 ret = rte_errno; /* Save rte_errno before cleanup. */ 5754 flow_mreg_del_copy_action(dev, flow); 5755 flow_drv_destroy(dev, flow); 5756 if (rss_desc->shared_rss) 5757 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 5758 mlx5_ipool_get 5759 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 5760 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 5761 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 5762 rte_errno = ret; /* Restore rte_errno. */ 5763 ret = rte_errno; 5764 rte_errno = ret; 5765 mlx5_flow_pop_thread_workspace(); 5766 error_before_hairpin_split: 5767 rte_free(translated_actions); 5768 return 0; 5769 } 5770 5771 /** 5772 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 5773 * incoming packets to table 1. 5774 * 5775 * Other flow rules, requested for group n, will be created in 5776 * e-switch table n+1. 5777 * Jump action to e-switch group n will be created to group n+1. 5778 * 5779 * Used when working in switchdev mode, to utilise advantages of table 1 5780 * and above. 5781 * 5782 * @param dev 5783 * Pointer to Ethernet device. 5784 * 5785 * @return 5786 * Pointer to flow on success, NULL otherwise and rte_errno is set. 5787 */ 5788 struct rte_flow * 5789 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 5790 { 5791 const struct rte_flow_attr attr = { 5792 .group = 0, 5793 .priority = 0, 5794 .ingress = 1, 5795 .egress = 0, 5796 .transfer = 1, 5797 }; 5798 const struct rte_flow_item pattern = { 5799 .type = RTE_FLOW_ITEM_TYPE_END, 5800 }; 5801 struct rte_flow_action_jump jump = { 5802 .group = 1, 5803 }; 5804 const struct rte_flow_action actions[] = { 5805 { 5806 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5807 .conf = &jump, 5808 }, 5809 { 5810 .type = RTE_FLOW_ACTION_TYPE_END, 5811 }, 5812 }; 5813 struct mlx5_priv *priv = dev->data->dev_private; 5814 struct rte_flow_error error; 5815 5816 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 5817 &attr, &pattern, 5818 actions, false, &error); 5819 } 5820 5821 /** 5822 * Validate a flow supported by the NIC. 5823 * 5824 * @see rte_flow_validate() 5825 * @see rte_flow_ops 5826 */ 5827 int 5828 mlx5_flow_validate(struct rte_eth_dev *dev, 5829 const struct rte_flow_attr *attr, 5830 const struct rte_flow_item items[], 5831 const struct rte_flow_action original_actions[], 5832 struct rte_flow_error *error) 5833 { 5834 int hairpin_flow; 5835 struct mlx5_translated_shared_action 5836 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5837 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5838 const struct rte_flow_action *actions; 5839 struct rte_flow_action *translated_actions = NULL; 5840 int ret = flow_shared_actions_translate(dev, original_actions, 5841 shared_actions, 5842 &shared_actions_n, 5843 &translated_actions, error); 5844 5845 if (ret) 5846 return ret; 5847 actions = translated_actions ? translated_actions : original_actions; 5848 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5849 ret = flow_drv_validate(dev, attr, items, actions, 5850 true, hairpin_flow, error); 5851 rte_free(translated_actions); 5852 return ret; 5853 } 5854 5855 /** 5856 * Create a flow. 5857 * 5858 * @see rte_flow_create() 5859 * @see rte_flow_ops 5860 */ 5861 struct rte_flow * 5862 mlx5_flow_create(struct rte_eth_dev *dev, 5863 const struct rte_flow_attr *attr, 5864 const struct rte_flow_item items[], 5865 const struct rte_flow_action actions[], 5866 struct rte_flow_error *error) 5867 { 5868 struct mlx5_priv *priv = dev->data->dev_private; 5869 5870 /* 5871 * If the device is not started yet, it is not allowed to created a 5872 * flow from application. PMD default flows and traffic control flows 5873 * are not affected. 5874 */ 5875 if (unlikely(!dev->data->dev_started)) { 5876 DRV_LOG(DEBUG, "port %u is not started when " 5877 "inserting a flow", dev->data->port_id); 5878 rte_flow_error_set(error, ENODEV, 5879 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5880 NULL, 5881 "port not started"); 5882 return NULL; 5883 } 5884 5885 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 5886 attr, items, actions, true, error); 5887 } 5888 5889 /** 5890 * Destroy a flow in a list. 5891 * 5892 * @param dev 5893 * Pointer to Ethernet device. 5894 * @param list 5895 * Pointer to the Indexed flow list. If this parameter NULL, 5896 * there is no flow removal from the list. Be noted that as 5897 * flow is add to the indexed list, memory of the indexed 5898 * list points to maybe changed as flow destroyed. 5899 * @param[in] flow_idx 5900 * Index of flow to destroy. 5901 */ 5902 static void 5903 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 5904 uint32_t flow_idx) 5905 { 5906 struct mlx5_priv *priv = dev->data->dev_private; 5907 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5908 [MLX5_IPOOL_RTE_FLOW], flow_idx); 5909 5910 if (!flow) 5911 return; 5912 /* 5913 * Update RX queue flags only if port is started, otherwise it is 5914 * already clean. 5915 */ 5916 if (dev->data->dev_started) 5917 flow_rxq_flags_trim(dev, flow); 5918 flow_drv_destroy(dev, flow); 5919 if (list) { 5920 rte_spinlock_lock(&priv->flow_list_lock); 5921 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 5922 flow_idx, flow, next); 5923 rte_spinlock_unlock(&priv->flow_list_lock); 5924 } 5925 if (flow->tunnel) { 5926 struct mlx5_flow_tunnel *tunnel; 5927 5928 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 5929 RTE_VERIFY(tunnel); 5930 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 5931 mlx5_flow_tunnel_free(dev, tunnel); 5932 } 5933 flow_mreg_del_copy_action(dev, flow); 5934 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5935 } 5936 5937 /** 5938 * Destroy all flows. 5939 * 5940 * @param dev 5941 * Pointer to Ethernet device. 5942 * @param list 5943 * Pointer to the Indexed flow list. 5944 * @param active 5945 * If flushing is called avtively. 5946 */ 5947 void 5948 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 5949 { 5950 uint32_t num_flushed = 0; 5951 5952 while (*list) { 5953 flow_list_destroy(dev, list, *list); 5954 num_flushed++; 5955 } 5956 if (active) { 5957 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 5958 dev->data->port_id, num_flushed); 5959 } 5960 } 5961 5962 /** 5963 * Stop all default actions for flows. 5964 * 5965 * @param dev 5966 * Pointer to Ethernet device. 5967 */ 5968 void 5969 mlx5_flow_stop_default(struct rte_eth_dev *dev) 5970 { 5971 flow_mreg_del_default_copy_action(dev); 5972 flow_rxq_flags_clear(dev); 5973 } 5974 5975 /** 5976 * Start all default actions for flows. 5977 * 5978 * @param dev 5979 * Pointer to Ethernet device. 5980 * @return 5981 * 0 on success, a negative errno value otherwise and rte_errno is set. 5982 */ 5983 int 5984 mlx5_flow_start_default(struct rte_eth_dev *dev) 5985 { 5986 struct rte_flow_error error; 5987 5988 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 5989 return flow_mreg_add_default_copy_action(dev, &error); 5990 } 5991 5992 /** 5993 * Release key of thread specific flow workspace data. 5994 */ 5995 void 5996 flow_release_workspace(void *data) 5997 { 5998 struct mlx5_flow_workspace *wks = data; 5999 struct mlx5_flow_workspace *next; 6000 6001 while (wks) { 6002 next = wks->next; 6003 free(wks->rss_desc.queue); 6004 free(wks); 6005 wks = next; 6006 } 6007 } 6008 6009 /** 6010 * Get thread specific current flow workspace. 6011 * 6012 * @return pointer to thread specific flow workspace data, NULL on error. 6013 */ 6014 struct mlx5_flow_workspace* 6015 mlx5_flow_get_thread_workspace(void) 6016 { 6017 struct mlx5_flow_workspace *data; 6018 6019 data = mlx5_flow_os_get_specific_workspace(); 6020 MLX5_ASSERT(data && data->inuse); 6021 if (!data || !data->inuse) 6022 DRV_LOG(ERR, "flow workspace not initialized."); 6023 return data; 6024 } 6025 6026 /** 6027 * Allocate and init new flow workspace. 6028 * 6029 * @return pointer to flow workspace data, NULL on error. 6030 */ 6031 static struct mlx5_flow_workspace* 6032 flow_alloc_thread_workspace(void) 6033 { 6034 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6035 6036 if (!data) { 6037 DRV_LOG(ERR, "Failed to allocate flow workspace " 6038 "memory."); 6039 return NULL; 6040 } 6041 data->rss_desc.queue = calloc(1, 6042 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6043 if (!data->rss_desc.queue) 6044 goto err; 6045 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6046 return data; 6047 err: 6048 if (data->rss_desc.queue) 6049 free(data->rss_desc.queue); 6050 free(data); 6051 return NULL; 6052 } 6053 6054 /** 6055 * Get new thread specific flow workspace. 6056 * 6057 * If current workspace inuse, create new one and set as current. 6058 * 6059 * @return pointer to thread specific flow workspace data, NULL on error. 6060 */ 6061 static struct mlx5_flow_workspace* 6062 mlx5_flow_push_thread_workspace(void) 6063 { 6064 struct mlx5_flow_workspace *curr; 6065 struct mlx5_flow_workspace *data; 6066 6067 curr = mlx5_flow_os_get_specific_workspace(); 6068 if (!curr) { 6069 data = flow_alloc_thread_workspace(); 6070 if (!data) 6071 return NULL; 6072 } else if (!curr->inuse) { 6073 data = curr; 6074 } else if (curr->next) { 6075 data = curr->next; 6076 } else { 6077 data = flow_alloc_thread_workspace(); 6078 if (!data) 6079 return NULL; 6080 curr->next = data; 6081 data->prev = curr; 6082 } 6083 data->inuse = 1; 6084 data->flow_idx = 0; 6085 /* Set as current workspace */ 6086 if (mlx5_flow_os_set_specific_workspace(data)) 6087 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6088 return data; 6089 } 6090 6091 /** 6092 * Close current thread specific flow workspace. 6093 * 6094 * If previous workspace available, set it as current. 6095 * 6096 * @return pointer to thread specific flow workspace data, NULL on error. 6097 */ 6098 static void 6099 mlx5_flow_pop_thread_workspace(void) 6100 { 6101 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6102 6103 if (!data) 6104 return; 6105 if (!data->inuse) { 6106 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6107 return; 6108 } 6109 data->inuse = 0; 6110 if (!data->prev) 6111 return; 6112 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6113 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6114 } 6115 6116 /** 6117 * Verify the flow list is empty 6118 * 6119 * @param dev 6120 * Pointer to Ethernet device. 6121 * 6122 * @return the number of flows not released. 6123 */ 6124 int 6125 mlx5_flow_verify(struct rte_eth_dev *dev) 6126 { 6127 struct mlx5_priv *priv = dev->data->dev_private; 6128 struct rte_flow *flow; 6129 uint32_t idx; 6130 int ret = 0; 6131 6132 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 6133 flow, next) { 6134 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6135 dev->data->port_id, (void *)flow); 6136 ++ret; 6137 } 6138 return ret; 6139 } 6140 6141 /** 6142 * Enable default hairpin egress flow. 6143 * 6144 * @param dev 6145 * Pointer to Ethernet device. 6146 * @param queue 6147 * The queue index. 6148 * 6149 * @return 6150 * 0 on success, a negative errno value otherwise and rte_errno is set. 6151 */ 6152 int 6153 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6154 uint32_t queue) 6155 { 6156 struct mlx5_priv *priv = dev->data->dev_private; 6157 const struct rte_flow_attr attr = { 6158 .egress = 1, 6159 .priority = 0, 6160 }; 6161 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6162 .queue = queue, 6163 }; 6164 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6165 .queue = UINT32_MAX, 6166 }; 6167 struct rte_flow_item items[] = { 6168 { 6169 .type = (enum rte_flow_item_type) 6170 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6171 .spec = &queue_spec, 6172 .last = NULL, 6173 .mask = &queue_mask, 6174 }, 6175 { 6176 .type = RTE_FLOW_ITEM_TYPE_END, 6177 }, 6178 }; 6179 struct rte_flow_action_jump jump = { 6180 .group = MLX5_HAIRPIN_TX_TABLE, 6181 }; 6182 struct rte_flow_action actions[2]; 6183 uint32_t flow_idx; 6184 struct rte_flow_error error; 6185 6186 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6187 actions[0].conf = &jump; 6188 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6189 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6190 &attr, items, actions, false, &error); 6191 if (!flow_idx) { 6192 DRV_LOG(DEBUG, 6193 "Failed to create ctrl flow: rte_errno(%d)," 6194 " type(%d), message(%s)", 6195 rte_errno, error.type, 6196 error.message ? error.message : " (no stated reason)"); 6197 return -rte_errno; 6198 } 6199 return 0; 6200 } 6201 6202 /** 6203 * Enable a control flow configured from the control plane. 6204 * 6205 * @param dev 6206 * Pointer to Ethernet device. 6207 * @param eth_spec 6208 * An Ethernet flow spec to apply. 6209 * @param eth_mask 6210 * An Ethernet flow mask to apply. 6211 * @param vlan_spec 6212 * A VLAN flow spec to apply. 6213 * @param vlan_mask 6214 * A VLAN flow mask to apply. 6215 * 6216 * @return 6217 * 0 on success, a negative errno value otherwise and rte_errno is set. 6218 */ 6219 int 6220 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6221 struct rte_flow_item_eth *eth_spec, 6222 struct rte_flow_item_eth *eth_mask, 6223 struct rte_flow_item_vlan *vlan_spec, 6224 struct rte_flow_item_vlan *vlan_mask) 6225 { 6226 struct mlx5_priv *priv = dev->data->dev_private; 6227 const struct rte_flow_attr attr = { 6228 .ingress = 1, 6229 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6230 }; 6231 struct rte_flow_item items[] = { 6232 { 6233 .type = RTE_FLOW_ITEM_TYPE_ETH, 6234 .spec = eth_spec, 6235 .last = NULL, 6236 .mask = eth_mask, 6237 }, 6238 { 6239 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6240 RTE_FLOW_ITEM_TYPE_END, 6241 .spec = vlan_spec, 6242 .last = NULL, 6243 .mask = vlan_mask, 6244 }, 6245 { 6246 .type = RTE_FLOW_ITEM_TYPE_END, 6247 }, 6248 }; 6249 uint16_t queue[priv->reta_idx_n]; 6250 struct rte_flow_action_rss action_rss = { 6251 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6252 .level = 0, 6253 .types = priv->rss_conf.rss_hf, 6254 .key_len = priv->rss_conf.rss_key_len, 6255 .queue_num = priv->reta_idx_n, 6256 .key = priv->rss_conf.rss_key, 6257 .queue = queue, 6258 }; 6259 struct rte_flow_action actions[] = { 6260 { 6261 .type = RTE_FLOW_ACTION_TYPE_RSS, 6262 .conf = &action_rss, 6263 }, 6264 { 6265 .type = RTE_FLOW_ACTION_TYPE_END, 6266 }, 6267 }; 6268 uint32_t flow_idx; 6269 struct rte_flow_error error; 6270 unsigned int i; 6271 6272 if (!priv->reta_idx_n || !priv->rxqs_n) { 6273 return 0; 6274 } 6275 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6276 action_rss.types = 0; 6277 for (i = 0; i != priv->reta_idx_n; ++i) 6278 queue[i] = (*priv->reta_idx)[i]; 6279 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6280 &attr, items, actions, false, &error); 6281 if (!flow_idx) 6282 return -rte_errno; 6283 return 0; 6284 } 6285 6286 /** 6287 * Enable a flow control configured from the control plane. 6288 * 6289 * @param dev 6290 * Pointer to Ethernet device. 6291 * @param eth_spec 6292 * An Ethernet flow spec to apply. 6293 * @param eth_mask 6294 * An Ethernet flow mask to apply. 6295 * 6296 * @return 6297 * 0 on success, a negative errno value otherwise and rte_errno is set. 6298 */ 6299 int 6300 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6301 struct rte_flow_item_eth *eth_spec, 6302 struct rte_flow_item_eth *eth_mask) 6303 { 6304 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6305 } 6306 6307 /** 6308 * Create default miss flow rule matching lacp traffic 6309 * 6310 * @param dev 6311 * Pointer to Ethernet device. 6312 * @param eth_spec 6313 * An Ethernet flow spec to apply. 6314 * 6315 * @return 6316 * 0 on success, a negative errno value otherwise and rte_errno is set. 6317 */ 6318 int 6319 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6320 { 6321 struct mlx5_priv *priv = dev->data->dev_private; 6322 /* 6323 * The LACP matching is done by only using ether type since using 6324 * a multicast dst mac causes kernel to give low priority to this flow. 6325 */ 6326 static const struct rte_flow_item_eth lacp_spec = { 6327 .type = RTE_BE16(0x8809), 6328 }; 6329 static const struct rte_flow_item_eth lacp_mask = { 6330 .type = 0xffff, 6331 }; 6332 const struct rte_flow_attr attr = { 6333 .ingress = 1, 6334 }; 6335 struct rte_flow_item items[] = { 6336 { 6337 .type = RTE_FLOW_ITEM_TYPE_ETH, 6338 .spec = &lacp_spec, 6339 .mask = &lacp_mask, 6340 }, 6341 { 6342 .type = RTE_FLOW_ITEM_TYPE_END, 6343 }, 6344 }; 6345 struct rte_flow_action actions[] = { 6346 { 6347 .type = (enum rte_flow_action_type) 6348 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6349 }, 6350 { 6351 .type = RTE_FLOW_ACTION_TYPE_END, 6352 }, 6353 }; 6354 struct rte_flow_error error; 6355 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6356 &attr, items, actions, false, &error); 6357 6358 if (!flow_idx) 6359 return -rte_errno; 6360 return 0; 6361 } 6362 6363 /** 6364 * Destroy a flow. 6365 * 6366 * @see rte_flow_destroy() 6367 * @see rte_flow_ops 6368 */ 6369 int 6370 mlx5_flow_destroy(struct rte_eth_dev *dev, 6371 struct rte_flow *flow, 6372 struct rte_flow_error *error __rte_unused) 6373 { 6374 struct mlx5_priv *priv = dev->data->dev_private; 6375 6376 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6377 return 0; 6378 } 6379 6380 /** 6381 * Destroy all flows. 6382 * 6383 * @see rte_flow_flush() 6384 * @see rte_flow_ops 6385 */ 6386 int 6387 mlx5_flow_flush(struct rte_eth_dev *dev, 6388 struct rte_flow_error *error __rte_unused) 6389 { 6390 struct mlx5_priv *priv = dev->data->dev_private; 6391 6392 mlx5_flow_list_flush(dev, &priv->flows, false); 6393 return 0; 6394 } 6395 6396 /** 6397 * Isolated mode. 6398 * 6399 * @see rte_flow_isolate() 6400 * @see rte_flow_ops 6401 */ 6402 int 6403 mlx5_flow_isolate(struct rte_eth_dev *dev, 6404 int enable, 6405 struct rte_flow_error *error) 6406 { 6407 struct mlx5_priv *priv = dev->data->dev_private; 6408 6409 if (dev->data->dev_started) { 6410 rte_flow_error_set(error, EBUSY, 6411 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6412 NULL, 6413 "port must be stopped first"); 6414 return -rte_errno; 6415 } 6416 priv->isolated = !!enable; 6417 if (enable) 6418 dev->dev_ops = &mlx5_dev_ops_isolate; 6419 else 6420 dev->dev_ops = &mlx5_dev_ops; 6421 6422 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 6423 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 6424 6425 return 0; 6426 } 6427 6428 /** 6429 * Query a flow. 6430 * 6431 * @see rte_flow_query() 6432 * @see rte_flow_ops 6433 */ 6434 static int 6435 flow_drv_query(struct rte_eth_dev *dev, 6436 uint32_t flow_idx, 6437 const struct rte_flow_action *actions, 6438 void *data, 6439 struct rte_flow_error *error) 6440 { 6441 struct mlx5_priv *priv = dev->data->dev_private; 6442 const struct mlx5_flow_driver_ops *fops; 6443 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6444 [MLX5_IPOOL_RTE_FLOW], 6445 flow_idx); 6446 enum mlx5_flow_drv_type ftype; 6447 6448 if (!flow) { 6449 return rte_flow_error_set(error, ENOENT, 6450 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6451 NULL, 6452 "invalid flow handle"); 6453 } 6454 ftype = flow->drv_type; 6455 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 6456 fops = flow_get_drv_ops(ftype); 6457 6458 return fops->query(dev, flow, actions, data, error); 6459 } 6460 6461 /** 6462 * Query a flow. 6463 * 6464 * @see rte_flow_query() 6465 * @see rte_flow_ops 6466 */ 6467 int 6468 mlx5_flow_query(struct rte_eth_dev *dev, 6469 struct rte_flow *flow, 6470 const struct rte_flow_action *actions, 6471 void *data, 6472 struct rte_flow_error *error) 6473 { 6474 int ret; 6475 6476 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 6477 error); 6478 if (ret < 0) 6479 return ret; 6480 return 0; 6481 } 6482 6483 /** 6484 * Get rte_flow callbacks. 6485 * 6486 * @param dev 6487 * Pointer to Ethernet device structure. 6488 * @param ops 6489 * Pointer to operation-specific structure. 6490 * 6491 * @return 0 6492 */ 6493 int 6494 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 6495 const struct rte_flow_ops **ops) 6496 { 6497 *ops = &mlx5_flow_ops; 6498 return 0; 6499 } 6500 6501 /** 6502 * Create the needed meter and suffix tables. 6503 * 6504 * @param[in] dev 6505 * Pointer to Ethernet device. 6506 * @param[in] fm 6507 * Pointer to the flow meter. 6508 * 6509 * @return 6510 * Pointer to table set on success, NULL otherwise. 6511 */ 6512 struct mlx5_meter_domains_infos * 6513 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 6514 const struct mlx5_flow_meter *fm) 6515 { 6516 const struct mlx5_flow_driver_ops *fops; 6517 6518 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6519 return fops->create_mtr_tbls(dev, fm); 6520 } 6521 6522 /** 6523 * Destroy the meter table set. 6524 * 6525 * @param[in] dev 6526 * Pointer to Ethernet device. 6527 * @param[in] tbl 6528 * Pointer to the meter table set. 6529 * 6530 * @return 6531 * 0 on success. 6532 */ 6533 int 6534 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 6535 struct mlx5_meter_domains_infos *tbls) 6536 { 6537 const struct mlx5_flow_driver_ops *fops; 6538 6539 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6540 return fops->destroy_mtr_tbls(dev, tbls); 6541 } 6542 6543 /** 6544 * Create policer rules. 6545 * 6546 * @param[in] dev 6547 * Pointer to Ethernet device. 6548 * @param[in] fm 6549 * Pointer to flow meter structure. 6550 * @param[in] attr 6551 * Pointer to flow attributes. 6552 * 6553 * @return 6554 * 0 on success, -1 otherwise. 6555 */ 6556 int 6557 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 6558 struct mlx5_flow_meter *fm, 6559 const struct rte_flow_attr *attr) 6560 { 6561 const struct mlx5_flow_driver_ops *fops; 6562 6563 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6564 return fops->create_policer_rules(dev, fm, attr); 6565 } 6566 6567 /** 6568 * Destroy policer rules. 6569 * 6570 * @param[in] fm 6571 * Pointer to flow meter structure. 6572 * @param[in] attr 6573 * Pointer to flow attributes. 6574 * 6575 * @return 6576 * 0 on success, -1 otherwise. 6577 */ 6578 int 6579 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 6580 struct mlx5_flow_meter *fm, 6581 const struct rte_flow_attr *attr) 6582 { 6583 const struct mlx5_flow_driver_ops *fops; 6584 6585 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6586 return fops->destroy_policer_rules(dev, fm, attr); 6587 } 6588 6589 /** 6590 * Allocate a counter. 6591 * 6592 * @param[in] dev 6593 * Pointer to Ethernet device structure. 6594 * 6595 * @return 6596 * Index to allocated counter on success, 0 otherwise. 6597 */ 6598 uint32_t 6599 mlx5_counter_alloc(struct rte_eth_dev *dev) 6600 { 6601 const struct mlx5_flow_driver_ops *fops; 6602 struct rte_flow_attr attr = { .transfer = 0 }; 6603 6604 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6605 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6606 return fops->counter_alloc(dev); 6607 } 6608 DRV_LOG(ERR, 6609 "port %u counter allocate is not supported.", 6610 dev->data->port_id); 6611 return 0; 6612 } 6613 6614 /** 6615 * Free a counter. 6616 * 6617 * @param[in] dev 6618 * Pointer to Ethernet device structure. 6619 * @param[in] cnt 6620 * Index to counter to be free. 6621 */ 6622 void 6623 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 6624 { 6625 const struct mlx5_flow_driver_ops *fops; 6626 struct rte_flow_attr attr = { .transfer = 0 }; 6627 6628 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6629 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6630 fops->counter_free(dev, cnt); 6631 return; 6632 } 6633 DRV_LOG(ERR, 6634 "port %u counter free is not supported.", 6635 dev->data->port_id); 6636 } 6637 6638 /** 6639 * Query counter statistics. 6640 * 6641 * @param[in] dev 6642 * Pointer to Ethernet device structure. 6643 * @param[in] cnt 6644 * Index to counter to query. 6645 * @param[in] clear 6646 * Set to clear counter statistics. 6647 * @param[out] pkts 6648 * The counter hits packets number to save. 6649 * @param[out] bytes 6650 * The counter hits bytes number to save. 6651 * 6652 * @return 6653 * 0 on success, a negative errno value otherwise. 6654 */ 6655 int 6656 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 6657 bool clear, uint64_t *pkts, uint64_t *bytes) 6658 { 6659 const struct mlx5_flow_driver_ops *fops; 6660 struct rte_flow_attr attr = { .transfer = 0 }; 6661 6662 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6663 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6664 return fops->counter_query(dev, cnt, clear, pkts, bytes); 6665 } 6666 DRV_LOG(ERR, 6667 "port %u counter query is not supported.", 6668 dev->data->port_id); 6669 return -ENOTSUP; 6670 } 6671 6672 /** 6673 * Allocate a new memory for the counter values wrapped by all the needed 6674 * management. 6675 * 6676 * @param[in] sh 6677 * Pointer to mlx5_dev_ctx_shared object. 6678 * 6679 * @return 6680 * 0 on success, a negative errno value otherwise. 6681 */ 6682 static int 6683 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 6684 { 6685 struct mlx5_devx_mkey_attr mkey_attr; 6686 struct mlx5_counter_stats_mem_mng *mem_mng; 6687 volatile struct flow_counter_stats *raw_data; 6688 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 6689 int size = (sizeof(struct flow_counter_stats) * 6690 MLX5_COUNTERS_PER_POOL + 6691 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 6692 sizeof(struct mlx5_counter_stats_mem_mng); 6693 size_t pgsize = rte_mem_page_size(); 6694 uint8_t *mem; 6695 int i; 6696 6697 if (pgsize == (size_t)-1) { 6698 DRV_LOG(ERR, "Failed to get mem page size"); 6699 rte_errno = ENOMEM; 6700 return -ENOMEM; 6701 } 6702 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 6703 if (!mem) { 6704 rte_errno = ENOMEM; 6705 return -ENOMEM; 6706 } 6707 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 6708 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 6709 mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size, 6710 IBV_ACCESS_LOCAL_WRITE); 6711 if (!mem_mng->umem) { 6712 rte_errno = errno; 6713 mlx5_free(mem); 6714 return -rte_errno; 6715 } 6716 mkey_attr.addr = (uintptr_t)mem; 6717 mkey_attr.size = size; 6718 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 6719 mkey_attr.pd = sh->pdn; 6720 mkey_attr.log_entity_size = 0; 6721 mkey_attr.pg_access = 0; 6722 mkey_attr.klm_array = NULL; 6723 mkey_attr.klm_num = 0; 6724 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 6725 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 6726 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 6727 if (!mem_mng->dm) { 6728 mlx5_os_umem_dereg(mem_mng->umem); 6729 rte_errno = errno; 6730 mlx5_free(mem); 6731 return -rte_errno; 6732 } 6733 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 6734 raw_data = (volatile struct flow_counter_stats *)mem; 6735 for (i = 0; i < raws_n; ++i) { 6736 mem_mng->raws[i].mem_mng = mem_mng; 6737 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 6738 } 6739 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 6740 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 6741 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 6742 next); 6743 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 6744 sh->cmng.mem_mng = mem_mng; 6745 return 0; 6746 } 6747 6748 /** 6749 * Set the statistic memory to the new counter pool. 6750 * 6751 * @param[in] sh 6752 * Pointer to mlx5_dev_ctx_shared object. 6753 * @param[in] pool 6754 * Pointer to the pool to set the statistic memory. 6755 * 6756 * @return 6757 * 0 on success, a negative errno value otherwise. 6758 */ 6759 static int 6760 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 6761 struct mlx5_flow_counter_pool *pool) 6762 { 6763 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6764 /* Resize statistic memory once used out. */ 6765 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 6766 mlx5_flow_create_counter_stat_mem_mng(sh)) { 6767 DRV_LOG(ERR, "Cannot resize counter stat mem."); 6768 return -1; 6769 } 6770 rte_spinlock_lock(&pool->sl); 6771 pool->raw = cmng->mem_mng->raws + pool->index % 6772 MLX5_CNT_CONTAINER_RESIZE; 6773 rte_spinlock_unlock(&pool->sl); 6774 pool->raw_hw = NULL; 6775 return 0; 6776 } 6777 6778 #define MLX5_POOL_QUERY_FREQ_US 1000000 6779 6780 /** 6781 * Set the periodic procedure for triggering asynchronous batch queries for all 6782 * the counter pools. 6783 * 6784 * @param[in] sh 6785 * Pointer to mlx5_dev_ctx_shared object. 6786 */ 6787 void 6788 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 6789 { 6790 uint32_t pools_n, us; 6791 6792 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 6793 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 6794 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 6795 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 6796 sh->cmng.query_thread_on = 0; 6797 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 6798 } else { 6799 sh->cmng.query_thread_on = 1; 6800 } 6801 } 6802 6803 /** 6804 * The periodic procedure for triggering asynchronous batch queries for all the 6805 * counter pools. This function is probably called by the host thread. 6806 * 6807 * @param[in] arg 6808 * The parameter for the alarm process. 6809 */ 6810 void 6811 mlx5_flow_query_alarm(void *arg) 6812 { 6813 struct mlx5_dev_ctx_shared *sh = arg; 6814 int ret; 6815 uint16_t pool_index = sh->cmng.pool_index; 6816 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6817 struct mlx5_flow_counter_pool *pool; 6818 uint16_t n_valid; 6819 6820 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 6821 goto set_alarm; 6822 rte_spinlock_lock(&cmng->pool_update_sl); 6823 pool = cmng->pools[pool_index]; 6824 n_valid = cmng->n_valid; 6825 rte_spinlock_unlock(&cmng->pool_update_sl); 6826 /* Set the statistic memory to the new created pool. */ 6827 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 6828 goto set_alarm; 6829 if (pool->raw_hw) 6830 /* There is a pool query in progress. */ 6831 goto set_alarm; 6832 pool->raw_hw = 6833 LIST_FIRST(&sh->cmng.free_stat_raws); 6834 if (!pool->raw_hw) 6835 /* No free counter statistics raw memory. */ 6836 goto set_alarm; 6837 /* 6838 * Identify the counters released between query trigger and query 6839 * handle more efficiently. The counter released in this gap period 6840 * should wait for a new round of query as the new arrived packets 6841 * will not be taken into account. 6842 */ 6843 pool->query_gen++; 6844 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 6845 MLX5_COUNTERS_PER_POOL, 6846 NULL, NULL, 6847 pool->raw_hw->mem_mng->dm->id, 6848 (void *)(uintptr_t) 6849 pool->raw_hw->data, 6850 sh->devx_comp, 6851 (uint64_t)(uintptr_t)pool); 6852 if (ret) { 6853 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 6854 " %d", pool->min_dcs->id); 6855 pool->raw_hw = NULL; 6856 goto set_alarm; 6857 } 6858 LIST_REMOVE(pool->raw_hw, next); 6859 sh->cmng.pending_queries++; 6860 pool_index++; 6861 if (pool_index >= n_valid) 6862 pool_index = 0; 6863 set_alarm: 6864 sh->cmng.pool_index = pool_index; 6865 mlx5_set_query_alarm(sh); 6866 } 6867 6868 /** 6869 * Check and callback event for new aged flow in the counter pool 6870 * 6871 * @param[in] sh 6872 * Pointer to mlx5_dev_ctx_shared object. 6873 * @param[in] pool 6874 * Pointer to Current counter pool. 6875 */ 6876 static void 6877 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 6878 struct mlx5_flow_counter_pool *pool) 6879 { 6880 struct mlx5_priv *priv; 6881 struct mlx5_flow_counter *cnt; 6882 struct mlx5_age_info *age_info; 6883 struct mlx5_age_param *age_param; 6884 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 6885 struct mlx5_counter_stats_raw *prev = pool->raw; 6886 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 6887 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 6888 uint16_t expected = AGE_CANDIDATE; 6889 uint32_t i; 6890 6891 pool->time_of_last_age_check = curr_time; 6892 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 6893 cnt = MLX5_POOL_GET_CNT(pool, i); 6894 age_param = MLX5_CNT_TO_AGE(cnt); 6895 if (__atomic_load_n(&age_param->state, 6896 __ATOMIC_RELAXED) != AGE_CANDIDATE) 6897 continue; 6898 if (cur->data[i].hits != prev->data[i].hits) { 6899 __atomic_store_n(&age_param->sec_since_last_hit, 0, 6900 __ATOMIC_RELAXED); 6901 continue; 6902 } 6903 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 6904 time_delta, 6905 __ATOMIC_RELAXED) <= age_param->timeout) 6906 continue; 6907 /** 6908 * Hold the lock first, or if between the 6909 * state AGE_TMOUT and tailq operation the 6910 * release happened, the release procedure 6911 * may delete a non-existent tailq node. 6912 */ 6913 priv = rte_eth_devices[age_param->port_id].data->dev_private; 6914 age_info = GET_PORT_AGE_INFO(priv); 6915 rte_spinlock_lock(&age_info->aged_sl); 6916 if (__atomic_compare_exchange_n(&age_param->state, &expected, 6917 AGE_TMOUT, false, 6918 __ATOMIC_RELAXED, 6919 __ATOMIC_RELAXED)) { 6920 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 6921 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 6922 } 6923 rte_spinlock_unlock(&age_info->aged_sl); 6924 } 6925 mlx5_age_event_prepare(sh); 6926 } 6927 6928 /** 6929 * Handler for the HW respond about ready values from an asynchronous batch 6930 * query. This function is probably called by the host thread. 6931 * 6932 * @param[in] sh 6933 * The pointer to the shared device context. 6934 * @param[in] async_id 6935 * The Devx async ID. 6936 * @param[in] status 6937 * The status of the completion. 6938 */ 6939 void 6940 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 6941 uint64_t async_id, int status) 6942 { 6943 struct mlx5_flow_counter_pool *pool = 6944 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 6945 struct mlx5_counter_stats_raw *raw_to_free; 6946 uint8_t query_gen = pool->query_gen ^ 1; 6947 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6948 enum mlx5_counter_type cnt_type = 6949 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 6950 MLX5_COUNTER_TYPE_ORIGIN; 6951 6952 if (unlikely(status)) { 6953 raw_to_free = pool->raw_hw; 6954 } else { 6955 raw_to_free = pool->raw; 6956 if (pool->is_aged) 6957 mlx5_flow_aging_check(sh, pool); 6958 rte_spinlock_lock(&pool->sl); 6959 pool->raw = pool->raw_hw; 6960 rte_spinlock_unlock(&pool->sl); 6961 /* Be sure the new raw counters data is updated in memory. */ 6962 rte_io_wmb(); 6963 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 6964 rte_spinlock_lock(&cmng->csl[cnt_type]); 6965 TAILQ_CONCAT(&cmng->counters[cnt_type], 6966 &pool->counters[query_gen], next); 6967 rte_spinlock_unlock(&cmng->csl[cnt_type]); 6968 } 6969 } 6970 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 6971 pool->raw_hw = NULL; 6972 sh->cmng.pending_queries--; 6973 } 6974 6975 static int 6976 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 6977 const struct flow_grp_info *grp_info, 6978 struct rte_flow_error *error) 6979 { 6980 if (grp_info->transfer && grp_info->external && 6981 grp_info->fdb_def_rule) { 6982 if (group == UINT32_MAX) 6983 return rte_flow_error_set 6984 (error, EINVAL, 6985 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6986 NULL, 6987 "group index not supported"); 6988 *table = group + 1; 6989 } else { 6990 *table = group; 6991 } 6992 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 6993 return 0; 6994 } 6995 6996 /** 6997 * Translate the rte_flow group index to HW table value. 6998 * 6999 * If tunnel offload is disabled, all group ids converted to flow table 7000 * id using the standard method. 7001 * If tunnel offload is enabled, group id can be converted using the 7002 * standard or tunnel conversion method. Group conversion method 7003 * selection depends on flags in `grp_info` parameter: 7004 * - Internal (grp_info.external == 0) groups conversion uses the 7005 * standard method. 7006 * - Group ids in JUMP action converted with the tunnel conversion. 7007 * - Group id in rule attribute conversion depends on a rule type and 7008 * group id value: 7009 * ** non zero group attributes converted with the tunnel method 7010 * ** zero group attribute in non-tunnel rule is converted using the 7011 * standard method - there's only one root table 7012 * ** zero group attribute in steer tunnel rule is converted with the 7013 * standard method - single root table 7014 * ** zero group attribute in match tunnel rule is a special OvS 7015 * case: that value is used for portability reasons. That group 7016 * id is converted with the tunnel conversion method. 7017 * 7018 * @param[in] dev 7019 * Port device 7020 * @param[in] tunnel 7021 * PMD tunnel offload object 7022 * @param[in] group 7023 * rte_flow group index value. 7024 * @param[out] table 7025 * HW table value. 7026 * @param[in] grp_info 7027 * flags used for conversion 7028 * @param[out] error 7029 * Pointer to error structure. 7030 * 7031 * @return 7032 * 0 on success, a negative errno value otherwise and rte_errno is set. 7033 */ 7034 int 7035 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7036 const struct mlx5_flow_tunnel *tunnel, 7037 uint32_t group, uint32_t *table, 7038 const struct flow_grp_info *grp_info, 7039 struct rte_flow_error *error) 7040 { 7041 int ret; 7042 bool standard_translation; 7043 7044 if (!grp_info->skip_scale && grp_info->external && 7045 group < MLX5_MAX_TABLES_EXTERNAL) 7046 group *= MLX5_FLOW_TABLE_FACTOR; 7047 if (is_tunnel_offload_active(dev)) { 7048 standard_translation = !grp_info->external || 7049 grp_info->std_tbl_fix; 7050 } else { 7051 standard_translation = true; 7052 } 7053 DRV_LOG(DEBUG, 7054 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 7055 dev->data->port_id, group, grp_info->transfer, 7056 grp_info->external, grp_info->fdb_def_rule, 7057 standard_translation ? "STANDARD" : "TUNNEL"); 7058 if (standard_translation) 7059 ret = flow_group_to_table(dev->data->port_id, group, table, 7060 grp_info, error); 7061 else 7062 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7063 table, error); 7064 7065 return ret; 7066 } 7067 7068 /** 7069 * Discover availability of metadata reg_c's. 7070 * 7071 * Iteratively use test flows to check availability. 7072 * 7073 * @param[in] dev 7074 * Pointer to the Ethernet device structure. 7075 * 7076 * @return 7077 * 0 on success, a negative errno value otherwise and rte_errno is set. 7078 */ 7079 int 7080 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7081 { 7082 struct mlx5_priv *priv = dev->data->dev_private; 7083 struct mlx5_dev_config *config = &priv->config; 7084 enum modify_reg idx; 7085 int n = 0; 7086 7087 /* reg_c[0] and reg_c[1] are reserved. */ 7088 config->flow_mreg_c[n++] = REG_C_0; 7089 config->flow_mreg_c[n++] = REG_C_1; 7090 /* Discover availability of other reg_c's. */ 7091 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7092 struct rte_flow_attr attr = { 7093 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7094 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7095 .ingress = 1, 7096 }; 7097 struct rte_flow_item items[] = { 7098 [0] = { 7099 .type = RTE_FLOW_ITEM_TYPE_END, 7100 }, 7101 }; 7102 struct rte_flow_action actions[] = { 7103 [0] = { 7104 .type = (enum rte_flow_action_type) 7105 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7106 .conf = &(struct mlx5_flow_action_copy_mreg){ 7107 .src = REG_C_1, 7108 .dst = idx, 7109 }, 7110 }, 7111 [1] = { 7112 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7113 .conf = &(struct rte_flow_action_jump){ 7114 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7115 }, 7116 }, 7117 [2] = { 7118 .type = RTE_FLOW_ACTION_TYPE_END, 7119 }, 7120 }; 7121 uint32_t flow_idx; 7122 struct rte_flow *flow; 7123 struct rte_flow_error error; 7124 7125 if (!config->dv_flow_en) 7126 break; 7127 /* Create internal flow, validation skips copy action. */ 7128 flow_idx = flow_list_create(dev, NULL, &attr, items, 7129 actions, false, &error); 7130 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 7131 flow_idx); 7132 if (!flow) 7133 continue; 7134 config->flow_mreg_c[n++] = idx; 7135 flow_list_destroy(dev, NULL, flow_idx); 7136 } 7137 for (; n < MLX5_MREG_C_NUM; ++n) 7138 config->flow_mreg_c[n] = REG_NON; 7139 return 0; 7140 } 7141 7142 /** 7143 * Dump flow raw hw data to file 7144 * 7145 * @param[in] dev 7146 * The pointer to Ethernet device. 7147 * @param[in] file 7148 * A pointer to a file for output. 7149 * @param[out] error 7150 * Perform verbose error reporting if not NULL. PMDs initialize this 7151 * structure in case of error only. 7152 * @return 7153 * 0 on success, a nagative value otherwise. 7154 */ 7155 int 7156 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 7157 FILE *file, 7158 struct rte_flow_error *error __rte_unused) 7159 { 7160 struct mlx5_priv *priv = dev->data->dev_private; 7161 struct mlx5_dev_ctx_shared *sh = priv->sh; 7162 7163 if (!priv->config.dv_flow_en) { 7164 if (fputs("device dv flow disabled\n", file) <= 0) 7165 return -errno; 7166 return -ENOTSUP; 7167 } 7168 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 7169 sh->tx_domain, file); 7170 } 7171 7172 /** 7173 * Get aged-out flows. 7174 * 7175 * @param[in] dev 7176 * Pointer to the Ethernet device structure. 7177 * @param[in] context 7178 * The address of an array of pointers to the aged-out flows contexts. 7179 * @param[in] nb_countexts 7180 * The length of context array pointers. 7181 * @param[out] error 7182 * Perform verbose error reporting if not NULL. Initialized in case of 7183 * error only. 7184 * 7185 * @return 7186 * how many contexts get in success, otherwise negative errno value. 7187 * if nb_contexts is 0, return the amount of all aged contexts. 7188 * if nb_contexts is not 0 , return the amount of aged flows reported 7189 * in the context array. 7190 */ 7191 int 7192 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 7193 uint32_t nb_contexts, struct rte_flow_error *error) 7194 { 7195 const struct mlx5_flow_driver_ops *fops; 7196 struct rte_flow_attr attr = { .transfer = 0 }; 7197 7198 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7199 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7200 return fops->get_aged_flows(dev, contexts, nb_contexts, 7201 error); 7202 } 7203 DRV_LOG(ERR, 7204 "port %u get aged flows is not supported.", 7205 dev->data->port_id); 7206 return -ENOTSUP; 7207 } 7208 7209 /* Wrapper for driver action_validate op callback */ 7210 static int 7211 flow_drv_action_validate(struct rte_eth_dev *dev, 7212 const struct rte_flow_shared_action_conf *conf, 7213 const struct rte_flow_action *action, 7214 const struct mlx5_flow_driver_ops *fops, 7215 struct rte_flow_error *error) 7216 { 7217 static const char err_msg[] = "shared action validation unsupported"; 7218 7219 if (!fops->action_validate) { 7220 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7221 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7222 NULL, err_msg); 7223 return -rte_errno; 7224 } 7225 return fops->action_validate(dev, conf, action, error); 7226 } 7227 7228 /** 7229 * Destroys the shared action by handle. 7230 * 7231 * @param dev 7232 * Pointer to Ethernet device structure. 7233 * @param[in] action 7234 * Handle for the shared action to be destroyed. 7235 * @param[out] error 7236 * Perform verbose error reporting if not NULL. PMDs initialize this 7237 * structure in case of error only. 7238 * 7239 * @return 7240 * 0 on success, a negative errno value otherwise and rte_errno is set. 7241 * 7242 * @note: wrapper for driver action_create op callback. 7243 */ 7244 static int 7245 mlx5_shared_action_destroy(struct rte_eth_dev *dev, 7246 struct rte_flow_shared_action *action, 7247 struct rte_flow_error *error) 7248 { 7249 static const char err_msg[] = "shared action destruction unsupported"; 7250 struct rte_flow_attr attr = { .transfer = 0 }; 7251 const struct mlx5_flow_driver_ops *fops = 7252 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7253 7254 if (!fops->action_destroy) { 7255 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7256 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7257 NULL, err_msg); 7258 return -rte_errno; 7259 } 7260 return fops->action_destroy(dev, action, error); 7261 } 7262 7263 /* Wrapper for driver action_destroy op callback */ 7264 static int 7265 flow_drv_action_update(struct rte_eth_dev *dev, 7266 struct rte_flow_shared_action *action, 7267 const void *action_conf, 7268 const struct mlx5_flow_driver_ops *fops, 7269 struct rte_flow_error *error) 7270 { 7271 static const char err_msg[] = "shared action update unsupported"; 7272 7273 if (!fops->action_update) { 7274 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7275 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7276 NULL, err_msg); 7277 return -rte_errno; 7278 } 7279 return fops->action_update(dev, action, action_conf, error); 7280 } 7281 7282 /* Wrapper for driver action_destroy op callback */ 7283 static int 7284 flow_drv_action_query(struct rte_eth_dev *dev, 7285 const struct rte_flow_shared_action *action, 7286 void *data, 7287 const struct mlx5_flow_driver_ops *fops, 7288 struct rte_flow_error *error) 7289 { 7290 static const char err_msg[] = "shared action query unsupported"; 7291 7292 if (!fops->action_query) { 7293 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7294 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7295 NULL, err_msg); 7296 return -rte_errno; 7297 } 7298 return fops->action_query(dev, action, data, error); 7299 } 7300 7301 /** 7302 * Create shared action for reuse in multiple flow rules. 7303 * 7304 * @param dev 7305 * Pointer to Ethernet device structure. 7306 * @param[in] action 7307 * Action configuration for shared action creation. 7308 * @param[out] error 7309 * Perform verbose error reporting if not NULL. PMDs initialize this 7310 * structure in case of error only. 7311 * @return 7312 * A valid handle in case of success, NULL otherwise and rte_errno is set. 7313 */ 7314 static struct rte_flow_shared_action * 7315 mlx5_shared_action_create(struct rte_eth_dev *dev, 7316 const struct rte_flow_shared_action_conf *conf, 7317 const struct rte_flow_action *action, 7318 struct rte_flow_error *error) 7319 { 7320 static const char err_msg[] = "shared action creation unsupported"; 7321 struct rte_flow_attr attr = { .transfer = 0 }; 7322 const struct mlx5_flow_driver_ops *fops = 7323 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7324 7325 if (flow_drv_action_validate(dev, conf, action, fops, error)) 7326 return NULL; 7327 if (!fops->action_create) { 7328 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7329 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7330 NULL, err_msg); 7331 return NULL; 7332 } 7333 return fops->action_create(dev, conf, action, error); 7334 } 7335 7336 /** 7337 * Updates inplace the shared action configuration pointed by *action* handle 7338 * with the configuration provided as *action* argument. 7339 * The update of the shared action configuration effects all flow rules reusing 7340 * the action via handle. 7341 * 7342 * @param dev 7343 * Pointer to Ethernet device structure. 7344 * @param[in] shared_action 7345 * Handle for the shared action to be updated. 7346 * @param[in] action 7347 * Action specification used to modify the action pointed by handle. 7348 * *action* should be of same type with the action pointed by the *action* 7349 * handle argument, otherwise considered as invalid. 7350 * @param[out] error 7351 * Perform verbose error reporting if not NULL. PMDs initialize this 7352 * structure in case of error only. 7353 * 7354 * @return 7355 * 0 on success, a negative errno value otherwise and rte_errno is set. 7356 */ 7357 static int 7358 mlx5_shared_action_update(struct rte_eth_dev *dev, 7359 struct rte_flow_shared_action *shared_action, 7360 const struct rte_flow_action *action, 7361 struct rte_flow_error *error) 7362 { 7363 struct rte_flow_attr attr = { .transfer = 0 }; 7364 const struct mlx5_flow_driver_ops *fops = 7365 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7366 int ret; 7367 7368 ret = flow_drv_action_validate(dev, NULL, action, fops, error); 7369 if (ret) 7370 return ret; 7371 return flow_drv_action_update(dev, shared_action, action->conf, fops, 7372 error); 7373 } 7374 7375 /** 7376 * Query the shared action by handle. 7377 * 7378 * This function allows retrieving action-specific data such as counters. 7379 * Data is gathered by special action which may be present/referenced in 7380 * more than one flow rule definition. 7381 * 7382 * \see RTE_FLOW_ACTION_TYPE_COUNT 7383 * 7384 * @param dev 7385 * Pointer to Ethernet device structure. 7386 * @param[in] action 7387 * Handle for the shared action to query. 7388 * @param[in, out] data 7389 * Pointer to storage for the associated query data type. 7390 * @param[out] error 7391 * Perform verbose error reporting if not NULL. PMDs initialize this 7392 * structure in case of error only. 7393 * 7394 * @return 7395 * 0 on success, a negative errno value otherwise and rte_errno is set. 7396 */ 7397 static int 7398 mlx5_shared_action_query(struct rte_eth_dev *dev, 7399 const struct rte_flow_shared_action *action, 7400 void *data, 7401 struct rte_flow_error *error) 7402 { 7403 struct rte_flow_attr attr = { .transfer = 0 }; 7404 const struct mlx5_flow_driver_ops *fops = 7405 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7406 7407 return flow_drv_action_query(dev, action, data, fops, error); 7408 } 7409 7410 /** 7411 * Destroy all shared actions. 7412 * 7413 * @param dev 7414 * Pointer to Ethernet device. 7415 * 7416 * @return 7417 * 0 on success, a negative errno value otherwise and rte_errno is set. 7418 */ 7419 int 7420 mlx5_shared_action_flush(struct rte_eth_dev *dev) 7421 { 7422 struct rte_flow_error error; 7423 struct mlx5_priv *priv = dev->data->dev_private; 7424 struct mlx5_shared_action_rss *shared_rss; 7425 int ret = 0; 7426 uint32_t idx; 7427 7428 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7429 priv->rss_shared_actions, idx, shared_rss, next) { 7430 ret |= mlx5_shared_action_destroy(dev, 7431 (struct rte_flow_shared_action *)(uintptr_t)idx, &error); 7432 } 7433 return ret; 7434 } 7435 7436 #ifndef HAVE_MLX5DV_DR 7437 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 7438 #else 7439 #define MLX5_DOMAIN_SYNC_FLOW \ 7440 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 7441 #endif 7442 7443 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 7444 { 7445 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 7446 const struct mlx5_flow_driver_ops *fops; 7447 int ret; 7448 struct rte_flow_attr attr = { .transfer = 0 }; 7449 7450 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7451 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 7452 if (ret > 0) 7453 ret = -ret; 7454 return ret; 7455 } 7456 7457 /** 7458 * tunnel offload functionalilty is defined for DV environment only 7459 */ 7460 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7461 __extension__ 7462 union tunnel_offload_mark { 7463 uint32_t val; 7464 struct { 7465 uint32_t app_reserve:8; 7466 uint32_t table_id:15; 7467 uint32_t transfer:1; 7468 uint32_t _unused_:8; 7469 }; 7470 }; 7471 7472 static bool 7473 mlx5_access_tunnel_offload_db 7474 (struct rte_eth_dev *dev, 7475 bool (*match)(struct rte_eth_dev *, 7476 struct mlx5_flow_tunnel *, const void *), 7477 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 7478 void (*miss)(struct rte_eth_dev *, void *), 7479 void *ctx, bool lock_op); 7480 7481 static int 7482 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 7483 struct rte_flow *flow, 7484 const struct rte_flow_attr *attr, 7485 const struct rte_flow_action *app_actions, 7486 uint32_t flow_idx, 7487 struct tunnel_default_miss_ctx *ctx, 7488 struct rte_flow_error *error) 7489 { 7490 struct mlx5_priv *priv = dev->data->dev_private; 7491 struct mlx5_flow *dev_flow; 7492 struct rte_flow_attr miss_attr = *attr; 7493 const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf; 7494 const struct rte_flow_item miss_items[2] = { 7495 { 7496 .type = RTE_FLOW_ITEM_TYPE_ETH, 7497 .spec = NULL, 7498 .last = NULL, 7499 .mask = NULL 7500 }, 7501 { 7502 .type = RTE_FLOW_ITEM_TYPE_END, 7503 .spec = NULL, 7504 .last = NULL, 7505 .mask = NULL 7506 } 7507 }; 7508 union tunnel_offload_mark mark_id; 7509 struct rte_flow_action_mark miss_mark; 7510 struct rte_flow_action miss_actions[3] = { 7511 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 7512 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 7513 }; 7514 const struct rte_flow_action_jump *jump_data; 7515 uint32_t i, flow_table = 0; /* prevent compilation warning */ 7516 struct flow_grp_info grp_info = { 7517 .external = 1, 7518 .transfer = attr->transfer, 7519 .fdb_def_rule = !!priv->fdb_def_rule, 7520 .std_tbl_fix = 0, 7521 }; 7522 int ret; 7523 7524 if (!attr->transfer) { 7525 uint32_t q_size; 7526 7527 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 7528 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 7529 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 7530 0, SOCKET_ID_ANY); 7531 if (!ctx->queue) 7532 return rte_flow_error_set 7533 (error, ENOMEM, 7534 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7535 NULL, "invalid default miss RSS"); 7536 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7537 ctx->action_rss.level = 0, 7538 ctx->action_rss.types = priv->rss_conf.rss_hf, 7539 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 7540 ctx->action_rss.queue_num = priv->reta_idx_n, 7541 ctx->action_rss.key = priv->rss_conf.rss_key, 7542 ctx->action_rss.queue = ctx->queue; 7543 if (!priv->reta_idx_n || !priv->rxqs_n) 7544 return rte_flow_error_set 7545 (error, EINVAL, 7546 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7547 NULL, "invalid port configuration"); 7548 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 7549 ctx->action_rss.types = 0; 7550 for (i = 0; i != priv->reta_idx_n; ++i) 7551 ctx->queue[i] = (*priv->reta_idx)[i]; 7552 } else { 7553 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 7554 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 7555 } 7556 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 7557 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 7558 jump_data = app_actions->conf; 7559 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 7560 miss_attr.group = jump_data->group; 7561 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 7562 &flow_table, &grp_info, error); 7563 if (ret) 7564 return rte_flow_error_set(error, EINVAL, 7565 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7566 NULL, "invalid tunnel id"); 7567 mark_id.app_reserve = 0; 7568 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 7569 mark_id.transfer = !!attr->transfer; 7570 mark_id._unused_ = 0; 7571 miss_mark.id = mark_id.val; 7572 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 7573 miss_items, miss_actions, flow_idx, error); 7574 if (!dev_flow) 7575 return -rte_errno; 7576 dev_flow->flow = flow; 7577 dev_flow->external = true; 7578 dev_flow->tunnel = tunnel; 7579 /* Subflow object was created, we must include one in the list. */ 7580 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7581 dev_flow->handle, next); 7582 DRV_LOG(DEBUG, 7583 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 7584 dev->data->port_id, tunnel->app_tunnel.type, 7585 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 7586 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 7587 miss_actions, error); 7588 if (!ret) 7589 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 7590 error); 7591 7592 return ret; 7593 } 7594 7595 static const struct mlx5_flow_tbl_data_entry * 7596 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 7597 { 7598 struct mlx5_priv *priv = dev->data->dev_private; 7599 struct mlx5_dev_ctx_shared *sh = priv->sh; 7600 struct mlx5_hlist_entry *he; 7601 union tunnel_offload_mark mbits = { .val = mark }; 7602 union mlx5_flow_tbl_key table_key = { 7603 { 7604 .table_id = tunnel_id_to_flow_tbl(mbits.table_id), 7605 .dummy = 0, 7606 .domain = !!mbits.transfer, 7607 .direction = 0, 7608 } 7609 }; 7610 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 7611 return he ? 7612 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 7613 } 7614 7615 static void 7616 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 7617 struct mlx5_hlist_entry *entry) 7618 { 7619 struct mlx5_dev_ctx_shared *sh = list->ctx; 7620 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 7621 7622 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7623 tunnel_flow_tbl_to_id(tte->flow_table)); 7624 mlx5_free(tte); 7625 } 7626 7627 static int 7628 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused, 7629 struct mlx5_hlist_entry *entry, 7630 uint64_t key, void *cb_ctx __rte_unused) 7631 { 7632 union tunnel_tbl_key tbl = { 7633 .val = key, 7634 }; 7635 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 7636 7637 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 7638 } 7639 7640 static struct mlx5_hlist_entry * 7641 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key, 7642 void *ctx __rte_unused) 7643 { 7644 struct mlx5_dev_ctx_shared *sh = list->ctx; 7645 struct tunnel_tbl_entry *tte; 7646 union tunnel_tbl_key tbl = { 7647 .val = key, 7648 }; 7649 7650 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 7651 sizeof(*tte), 0, 7652 SOCKET_ID_ANY); 7653 if (!tte) 7654 goto err; 7655 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7656 &tte->flow_table); 7657 if (tte->flow_table >= MLX5_MAX_TABLES) { 7658 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 7659 tte->flow_table); 7660 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7661 tte->flow_table); 7662 goto err; 7663 } else if (!tte->flow_table) { 7664 goto err; 7665 } 7666 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 7667 tte->tunnel_id = tbl.tunnel_id; 7668 tte->group = tbl.group; 7669 return &tte->hash; 7670 err: 7671 if (tte) 7672 mlx5_free(tte); 7673 return NULL; 7674 } 7675 7676 static uint32_t 7677 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 7678 const struct mlx5_flow_tunnel *tunnel, 7679 uint32_t group, uint32_t *table, 7680 struct rte_flow_error *error) 7681 { 7682 struct mlx5_hlist_entry *he; 7683 struct tunnel_tbl_entry *tte; 7684 union tunnel_tbl_key key = { 7685 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 7686 .group = group 7687 }; 7688 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7689 struct mlx5_hlist *group_hash; 7690 7691 group_hash = tunnel ? tunnel->groups : thub->groups; 7692 he = mlx5_hlist_register(group_hash, key.val, NULL); 7693 if (!he) 7694 return rte_flow_error_set(error, EINVAL, 7695 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7696 NULL, 7697 "tunnel group index not supported"); 7698 tte = container_of(he, typeof(*tte), hash); 7699 *table = tte->flow_table; 7700 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 7701 dev->data->port_id, key.tunnel_id, group, *table); 7702 return 0; 7703 } 7704 7705 static void 7706 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 7707 struct mlx5_flow_tunnel *tunnel) 7708 { 7709 struct mlx5_priv *priv = dev->data->dev_private; 7710 struct mlx5_indexed_pool *ipool; 7711 7712 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 7713 dev->data->port_id, tunnel->tunnel_id); 7714 LIST_REMOVE(tunnel, chain); 7715 mlx5_hlist_destroy(tunnel->groups); 7716 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 7717 mlx5_ipool_free(ipool, tunnel->tunnel_id); 7718 } 7719 7720 static bool 7721 mlx5_access_tunnel_offload_db 7722 (struct rte_eth_dev *dev, 7723 bool (*match)(struct rte_eth_dev *, 7724 struct mlx5_flow_tunnel *, const void *), 7725 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 7726 void (*miss)(struct rte_eth_dev *, void *), 7727 void *ctx, bool lock_op) 7728 { 7729 bool verdict = false; 7730 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7731 struct mlx5_flow_tunnel *tunnel; 7732 7733 rte_spinlock_lock(&thub->sl); 7734 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 7735 verdict = match(dev, tunnel, (const void *)ctx); 7736 if (verdict) 7737 break; 7738 } 7739 if (!lock_op) 7740 rte_spinlock_unlock(&thub->sl); 7741 if (verdict && hit) 7742 hit(dev, tunnel, ctx); 7743 if (!verdict && miss) 7744 miss(dev, ctx); 7745 if (lock_op) 7746 rte_spinlock_unlock(&thub->sl); 7747 7748 return verdict; 7749 } 7750 7751 struct tunnel_db_find_tunnel_id_ctx { 7752 uint32_t tunnel_id; 7753 struct mlx5_flow_tunnel *tunnel; 7754 }; 7755 7756 static bool 7757 find_tunnel_id_match(struct rte_eth_dev *dev, 7758 struct mlx5_flow_tunnel *tunnel, const void *x) 7759 { 7760 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 7761 7762 RTE_SET_USED(dev); 7763 return tunnel->tunnel_id == ctx->tunnel_id; 7764 } 7765 7766 static void 7767 find_tunnel_id_hit(struct rte_eth_dev *dev, 7768 struct mlx5_flow_tunnel *tunnel, void *x) 7769 { 7770 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 7771 RTE_SET_USED(dev); 7772 ctx->tunnel = tunnel; 7773 } 7774 7775 static struct mlx5_flow_tunnel * 7776 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 7777 { 7778 struct tunnel_db_find_tunnel_id_ctx ctx = { 7779 .tunnel_id = id, 7780 }; 7781 7782 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 7783 find_tunnel_id_hit, NULL, &ctx, true); 7784 7785 return ctx.tunnel; 7786 } 7787 7788 static struct mlx5_flow_tunnel * 7789 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 7790 const struct rte_flow_tunnel *app_tunnel) 7791 { 7792 struct mlx5_priv *priv = dev->data->dev_private; 7793 struct mlx5_indexed_pool *ipool; 7794 struct mlx5_flow_tunnel *tunnel; 7795 uint32_t id; 7796 7797 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 7798 tunnel = mlx5_ipool_zmalloc(ipool, &id); 7799 if (!tunnel) 7800 return NULL; 7801 if (id >= MLX5_MAX_TUNNELS) { 7802 mlx5_ipool_free(ipool, id); 7803 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 7804 return NULL; 7805 } 7806 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 7807 mlx5_flow_tunnel_grp2tbl_create_cb, 7808 mlx5_flow_tunnel_grp2tbl_match_cb, 7809 mlx5_flow_tunnel_grp2tbl_remove_cb); 7810 if (!tunnel->groups) { 7811 mlx5_ipool_free(ipool, id); 7812 return NULL; 7813 } 7814 tunnel->groups->ctx = priv->sh; 7815 /* initiate new PMD tunnel */ 7816 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 7817 tunnel->tunnel_id = id; 7818 tunnel->action.type = (typeof(tunnel->action.type)) 7819 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 7820 tunnel->action.conf = tunnel; 7821 tunnel->item.type = (typeof(tunnel->item.type)) 7822 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 7823 tunnel->item.spec = tunnel; 7824 tunnel->item.last = NULL; 7825 tunnel->item.mask = NULL; 7826 7827 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 7828 dev->data->port_id, tunnel->tunnel_id); 7829 7830 return tunnel; 7831 } 7832 7833 struct tunnel_db_get_tunnel_ctx { 7834 const struct rte_flow_tunnel *app_tunnel; 7835 struct mlx5_flow_tunnel *tunnel; 7836 }; 7837 7838 static bool get_tunnel_match(struct rte_eth_dev *dev, 7839 struct mlx5_flow_tunnel *tunnel, const void *x) 7840 { 7841 const struct tunnel_db_get_tunnel_ctx *ctx = x; 7842 7843 RTE_SET_USED(dev); 7844 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 7845 sizeof(*ctx->app_tunnel)); 7846 } 7847 7848 static void get_tunnel_hit(struct rte_eth_dev *dev, 7849 struct mlx5_flow_tunnel *tunnel, void *x) 7850 { 7851 /* called under tunnel spinlock protection */ 7852 struct tunnel_db_get_tunnel_ctx *ctx = x; 7853 7854 RTE_SET_USED(dev); 7855 tunnel->refctn++; 7856 ctx->tunnel = tunnel; 7857 } 7858 7859 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 7860 { 7861 /* called under tunnel spinlock protection */ 7862 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7863 struct tunnel_db_get_tunnel_ctx *ctx = x; 7864 7865 rte_spinlock_unlock(&thub->sl); 7866 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 7867 rte_spinlock_lock(&thub->sl); 7868 if (ctx->tunnel) { 7869 ctx->tunnel->refctn = 1; 7870 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 7871 } 7872 } 7873 7874 7875 static int 7876 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 7877 const struct rte_flow_tunnel *app_tunnel, 7878 struct mlx5_flow_tunnel **tunnel) 7879 { 7880 struct tunnel_db_get_tunnel_ctx ctx = { 7881 .app_tunnel = app_tunnel, 7882 }; 7883 7884 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 7885 get_tunnel_miss, &ctx, true); 7886 *tunnel = ctx.tunnel; 7887 return ctx.tunnel ? 0 : -ENOMEM; 7888 } 7889 7890 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 7891 { 7892 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 7893 7894 if (!thub) 7895 return; 7896 if (!LIST_EMPTY(&thub->tunnels)) 7897 DRV_LOG(WARNING, "port %u tunnels present", port_id); 7898 mlx5_hlist_destroy(thub->groups); 7899 mlx5_free(thub); 7900 } 7901 7902 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 7903 { 7904 int err; 7905 struct mlx5_flow_tunnel_hub *thub; 7906 7907 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 7908 0, SOCKET_ID_ANY); 7909 if (!thub) 7910 return -ENOMEM; 7911 LIST_INIT(&thub->tunnels); 7912 rte_spinlock_init(&thub->sl); 7913 thub->groups = mlx5_hlist_create("flow groups", 7914 rte_align32pow2(MLX5_MAX_TABLES), 0, 7915 0, mlx5_flow_tunnel_grp2tbl_create_cb, 7916 mlx5_flow_tunnel_grp2tbl_match_cb, 7917 mlx5_flow_tunnel_grp2tbl_remove_cb); 7918 if (!thub->groups) { 7919 err = -rte_errno; 7920 goto err; 7921 } 7922 thub->groups->ctx = sh; 7923 sh->tunnel_hub = thub; 7924 7925 return 0; 7926 7927 err: 7928 if (thub->groups) 7929 mlx5_hlist_destroy(thub->groups); 7930 if (thub) 7931 mlx5_free(thub); 7932 return err; 7933 } 7934 7935 static inline bool 7936 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 7937 struct rte_flow_tunnel *tunnel, 7938 const char *err_msg) 7939 { 7940 err_msg = NULL; 7941 if (!is_tunnel_offload_active(dev)) { 7942 err_msg = "tunnel offload was not activated"; 7943 goto out; 7944 } else if (!tunnel) { 7945 err_msg = "no application tunnel"; 7946 goto out; 7947 } 7948 7949 switch (tunnel->type) { 7950 default: 7951 err_msg = "unsupported tunnel type"; 7952 goto out; 7953 case RTE_FLOW_ITEM_TYPE_VXLAN: 7954 break; 7955 } 7956 7957 out: 7958 return !err_msg; 7959 } 7960 7961 static int 7962 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 7963 struct rte_flow_tunnel *app_tunnel, 7964 struct rte_flow_action **actions, 7965 uint32_t *num_of_actions, 7966 struct rte_flow_error *error) 7967 { 7968 int ret; 7969 struct mlx5_flow_tunnel *tunnel; 7970 const char *err_msg = NULL; 7971 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 7972 7973 if (!verdict) 7974 return rte_flow_error_set(error, EINVAL, 7975 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 7976 err_msg); 7977 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 7978 if (ret < 0) { 7979 return rte_flow_error_set(error, ret, 7980 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 7981 "failed to initialize pmd tunnel"); 7982 } 7983 *actions = &tunnel->action; 7984 *num_of_actions = 1; 7985 return 0; 7986 } 7987 7988 static int 7989 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 7990 struct rte_flow_tunnel *app_tunnel, 7991 struct rte_flow_item **items, 7992 uint32_t *num_of_items, 7993 struct rte_flow_error *error) 7994 { 7995 int ret; 7996 struct mlx5_flow_tunnel *tunnel; 7997 const char *err_msg = NULL; 7998 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 7999 8000 if (!verdict) 8001 return rte_flow_error_set(error, EINVAL, 8002 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8003 err_msg); 8004 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8005 if (ret < 0) { 8006 return rte_flow_error_set(error, ret, 8007 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8008 "failed to initialize pmd tunnel"); 8009 } 8010 *items = &tunnel->item; 8011 *num_of_items = 1; 8012 return 0; 8013 } 8014 8015 struct tunnel_db_element_release_ctx { 8016 struct rte_flow_item *items; 8017 struct rte_flow_action *actions; 8018 uint32_t num_elements; 8019 struct rte_flow_error *error; 8020 int ret; 8021 }; 8022 8023 static bool 8024 tunnel_element_release_match(struct rte_eth_dev *dev, 8025 struct mlx5_flow_tunnel *tunnel, const void *x) 8026 { 8027 const struct tunnel_db_element_release_ctx *ctx = x; 8028 8029 RTE_SET_USED(dev); 8030 if (ctx->num_elements != 1) 8031 return false; 8032 else if (ctx->items) 8033 return ctx->items == &tunnel->item; 8034 else if (ctx->actions) 8035 return ctx->actions == &tunnel->action; 8036 8037 return false; 8038 } 8039 8040 static void 8041 tunnel_element_release_hit(struct rte_eth_dev *dev, 8042 struct mlx5_flow_tunnel *tunnel, void *x) 8043 { 8044 struct tunnel_db_element_release_ctx *ctx = x; 8045 ctx->ret = 0; 8046 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 8047 mlx5_flow_tunnel_free(dev, tunnel); 8048 } 8049 8050 static void 8051 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 8052 { 8053 struct tunnel_db_element_release_ctx *ctx = x; 8054 RTE_SET_USED(dev); 8055 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 8056 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8057 "invalid argument"); 8058 } 8059 8060 static int 8061 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 8062 struct rte_flow_item *pmd_items, 8063 uint32_t num_items, struct rte_flow_error *err) 8064 { 8065 struct tunnel_db_element_release_ctx ctx = { 8066 .items = pmd_items, 8067 .actions = NULL, 8068 .num_elements = num_items, 8069 .error = err, 8070 }; 8071 8072 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 8073 tunnel_element_release_hit, 8074 tunnel_element_release_miss, &ctx, false); 8075 8076 return ctx.ret; 8077 } 8078 8079 static int 8080 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 8081 struct rte_flow_action *pmd_actions, 8082 uint32_t num_actions, struct rte_flow_error *err) 8083 { 8084 struct tunnel_db_element_release_ctx ctx = { 8085 .items = NULL, 8086 .actions = pmd_actions, 8087 .num_elements = num_actions, 8088 .error = err, 8089 }; 8090 8091 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 8092 tunnel_element_release_hit, 8093 tunnel_element_release_miss, &ctx, false); 8094 8095 return ctx.ret; 8096 } 8097 8098 static int 8099 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 8100 struct rte_mbuf *m, 8101 struct rte_flow_restore_info *info, 8102 struct rte_flow_error *err) 8103 { 8104 uint64_t ol_flags = m->ol_flags; 8105 const struct mlx5_flow_tbl_data_entry *tble; 8106 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 8107 8108 if (!is_tunnel_offload_active(dev)) { 8109 info->flags = 0; 8110 return 0; 8111 } 8112 8113 if ((ol_flags & mask) != mask) 8114 goto err; 8115 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 8116 if (!tble) { 8117 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 8118 dev->data->port_id, m->hash.fdir.hi); 8119 goto err; 8120 } 8121 MLX5_ASSERT(tble->tunnel); 8122 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 8123 info->group_id = tble->group_id; 8124 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 8125 RTE_FLOW_RESTORE_INFO_GROUP_ID | 8126 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 8127 8128 return 0; 8129 8130 err: 8131 return rte_flow_error_set(err, EINVAL, 8132 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8133 "failed to get restore info"); 8134 } 8135 8136 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 8137 static int 8138 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 8139 __rte_unused struct rte_flow_tunnel *app_tunnel, 8140 __rte_unused struct rte_flow_action **actions, 8141 __rte_unused uint32_t *num_of_actions, 8142 __rte_unused struct rte_flow_error *error) 8143 { 8144 return -ENOTSUP; 8145 } 8146 8147 static int 8148 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 8149 __rte_unused struct rte_flow_tunnel *app_tunnel, 8150 __rte_unused struct rte_flow_item **items, 8151 __rte_unused uint32_t *num_of_items, 8152 __rte_unused struct rte_flow_error *error) 8153 { 8154 return -ENOTSUP; 8155 } 8156 8157 static int 8158 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 8159 __rte_unused struct rte_flow_item *pmd_items, 8160 __rte_unused uint32_t num_items, 8161 __rte_unused struct rte_flow_error *err) 8162 { 8163 return -ENOTSUP; 8164 } 8165 8166 static int 8167 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 8168 __rte_unused struct rte_flow_action *pmd_action, 8169 __rte_unused uint32_t num_actions, 8170 __rte_unused struct rte_flow_error *err) 8171 { 8172 return -ENOTSUP; 8173 } 8174 8175 static int 8176 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 8177 __rte_unused struct rte_mbuf *m, 8178 __rte_unused struct rte_flow_restore_info *i, 8179 __rte_unused struct rte_flow_error *err) 8180 { 8181 return -ENOTSUP; 8182 } 8183 8184 static int 8185 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 8186 __rte_unused struct rte_flow *flow, 8187 __rte_unused const struct rte_flow_attr *attr, 8188 __rte_unused const struct rte_flow_action *actions, 8189 __rte_unused uint32_t flow_idx, 8190 __rte_unused struct tunnel_default_miss_ctx *ctx, 8191 __rte_unused struct rte_flow_error *error) 8192 { 8193 return -ENOTSUP; 8194 } 8195 8196 static struct mlx5_flow_tunnel * 8197 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 8198 __rte_unused uint32_t id) 8199 { 8200 return NULL; 8201 } 8202 8203 static void 8204 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 8205 __rte_unused struct mlx5_flow_tunnel *tunnel) 8206 { 8207 } 8208 8209 static uint32_t 8210 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 8211 __rte_unused const struct mlx5_flow_tunnel *t, 8212 __rte_unused uint32_t group, 8213 __rte_unused uint32_t *table, 8214 struct rte_flow_error *error) 8215 { 8216 return rte_flow_error_set(error, ENOTSUP, 8217 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8218 "tunnel offload requires DV support"); 8219 } 8220 8221 void 8222 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 8223 __rte_unused uint16_t port_id) 8224 { 8225 } 8226 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 8227