1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see ETH_RSS_* definitions). 102 */ 103 uint8_t optional; 104 /**< optional expand field. Default 0 to expand, 1 not go deeper. */ 105 }; 106 107 /** Object returned by mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_rss { 109 uint32_t entries; 110 /**< Number of entries @p patterns and @p priorities. */ 111 struct { 112 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 113 uint32_t priority; /**< Priority offset for each expansion. */ 114 } entry[]; 115 }; 116 117 static void 118 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 119 120 static bool 121 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 122 { 123 switch (item->type) { 124 case RTE_FLOW_ITEM_TYPE_ETH: 125 case RTE_FLOW_ITEM_TYPE_VLAN: 126 case RTE_FLOW_ITEM_TYPE_IPV4: 127 case RTE_FLOW_ITEM_TYPE_IPV6: 128 case RTE_FLOW_ITEM_TYPE_UDP: 129 case RTE_FLOW_ITEM_TYPE_TCP: 130 case RTE_FLOW_ITEM_TYPE_VXLAN: 131 case RTE_FLOW_ITEM_TYPE_NVGRE: 132 case RTE_FLOW_ITEM_TYPE_GRE: 133 case RTE_FLOW_ITEM_TYPE_GENEVE: 134 return true; 135 default: 136 break; 137 } 138 return false; 139 } 140 141 static enum rte_flow_item_type 142 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 143 { 144 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 145 uint16_t ether_type = 0; 146 uint16_t ether_type_m; 147 uint8_t ip_next_proto = 0; 148 uint8_t ip_next_proto_m; 149 150 if (item == NULL || item->spec == NULL) 151 return ret; 152 switch (item->type) { 153 case RTE_FLOW_ITEM_TYPE_ETH: 154 if (item->mask) 155 ether_type_m = ((const struct rte_flow_item_eth *) 156 (item->mask))->type; 157 else 158 ether_type_m = rte_flow_item_eth_mask.type; 159 if (ether_type_m != RTE_BE16(0xFFFF)) 160 break; 161 ether_type = ((const struct rte_flow_item_eth *) 162 (item->spec))->type; 163 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 164 ret = RTE_FLOW_ITEM_TYPE_IPV4; 165 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 166 ret = RTE_FLOW_ITEM_TYPE_IPV6; 167 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 168 ret = RTE_FLOW_ITEM_TYPE_VLAN; 169 else 170 ret = RTE_FLOW_ITEM_TYPE_END; 171 break; 172 case RTE_FLOW_ITEM_TYPE_VLAN: 173 if (item->mask) 174 ether_type_m = ((const struct rte_flow_item_vlan *) 175 (item->mask))->inner_type; 176 else 177 ether_type_m = rte_flow_item_vlan_mask.inner_type; 178 if (ether_type_m != RTE_BE16(0xFFFF)) 179 break; 180 ether_type = ((const struct rte_flow_item_vlan *) 181 (item->spec))->inner_type; 182 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 183 ret = RTE_FLOW_ITEM_TYPE_IPV4; 184 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 185 ret = RTE_FLOW_ITEM_TYPE_IPV6; 186 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 187 ret = RTE_FLOW_ITEM_TYPE_VLAN; 188 else 189 ret = RTE_FLOW_ITEM_TYPE_END; 190 break; 191 case RTE_FLOW_ITEM_TYPE_IPV4: 192 if (item->mask) 193 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 194 (item->mask))->hdr.next_proto_id; 195 else 196 ip_next_proto_m = 197 rte_flow_item_ipv4_mask.hdr.next_proto_id; 198 if (ip_next_proto_m != 0xFF) 199 break; 200 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 201 (item->spec))->hdr.next_proto_id; 202 if (ip_next_proto == IPPROTO_UDP) 203 ret = RTE_FLOW_ITEM_TYPE_UDP; 204 else if (ip_next_proto == IPPROTO_TCP) 205 ret = RTE_FLOW_ITEM_TYPE_TCP; 206 else if (ip_next_proto == IPPROTO_IP) 207 ret = RTE_FLOW_ITEM_TYPE_IPV4; 208 else if (ip_next_proto == IPPROTO_IPV6) 209 ret = RTE_FLOW_ITEM_TYPE_IPV6; 210 else 211 ret = RTE_FLOW_ITEM_TYPE_END; 212 break; 213 case RTE_FLOW_ITEM_TYPE_IPV6: 214 if (item->mask) 215 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 216 (item->mask))->hdr.proto; 217 else 218 ip_next_proto_m = 219 rte_flow_item_ipv6_mask.hdr.proto; 220 if (ip_next_proto_m != 0xFF) 221 break; 222 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 223 (item->spec))->hdr.proto; 224 if (ip_next_proto == IPPROTO_UDP) 225 ret = RTE_FLOW_ITEM_TYPE_UDP; 226 else if (ip_next_proto == IPPROTO_TCP) 227 ret = RTE_FLOW_ITEM_TYPE_TCP; 228 else if (ip_next_proto == IPPROTO_IP) 229 ret = RTE_FLOW_ITEM_TYPE_IPV4; 230 else if (ip_next_proto == IPPROTO_IPV6) 231 ret = RTE_FLOW_ITEM_TYPE_IPV6; 232 else 233 ret = RTE_FLOW_ITEM_TYPE_END; 234 break; 235 default: 236 ret = RTE_FLOW_ITEM_TYPE_VOID; 237 break; 238 } 239 return ret; 240 } 241 242 #define MLX5_RSS_EXP_ELT_N 16 243 244 /** 245 * Expand RSS flows into several possible flows according to the RSS hash 246 * fields requested and the driver capabilities. 247 * 248 * @param[out] buf 249 * Buffer to store the result expansion. 250 * @param[in] size 251 * Buffer size in bytes. If 0, @p buf can be NULL. 252 * @param[in] pattern 253 * User flow pattern. 254 * @param[in] types 255 * RSS types to expand (see ETH_RSS_* definitions). 256 * @param[in] graph 257 * Input graph to expand @p pattern according to @p types. 258 * @param[in] graph_root_index 259 * Index of root node in @p graph, typically 0. 260 * 261 * @return 262 * A positive value representing the size of @p buf in bytes regardless of 263 * @p size on success, a negative errno value otherwise and rte_errno is 264 * set, the following errors are defined: 265 * 266 * -E2BIG: graph-depth @p graph is too deep. 267 */ 268 static int 269 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 270 const struct rte_flow_item *pattern, uint64_t types, 271 const struct mlx5_flow_expand_node graph[], 272 int graph_root_index) 273 { 274 const struct rte_flow_item *item; 275 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 276 const int *next_node; 277 const int *stack[MLX5_RSS_EXP_ELT_N]; 278 int stack_pos = 0; 279 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 280 unsigned int i; 281 size_t lsize; 282 size_t user_pattern_size = 0; 283 void *addr = NULL; 284 const struct mlx5_flow_expand_node *next = NULL; 285 struct rte_flow_item missed_item; 286 int missed = 0; 287 int elt = 0; 288 const struct rte_flow_item *last_item = NULL; 289 290 memset(&missed_item, 0, sizeof(missed_item)); 291 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 292 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 293 if (lsize <= size) { 294 buf->entry[0].priority = 0; 295 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 296 buf->entries = 0; 297 addr = buf->entry[0].pattern; 298 } 299 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 300 if (!mlx5_flow_is_rss_expandable_item(item)) { 301 user_pattern_size += sizeof(*item); 302 continue; 303 } 304 last_item = item; 305 for (i = 0; node->next && node->next[i]; ++i) { 306 next = &graph[node->next[i]]; 307 if (next->type == item->type) 308 break; 309 } 310 if (next) 311 node = next; 312 user_pattern_size += sizeof(*item); 313 } 314 user_pattern_size += sizeof(*item); /* Handle END item. */ 315 lsize += user_pattern_size; 316 /* Copy the user pattern in the first entry of the buffer. */ 317 if (lsize <= size) { 318 rte_memcpy(addr, pattern, user_pattern_size); 319 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 320 buf->entries = 1; 321 } 322 /* Start expanding. */ 323 memset(flow_items, 0, sizeof(flow_items)); 324 user_pattern_size -= sizeof(*item); 325 /* 326 * Check if the last valid item has spec set, need complete pattern, 327 * and the pattern can be used for expansion. 328 */ 329 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 330 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 331 /* Item type END indicates expansion is not required. */ 332 return lsize; 333 } 334 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 335 next = NULL; 336 missed = 1; 337 for (i = 0; node->next && node->next[i]; ++i) { 338 next = &graph[node->next[i]]; 339 if (next->type == missed_item.type) { 340 flow_items[0].type = missed_item.type; 341 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 342 break; 343 } 344 next = NULL; 345 } 346 } 347 if (next && missed) { 348 elt = 2; /* missed item + item end. */ 349 node = next; 350 lsize += elt * sizeof(*item) + user_pattern_size; 351 if ((node->rss_types & types) && lsize <= size) { 352 buf->entry[buf->entries].priority = 1; 353 buf->entry[buf->entries].pattern = addr; 354 buf->entries++; 355 rte_memcpy(addr, buf->entry[0].pattern, 356 user_pattern_size); 357 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 358 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 359 addr = (void *)(((uintptr_t)addr) + 360 elt * sizeof(*item)); 361 } 362 } 363 memset(flow_items, 0, sizeof(flow_items)); 364 next_node = node->next; 365 stack[stack_pos] = next_node; 366 node = next_node ? &graph[*next_node] : NULL; 367 while (node) { 368 flow_items[stack_pos].type = node->type; 369 if (node->rss_types & types) { 370 /* 371 * compute the number of items to copy from the 372 * expansion and copy it. 373 * When the stack_pos is 0, there are 1 element in it, 374 * plus the addition END item. 375 */ 376 elt = stack_pos + 2; 377 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 378 lsize += elt * sizeof(*item) + user_pattern_size; 379 if (lsize <= size) { 380 size_t n = elt * sizeof(*item); 381 382 buf->entry[buf->entries].priority = 383 stack_pos + 1 + missed; 384 buf->entry[buf->entries].pattern = addr; 385 buf->entries++; 386 rte_memcpy(addr, buf->entry[0].pattern, 387 user_pattern_size); 388 addr = (void *)(((uintptr_t)addr) + 389 user_pattern_size); 390 rte_memcpy(addr, &missed_item, 391 missed * sizeof(*item)); 392 addr = (void *)(((uintptr_t)addr) + 393 missed * sizeof(*item)); 394 rte_memcpy(addr, flow_items, n); 395 addr = (void *)(((uintptr_t)addr) + n); 396 } 397 } 398 /* Go deeper. */ 399 if (!node->optional && node->next) { 400 next_node = node->next; 401 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 402 rte_errno = E2BIG; 403 return -rte_errno; 404 } 405 stack[stack_pos] = next_node; 406 } else if (*(next_node + 1)) { 407 /* Follow up with the next possibility. */ 408 ++next_node; 409 } else { 410 /* Move to the next path. */ 411 if (stack_pos) 412 next_node = stack[--stack_pos]; 413 next_node++; 414 stack[stack_pos] = next_node; 415 } 416 node = *next_node ? &graph[*next_node] : NULL; 417 }; 418 return lsize; 419 } 420 421 enum mlx5_expansion { 422 MLX5_EXPANSION_ROOT, 423 MLX5_EXPANSION_ROOT_OUTER, 424 MLX5_EXPANSION_ROOT_ETH_VLAN, 425 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 426 MLX5_EXPANSION_OUTER_ETH, 427 MLX5_EXPANSION_OUTER_ETH_VLAN, 428 MLX5_EXPANSION_OUTER_VLAN, 429 MLX5_EXPANSION_OUTER_IPV4, 430 MLX5_EXPANSION_OUTER_IPV4_UDP, 431 MLX5_EXPANSION_OUTER_IPV4_TCP, 432 MLX5_EXPANSION_OUTER_IPV6, 433 MLX5_EXPANSION_OUTER_IPV6_UDP, 434 MLX5_EXPANSION_OUTER_IPV6_TCP, 435 MLX5_EXPANSION_VXLAN, 436 MLX5_EXPANSION_VXLAN_GPE, 437 MLX5_EXPANSION_GRE, 438 MLX5_EXPANSION_NVGRE, 439 MLX5_EXPANSION_GRE_KEY, 440 MLX5_EXPANSION_MPLS, 441 MLX5_EXPANSION_ETH, 442 MLX5_EXPANSION_ETH_VLAN, 443 MLX5_EXPANSION_VLAN, 444 MLX5_EXPANSION_IPV4, 445 MLX5_EXPANSION_IPV4_UDP, 446 MLX5_EXPANSION_IPV4_TCP, 447 MLX5_EXPANSION_IPV6, 448 MLX5_EXPANSION_IPV6_UDP, 449 MLX5_EXPANSION_IPV6_TCP, 450 }; 451 452 /** Supported expansion of items. */ 453 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 454 [MLX5_EXPANSION_ROOT] = { 455 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 456 MLX5_EXPANSION_IPV4, 457 MLX5_EXPANSION_IPV6), 458 .type = RTE_FLOW_ITEM_TYPE_END, 459 }, 460 [MLX5_EXPANSION_ROOT_OUTER] = { 461 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 462 MLX5_EXPANSION_OUTER_IPV4, 463 MLX5_EXPANSION_OUTER_IPV6), 464 .type = RTE_FLOW_ITEM_TYPE_END, 465 }, 466 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 467 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 468 .type = RTE_FLOW_ITEM_TYPE_END, 469 }, 470 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 471 .next = MLX5_FLOW_EXPAND_RSS_NEXT 472 (MLX5_EXPANSION_OUTER_ETH_VLAN), 473 .type = RTE_FLOW_ITEM_TYPE_END, 474 }, 475 [MLX5_EXPANSION_OUTER_ETH] = { 476 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 477 MLX5_EXPANSION_OUTER_IPV6, 478 MLX5_EXPANSION_MPLS), 479 .type = RTE_FLOW_ITEM_TYPE_ETH, 480 .rss_types = 0, 481 }, 482 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 483 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 484 .type = RTE_FLOW_ITEM_TYPE_ETH, 485 .rss_types = 0, 486 }, 487 [MLX5_EXPANSION_OUTER_VLAN] = { 488 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 489 MLX5_EXPANSION_OUTER_IPV6), 490 .type = RTE_FLOW_ITEM_TYPE_VLAN, 491 }, 492 [MLX5_EXPANSION_OUTER_IPV4] = { 493 .next = MLX5_FLOW_EXPAND_RSS_NEXT 494 (MLX5_EXPANSION_OUTER_IPV4_UDP, 495 MLX5_EXPANSION_OUTER_IPV4_TCP, 496 MLX5_EXPANSION_GRE, 497 MLX5_EXPANSION_NVGRE, 498 MLX5_EXPANSION_IPV4, 499 MLX5_EXPANSION_IPV6), 500 .type = RTE_FLOW_ITEM_TYPE_IPV4, 501 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 502 ETH_RSS_NONFRAG_IPV4_OTHER, 503 }, 504 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 505 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 506 MLX5_EXPANSION_VXLAN_GPE), 507 .type = RTE_FLOW_ITEM_TYPE_UDP, 508 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 509 }, 510 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 511 .type = RTE_FLOW_ITEM_TYPE_TCP, 512 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 513 }, 514 [MLX5_EXPANSION_OUTER_IPV6] = { 515 .next = MLX5_FLOW_EXPAND_RSS_NEXT 516 (MLX5_EXPANSION_OUTER_IPV6_UDP, 517 MLX5_EXPANSION_OUTER_IPV6_TCP, 518 MLX5_EXPANSION_IPV4, 519 MLX5_EXPANSION_IPV6, 520 MLX5_EXPANSION_GRE, 521 MLX5_EXPANSION_NVGRE), 522 .type = RTE_FLOW_ITEM_TYPE_IPV6, 523 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 524 ETH_RSS_NONFRAG_IPV6_OTHER, 525 }, 526 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 527 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 528 MLX5_EXPANSION_VXLAN_GPE), 529 .type = RTE_FLOW_ITEM_TYPE_UDP, 530 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 531 }, 532 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 533 .type = RTE_FLOW_ITEM_TYPE_TCP, 534 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 535 }, 536 [MLX5_EXPANSION_VXLAN] = { 537 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 538 MLX5_EXPANSION_IPV4, 539 MLX5_EXPANSION_IPV6), 540 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 541 }, 542 [MLX5_EXPANSION_VXLAN_GPE] = { 543 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 544 MLX5_EXPANSION_IPV4, 545 MLX5_EXPANSION_IPV6), 546 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 547 }, 548 [MLX5_EXPANSION_GRE] = { 549 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 550 MLX5_EXPANSION_IPV6, 551 MLX5_EXPANSION_GRE_KEY), 552 .type = RTE_FLOW_ITEM_TYPE_GRE, 553 }, 554 [MLX5_EXPANSION_GRE_KEY] = { 555 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 556 MLX5_EXPANSION_IPV6), 557 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 558 .optional = 1, 559 }, 560 [MLX5_EXPANSION_NVGRE] = { 561 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 562 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 563 }, 564 [MLX5_EXPANSION_MPLS] = { 565 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 566 MLX5_EXPANSION_IPV6), 567 .type = RTE_FLOW_ITEM_TYPE_MPLS, 568 }, 569 [MLX5_EXPANSION_ETH] = { 570 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 571 MLX5_EXPANSION_IPV6), 572 .type = RTE_FLOW_ITEM_TYPE_ETH, 573 }, 574 [MLX5_EXPANSION_ETH_VLAN] = { 575 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 576 .type = RTE_FLOW_ITEM_TYPE_ETH, 577 }, 578 [MLX5_EXPANSION_VLAN] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 580 MLX5_EXPANSION_IPV6), 581 .type = RTE_FLOW_ITEM_TYPE_VLAN, 582 }, 583 [MLX5_EXPANSION_IPV4] = { 584 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 585 MLX5_EXPANSION_IPV4_TCP), 586 .type = RTE_FLOW_ITEM_TYPE_IPV4, 587 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 588 ETH_RSS_NONFRAG_IPV4_OTHER, 589 }, 590 [MLX5_EXPANSION_IPV4_UDP] = { 591 .type = RTE_FLOW_ITEM_TYPE_UDP, 592 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 593 }, 594 [MLX5_EXPANSION_IPV4_TCP] = { 595 .type = RTE_FLOW_ITEM_TYPE_TCP, 596 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 597 }, 598 [MLX5_EXPANSION_IPV6] = { 599 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 600 MLX5_EXPANSION_IPV6_TCP), 601 .type = RTE_FLOW_ITEM_TYPE_IPV6, 602 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 603 ETH_RSS_NONFRAG_IPV6_OTHER, 604 }, 605 [MLX5_EXPANSION_IPV6_UDP] = { 606 .type = RTE_FLOW_ITEM_TYPE_UDP, 607 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 608 }, 609 [MLX5_EXPANSION_IPV6_TCP] = { 610 .type = RTE_FLOW_ITEM_TYPE_TCP, 611 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 612 }, 613 }; 614 615 static struct rte_flow_action_handle * 616 mlx5_action_handle_create(struct rte_eth_dev *dev, 617 const struct rte_flow_indir_action_conf *conf, 618 const struct rte_flow_action *action, 619 struct rte_flow_error *error); 620 static int mlx5_action_handle_destroy 621 (struct rte_eth_dev *dev, 622 struct rte_flow_action_handle *handle, 623 struct rte_flow_error *error); 624 static int mlx5_action_handle_update 625 (struct rte_eth_dev *dev, 626 struct rte_flow_action_handle *handle, 627 const void *update, 628 struct rte_flow_error *error); 629 static int mlx5_action_handle_query 630 (struct rte_eth_dev *dev, 631 const struct rte_flow_action_handle *handle, 632 void *data, 633 struct rte_flow_error *error); 634 static int 635 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 636 struct rte_flow_tunnel *app_tunnel, 637 struct rte_flow_action **actions, 638 uint32_t *num_of_actions, 639 struct rte_flow_error *error); 640 static int 641 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 642 struct rte_flow_tunnel *app_tunnel, 643 struct rte_flow_item **items, 644 uint32_t *num_of_items, 645 struct rte_flow_error *error); 646 static int 647 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 648 struct rte_flow_item *pmd_items, 649 uint32_t num_items, struct rte_flow_error *err); 650 static int 651 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 652 struct rte_flow_action *pmd_actions, 653 uint32_t num_actions, 654 struct rte_flow_error *err); 655 static int 656 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 657 struct rte_mbuf *m, 658 struct rte_flow_restore_info *info, 659 struct rte_flow_error *err); 660 661 static const struct rte_flow_ops mlx5_flow_ops = { 662 .validate = mlx5_flow_validate, 663 .create = mlx5_flow_create, 664 .destroy = mlx5_flow_destroy, 665 .flush = mlx5_flow_flush, 666 .isolate = mlx5_flow_isolate, 667 .query = mlx5_flow_query, 668 .dev_dump = mlx5_flow_dev_dump, 669 .get_aged_flows = mlx5_flow_get_aged_flows, 670 .action_handle_create = mlx5_action_handle_create, 671 .action_handle_destroy = mlx5_action_handle_destroy, 672 .action_handle_update = mlx5_action_handle_update, 673 .action_handle_query = mlx5_action_handle_query, 674 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 675 .tunnel_match = mlx5_flow_tunnel_match, 676 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 677 .tunnel_item_release = mlx5_flow_tunnel_item_release, 678 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 679 }; 680 681 /* Tunnel information. */ 682 struct mlx5_flow_tunnel_info { 683 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 684 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 685 }; 686 687 static struct mlx5_flow_tunnel_info tunnels_info[] = { 688 { 689 .tunnel = MLX5_FLOW_LAYER_VXLAN, 690 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 691 }, 692 { 693 .tunnel = MLX5_FLOW_LAYER_GENEVE, 694 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 695 }, 696 { 697 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 698 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 699 }, 700 { 701 .tunnel = MLX5_FLOW_LAYER_GRE, 702 .ptype = RTE_PTYPE_TUNNEL_GRE, 703 }, 704 { 705 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 706 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 707 }, 708 { 709 .tunnel = MLX5_FLOW_LAYER_MPLS, 710 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 711 }, 712 { 713 .tunnel = MLX5_FLOW_LAYER_NVGRE, 714 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 715 }, 716 { 717 .tunnel = MLX5_FLOW_LAYER_IPIP, 718 .ptype = RTE_PTYPE_TUNNEL_IP, 719 }, 720 { 721 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 722 .ptype = RTE_PTYPE_TUNNEL_IP, 723 }, 724 { 725 .tunnel = MLX5_FLOW_LAYER_GTP, 726 .ptype = RTE_PTYPE_TUNNEL_GTPU, 727 }, 728 }; 729 730 731 732 /** 733 * Translate tag ID to register. 734 * 735 * @param[in] dev 736 * Pointer to the Ethernet device structure. 737 * @param[in] feature 738 * The feature that request the register. 739 * @param[in] id 740 * The request register ID. 741 * @param[out] error 742 * Error description in case of any. 743 * 744 * @return 745 * The request register on success, a negative errno 746 * value otherwise and rte_errno is set. 747 */ 748 int 749 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 750 enum mlx5_feature_name feature, 751 uint32_t id, 752 struct rte_flow_error *error) 753 { 754 struct mlx5_priv *priv = dev->data->dev_private; 755 struct mlx5_dev_config *config = &priv->config; 756 enum modify_reg start_reg; 757 bool skip_mtr_reg = false; 758 759 switch (feature) { 760 case MLX5_HAIRPIN_RX: 761 return REG_B; 762 case MLX5_HAIRPIN_TX: 763 return REG_A; 764 case MLX5_METADATA_RX: 765 switch (config->dv_xmeta_en) { 766 case MLX5_XMETA_MODE_LEGACY: 767 return REG_B; 768 case MLX5_XMETA_MODE_META16: 769 return REG_C_0; 770 case MLX5_XMETA_MODE_META32: 771 return REG_C_1; 772 } 773 break; 774 case MLX5_METADATA_TX: 775 return REG_A; 776 case MLX5_METADATA_FDB: 777 switch (config->dv_xmeta_en) { 778 case MLX5_XMETA_MODE_LEGACY: 779 return REG_NON; 780 case MLX5_XMETA_MODE_META16: 781 return REG_C_0; 782 case MLX5_XMETA_MODE_META32: 783 return REG_C_1; 784 } 785 break; 786 case MLX5_FLOW_MARK: 787 switch (config->dv_xmeta_en) { 788 case MLX5_XMETA_MODE_LEGACY: 789 return REG_NON; 790 case MLX5_XMETA_MODE_META16: 791 return REG_C_1; 792 case MLX5_XMETA_MODE_META32: 793 return REG_C_0; 794 } 795 break; 796 case MLX5_MTR_ID: 797 /* 798 * If meter color and meter id share one register, flow match 799 * should use the meter color register for match. 800 */ 801 if (priv->mtr_reg_share) 802 return priv->mtr_color_reg; 803 else 804 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 805 REG_C_3; 806 case MLX5_MTR_COLOR: 807 case MLX5_ASO_FLOW_HIT: 808 case MLX5_ASO_CONNTRACK: 809 /* All features use the same REG_C. */ 810 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 811 return priv->mtr_color_reg; 812 case MLX5_COPY_MARK: 813 /* 814 * Metadata COPY_MARK register using is in meter suffix sub 815 * flow while with meter. It's safe to share the same register. 816 */ 817 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 818 case MLX5_APP_TAG: 819 /* 820 * If meter is enable, it will engage the register for color 821 * match and flow match. If meter color match is not using the 822 * REG_C_2, need to skip the REG_C_x be used by meter color 823 * match. 824 * If meter is disable, free to use all available registers. 825 */ 826 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 827 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 828 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 829 if (id > (uint32_t)(REG_C_7 - start_reg)) 830 return rte_flow_error_set(error, EINVAL, 831 RTE_FLOW_ERROR_TYPE_ITEM, 832 NULL, "invalid tag id"); 833 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 834 return rte_flow_error_set(error, ENOTSUP, 835 RTE_FLOW_ERROR_TYPE_ITEM, 836 NULL, "unsupported tag id"); 837 /* 838 * This case means meter is using the REG_C_x great than 2. 839 * Take care not to conflict with meter color REG_C_x. 840 * If the available index REG_C_y >= REG_C_x, skip the 841 * color register. 842 */ 843 if (skip_mtr_reg && config->flow_mreg_c 844 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 845 if (id >= (uint32_t)(REG_C_7 - start_reg)) 846 return rte_flow_error_set(error, EINVAL, 847 RTE_FLOW_ERROR_TYPE_ITEM, 848 NULL, "invalid tag id"); 849 if (config->flow_mreg_c 850 [id + 1 + start_reg - REG_C_0] != REG_NON) 851 return config->flow_mreg_c 852 [id + 1 + start_reg - REG_C_0]; 853 return rte_flow_error_set(error, ENOTSUP, 854 RTE_FLOW_ERROR_TYPE_ITEM, 855 NULL, "unsupported tag id"); 856 } 857 return config->flow_mreg_c[id + start_reg - REG_C_0]; 858 } 859 MLX5_ASSERT(false); 860 return rte_flow_error_set(error, EINVAL, 861 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 862 NULL, "invalid feature name"); 863 } 864 865 /** 866 * Check extensive flow metadata register support. 867 * 868 * @param dev 869 * Pointer to rte_eth_dev structure. 870 * 871 * @return 872 * True if device supports extensive flow metadata register, otherwise false. 873 */ 874 bool 875 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 876 { 877 struct mlx5_priv *priv = dev->data->dev_private; 878 struct mlx5_dev_config *config = &priv->config; 879 880 /* 881 * Having available reg_c can be regarded inclusively as supporting 882 * extensive flow metadata register, which could mean, 883 * - metadata register copy action by modify header. 884 * - 16 modify header actions is supported. 885 * - reg_c's are preserved across different domain (FDB and NIC) on 886 * packet loopback by flow lookup miss. 887 */ 888 return config->flow_mreg_c[2] != REG_NON; 889 } 890 891 /** 892 * Get the lowest priority. 893 * 894 * @param[in] dev 895 * Pointer to the Ethernet device structure. 896 * @param[in] attributes 897 * Pointer to device flow rule attributes. 898 * 899 * @return 900 * The value of lowest priority of flow. 901 */ 902 uint32_t 903 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 904 const struct rte_flow_attr *attr) 905 { 906 struct mlx5_priv *priv = dev->data->dev_private; 907 908 if (!attr->group && !attr->transfer) 909 return priv->config.flow_prio - 2; 910 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 911 } 912 913 /** 914 * Calculate matcher priority of the flow. 915 * 916 * @param[in] dev 917 * Pointer to the Ethernet device structure. 918 * @param[in] attr 919 * Pointer to device flow rule attributes. 920 * @param[in] subpriority 921 * The priority based on the items. 922 * @return 923 * The matcher priority of the flow. 924 */ 925 uint16_t 926 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 927 const struct rte_flow_attr *attr, 928 uint32_t subpriority) 929 { 930 uint16_t priority = (uint16_t)attr->priority; 931 struct mlx5_priv *priv = dev->data->dev_private; 932 933 if (!attr->group && !attr->transfer) { 934 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 935 priority = priv->config.flow_prio - 1; 936 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 937 } 938 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 939 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 940 return priority * 3 + subpriority; 941 } 942 943 /** 944 * Verify the @p item specifications (spec, last, mask) are compatible with the 945 * NIC capabilities. 946 * 947 * @param[in] item 948 * Item specification. 949 * @param[in] mask 950 * @p item->mask or flow default bit-masks. 951 * @param[in] nic_mask 952 * Bit-masks covering supported fields by the NIC to compare with user mask. 953 * @param[in] size 954 * Bit-masks size in bytes. 955 * @param[in] range_accepted 956 * True if range of values is accepted for specific fields, false otherwise. 957 * @param[out] error 958 * Pointer to error structure. 959 * 960 * @return 961 * 0 on success, a negative errno value otherwise and rte_errno is set. 962 */ 963 int 964 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 965 const uint8_t *mask, 966 const uint8_t *nic_mask, 967 unsigned int size, 968 bool range_accepted, 969 struct rte_flow_error *error) 970 { 971 unsigned int i; 972 973 MLX5_ASSERT(nic_mask); 974 for (i = 0; i < size; ++i) 975 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 976 return rte_flow_error_set(error, ENOTSUP, 977 RTE_FLOW_ERROR_TYPE_ITEM, 978 item, 979 "mask enables non supported" 980 " bits"); 981 if (!item->spec && (item->mask || item->last)) 982 return rte_flow_error_set(error, EINVAL, 983 RTE_FLOW_ERROR_TYPE_ITEM, item, 984 "mask/last without a spec is not" 985 " supported"); 986 if (item->spec && item->last && !range_accepted) { 987 uint8_t spec[size]; 988 uint8_t last[size]; 989 unsigned int i; 990 int ret; 991 992 for (i = 0; i < size; ++i) { 993 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 994 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 995 } 996 ret = memcmp(spec, last, size); 997 if (ret != 0) 998 return rte_flow_error_set(error, EINVAL, 999 RTE_FLOW_ERROR_TYPE_ITEM, 1000 item, 1001 "range is not valid"); 1002 } 1003 return 0; 1004 } 1005 1006 /** 1007 * Adjust the hash fields according to the @p flow information. 1008 * 1009 * @param[in] dev_flow. 1010 * Pointer to the mlx5_flow. 1011 * @param[in] tunnel 1012 * 1 when the hash field is for a tunnel item. 1013 * @param[in] layer_types 1014 * ETH_RSS_* types. 1015 * @param[in] hash_fields 1016 * Item hash fields. 1017 * 1018 * @return 1019 * The hash fields that should be used. 1020 */ 1021 uint64_t 1022 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1023 int tunnel __rte_unused, uint64_t layer_types, 1024 uint64_t hash_fields) 1025 { 1026 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1027 int rss_request_inner = rss_desc->level >= 2; 1028 1029 /* Check RSS hash level for tunnel. */ 1030 if (tunnel && rss_request_inner) 1031 hash_fields |= IBV_RX_HASH_INNER; 1032 else if (tunnel || rss_request_inner) 1033 return 0; 1034 #endif 1035 /* Check if requested layer matches RSS hash fields. */ 1036 if (!(rss_desc->types & layer_types)) 1037 return 0; 1038 return hash_fields; 1039 } 1040 1041 /** 1042 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1043 * if several tunnel rules are used on this queue, the tunnel ptype will be 1044 * cleared. 1045 * 1046 * @param rxq_ctrl 1047 * Rx queue to update. 1048 */ 1049 static void 1050 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1051 { 1052 unsigned int i; 1053 uint32_t tunnel_ptype = 0; 1054 1055 /* Look up for the ptype to use. */ 1056 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1057 if (!rxq_ctrl->flow_tunnels_n[i]) 1058 continue; 1059 if (!tunnel_ptype) { 1060 tunnel_ptype = tunnels_info[i].ptype; 1061 } else { 1062 tunnel_ptype = 0; 1063 break; 1064 } 1065 } 1066 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1067 } 1068 1069 /** 1070 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1071 * flow. 1072 * 1073 * @param[in] dev 1074 * Pointer to the Ethernet device structure. 1075 * @param[in] dev_handle 1076 * Pointer to device flow handle structure. 1077 */ 1078 void 1079 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1080 struct mlx5_flow_handle *dev_handle) 1081 { 1082 struct mlx5_priv *priv = dev->data->dev_private; 1083 const int mark = dev_handle->mark; 1084 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1085 struct mlx5_ind_table_obj *ind_tbl = NULL; 1086 unsigned int i; 1087 1088 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1089 struct mlx5_hrxq *hrxq; 1090 1091 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1092 dev_handle->rix_hrxq); 1093 if (hrxq) 1094 ind_tbl = hrxq->ind_table; 1095 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1096 struct mlx5_shared_action_rss *shared_rss; 1097 1098 shared_rss = mlx5_ipool_get 1099 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1100 dev_handle->rix_srss); 1101 if (shared_rss) 1102 ind_tbl = shared_rss->ind_tbl; 1103 } 1104 if (!ind_tbl) 1105 return; 1106 for (i = 0; i != ind_tbl->queues_n; ++i) { 1107 int idx = ind_tbl->queues[i]; 1108 struct mlx5_rxq_ctrl *rxq_ctrl = 1109 container_of((*priv->rxqs)[idx], 1110 struct mlx5_rxq_ctrl, rxq); 1111 1112 /* 1113 * To support metadata register copy on Tx loopback, 1114 * this must be always enabled (metadata may arive 1115 * from other port - not from local flows only. 1116 */ 1117 if (priv->config.dv_flow_en && 1118 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1119 mlx5_flow_ext_mreg_supported(dev)) { 1120 rxq_ctrl->rxq.mark = 1; 1121 rxq_ctrl->flow_mark_n = 1; 1122 } else if (mark) { 1123 rxq_ctrl->rxq.mark = 1; 1124 rxq_ctrl->flow_mark_n++; 1125 } 1126 if (tunnel) { 1127 unsigned int j; 1128 1129 /* Increase the counter matching the flow. */ 1130 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1131 if ((tunnels_info[j].tunnel & 1132 dev_handle->layers) == 1133 tunnels_info[j].tunnel) { 1134 rxq_ctrl->flow_tunnels_n[j]++; 1135 break; 1136 } 1137 } 1138 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1139 } 1140 } 1141 } 1142 1143 /** 1144 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1145 * 1146 * @param[in] dev 1147 * Pointer to the Ethernet device structure. 1148 * @param[in] flow 1149 * Pointer to flow structure. 1150 */ 1151 static void 1152 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1153 { 1154 struct mlx5_priv *priv = dev->data->dev_private; 1155 uint32_t handle_idx; 1156 struct mlx5_flow_handle *dev_handle; 1157 1158 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1159 handle_idx, dev_handle, next) 1160 flow_drv_rxq_flags_set(dev, dev_handle); 1161 } 1162 1163 /** 1164 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1165 * device flow if no other flow uses it with the same kind of request. 1166 * 1167 * @param dev 1168 * Pointer to Ethernet device. 1169 * @param[in] dev_handle 1170 * Pointer to the device flow handle structure. 1171 */ 1172 static void 1173 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1174 struct mlx5_flow_handle *dev_handle) 1175 { 1176 struct mlx5_priv *priv = dev->data->dev_private; 1177 const int mark = dev_handle->mark; 1178 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1179 struct mlx5_ind_table_obj *ind_tbl = NULL; 1180 unsigned int i; 1181 1182 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1183 struct mlx5_hrxq *hrxq; 1184 1185 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1186 dev_handle->rix_hrxq); 1187 if (hrxq) 1188 ind_tbl = hrxq->ind_table; 1189 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1190 struct mlx5_shared_action_rss *shared_rss; 1191 1192 shared_rss = mlx5_ipool_get 1193 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1194 dev_handle->rix_srss); 1195 if (shared_rss) 1196 ind_tbl = shared_rss->ind_tbl; 1197 } 1198 if (!ind_tbl) 1199 return; 1200 MLX5_ASSERT(dev->data->dev_started); 1201 for (i = 0; i != ind_tbl->queues_n; ++i) { 1202 int idx = ind_tbl->queues[i]; 1203 struct mlx5_rxq_ctrl *rxq_ctrl = 1204 container_of((*priv->rxqs)[idx], 1205 struct mlx5_rxq_ctrl, rxq); 1206 1207 if (priv->config.dv_flow_en && 1208 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1209 mlx5_flow_ext_mreg_supported(dev)) { 1210 rxq_ctrl->rxq.mark = 1; 1211 rxq_ctrl->flow_mark_n = 1; 1212 } else if (mark) { 1213 rxq_ctrl->flow_mark_n--; 1214 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1215 } 1216 if (tunnel) { 1217 unsigned int j; 1218 1219 /* Decrease the counter matching the flow. */ 1220 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1221 if ((tunnels_info[j].tunnel & 1222 dev_handle->layers) == 1223 tunnels_info[j].tunnel) { 1224 rxq_ctrl->flow_tunnels_n[j]--; 1225 break; 1226 } 1227 } 1228 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1229 } 1230 } 1231 } 1232 1233 /** 1234 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1235 * @p flow if no other flow uses it with the same kind of request. 1236 * 1237 * @param dev 1238 * Pointer to Ethernet device. 1239 * @param[in] flow 1240 * Pointer to the flow. 1241 */ 1242 static void 1243 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1244 { 1245 struct mlx5_priv *priv = dev->data->dev_private; 1246 uint32_t handle_idx; 1247 struct mlx5_flow_handle *dev_handle; 1248 1249 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1250 handle_idx, dev_handle, next) 1251 flow_drv_rxq_flags_trim(dev, dev_handle); 1252 } 1253 1254 /** 1255 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1256 * 1257 * @param dev 1258 * Pointer to Ethernet device. 1259 */ 1260 static void 1261 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1262 { 1263 struct mlx5_priv *priv = dev->data->dev_private; 1264 unsigned int i; 1265 1266 for (i = 0; i != priv->rxqs_n; ++i) { 1267 struct mlx5_rxq_ctrl *rxq_ctrl; 1268 unsigned int j; 1269 1270 if (!(*priv->rxqs)[i]) 1271 continue; 1272 rxq_ctrl = container_of((*priv->rxqs)[i], 1273 struct mlx5_rxq_ctrl, rxq); 1274 rxq_ctrl->flow_mark_n = 0; 1275 rxq_ctrl->rxq.mark = 0; 1276 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1277 rxq_ctrl->flow_tunnels_n[j] = 0; 1278 rxq_ctrl->rxq.tunnel = 0; 1279 } 1280 } 1281 1282 /** 1283 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1284 * 1285 * @param[in] dev 1286 * Pointer to the Ethernet device structure. 1287 */ 1288 void 1289 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1290 { 1291 struct mlx5_priv *priv = dev->data->dev_private; 1292 struct mlx5_rxq_data *data; 1293 unsigned int i; 1294 1295 for (i = 0; i != priv->rxqs_n; ++i) { 1296 if (!(*priv->rxqs)[i]) 1297 continue; 1298 data = (*priv->rxqs)[i]; 1299 if (!rte_flow_dynf_metadata_avail()) { 1300 data->dynf_meta = 0; 1301 data->flow_meta_mask = 0; 1302 data->flow_meta_offset = -1; 1303 data->flow_meta_port_mask = 0; 1304 } else { 1305 data->dynf_meta = 1; 1306 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1307 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1308 data->flow_meta_port_mask = (uint32_t)~0; 1309 if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16) 1310 data->flow_meta_port_mask >>= 16; 1311 } 1312 } 1313 } 1314 1315 /* 1316 * return a pointer to the desired action in the list of actions. 1317 * 1318 * @param[in] actions 1319 * The list of actions to search the action in. 1320 * @param[in] action 1321 * The action to find. 1322 * 1323 * @return 1324 * Pointer to the action in the list, if found. NULL otherwise. 1325 */ 1326 const struct rte_flow_action * 1327 mlx5_flow_find_action(const struct rte_flow_action *actions, 1328 enum rte_flow_action_type action) 1329 { 1330 if (actions == NULL) 1331 return NULL; 1332 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1333 if (actions->type == action) 1334 return actions; 1335 return NULL; 1336 } 1337 1338 /* 1339 * Validate the flag action. 1340 * 1341 * @param[in] action_flags 1342 * Bit-fields that holds the actions detected until now. 1343 * @param[in] attr 1344 * Attributes of flow that includes this action. 1345 * @param[out] error 1346 * Pointer to error structure. 1347 * 1348 * @return 1349 * 0 on success, a negative errno value otherwise and rte_errno is set. 1350 */ 1351 int 1352 mlx5_flow_validate_action_flag(uint64_t action_flags, 1353 const struct rte_flow_attr *attr, 1354 struct rte_flow_error *error) 1355 { 1356 if (action_flags & MLX5_FLOW_ACTION_MARK) 1357 return rte_flow_error_set(error, EINVAL, 1358 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1359 "can't mark and flag in same flow"); 1360 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1361 return rte_flow_error_set(error, EINVAL, 1362 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1363 "can't have 2 flag" 1364 " actions in same flow"); 1365 if (attr->egress) 1366 return rte_flow_error_set(error, ENOTSUP, 1367 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1368 "flag action not supported for " 1369 "egress"); 1370 return 0; 1371 } 1372 1373 /* 1374 * Validate the mark action. 1375 * 1376 * @param[in] action 1377 * Pointer to the queue action. 1378 * @param[in] action_flags 1379 * Bit-fields that holds the actions detected until now. 1380 * @param[in] attr 1381 * Attributes of flow that includes this action. 1382 * @param[out] error 1383 * Pointer to error structure. 1384 * 1385 * @return 1386 * 0 on success, a negative errno value otherwise and rte_errno is set. 1387 */ 1388 int 1389 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1390 uint64_t action_flags, 1391 const struct rte_flow_attr *attr, 1392 struct rte_flow_error *error) 1393 { 1394 const struct rte_flow_action_mark *mark = action->conf; 1395 1396 if (!mark) 1397 return rte_flow_error_set(error, EINVAL, 1398 RTE_FLOW_ERROR_TYPE_ACTION, 1399 action, 1400 "configuration cannot be null"); 1401 if (mark->id >= MLX5_FLOW_MARK_MAX) 1402 return rte_flow_error_set(error, EINVAL, 1403 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1404 &mark->id, 1405 "mark id must in 0 <= id < " 1406 RTE_STR(MLX5_FLOW_MARK_MAX)); 1407 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1408 return rte_flow_error_set(error, EINVAL, 1409 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1410 "can't flag and mark in same flow"); 1411 if (action_flags & MLX5_FLOW_ACTION_MARK) 1412 return rte_flow_error_set(error, EINVAL, 1413 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1414 "can't have 2 mark actions in same" 1415 " flow"); 1416 if (attr->egress) 1417 return rte_flow_error_set(error, ENOTSUP, 1418 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1419 "mark action not supported for " 1420 "egress"); 1421 return 0; 1422 } 1423 1424 /* 1425 * Validate the drop action. 1426 * 1427 * @param[in] action_flags 1428 * Bit-fields that holds the actions detected until now. 1429 * @param[in] attr 1430 * Attributes of flow that includes this action. 1431 * @param[out] error 1432 * Pointer to error structure. 1433 * 1434 * @return 1435 * 0 on success, a negative errno value otherwise and rte_errno is set. 1436 */ 1437 int 1438 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1439 const struct rte_flow_attr *attr, 1440 struct rte_flow_error *error) 1441 { 1442 if (attr->egress) 1443 return rte_flow_error_set(error, ENOTSUP, 1444 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1445 "drop action not supported for " 1446 "egress"); 1447 return 0; 1448 } 1449 1450 /* 1451 * Validate the queue action. 1452 * 1453 * @param[in] action 1454 * Pointer to the queue action. 1455 * @param[in] action_flags 1456 * Bit-fields that holds the actions detected until now. 1457 * @param[in] dev 1458 * Pointer to the Ethernet device structure. 1459 * @param[in] attr 1460 * Attributes of flow that includes this action. 1461 * @param[out] error 1462 * Pointer to error structure. 1463 * 1464 * @return 1465 * 0 on success, a negative errno value otherwise and rte_errno is set. 1466 */ 1467 int 1468 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1469 uint64_t action_flags, 1470 struct rte_eth_dev *dev, 1471 const struct rte_flow_attr *attr, 1472 struct rte_flow_error *error) 1473 { 1474 struct mlx5_priv *priv = dev->data->dev_private; 1475 const struct rte_flow_action_queue *queue = action->conf; 1476 1477 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1478 return rte_flow_error_set(error, EINVAL, 1479 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1480 "can't have 2 fate actions in" 1481 " same flow"); 1482 if (!priv->rxqs_n) 1483 return rte_flow_error_set(error, EINVAL, 1484 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1485 NULL, "No Rx queues configured"); 1486 if (queue->index >= priv->rxqs_n) 1487 return rte_flow_error_set(error, EINVAL, 1488 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1489 &queue->index, 1490 "queue index out of range"); 1491 if (!(*priv->rxqs)[queue->index]) 1492 return rte_flow_error_set(error, EINVAL, 1493 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1494 &queue->index, 1495 "queue is not configured"); 1496 if (attr->egress) 1497 return rte_flow_error_set(error, ENOTSUP, 1498 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1499 "queue action not supported for " 1500 "egress"); 1501 return 0; 1502 } 1503 1504 /* 1505 * Validate the rss action. 1506 * 1507 * @param[in] dev 1508 * Pointer to the Ethernet device structure. 1509 * @param[in] action 1510 * Pointer to the queue action. 1511 * @param[out] error 1512 * Pointer to error structure. 1513 * 1514 * @return 1515 * 0 on success, a negative errno value otherwise and rte_errno is set. 1516 */ 1517 int 1518 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1519 const struct rte_flow_action *action, 1520 struct rte_flow_error *error) 1521 { 1522 struct mlx5_priv *priv = dev->data->dev_private; 1523 const struct rte_flow_action_rss *rss = action->conf; 1524 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1525 unsigned int i; 1526 1527 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1528 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1529 return rte_flow_error_set(error, ENOTSUP, 1530 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1531 &rss->func, 1532 "RSS hash function not supported"); 1533 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1534 if (rss->level > 2) 1535 #else 1536 if (rss->level > 1) 1537 #endif 1538 return rte_flow_error_set(error, ENOTSUP, 1539 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1540 &rss->level, 1541 "tunnel RSS is not supported"); 1542 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1543 if (rss->key_len == 0 && rss->key != NULL) 1544 return rte_flow_error_set(error, ENOTSUP, 1545 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1546 &rss->key_len, 1547 "RSS hash key length 0"); 1548 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1549 return rte_flow_error_set(error, ENOTSUP, 1550 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1551 &rss->key_len, 1552 "RSS hash key too small"); 1553 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1554 return rte_flow_error_set(error, ENOTSUP, 1555 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1556 &rss->key_len, 1557 "RSS hash key too large"); 1558 if (rss->queue_num > priv->config.ind_table_max_size) 1559 return rte_flow_error_set(error, ENOTSUP, 1560 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1561 &rss->queue_num, 1562 "number of queues too large"); 1563 if (rss->types & MLX5_RSS_HF_MASK) 1564 return rte_flow_error_set(error, ENOTSUP, 1565 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1566 &rss->types, 1567 "some RSS protocols are not" 1568 " supported"); 1569 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1570 !(rss->types & ETH_RSS_IP)) 1571 return rte_flow_error_set(error, EINVAL, 1572 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1573 "L3 partial RSS requested but L3 RSS" 1574 " type not specified"); 1575 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1576 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1577 return rte_flow_error_set(error, EINVAL, 1578 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1579 "L4 partial RSS requested but L4 RSS" 1580 " type not specified"); 1581 if (!priv->rxqs_n) 1582 return rte_flow_error_set(error, EINVAL, 1583 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1584 NULL, "No Rx queues configured"); 1585 if (!rss->queue_num) 1586 return rte_flow_error_set(error, EINVAL, 1587 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1588 NULL, "No queues configured"); 1589 for (i = 0; i != rss->queue_num; ++i) { 1590 struct mlx5_rxq_ctrl *rxq_ctrl; 1591 1592 if (rss->queue[i] >= priv->rxqs_n) 1593 return rte_flow_error_set 1594 (error, EINVAL, 1595 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1596 &rss->queue[i], "queue index out of range"); 1597 if (!(*priv->rxqs)[rss->queue[i]]) 1598 return rte_flow_error_set 1599 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1600 &rss->queue[i], "queue is not configured"); 1601 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1602 struct mlx5_rxq_ctrl, rxq); 1603 if (i == 0) 1604 rxq_type = rxq_ctrl->type; 1605 if (rxq_type != rxq_ctrl->type) 1606 return rte_flow_error_set 1607 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1608 &rss->queue[i], 1609 "combining hairpin and regular RSS queues is not supported"); 1610 } 1611 return 0; 1612 } 1613 1614 /* 1615 * Validate the rss action. 1616 * 1617 * @param[in] action 1618 * Pointer to the queue action. 1619 * @param[in] action_flags 1620 * Bit-fields that holds the actions detected until now. 1621 * @param[in] dev 1622 * Pointer to the Ethernet device structure. 1623 * @param[in] attr 1624 * Attributes of flow that includes this action. 1625 * @param[in] item_flags 1626 * Items that were detected. 1627 * @param[out] error 1628 * Pointer to error structure. 1629 * 1630 * @return 1631 * 0 on success, a negative errno value otherwise and rte_errno is set. 1632 */ 1633 int 1634 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1635 uint64_t action_flags, 1636 struct rte_eth_dev *dev, 1637 const struct rte_flow_attr *attr, 1638 uint64_t item_flags, 1639 struct rte_flow_error *error) 1640 { 1641 const struct rte_flow_action_rss *rss = action->conf; 1642 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1643 int ret; 1644 1645 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1646 return rte_flow_error_set(error, EINVAL, 1647 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1648 "can't have 2 fate actions" 1649 " in same flow"); 1650 ret = mlx5_validate_action_rss(dev, action, error); 1651 if (ret) 1652 return ret; 1653 if (attr->egress) 1654 return rte_flow_error_set(error, ENOTSUP, 1655 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1656 "rss action not supported for " 1657 "egress"); 1658 if (rss->level > 1 && !tunnel) 1659 return rte_flow_error_set(error, EINVAL, 1660 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1661 "inner RSS is not supported for " 1662 "non-tunnel flows"); 1663 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1664 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1665 return rte_flow_error_set(error, EINVAL, 1666 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1667 "RSS on eCPRI is not supported now"); 1668 } 1669 return 0; 1670 } 1671 1672 /* 1673 * Validate the default miss action. 1674 * 1675 * @param[in] action_flags 1676 * Bit-fields that holds the actions detected until now. 1677 * @param[out] error 1678 * Pointer to error structure. 1679 * 1680 * @return 1681 * 0 on success, a negative errno value otherwise and rte_errno is set. 1682 */ 1683 int 1684 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1685 const struct rte_flow_attr *attr, 1686 struct rte_flow_error *error) 1687 { 1688 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1689 return rte_flow_error_set(error, EINVAL, 1690 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1691 "can't have 2 fate actions in" 1692 " same flow"); 1693 if (attr->egress) 1694 return rte_flow_error_set(error, ENOTSUP, 1695 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1696 "default miss action not supported " 1697 "for egress"); 1698 if (attr->group) 1699 return rte_flow_error_set(error, ENOTSUP, 1700 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1701 "only group 0 is supported"); 1702 if (attr->transfer) 1703 return rte_flow_error_set(error, ENOTSUP, 1704 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1705 NULL, "transfer is not supported"); 1706 return 0; 1707 } 1708 1709 /* 1710 * Validate the count action. 1711 * 1712 * @param[in] dev 1713 * Pointer to the Ethernet device structure. 1714 * @param[in] attr 1715 * Attributes of flow that includes this action. 1716 * @param[out] error 1717 * Pointer to error structure. 1718 * 1719 * @return 1720 * 0 on success, a negative errno value otherwise and rte_errno is set. 1721 */ 1722 int 1723 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1724 const struct rte_flow_attr *attr, 1725 struct rte_flow_error *error) 1726 { 1727 if (attr->egress) 1728 return rte_flow_error_set(error, ENOTSUP, 1729 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1730 "count action not supported for " 1731 "egress"); 1732 return 0; 1733 } 1734 1735 /* 1736 * Validate the ASO CT action. 1737 * 1738 * @param[in] dev 1739 * Pointer to the Ethernet device structure. 1740 * @param[in] conntrack 1741 * Pointer to the CT action profile. 1742 * @param[out] error 1743 * Pointer to error structure. 1744 * 1745 * @return 1746 * 0 on success, a negative errno value otherwise and rte_errno is set. 1747 */ 1748 int 1749 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1750 const struct rte_flow_action_conntrack *conntrack, 1751 struct rte_flow_error *error) 1752 { 1753 RTE_SET_USED(dev); 1754 1755 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1756 return rte_flow_error_set(error, EINVAL, 1757 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1758 "Invalid CT state"); 1759 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1760 return rte_flow_error_set(error, EINVAL, 1761 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1762 "Invalid last TCP packet flag"); 1763 return 0; 1764 } 1765 1766 /** 1767 * Verify the @p attributes will be correctly understood by the NIC and store 1768 * them in the @p flow if everything is correct. 1769 * 1770 * @param[in] dev 1771 * Pointer to the Ethernet device structure. 1772 * @param[in] attributes 1773 * Pointer to flow attributes 1774 * @param[out] error 1775 * Pointer to error structure. 1776 * 1777 * @return 1778 * 0 on success, a negative errno value otherwise and rte_errno is set. 1779 */ 1780 int 1781 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1782 const struct rte_flow_attr *attributes, 1783 struct rte_flow_error *error) 1784 { 1785 struct mlx5_priv *priv = dev->data->dev_private; 1786 uint32_t priority_max = priv->config.flow_prio - 1; 1787 1788 if (attributes->group) 1789 return rte_flow_error_set(error, ENOTSUP, 1790 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1791 NULL, "groups is not supported"); 1792 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1793 attributes->priority >= priority_max) 1794 return rte_flow_error_set(error, ENOTSUP, 1795 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1796 NULL, "priority out of range"); 1797 if (attributes->egress) 1798 return rte_flow_error_set(error, ENOTSUP, 1799 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1800 "egress is not supported"); 1801 if (attributes->transfer && !priv->config.dv_esw_en) 1802 return rte_flow_error_set(error, ENOTSUP, 1803 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1804 NULL, "transfer is not supported"); 1805 if (!attributes->ingress) 1806 return rte_flow_error_set(error, EINVAL, 1807 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1808 NULL, 1809 "ingress attribute is mandatory"); 1810 return 0; 1811 } 1812 1813 /** 1814 * Validate ICMP6 item. 1815 * 1816 * @param[in] item 1817 * Item specification. 1818 * @param[in] item_flags 1819 * Bit-fields that holds the items detected until now. 1820 * @param[in] ext_vlan_sup 1821 * Whether extended VLAN features are supported or not. 1822 * @param[out] error 1823 * Pointer to error structure. 1824 * 1825 * @return 1826 * 0 on success, a negative errno value otherwise and rte_errno is set. 1827 */ 1828 int 1829 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1830 uint64_t item_flags, 1831 uint8_t target_protocol, 1832 struct rte_flow_error *error) 1833 { 1834 const struct rte_flow_item_icmp6 *mask = item->mask; 1835 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1836 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1837 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1838 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1839 MLX5_FLOW_LAYER_OUTER_L4; 1840 int ret; 1841 1842 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1843 return rte_flow_error_set(error, EINVAL, 1844 RTE_FLOW_ERROR_TYPE_ITEM, item, 1845 "protocol filtering not compatible" 1846 " with ICMP6 layer"); 1847 if (!(item_flags & l3m)) 1848 return rte_flow_error_set(error, EINVAL, 1849 RTE_FLOW_ERROR_TYPE_ITEM, item, 1850 "IPv6 is mandatory to filter on" 1851 " ICMP6"); 1852 if (item_flags & l4m) 1853 return rte_flow_error_set(error, EINVAL, 1854 RTE_FLOW_ERROR_TYPE_ITEM, item, 1855 "multiple L4 layers not supported"); 1856 if (!mask) 1857 mask = &rte_flow_item_icmp6_mask; 1858 ret = mlx5_flow_item_acceptable 1859 (item, (const uint8_t *)mask, 1860 (const uint8_t *)&rte_flow_item_icmp6_mask, 1861 sizeof(struct rte_flow_item_icmp6), 1862 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1863 if (ret < 0) 1864 return ret; 1865 return 0; 1866 } 1867 1868 /** 1869 * Validate ICMP item. 1870 * 1871 * @param[in] item 1872 * Item specification. 1873 * @param[in] item_flags 1874 * Bit-fields that holds the items detected until now. 1875 * @param[out] error 1876 * Pointer to error structure. 1877 * 1878 * @return 1879 * 0 on success, a negative errno value otherwise and rte_errno is set. 1880 */ 1881 int 1882 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1883 uint64_t item_flags, 1884 uint8_t target_protocol, 1885 struct rte_flow_error *error) 1886 { 1887 const struct rte_flow_item_icmp *mask = item->mask; 1888 const struct rte_flow_item_icmp nic_mask = { 1889 .hdr.icmp_type = 0xff, 1890 .hdr.icmp_code = 0xff, 1891 .hdr.icmp_ident = RTE_BE16(0xffff), 1892 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1893 }; 1894 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1895 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1896 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1897 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1898 MLX5_FLOW_LAYER_OUTER_L4; 1899 int ret; 1900 1901 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1902 return rte_flow_error_set(error, EINVAL, 1903 RTE_FLOW_ERROR_TYPE_ITEM, item, 1904 "protocol filtering not compatible" 1905 " with ICMP layer"); 1906 if (!(item_flags & l3m)) 1907 return rte_flow_error_set(error, EINVAL, 1908 RTE_FLOW_ERROR_TYPE_ITEM, item, 1909 "IPv4 is mandatory to filter" 1910 " on ICMP"); 1911 if (item_flags & l4m) 1912 return rte_flow_error_set(error, EINVAL, 1913 RTE_FLOW_ERROR_TYPE_ITEM, item, 1914 "multiple L4 layers not supported"); 1915 if (!mask) 1916 mask = &nic_mask; 1917 ret = mlx5_flow_item_acceptable 1918 (item, (const uint8_t *)mask, 1919 (const uint8_t *)&nic_mask, 1920 sizeof(struct rte_flow_item_icmp), 1921 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1922 if (ret < 0) 1923 return ret; 1924 return 0; 1925 } 1926 1927 /** 1928 * Validate Ethernet item. 1929 * 1930 * @param[in] item 1931 * Item specification. 1932 * @param[in] item_flags 1933 * Bit-fields that holds the items detected until now. 1934 * @param[out] error 1935 * Pointer to error structure. 1936 * 1937 * @return 1938 * 0 on success, a negative errno value otherwise and rte_errno is set. 1939 */ 1940 int 1941 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1942 uint64_t item_flags, bool ext_vlan_sup, 1943 struct rte_flow_error *error) 1944 { 1945 const struct rte_flow_item_eth *mask = item->mask; 1946 const struct rte_flow_item_eth nic_mask = { 1947 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1948 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1949 .type = RTE_BE16(0xffff), 1950 .has_vlan = ext_vlan_sup ? 1 : 0, 1951 }; 1952 int ret; 1953 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1954 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1955 MLX5_FLOW_LAYER_OUTER_L2; 1956 1957 if (item_flags & ethm) 1958 return rte_flow_error_set(error, ENOTSUP, 1959 RTE_FLOW_ERROR_TYPE_ITEM, item, 1960 "multiple L2 layers not supported"); 1961 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1962 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1963 return rte_flow_error_set(error, EINVAL, 1964 RTE_FLOW_ERROR_TYPE_ITEM, item, 1965 "L2 layer should not follow " 1966 "L3 layers"); 1967 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1968 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1969 return rte_flow_error_set(error, EINVAL, 1970 RTE_FLOW_ERROR_TYPE_ITEM, item, 1971 "L2 layer should not follow VLAN"); 1972 if (!mask) 1973 mask = &rte_flow_item_eth_mask; 1974 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1975 (const uint8_t *)&nic_mask, 1976 sizeof(struct rte_flow_item_eth), 1977 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1978 return ret; 1979 } 1980 1981 /** 1982 * Validate VLAN item. 1983 * 1984 * @param[in] item 1985 * Item specification. 1986 * @param[in] item_flags 1987 * Bit-fields that holds the items detected until now. 1988 * @param[in] dev 1989 * Ethernet device flow is being created on. 1990 * @param[out] error 1991 * Pointer to error structure. 1992 * 1993 * @return 1994 * 0 on success, a negative errno value otherwise and rte_errno is set. 1995 */ 1996 int 1997 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1998 uint64_t item_flags, 1999 struct rte_eth_dev *dev, 2000 struct rte_flow_error *error) 2001 { 2002 const struct rte_flow_item_vlan *spec = item->spec; 2003 const struct rte_flow_item_vlan *mask = item->mask; 2004 const struct rte_flow_item_vlan nic_mask = { 2005 .tci = RTE_BE16(UINT16_MAX), 2006 .inner_type = RTE_BE16(UINT16_MAX), 2007 }; 2008 uint16_t vlan_tag = 0; 2009 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2010 int ret; 2011 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2012 MLX5_FLOW_LAYER_INNER_L4) : 2013 (MLX5_FLOW_LAYER_OUTER_L3 | 2014 MLX5_FLOW_LAYER_OUTER_L4); 2015 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2016 MLX5_FLOW_LAYER_OUTER_VLAN; 2017 2018 if (item_flags & vlanm) 2019 return rte_flow_error_set(error, EINVAL, 2020 RTE_FLOW_ERROR_TYPE_ITEM, item, 2021 "multiple VLAN layers not supported"); 2022 else if ((item_flags & l34m) != 0) 2023 return rte_flow_error_set(error, EINVAL, 2024 RTE_FLOW_ERROR_TYPE_ITEM, item, 2025 "VLAN cannot follow L3/L4 layer"); 2026 if (!mask) 2027 mask = &rte_flow_item_vlan_mask; 2028 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2029 (const uint8_t *)&nic_mask, 2030 sizeof(struct rte_flow_item_vlan), 2031 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2032 if (ret) 2033 return ret; 2034 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2035 struct mlx5_priv *priv = dev->data->dev_private; 2036 2037 if (priv->vmwa_context) { 2038 /* 2039 * Non-NULL context means we have a virtual machine 2040 * and SR-IOV enabled, we have to create VLAN interface 2041 * to make hypervisor to setup E-Switch vport 2042 * context correctly. We avoid creating the multiple 2043 * VLAN interfaces, so we cannot support VLAN tag mask. 2044 */ 2045 return rte_flow_error_set(error, EINVAL, 2046 RTE_FLOW_ERROR_TYPE_ITEM, 2047 item, 2048 "VLAN tag mask is not" 2049 " supported in virtual" 2050 " environment"); 2051 } 2052 } 2053 if (spec) { 2054 vlan_tag = spec->tci; 2055 vlan_tag &= mask->tci; 2056 } 2057 /* 2058 * From verbs perspective an empty VLAN is equivalent 2059 * to a packet without VLAN layer. 2060 */ 2061 if (!vlan_tag) 2062 return rte_flow_error_set(error, EINVAL, 2063 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2064 item->spec, 2065 "VLAN cannot be empty"); 2066 return 0; 2067 } 2068 2069 /** 2070 * Validate IPV4 item. 2071 * 2072 * @param[in] item 2073 * Item specification. 2074 * @param[in] item_flags 2075 * Bit-fields that holds the items detected until now. 2076 * @param[in] last_item 2077 * Previous validated item in the pattern items. 2078 * @param[in] ether_type 2079 * Type in the ethernet layer header (including dot1q). 2080 * @param[in] acc_mask 2081 * Acceptable mask, if NULL default internal default mask 2082 * will be used to check whether item fields are supported. 2083 * @param[in] range_accepted 2084 * True if range of values is accepted for specific fields, false otherwise. 2085 * @param[out] error 2086 * Pointer to error structure. 2087 * 2088 * @return 2089 * 0 on success, a negative errno value otherwise and rte_errno is set. 2090 */ 2091 int 2092 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2093 uint64_t item_flags, 2094 uint64_t last_item, 2095 uint16_t ether_type, 2096 const struct rte_flow_item_ipv4 *acc_mask, 2097 bool range_accepted, 2098 struct rte_flow_error *error) 2099 { 2100 const struct rte_flow_item_ipv4 *mask = item->mask; 2101 const struct rte_flow_item_ipv4 *spec = item->spec; 2102 const struct rte_flow_item_ipv4 nic_mask = { 2103 .hdr = { 2104 .src_addr = RTE_BE32(0xffffffff), 2105 .dst_addr = RTE_BE32(0xffffffff), 2106 .type_of_service = 0xff, 2107 .next_proto_id = 0xff, 2108 }, 2109 }; 2110 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2111 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2112 MLX5_FLOW_LAYER_OUTER_L3; 2113 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2114 MLX5_FLOW_LAYER_OUTER_L4; 2115 int ret; 2116 uint8_t next_proto = 0xFF; 2117 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2118 MLX5_FLOW_LAYER_OUTER_VLAN | 2119 MLX5_FLOW_LAYER_INNER_VLAN); 2120 2121 if ((last_item & l2_vlan) && ether_type && 2122 ether_type != RTE_ETHER_TYPE_IPV4) 2123 return rte_flow_error_set(error, EINVAL, 2124 RTE_FLOW_ERROR_TYPE_ITEM, item, 2125 "IPv4 cannot follow L2/VLAN layer " 2126 "which ether type is not IPv4"); 2127 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2128 if (mask && spec) 2129 next_proto = mask->hdr.next_proto_id & 2130 spec->hdr.next_proto_id; 2131 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2132 return rte_flow_error_set(error, EINVAL, 2133 RTE_FLOW_ERROR_TYPE_ITEM, 2134 item, 2135 "multiple tunnel " 2136 "not supported"); 2137 } 2138 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2139 return rte_flow_error_set(error, EINVAL, 2140 RTE_FLOW_ERROR_TYPE_ITEM, item, 2141 "wrong tunnel type - IPv6 specified " 2142 "but IPv4 item provided"); 2143 if (item_flags & l3m) 2144 return rte_flow_error_set(error, ENOTSUP, 2145 RTE_FLOW_ERROR_TYPE_ITEM, item, 2146 "multiple L3 layers not supported"); 2147 else if (item_flags & l4m) 2148 return rte_flow_error_set(error, EINVAL, 2149 RTE_FLOW_ERROR_TYPE_ITEM, item, 2150 "L3 cannot follow an L4 layer."); 2151 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2152 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2153 return rte_flow_error_set(error, EINVAL, 2154 RTE_FLOW_ERROR_TYPE_ITEM, item, 2155 "L3 cannot follow an NVGRE layer."); 2156 if (!mask) 2157 mask = &rte_flow_item_ipv4_mask; 2158 else if (mask->hdr.next_proto_id != 0 && 2159 mask->hdr.next_proto_id != 0xff) 2160 return rte_flow_error_set(error, EINVAL, 2161 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2162 "partial mask is not supported" 2163 " for protocol"); 2164 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2165 acc_mask ? (const uint8_t *)acc_mask 2166 : (const uint8_t *)&nic_mask, 2167 sizeof(struct rte_flow_item_ipv4), 2168 range_accepted, error); 2169 if (ret < 0) 2170 return ret; 2171 return 0; 2172 } 2173 2174 /** 2175 * Validate IPV6 item. 2176 * 2177 * @param[in] item 2178 * Item specification. 2179 * @param[in] item_flags 2180 * Bit-fields that holds the items detected until now. 2181 * @param[in] last_item 2182 * Previous validated item in the pattern items. 2183 * @param[in] ether_type 2184 * Type in the ethernet layer header (including dot1q). 2185 * @param[in] acc_mask 2186 * Acceptable mask, if NULL default internal default mask 2187 * will be used to check whether item fields are supported. 2188 * @param[out] error 2189 * Pointer to error structure. 2190 * 2191 * @return 2192 * 0 on success, a negative errno value otherwise and rte_errno is set. 2193 */ 2194 int 2195 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2196 uint64_t item_flags, 2197 uint64_t last_item, 2198 uint16_t ether_type, 2199 const struct rte_flow_item_ipv6 *acc_mask, 2200 struct rte_flow_error *error) 2201 { 2202 const struct rte_flow_item_ipv6 *mask = item->mask; 2203 const struct rte_flow_item_ipv6 *spec = item->spec; 2204 const struct rte_flow_item_ipv6 nic_mask = { 2205 .hdr = { 2206 .src_addr = 2207 "\xff\xff\xff\xff\xff\xff\xff\xff" 2208 "\xff\xff\xff\xff\xff\xff\xff\xff", 2209 .dst_addr = 2210 "\xff\xff\xff\xff\xff\xff\xff\xff" 2211 "\xff\xff\xff\xff\xff\xff\xff\xff", 2212 .vtc_flow = RTE_BE32(0xffffffff), 2213 .proto = 0xff, 2214 }, 2215 }; 2216 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2217 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2218 MLX5_FLOW_LAYER_OUTER_L3; 2219 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2220 MLX5_FLOW_LAYER_OUTER_L4; 2221 int ret; 2222 uint8_t next_proto = 0xFF; 2223 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2224 MLX5_FLOW_LAYER_OUTER_VLAN | 2225 MLX5_FLOW_LAYER_INNER_VLAN); 2226 2227 if ((last_item & l2_vlan) && ether_type && 2228 ether_type != RTE_ETHER_TYPE_IPV6) 2229 return rte_flow_error_set(error, EINVAL, 2230 RTE_FLOW_ERROR_TYPE_ITEM, item, 2231 "IPv6 cannot follow L2/VLAN layer " 2232 "which ether type is not IPv6"); 2233 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2234 next_proto = spec->hdr.proto; 2235 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2236 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2237 return rte_flow_error_set(error, EINVAL, 2238 RTE_FLOW_ERROR_TYPE_ITEM, 2239 item, 2240 "multiple tunnel " 2241 "not supported"); 2242 } 2243 if (next_proto == IPPROTO_HOPOPTS || 2244 next_proto == IPPROTO_ROUTING || 2245 next_proto == IPPROTO_FRAGMENT || 2246 next_proto == IPPROTO_ESP || 2247 next_proto == IPPROTO_AH || 2248 next_proto == IPPROTO_DSTOPTS) 2249 return rte_flow_error_set(error, EINVAL, 2250 RTE_FLOW_ERROR_TYPE_ITEM, item, 2251 "IPv6 proto (next header) should " 2252 "not be set as extension header"); 2253 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2254 return rte_flow_error_set(error, EINVAL, 2255 RTE_FLOW_ERROR_TYPE_ITEM, item, 2256 "wrong tunnel type - IPv4 specified " 2257 "but IPv6 item provided"); 2258 if (item_flags & l3m) 2259 return rte_flow_error_set(error, ENOTSUP, 2260 RTE_FLOW_ERROR_TYPE_ITEM, item, 2261 "multiple L3 layers not supported"); 2262 else if (item_flags & l4m) 2263 return rte_flow_error_set(error, EINVAL, 2264 RTE_FLOW_ERROR_TYPE_ITEM, item, 2265 "L3 cannot follow an L4 layer."); 2266 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2267 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2268 return rte_flow_error_set(error, EINVAL, 2269 RTE_FLOW_ERROR_TYPE_ITEM, item, 2270 "L3 cannot follow an NVGRE layer."); 2271 if (!mask) 2272 mask = &rte_flow_item_ipv6_mask; 2273 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2274 acc_mask ? (const uint8_t *)acc_mask 2275 : (const uint8_t *)&nic_mask, 2276 sizeof(struct rte_flow_item_ipv6), 2277 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2278 if (ret < 0) 2279 return ret; 2280 return 0; 2281 } 2282 2283 /** 2284 * Validate UDP item. 2285 * 2286 * @param[in] item 2287 * Item specification. 2288 * @param[in] item_flags 2289 * Bit-fields that holds the items detected until now. 2290 * @param[in] target_protocol 2291 * The next protocol in the previous item. 2292 * @param[in] flow_mask 2293 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2294 * @param[out] error 2295 * Pointer to error structure. 2296 * 2297 * @return 2298 * 0 on success, a negative errno value otherwise and rte_errno is set. 2299 */ 2300 int 2301 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2302 uint64_t item_flags, 2303 uint8_t target_protocol, 2304 struct rte_flow_error *error) 2305 { 2306 const struct rte_flow_item_udp *mask = item->mask; 2307 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2308 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2309 MLX5_FLOW_LAYER_OUTER_L3; 2310 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2311 MLX5_FLOW_LAYER_OUTER_L4; 2312 int ret; 2313 2314 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2315 return rte_flow_error_set(error, EINVAL, 2316 RTE_FLOW_ERROR_TYPE_ITEM, item, 2317 "protocol filtering not compatible" 2318 " with UDP layer"); 2319 if (!(item_flags & l3m)) 2320 return rte_flow_error_set(error, EINVAL, 2321 RTE_FLOW_ERROR_TYPE_ITEM, item, 2322 "L3 is mandatory to filter on L4"); 2323 if (item_flags & l4m) 2324 return rte_flow_error_set(error, EINVAL, 2325 RTE_FLOW_ERROR_TYPE_ITEM, item, 2326 "multiple L4 layers not supported"); 2327 if (!mask) 2328 mask = &rte_flow_item_udp_mask; 2329 ret = mlx5_flow_item_acceptable 2330 (item, (const uint8_t *)mask, 2331 (const uint8_t *)&rte_flow_item_udp_mask, 2332 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2333 error); 2334 if (ret < 0) 2335 return ret; 2336 return 0; 2337 } 2338 2339 /** 2340 * Validate TCP item. 2341 * 2342 * @param[in] item 2343 * Item specification. 2344 * @param[in] item_flags 2345 * Bit-fields that holds the items detected until now. 2346 * @param[in] target_protocol 2347 * The next protocol in the previous item. 2348 * @param[out] error 2349 * Pointer to error structure. 2350 * 2351 * @return 2352 * 0 on success, a negative errno value otherwise and rte_errno is set. 2353 */ 2354 int 2355 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2356 uint64_t item_flags, 2357 uint8_t target_protocol, 2358 const struct rte_flow_item_tcp *flow_mask, 2359 struct rte_flow_error *error) 2360 { 2361 const struct rte_flow_item_tcp *mask = item->mask; 2362 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2363 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2364 MLX5_FLOW_LAYER_OUTER_L3; 2365 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2366 MLX5_FLOW_LAYER_OUTER_L4; 2367 int ret; 2368 2369 MLX5_ASSERT(flow_mask); 2370 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2371 return rte_flow_error_set(error, EINVAL, 2372 RTE_FLOW_ERROR_TYPE_ITEM, item, 2373 "protocol filtering not compatible" 2374 " with TCP layer"); 2375 if (!(item_flags & l3m)) 2376 return rte_flow_error_set(error, EINVAL, 2377 RTE_FLOW_ERROR_TYPE_ITEM, item, 2378 "L3 is mandatory to filter on L4"); 2379 if (item_flags & l4m) 2380 return rte_flow_error_set(error, EINVAL, 2381 RTE_FLOW_ERROR_TYPE_ITEM, item, 2382 "multiple L4 layers not supported"); 2383 if (!mask) 2384 mask = &rte_flow_item_tcp_mask; 2385 ret = mlx5_flow_item_acceptable 2386 (item, (const uint8_t *)mask, 2387 (const uint8_t *)flow_mask, 2388 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2389 error); 2390 if (ret < 0) 2391 return ret; 2392 return 0; 2393 } 2394 2395 /** 2396 * Validate VXLAN item. 2397 * 2398 * @param[in] item 2399 * Item specification. 2400 * @param[in] item_flags 2401 * Bit-fields that holds the items detected until now. 2402 * @param[in] target_protocol 2403 * The next protocol in the previous item. 2404 * @param[out] error 2405 * Pointer to error structure. 2406 * 2407 * @return 2408 * 0 on success, a negative errno value otherwise and rte_errno is set. 2409 */ 2410 int 2411 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2412 uint64_t item_flags, 2413 struct rte_flow_error *error) 2414 { 2415 const struct rte_flow_item_vxlan *spec = item->spec; 2416 const struct rte_flow_item_vxlan *mask = item->mask; 2417 int ret; 2418 union vni { 2419 uint32_t vlan_id; 2420 uint8_t vni[4]; 2421 } id = { .vlan_id = 0, }; 2422 2423 2424 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2425 return rte_flow_error_set(error, ENOTSUP, 2426 RTE_FLOW_ERROR_TYPE_ITEM, item, 2427 "multiple tunnel layers not" 2428 " supported"); 2429 /* 2430 * Verify only UDPv4 is present as defined in 2431 * https://tools.ietf.org/html/rfc7348 2432 */ 2433 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2434 return rte_flow_error_set(error, EINVAL, 2435 RTE_FLOW_ERROR_TYPE_ITEM, item, 2436 "no outer UDP layer found"); 2437 if (!mask) 2438 mask = &rte_flow_item_vxlan_mask; 2439 ret = mlx5_flow_item_acceptable 2440 (item, (const uint8_t *)mask, 2441 (const uint8_t *)&rte_flow_item_vxlan_mask, 2442 sizeof(struct rte_flow_item_vxlan), 2443 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2444 if (ret < 0) 2445 return ret; 2446 if (spec) { 2447 memcpy(&id.vni[1], spec->vni, 3); 2448 memcpy(&id.vni[1], mask->vni, 3); 2449 } 2450 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2451 return rte_flow_error_set(error, ENOTSUP, 2452 RTE_FLOW_ERROR_TYPE_ITEM, item, 2453 "VXLAN tunnel must be fully defined"); 2454 return 0; 2455 } 2456 2457 /** 2458 * Validate VXLAN_GPE item. 2459 * 2460 * @param[in] item 2461 * Item specification. 2462 * @param[in] item_flags 2463 * Bit-fields that holds the items detected until now. 2464 * @param[in] priv 2465 * Pointer to the private data structure. 2466 * @param[in] target_protocol 2467 * The next protocol in the previous item. 2468 * @param[out] error 2469 * Pointer to error structure. 2470 * 2471 * @return 2472 * 0 on success, a negative errno value otherwise and rte_errno is set. 2473 */ 2474 int 2475 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2476 uint64_t item_flags, 2477 struct rte_eth_dev *dev, 2478 struct rte_flow_error *error) 2479 { 2480 struct mlx5_priv *priv = dev->data->dev_private; 2481 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2482 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2483 int ret; 2484 union vni { 2485 uint32_t vlan_id; 2486 uint8_t vni[4]; 2487 } id = { .vlan_id = 0, }; 2488 2489 if (!priv->config.l3_vxlan_en) 2490 return rte_flow_error_set(error, ENOTSUP, 2491 RTE_FLOW_ERROR_TYPE_ITEM, item, 2492 "L3 VXLAN is not enabled by device" 2493 " parameter and/or not configured in" 2494 " firmware"); 2495 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2496 return rte_flow_error_set(error, ENOTSUP, 2497 RTE_FLOW_ERROR_TYPE_ITEM, item, 2498 "multiple tunnel layers not" 2499 " supported"); 2500 /* 2501 * Verify only UDPv4 is present as defined in 2502 * https://tools.ietf.org/html/rfc7348 2503 */ 2504 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2505 return rte_flow_error_set(error, EINVAL, 2506 RTE_FLOW_ERROR_TYPE_ITEM, item, 2507 "no outer UDP layer found"); 2508 if (!mask) 2509 mask = &rte_flow_item_vxlan_gpe_mask; 2510 ret = mlx5_flow_item_acceptable 2511 (item, (const uint8_t *)mask, 2512 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2513 sizeof(struct rte_flow_item_vxlan_gpe), 2514 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2515 if (ret < 0) 2516 return ret; 2517 if (spec) { 2518 if (spec->protocol) 2519 return rte_flow_error_set(error, ENOTSUP, 2520 RTE_FLOW_ERROR_TYPE_ITEM, 2521 item, 2522 "VxLAN-GPE protocol" 2523 " not supported"); 2524 memcpy(&id.vni[1], spec->vni, 3); 2525 memcpy(&id.vni[1], mask->vni, 3); 2526 } 2527 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2528 return rte_flow_error_set(error, ENOTSUP, 2529 RTE_FLOW_ERROR_TYPE_ITEM, item, 2530 "VXLAN-GPE tunnel must be fully" 2531 " defined"); 2532 return 0; 2533 } 2534 /** 2535 * Validate GRE Key item. 2536 * 2537 * @param[in] item 2538 * Item specification. 2539 * @param[in] item_flags 2540 * Bit flags to mark detected items. 2541 * @param[in] gre_item 2542 * Pointer to gre_item 2543 * @param[out] error 2544 * Pointer to error structure. 2545 * 2546 * @return 2547 * 0 on success, a negative errno value otherwise and rte_errno is set. 2548 */ 2549 int 2550 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2551 uint64_t item_flags, 2552 const struct rte_flow_item *gre_item, 2553 struct rte_flow_error *error) 2554 { 2555 const rte_be32_t *mask = item->mask; 2556 int ret = 0; 2557 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2558 const struct rte_flow_item_gre *gre_spec; 2559 const struct rte_flow_item_gre *gre_mask; 2560 2561 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2562 return rte_flow_error_set(error, ENOTSUP, 2563 RTE_FLOW_ERROR_TYPE_ITEM, item, 2564 "Multiple GRE key not support"); 2565 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2566 return rte_flow_error_set(error, ENOTSUP, 2567 RTE_FLOW_ERROR_TYPE_ITEM, item, 2568 "No preceding GRE header"); 2569 if (item_flags & MLX5_FLOW_LAYER_INNER) 2570 return rte_flow_error_set(error, ENOTSUP, 2571 RTE_FLOW_ERROR_TYPE_ITEM, item, 2572 "GRE key following a wrong item"); 2573 gre_mask = gre_item->mask; 2574 if (!gre_mask) 2575 gre_mask = &rte_flow_item_gre_mask; 2576 gre_spec = gre_item->spec; 2577 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2578 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2579 return rte_flow_error_set(error, EINVAL, 2580 RTE_FLOW_ERROR_TYPE_ITEM, item, 2581 "Key bit must be on"); 2582 2583 if (!mask) 2584 mask = &gre_key_default_mask; 2585 ret = mlx5_flow_item_acceptable 2586 (item, (const uint8_t *)mask, 2587 (const uint8_t *)&gre_key_default_mask, 2588 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2589 return ret; 2590 } 2591 2592 /** 2593 * Validate GRE item. 2594 * 2595 * @param[in] item 2596 * Item specification. 2597 * @param[in] item_flags 2598 * Bit flags to mark detected items. 2599 * @param[in] target_protocol 2600 * The next protocol in the previous item. 2601 * @param[out] error 2602 * Pointer to error structure. 2603 * 2604 * @return 2605 * 0 on success, a negative errno value otherwise and rte_errno is set. 2606 */ 2607 int 2608 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2609 uint64_t item_flags, 2610 uint8_t target_protocol, 2611 struct rte_flow_error *error) 2612 { 2613 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2614 const struct rte_flow_item_gre *mask = item->mask; 2615 int ret; 2616 const struct rte_flow_item_gre nic_mask = { 2617 .c_rsvd0_ver = RTE_BE16(0xB000), 2618 .protocol = RTE_BE16(UINT16_MAX), 2619 }; 2620 2621 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2622 return rte_flow_error_set(error, EINVAL, 2623 RTE_FLOW_ERROR_TYPE_ITEM, item, 2624 "protocol filtering not compatible" 2625 " with this GRE layer"); 2626 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2627 return rte_flow_error_set(error, ENOTSUP, 2628 RTE_FLOW_ERROR_TYPE_ITEM, item, 2629 "multiple tunnel layers not" 2630 " supported"); 2631 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2632 return rte_flow_error_set(error, ENOTSUP, 2633 RTE_FLOW_ERROR_TYPE_ITEM, item, 2634 "L3 Layer is missing"); 2635 if (!mask) 2636 mask = &rte_flow_item_gre_mask; 2637 ret = mlx5_flow_item_acceptable 2638 (item, (const uint8_t *)mask, 2639 (const uint8_t *)&nic_mask, 2640 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2641 error); 2642 if (ret < 0) 2643 return ret; 2644 #ifndef HAVE_MLX5DV_DR 2645 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2646 if (spec && (spec->protocol & mask->protocol)) 2647 return rte_flow_error_set(error, ENOTSUP, 2648 RTE_FLOW_ERROR_TYPE_ITEM, item, 2649 "without MPLS support the" 2650 " specification cannot be used for" 2651 " filtering"); 2652 #endif 2653 #endif 2654 return 0; 2655 } 2656 2657 /** 2658 * Validate Geneve item. 2659 * 2660 * @param[in] item 2661 * Item specification. 2662 * @param[in] itemFlags 2663 * Bit-fields that holds the items detected until now. 2664 * @param[in] enPriv 2665 * Pointer to the private data structure. 2666 * @param[out] error 2667 * Pointer to error structure. 2668 * 2669 * @return 2670 * 0 on success, a negative errno value otherwise and rte_errno is set. 2671 */ 2672 2673 int 2674 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2675 uint64_t item_flags, 2676 struct rte_eth_dev *dev, 2677 struct rte_flow_error *error) 2678 { 2679 struct mlx5_priv *priv = dev->data->dev_private; 2680 const struct rte_flow_item_geneve *spec = item->spec; 2681 const struct rte_flow_item_geneve *mask = item->mask; 2682 int ret; 2683 uint16_t gbhdr; 2684 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2685 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2686 const struct rte_flow_item_geneve nic_mask = { 2687 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2688 .vni = "\xff\xff\xff", 2689 .protocol = RTE_BE16(UINT16_MAX), 2690 }; 2691 2692 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2693 return rte_flow_error_set(error, ENOTSUP, 2694 RTE_FLOW_ERROR_TYPE_ITEM, item, 2695 "L3 Geneve is not enabled by device" 2696 " parameter and/or not configured in" 2697 " firmware"); 2698 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2699 return rte_flow_error_set(error, ENOTSUP, 2700 RTE_FLOW_ERROR_TYPE_ITEM, item, 2701 "multiple tunnel layers not" 2702 " supported"); 2703 /* 2704 * Verify only UDPv4 is present as defined in 2705 * https://tools.ietf.org/html/rfc7348 2706 */ 2707 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2708 return rte_flow_error_set(error, EINVAL, 2709 RTE_FLOW_ERROR_TYPE_ITEM, item, 2710 "no outer UDP layer found"); 2711 if (!mask) 2712 mask = &rte_flow_item_geneve_mask; 2713 ret = mlx5_flow_item_acceptable 2714 (item, (const uint8_t *)mask, 2715 (const uint8_t *)&nic_mask, 2716 sizeof(struct rte_flow_item_geneve), 2717 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2718 if (ret) 2719 return ret; 2720 if (spec) { 2721 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2722 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2723 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2724 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2725 return rte_flow_error_set(error, ENOTSUP, 2726 RTE_FLOW_ERROR_TYPE_ITEM, 2727 item, 2728 "Geneve protocol unsupported" 2729 " fields are being used"); 2730 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2731 return rte_flow_error_set 2732 (error, ENOTSUP, 2733 RTE_FLOW_ERROR_TYPE_ITEM, 2734 item, 2735 "Unsupported Geneve options length"); 2736 } 2737 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2738 return rte_flow_error_set 2739 (error, ENOTSUP, 2740 RTE_FLOW_ERROR_TYPE_ITEM, item, 2741 "Geneve tunnel must be fully defined"); 2742 return 0; 2743 } 2744 2745 /** 2746 * Validate Geneve TLV option item. 2747 * 2748 * @param[in] item 2749 * Item specification. 2750 * @param[in] last_item 2751 * Previous validated item in the pattern items. 2752 * @param[in] geneve_item 2753 * Previous GENEVE item specification. 2754 * @param[in] dev 2755 * Pointer to the rte_eth_dev structure. 2756 * @param[out] error 2757 * Pointer to error structure. 2758 * 2759 * @return 2760 * 0 on success, a negative errno value otherwise and rte_errno is set. 2761 */ 2762 int 2763 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2764 uint64_t last_item, 2765 const struct rte_flow_item *geneve_item, 2766 struct rte_eth_dev *dev, 2767 struct rte_flow_error *error) 2768 { 2769 struct mlx5_priv *priv = dev->data->dev_private; 2770 struct mlx5_dev_ctx_shared *sh = priv->sh; 2771 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2772 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2773 uint8_t data_max_supported = 2774 hca_attr->max_geneve_tlv_option_data_len * 4; 2775 struct mlx5_dev_config *config = &priv->config; 2776 const struct rte_flow_item_geneve *geneve_spec; 2777 const struct rte_flow_item_geneve *geneve_mask; 2778 const struct rte_flow_item_geneve_opt *spec = item->spec; 2779 const struct rte_flow_item_geneve_opt *mask = item->mask; 2780 unsigned int i; 2781 unsigned int data_len; 2782 uint8_t tlv_option_len; 2783 uint16_t optlen_m, optlen_v; 2784 const struct rte_flow_item_geneve_opt full_mask = { 2785 .option_class = RTE_BE16(0xffff), 2786 .option_type = 0xff, 2787 .option_len = 0x1f, 2788 }; 2789 2790 if (!mask) 2791 mask = &rte_flow_item_geneve_opt_mask; 2792 if (!spec) 2793 return rte_flow_error_set 2794 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2795 "Geneve TLV opt class/type/length must be specified"); 2796 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2797 return rte_flow_error_set 2798 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2799 "Geneve TLV opt length exceeeds the limit (31)"); 2800 /* Check if class type and length masks are full. */ 2801 if (full_mask.option_class != mask->option_class || 2802 full_mask.option_type != mask->option_type || 2803 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2804 return rte_flow_error_set 2805 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2806 "Geneve TLV opt class/type/length masks must be full"); 2807 /* Check if length is supported */ 2808 if ((uint32_t)spec->option_len > 2809 config->hca_attr.max_geneve_tlv_option_data_len) 2810 return rte_flow_error_set 2811 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2812 "Geneve TLV opt length not supported"); 2813 if (config->hca_attr.max_geneve_tlv_options > 1) 2814 DRV_LOG(DEBUG, 2815 "max_geneve_tlv_options supports more than 1 option"); 2816 /* Check GENEVE item preceding. */ 2817 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2818 return rte_flow_error_set 2819 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2820 "Geneve opt item must be preceded with Geneve item"); 2821 geneve_spec = geneve_item->spec; 2822 geneve_mask = geneve_item->mask ? geneve_item->mask : 2823 &rte_flow_item_geneve_mask; 2824 /* Check if GENEVE TLV option size doesn't exceed option length */ 2825 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2826 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2827 tlv_option_len = spec->option_len & mask->option_len; 2828 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2829 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2830 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2831 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2832 if ((optlen_v & optlen_m) <= tlv_option_len) 2833 return rte_flow_error_set 2834 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2835 "GENEVE TLV option length exceeds optlen"); 2836 } 2837 /* Check if length is 0 or data is 0. */ 2838 if (spec->data == NULL || spec->option_len == 0) 2839 return rte_flow_error_set 2840 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2841 "Geneve TLV opt with zero data/length not supported"); 2842 /* Check not all data & mask are 0. */ 2843 data_len = spec->option_len * 4; 2844 if (mask->data == NULL) { 2845 for (i = 0; i < data_len; i++) 2846 if (spec->data[i]) 2847 break; 2848 if (i == data_len) 2849 return rte_flow_error_set(error, ENOTSUP, 2850 RTE_FLOW_ERROR_TYPE_ITEM, item, 2851 "Can't match on Geneve option data 0"); 2852 } else { 2853 for (i = 0; i < data_len; i++) 2854 if (spec->data[i] & mask->data[i]) 2855 break; 2856 if (i == data_len) 2857 return rte_flow_error_set(error, ENOTSUP, 2858 RTE_FLOW_ERROR_TYPE_ITEM, item, 2859 "Can't match on Geneve option data and mask 0"); 2860 /* Check data mask supported. */ 2861 for (i = data_max_supported; i < data_len ; i++) 2862 if (mask->data[i]) 2863 return rte_flow_error_set(error, ENOTSUP, 2864 RTE_FLOW_ERROR_TYPE_ITEM, item, 2865 "Data mask is of unsupported size"); 2866 } 2867 /* Check GENEVE option is supported in NIC. */ 2868 if (!config->hca_attr.geneve_tlv_opt) 2869 return rte_flow_error_set 2870 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2871 "Geneve TLV opt not supported"); 2872 /* Check if we already have geneve option with different type/class. */ 2873 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2874 geneve_opt_resource = sh->geneve_tlv_option_resource; 2875 if (geneve_opt_resource != NULL) 2876 if (geneve_opt_resource->option_class != spec->option_class || 2877 geneve_opt_resource->option_type != spec->option_type || 2878 geneve_opt_resource->length != spec->option_len) { 2879 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2880 return rte_flow_error_set(error, ENOTSUP, 2881 RTE_FLOW_ERROR_TYPE_ITEM, item, 2882 "Only one Geneve TLV option supported"); 2883 } 2884 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2885 return 0; 2886 } 2887 2888 /** 2889 * Validate MPLS item. 2890 * 2891 * @param[in] dev 2892 * Pointer to the rte_eth_dev structure. 2893 * @param[in] item 2894 * Item specification. 2895 * @param[in] item_flags 2896 * Bit-fields that holds the items detected until now. 2897 * @param[in] prev_layer 2898 * The protocol layer indicated in previous item. 2899 * @param[out] error 2900 * Pointer to error structure. 2901 * 2902 * @return 2903 * 0 on success, a negative errno value otherwise and rte_errno is set. 2904 */ 2905 int 2906 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2907 const struct rte_flow_item *item __rte_unused, 2908 uint64_t item_flags __rte_unused, 2909 uint64_t prev_layer __rte_unused, 2910 struct rte_flow_error *error) 2911 { 2912 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2913 const struct rte_flow_item_mpls *mask = item->mask; 2914 struct mlx5_priv *priv = dev->data->dev_private; 2915 int ret; 2916 2917 if (!priv->config.mpls_en) 2918 return rte_flow_error_set(error, ENOTSUP, 2919 RTE_FLOW_ERROR_TYPE_ITEM, item, 2920 "MPLS not supported or" 2921 " disabled in firmware" 2922 " configuration."); 2923 /* MPLS over IP, UDP, GRE is allowed */ 2924 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2925 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2926 MLX5_FLOW_LAYER_GRE | 2927 MLX5_FLOW_LAYER_GRE_KEY))) 2928 return rte_flow_error_set(error, EINVAL, 2929 RTE_FLOW_ERROR_TYPE_ITEM, item, 2930 "protocol filtering not compatible" 2931 " with MPLS layer"); 2932 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2933 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2934 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2935 return rte_flow_error_set(error, ENOTSUP, 2936 RTE_FLOW_ERROR_TYPE_ITEM, item, 2937 "multiple tunnel layers not" 2938 " supported"); 2939 if (!mask) 2940 mask = &rte_flow_item_mpls_mask; 2941 ret = mlx5_flow_item_acceptable 2942 (item, (const uint8_t *)mask, 2943 (const uint8_t *)&rte_flow_item_mpls_mask, 2944 sizeof(struct rte_flow_item_mpls), 2945 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2946 if (ret < 0) 2947 return ret; 2948 return 0; 2949 #else 2950 return rte_flow_error_set(error, ENOTSUP, 2951 RTE_FLOW_ERROR_TYPE_ITEM, item, 2952 "MPLS is not supported by Verbs, please" 2953 " update."); 2954 #endif 2955 } 2956 2957 /** 2958 * Validate NVGRE item. 2959 * 2960 * @param[in] item 2961 * Item specification. 2962 * @param[in] item_flags 2963 * Bit flags to mark detected items. 2964 * @param[in] target_protocol 2965 * The next protocol in the previous item. 2966 * @param[out] error 2967 * Pointer to error structure. 2968 * 2969 * @return 2970 * 0 on success, a negative errno value otherwise and rte_errno is set. 2971 */ 2972 int 2973 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2974 uint64_t item_flags, 2975 uint8_t target_protocol, 2976 struct rte_flow_error *error) 2977 { 2978 const struct rte_flow_item_nvgre *mask = item->mask; 2979 int ret; 2980 2981 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2982 return rte_flow_error_set(error, EINVAL, 2983 RTE_FLOW_ERROR_TYPE_ITEM, item, 2984 "protocol filtering not compatible" 2985 " with this GRE layer"); 2986 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2987 return rte_flow_error_set(error, ENOTSUP, 2988 RTE_FLOW_ERROR_TYPE_ITEM, item, 2989 "multiple tunnel layers not" 2990 " supported"); 2991 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2992 return rte_flow_error_set(error, ENOTSUP, 2993 RTE_FLOW_ERROR_TYPE_ITEM, item, 2994 "L3 Layer is missing"); 2995 if (!mask) 2996 mask = &rte_flow_item_nvgre_mask; 2997 ret = mlx5_flow_item_acceptable 2998 (item, (const uint8_t *)mask, 2999 (const uint8_t *)&rte_flow_item_nvgre_mask, 3000 sizeof(struct rte_flow_item_nvgre), 3001 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3002 if (ret < 0) 3003 return ret; 3004 return 0; 3005 } 3006 3007 /** 3008 * Validate eCPRI item. 3009 * 3010 * @param[in] item 3011 * Item specification. 3012 * @param[in] item_flags 3013 * Bit-fields that holds the items detected until now. 3014 * @param[in] last_item 3015 * Previous validated item in the pattern items. 3016 * @param[in] ether_type 3017 * Type in the ethernet layer header (including dot1q). 3018 * @param[in] acc_mask 3019 * Acceptable mask, if NULL default internal default mask 3020 * will be used to check whether item fields are supported. 3021 * @param[out] error 3022 * Pointer to error structure. 3023 * 3024 * @return 3025 * 0 on success, a negative errno value otherwise and rte_errno is set. 3026 */ 3027 int 3028 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3029 uint64_t item_flags, 3030 uint64_t last_item, 3031 uint16_t ether_type, 3032 const struct rte_flow_item_ecpri *acc_mask, 3033 struct rte_flow_error *error) 3034 { 3035 const struct rte_flow_item_ecpri *mask = item->mask; 3036 const struct rte_flow_item_ecpri nic_mask = { 3037 .hdr = { 3038 .common = { 3039 .u32 = 3040 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3041 .type = 0xFF, 3042 }).u32), 3043 }, 3044 .dummy[0] = 0xFFFFFFFF, 3045 }, 3046 }; 3047 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3048 MLX5_FLOW_LAYER_OUTER_VLAN); 3049 struct rte_flow_item_ecpri mask_lo; 3050 3051 if (!(last_item & outer_l2_vlan) && 3052 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3053 return rte_flow_error_set(error, EINVAL, 3054 RTE_FLOW_ERROR_TYPE_ITEM, item, 3055 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3056 if ((last_item & outer_l2_vlan) && ether_type && 3057 ether_type != RTE_ETHER_TYPE_ECPRI) 3058 return rte_flow_error_set(error, EINVAL, 3059 RTE_FLOW_ERROR_TYPE_ITEM, item, 3060 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3061 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3062 return rte_flow_error_set(error, EINVAL, 3063 RTE_FLOW_ERROR_TYPE_ITEM, item, 3064 "eCPRI with tunnel is not supported right now"); 3065 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3066 return rte_flow_error_set(error, ENOTSUP, 3067 RTE_FLOW_ERROR_TYPE_ITEM, item, 3068 "multiple L3 layers not supported"); 3069 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3070 return rte_flow_error_set(error, EINVAL, 3071 RTE_FLOW_ERROR_TYPE_ITEM, item, 3072 "eCPRI cannot coexist with a TCP layer"); 3073 /* In specification, eCPRI could be over UDP layer. */ 3074 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3075 return rte_flow_error_set(error, EINVAL, 3076 RTE_FLOW_ERROR_TYPE_ITEM, item, 3077 "eCPRI over UDP layer is not yet supported right now"); 3078 /* Mask for type field in common header could be zero. */ 3079 if (!mask) 3080 mask = &rte_flow_item_ecpri_mask; 3081 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3082 /* Input mask is in big-endian format. */ 3083 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3084 return rte_flow_error_set(error, EINVAL, 3085 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3086 "partial mask is not supported for protocol"); 3087 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3088 return rte_flow_error_set(error, EINVAL, 3089 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3090 "message header mask must be after a type mask"); 3091 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3092 acc_mask ? (const uint8_t *)acc_mask 3093 : (const uint8_t *)&nic_mask, 3094 sizeof(struct rte_flow_item_ecpri), 3095 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3096 } 3097 3098 /** 3099 * Release resource related QUEUE/RSS action split. 3100 * 3101 * @param dev 3102 * Pointer to Ethernet device. 3103 * @param flow 3104 * Flow to release id's from. 3105 */ 3106 static void 3107 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 3108 struct rte_flow *flow) 3109 { 3110 struct mlx5_priv *priv = dev->data->dev_private; 3111 uint32_t handle_idx; 3112 struct mlx5_flow_handle *dev_handle; 3113 3114 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 3115 handle_idx, dev_handle, next) 3116 if (dev_handle->split_flow_id && 3117 !dev_handle->is_meter_flow_id) 3118 mlx5_ipool_free(priv->sh->ipool 3119 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 3120 dev_handle->split_flow_id); 3121 } 3122 3123 static int 3124 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3125 const struct rte_flow_attr *attr __rte_unused, 3126 const struct rte_flow_item items[] __rte_unused, 3127 const struct rte_flow_action actions[] __rte_unused, 3128 bool external __rte_unused, 3129 int hairpin __rte_unused, 3130 struct rte_flow_error *error) 3131 { 3132 return rte_flow_error_set(error, ENOTSUP, 3133 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3134 } 3135 3136 static struct mlx5_flow * 3137 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3138 const struct rte_flow_attr *attr __rte_unused, 3139 const struct rte_flow_item items[] __rte_unused, 3140 const struct rte_flow_action actions[] __rte_unused, 3141 struct rte_flow_error *error) 3142 { 3143 rte_flow_error_set(error, ENOTSUP, 3144 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3145 return NULL; 3146 } 3147 3148 static int 3149 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3150 struct mlx5_flow *dev_flow __rte_unused, 3151 const struct rte_flow_attr *attr __rte_unused, 3152 const struct rte_flow_item items[] __rte_unused, 3153 const struct rte_flow_action actions[] __rte_unused, 3154 struct rte_flow_error *error) 3155 { 3156 return rte_flow_error_set(error, ENOTSUP, 3157 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3158 } 3159 3160 static int 3161 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3162 struct rte_flow *flow __rte_unused, 3163 struct rte_flow_error *error) 3164 { 3165 return rte_flow_error_set(error, ENOTSUP, 3166 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3167 } 3168 3169 static void 3170 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3171 struct rte_flow *flow __rte_unused) 3172 { 3173 } 3174 3175 static void 3176 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3177 struct rte_flow *flow __rte_unused) 3178 { 3179 } 3180 3181 static int 3182 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3183 struct rte_flow *flow __rte_unused, 3184 const struct rte_flow_action *actions __rte_unused, 3185 void *data __rte_unused, 3186 struct rte_flow_error *error) 3187 { 3188 return rte_flow_error_set(error, ENOTSUP, 3189 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3190 } 3191 3192 static int 3193 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3194 uint32_t domains __rte_unused, 3195 uint32_t flags __rte_unused) 3196 { 3197 return 0; 3198 } 3199 3200 /* Void driver to protect from null pointer reference. */ 3201 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3202 .validate = flow_null_validate, 3203 .prepare = flow_null_prepare, 3204 .translate = flow_null_translate, 3205 .apply = flow_null_apply, 3206 .remove = flow_null_remove, 3207 .destroy = flow_null_destroy, 3208 .query = flow_null_query, 3209 .sync_domain = flow_null_sync_domain, 3210 }; 3211 3212 /** 3213 * Select flow driver type according to flow attributes and device 3214 * configuration. 3215 * 3216 * @param[in] dev 3217 * Pointer to the dev structure. 3218 * @param[in] attr 3219 * Pointer to the flow attributes. 3220 * 3221 * @return 3222 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3223 */ 3224 static enum mlx5_flow_drv_type 3225 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3226 { 3227 struct mlx5_priv *priv = dev->data->dev_private; 3228 /* The OS can determine first a specific flow type (DV, VERBS) */ 3229 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3230 3231 if (type != MLX5_FLOW_TYPE_MAX) 3232 return type; 3233 /* If no OS specific type - continue with DV/VERBS selection */ 3234 if (attr->transfer && priv->config.dv_esw_en) 3235 type = MLX5_FLOW_TYPE_DV; 3236 if (!attr->transfer) 3237 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3238 MLX5_FLOW_TYPE_VERBS; 3239 return type; 3240 } 3241 3242 #define flow_get_drv_ops(type) flow_drv_ops[type] 3243 3244 /** 3245 * Flow driver validation API. This abstracts calling driver specific functions. 3246 * The type of flow driver is determined according to flow attributes. 3247 * 3248 * @param[in] dev 3249 * Pointer to the dev structure. 3250 * @param[in] attr 3251 * Pointer to the flow attributes. 3252 * @param[in] items 3253 * Pointer to the list of items. 3254 * @param[in] actions 3255 * Pointer to the list of actions. 3256 * @param[in] external 3257 * This flow rule is created by request external to PMD. 3258 * @param[in] hairpin 3259 * Number of hairpin TX actions, 0 means classic flow. 3260 * @param[out] error 3261 * Pointer to the error structure. 3262 * 3263 * @return 3264 * 0 on success, a negative errno value otherwise and rte_errno is set. 3265 */ 3266 static inline int 3267 flow_drv_validate(struct rte_eth_dev *dev, 3268 const struct rte_flow_attr *attr, 3269 const struct rte_flow_item items[], 3270 const struct rte_flow_action actions[], 3271 bool external, int hairpin, struct rte_flow_error *error) 3272 { 3273 const struct mlx5_flow_driver_ops *fops; 3274 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3275 3276 fops = flow_get_drv_ops(type); 3277 return fops->validate(dev, attr, items, actions, external, 3278 hairpin, error); 3279 } 3280 3281 /** 3282 * Flow driver preparation API. This abstracts calling driver specific 3283 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3284 * calculates the size of memory required for device flow, allocates the memory, 3285 * initializes the device flow and returns the pointer. 3286 * 3287 * @note 3288 * This function initializes device flow structure such as dv or verbs in 3289 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3290 * rest. For example, adding returning device flow to flow->dev_flow list and 3291 * setting backward reference to the flow should be done out of this function. 3292 * layers field is not filled either. 3293 * 3294 * @param[in] dev 3295 * Pointer to the dev structure. 3296 * @param[in] attr 3297 * Pointer to the flow attributes. 3298 * @param[in] items 3299 * Pointer to the list of items. 3300 * @param[in] actions 3301 * Pointer to the list of actions. 3302 * @param[in] flow_idx 3303 * This memory pool index to the flow. 3304 * @param[out] error 3305 * Pointer to the error structure. 3306 * 3307 * @return 3308 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3309 */ 3310 static inline struct mlx5_flow * 3311 flow_drv_prepare(struct rte_eth_dev *dev, 3312 const struct rte_flow *flow, 3313 const struct rte_flow_attr *attr, 3314 const struct rte_flow_item items[], 3315 const struct rte_flow_action actions[], 3316 uint32_t flow_idx, 3317 struct rte_flow_error *error) 3318 { 3319 const struct mlx5_flow_driver_ops *fops; 3320 enum mlx5_flow_drv_type type = flow->drv_type; 3321 struct mlx5_flow *mlx5_flow = NULL; 3322 3323 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3324 fops = flow_get_drv_ops(type); 3325 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3326 if (mlx5_flow) 3327 mlx5_flow->flow_idx = flow_idx; 3328 return mlx5_flow; 3329 } 3330 3331 /** 3332 * Flow driver translation API. This abstracts calling driver specific 3333 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3334 * translates a generic flow into a driver flow. flow_drv_prepare() must 3335 * precede. 3336 * 3337 * @note 3338 * dev_flow->layers could be filled as a result of parsing during translation 3339 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3340 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3341 * flow->actions could be overwritten even though all the expanded dev_flows 3342 * have the same actions. 3343 * 3344 * @param[in] dev 3345 * Pointer to the rte dev structure. 3346 * @param[in, out] dev_flow 3347 * Pointer to the mlx5 flow. 3348 * @param[in] attr 3349 * Pointer to the flow attributes. 3350 * @param[in] items 3351 * Pointer to the list of items. 3352 * @param[in] actions 3353 * Pointer to the list of actions. 3354 * @param[out] error 3355 * Pointer to the error structure. 3356 * 3357 * @return 3358 * 0 on success, a negative errno value otherwise and rte_errno is set. 3359 */ 3360 static inline int 3361 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3362 const struct rte_flow_attr *attr, 3363 const struct rte_flow_item items[], 3364 const struct rte_flow_action actions[], 3365 struct rte_flow_error *error) 3366 { 3367 const struct mlx5_flow_driver_ops *fops; 3368 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3369 3370 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3371 fops = flow_get_drv_ops(type); 3372 return fops->translate(dev, dev_flow, attr, items, actions, error); 3373 } 3374 3375 /** 3376 * Flow driver apply API. This abstracts calling driver specific functions. 3377 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3378 * translated driver flows on to device. flow_drv_translate() must precede. 3379 * 3380 * @param[in] dev 3381 * Pointer to Ethernet device structure. 3382 * @param[in, out] flow 3383 * Pointer to flow structure. 3384 * @param[out] error 3385 * Pointer to error structure. 3386 * 3387 * @return 3388 * 0 on success, a negative errno value otherwise and rte_errno is set. 3389 */ 3390 static inline int 3391 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3392 struct rte_flow_error *error) 3393 { 3394 const struct mlx5_flow_driver_ops *fops; 3395 enum mlx5_flow_drv_type type = flow->drv_type; 3396 3397 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3398 fops = flow_get_drv_ops(type); 3399 return fops->apply(dev, flow, error); 3400 } 3401 3402 /** 3403 * Flow driver destroy API. This abstracts calling driver specific functions. 3404 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3405 * on device and releases resources of the flow. 3406 * 3407 * @param[in] dev 3408 * Pointer to Ethernet device. 3409 * @param[in, out] flow 3410 * Pointer to flow structure. 3411 */ 3412 static inline void 3413 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3414 { 3415 const struct mlx5_flow_driver_ops *fops; 3416 enum mlx5_flow_drv_type type = flow->drv_type; 3417 3418 flow_mreg_split_qrss_release(dev, flow); 3419 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3420 fops = flow_get_drv_ops(type); 3421 fops->destroy(dev, flow); 3422 } 3423 3424 /** 3425 * Flow driver find RSS policy tbl API. This abstracts calling driver 3426 * specific functions. Parent flow (rte_flow) should have driver 3427 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3428 * 3429 * @param[in] dev 3430 * Pointer to Ethernet device. 3431 * @param[in, out] flow 3432 * Pointer to flow structure. 3433 * @param[in] policy 3434 * Pointer to meter policy table. 3435 * @param[in] rss_desc 3436 * Pointer to rss_desc 3437 */ 3438 static struct mlx5_flow_meter_sub_policy * 3439 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3440 struct rte_flow *flow, 3441 struct mlx5_flow_meter_policy *policy, 3442 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3443 { 3444 const struct mlx5_flow_driver_ops *fops; 3445 enum mlx5_flow_drv_type type = flow->drv_type; 3446 3447 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3448 fops = flow_get_drv_ops(type); 3449 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3450 } 3451 3452 /** 3453 * Get RSS action from the action list. 3454 * 3455 * @param[in] dev 3456 * Pointer to Ethernet device. 3457 * @param[in] actions 3458 * Pointer to the list of actions. 3459 * @param[in] flow 3460 * Parent flow structure pointer. 3461 * 3462 * @return 3463 * Pointer to the RSS action if exist, else return NULL. 3464 */ 3465 static const struct rte_flow_action_rss* 3466 flow_get_rss_action(struct rte_eth_dev *dev, 3467 const struct rte_flow_action actions[]) 3468 { 3469 struct mlx5_priv *priv = dev->data->dev_private; 3470 const struct rte_flow_action_rss *rss = NULL; 3471 3472 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3473 switch (actions->type) { 3474 case RTE_FLOW_ACTION_TYPE_RSS: 3475 rss = actions->conf; 3476 break; 3477 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3478 { 3479 const struct rte_flow_action_sample *sample = 3480 actions->conf; 3481 const struct rte_flow_action *act = sample->actions; 3482 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3483 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3484 rss = act->conf; 3485 break; 3486 } 3487 case RTE_FLOW_ACTION_TYPE_METER: 3488 { 3489 uint32_t mtr_idx; 3490 struct mlx5_flow_meter_info *fm; 3491 struct mlx5_flow_meter_policy *policy; 3492 const struct rte_flow_action_meter *mtr = actions->conf; 3493 3494 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3495 if (fm) { 3496 policy = mlx5_flow_meter_policy_find(dev, 3497 fm->policy_id, NULL); 3498 if (policy && policy->is_rss) 3499 rss = 3500 policy->act_cnt[RTE_COLOR_GREEN].rss->conf; 3501 } 3502 break; 3503 } 3504 default: 3505 break; 3506 } 3507 } 3508 return rss; 3509 } 3510 3511 /** 3512 * Get ASO age action by index. 3513 * 3514 * @param[in] dev 3515 * Pointer to the Ethernet device structure. 3516 * @param[in] age_idx 3517 * Index to the ASO age action. 3518 * 3519 * @return 3520 * The specified ASO age action. 3521 */ 3522 struct mlx5_aso_age_action* 3523 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3524 { 3525 uint16_t pool_idx = age_idx & UINT16_MAX; 3526 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3527 struct mlx5_priv *priv = dev->data->dev_private; 3528 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3529 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3530 3531 return &pool->actions[offset - 1]; 3532 } 3533 3534 /* maps indirect action to translated direct in some actions array */ 3535 struct mlx5_translated_action_handle { 3536 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3537 int index; /**< Index in related array of rte_flow_action. */ 3538 }; 3539 3540 /** 3541 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3542 * direct action if translation possible. 3543 * This functionality used to run same execution path for both direct and 3544 * indirect actions on flow create. All necessary preparations for indirect 3545 * action handling should be performed on *handle* actions list returned 3546 * from this call. 3547 * 3548 * @param[in] dev 3549 * Pointer to Ethernet device. 3550 * @param[in] actions 3551 * List of actions to translate. 3552 * @param[out] handle 3553 * List to store translated indirect action object handles. 3554 * @param[in, out] indir_n 3555 * Size of *handle* array. On return should be updated with number of 3556 * indirect actions retrieved from the *actions* list. 3557 * @param[out] translated_actions 3558 * List of actions where all indirect actions were translated to direct 3559 * if possible. NULL if no translation took place. 3560 * @param[out] error 3561 * Pointer to the error structure. 3562 * 3563 * @return 3564 * 0 on success, a negative errno value otherwise and rte_errno is set. 3565 */ 3566 static int 3567 flow_action_handles_translate(struct rte_eth_dev *dev, 3568 const struct rte_flow_action actions[], 3569 struct mlx5_translated_action_handle *handle, 3570 int *indir_n, 3571 struct rte_flow_action **translated_actions, 3572 struct rte_flow_error *error) 3573 { 3574 struct mlx5_priv *priv = dev->data->dev_private; 3575 struct rte_flow_action *translated = NULL; 3576 size_t actions_size; 3577 int n; 3578 int copied_n = 0; 3579 struct mlx5_translated_action_handle *handle_end = NULL; 3580 3581 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3582 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3583 continue; 3584 if (copied_n == *indir_n) { 3585 return rte_flow_error_set 3586 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3587 NULL, "too many shared actions"); 3588 } 3589 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3590 sizeof(actions[n].conf)); 3591 handle[copied_n].index = n; 3592 copied_n++; 3593 } 3594 n++; 3595 *indir_n = copied_n; 3596 if (!copied_n) 3597 return 0; 3598 actions_size = sizeof(struct rte_flow_action) * n; 3599 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3600 if (!translated) { 3601 rte_errno = ENOMEM; 3602 return -ENOMEM; 3603 } 3604 memcpy(translated, actions, actions_size); 3605 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3606 struct mlx5_shared_action_rss *shared_rss; 3607 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3608 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3609 uint32_t idx = act_idx & 3610 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3611 3612 switch (type) { 3613 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3614 shared_rss = mlx5_ipool_get 3615 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3616 translated[handle->index].type = 3617 RTE_FLOW_ACTION_TYPE_RSS; 3618 translated[handle->index].conf = 3619 &shared_rss->origin; 3620 break; 3621 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3622 translated[handle->index].type = 3623 (enum rte_flow_action_type) 3624 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3625 translated[handle->index].conf = (void *)(uintptr_t)idx; 3626 break; 3627 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3628 if (priv->sh->flow_hit_aso_en) { 3629 translated[handle->index].type = 3630 (enum rte_flow_action_type) 3631 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3632 translated[handle->index].conf = 3633 (void *)(uintptr_t)idx; 3634 break; 3635 } 3636 /* Fall-through */ 3637 case MLX5_INDIRECT_ACTION_TYPE_CT: 3638 if (priv->sh->ct_aso_en) { 3639 translated[handle->index].type = 3640 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3641 translated[handle->index].conf = 3642 (void *)(uintptr_t)idx; 3643 break; 3644 } 3645 /* Fall-through */ 3646 default: 3647 mlx5_free(translated); 3648 return rte_flow_error_set 3649 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3650 NULL, "invalid indirect action type"); 3651 } 3652 } 3653 *translated_actions = translated; 3654 return 0; 3655 } 3656 3657 /** 3658 * Get Shared RSS action from the action list. 3659 * 3660 * @param[in] dev 3661 * Pointer to Ethernet device. 3662 * @param[in] shared 3663 * Pointer to the list of actions. 3664 * @param[in] shared_n 3665 * Actions list length. 3666 * 3667 * @return 3668 * The MLX5 RSS action ID if exists, otherwise return 0. 3669 */ 3670 static uint32_t 3671 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3672 struct mlx5_translated_action_handle *handle, 3673 int shared_n) 3674 { 3675 struct mlx5_translated_action_handle *handle_end; 3676 struct mlx5_priv *priv = dev->data->dev_private; 3677 struct mlx5_shared_action_rss *shared_rss; 3678 3679 3680 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3681 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3682 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3683 uint32_t idx = act_idx & 3684 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3685 switch (type) { 3686 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3687 shared_rss = mlx5_ipool_get 3688 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3689 idx); 3690 __atomic_add_fetch(&shared_rss->refcnt, 1, 3691 __ATOMIC_RELAXED); 3692 return idx; 3693 default: 3694 break; 3695 } 3696 } 3697 return 0; 3698 } 3699 3700 static unsigned int 3701 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3702 { 3703 const struct rte_flow_item *item; 3704 unsigned int has_vlan = 0; 3705 3706 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3707 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3708 has_vlan = 1; 3709 break; 3710 } 3711 } 3712 if (has_vlan) 3713 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3714 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3715 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3716 MLX5_EXPANSION_ROOT_OUTER; 3717 } 3718 3719 /** 3720 * Get layer flags from the prefix flow. 3721 * 3722 * Some flows may be split to several subflows, the prefix subflow gets the 3723 * match items and the suffix sub flow gets the actions. 3724 * Some actions need the user defined match item flags to get the detail for 3725 * the action. 3726 * This function helps the suffix flow to get the item layer flags from prefix 3727 * subflow. 3728 * 3729 * @param[in] dev_flow 3730 * Pointer the created preifx subflow. 3731 * 3732 * @return 3733 * The layers get from prefix subflow. 3734 */ 3735 static inline uint64_t 3736 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3737 { 3738 uint64_t layers = 0; 3739 3740 /* 3741 * Layers bits could be localization, but usually the compiler will 3742 * help to do the optimization work for source code. 3743 * If no decap actions, use the layers directly. 3744 */ 3745 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3746 return dev_flow->handle->layers; 3747 /* Convert L3 layers with decap action. */ 3748 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3749 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3750 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3751 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3752 /* Convert L4 layers with decap action. */ 3753 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3754 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3755 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3756 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3757 return layers; 3758 } 3759 3760 /** 3761 * Get metadata split action information. 3762 * 3763 * @param[in] actions 3764 * Pointer to the list of actions. 3765 * @param[out] qrss 3766 * Pointer to the return pointer. 3767 * @param[out] qrss_type 3768 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3769 * if no QUEUE/RSS is found. 3770 * @param[out] encap_idx 3771 * Pointer to the index of the encap action if exists, otherwise the last 3772 * action index. 3773 * 3774 * @return 3775 * Total number of actions. 3776 */ 3777 static int 3778 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3779 const struct rte_flow_action **qrss, 3780 int *encap_idx) 3781 { 3782 const struct rte_flow_action_raw_encap *raw_encap; 3783 int actions_n = 0; 3784 int raw_decap_idx = -1; 3785 3786 *encap_idx = -1; 3787 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3788 switch (actions->type) { 3789 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3790 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3791 *encap_idx = actions_n; 3792 break; 3793 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3794 raw_decap_idx = actions_n; 3795 break; 3796 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3797 raw_encap = actions->conf; 3798 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3799 *encap_idx = raw_decap_idx != -1 ? 3800 raw_decap_idx : actions_n; 3801 break; 3802 case RTE_FLOW_ACTION_TYPE_QUEUE: 3803 case RTE_FLOW_ACTION_TYPE_RSS: 3804 *qrss = actions; 3805 break; 3806 default: 3807 break; 3808 } 3809 actions_n++; 3810 } 3811 if (*encap_idx == -1) 3812 *encap_idx = actions_n; 3813 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3814 return actions_n + 1; 3815 } 3816 3817 /** 3818 * Check if the action will change packet. 3819 * 3820 * @param dev 3821 * Pointer to Ethernet device. 3822 * @param[in] type 3823 * action type. 3824 * 3825 * @return 3826 * true if action will change packet, false otherwise. 3827 */ 3828 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 3829 enum rte_flow_action_type type) 3830 { 3831 struct mlx5_priv *priv = dev->data->dev_private; 3832 3833 switch (type) { 3834 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3835 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3836 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3837 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3838 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3839 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3840 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3841 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3842 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3843 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3844 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3845 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3846 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3847 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3848 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 3849 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 3850 case RTE_FLOW_ACTION_TYPE_SET_META: 3851 case RTE_FLOW_ACTION_TYPE_SET_TAG: 3852 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 3853 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3854 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3855 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3856 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3857 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3858 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3859 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3860 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3861 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3862 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 3863 return true; 3864 case RTE_FLOW_ACTION_TYPE_FLAG: 3865 case RTE_FLOW_ACTION_TYPE_MARK: 3866 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 3867 return true; 3868 else 3869 return false; 3870 default: 3871 return false; 3872 } 3873 } 3874 3875 /** 3876 * Check meter action from the action list. 3877 * 3878 * @param dev 3879 * Pointer to Ethernet device. 3880 * @param[in] actions 3881 * Pointer to the list of actions. 3882 * @param[out] has_mtr 3883 * Pointer to the meter exist flag. 3884 * @param[out] has_modify 3885 * Pointer to the flag showing there's packet change action. 3886 * @param[out] meter_id 3887 * Pointer to the meter id. 3888 * 3889 * @return 3890 * Total number of actions. 3891 */ 3892 static int 3893 flow_check_meter_action(struct rte_eth_dev *dev, 3894 const struct rte_flow_action actions[], 3895 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 3896 { 3897 const struct rte_flow_action_meter *mtr = NULL; 3898 int actions_n = 0; 3899 3900 MLX5_ASSERT(has_mtr); 3901 *has_mtr = false; 3902 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3903 switch (actions->type) { 3904 case RTE_FLOW_ACTION_TYPE_METER: 3905 mtr = actions->conf; 3906 *meter_id = mtr->mtr_id; 3907 *has_mtr = true; 3908 break; 3909 default: 3910 break; 3911 } 3912 if (!*has_mtr) 3913 *has_modify |= flow_check_modify_action_type(dev, 3914 actions->type); 3915 actions_n++; 3916 } 3917 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3918 return actions_n + 1; 3919 } 3920 3921 /** 3922 * Check if the flow should be split due to hairpin. 3923 * The reason for the split is that in current HW we can't 3924 * support encap and push-vlan on Rx, so if a flow contains 3925 * these actions we move it to Tx. 3926 * 3927 * @param dev 3928 * Pointer to Ethernet device. 3929 * @param[in] attr 3930 * Flow rule attributes. 3931 * @param[in] actions 3932 * Associated actions (list terminated by the END action). 3933 * 3934 * @return 3935 * > 0 the number of actions and the flow should be split, 3936 * 0 when no split required. 3937 */ 3938 static int 3939 flow_check_hairpin_split(struct rte_eth_dev *dev, 3940 const struct rte_flow_attr *attr, 3941 const struct rte_flow_action actions[]) 3942 { 3943 int queue_action = 0; 3944 int action_n = 0; 3945 int split = 0; 3946 const struct rte_flow_action_queue *queue; 3947 const struct rte_flow_action_rss *rss; 3948 const struct rte_flow_action_raw_encap *raw_encap; 3949 const struct rte_eth_hairpin_conf *conf; 3950 3951 if (!attr->ingress) 3952 return 0; 3953 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3954 switch (actions->type) { 3955 case RTE_FLOW_ACTION_TYPE_QUEUE: 3956 queue = actions->conf; 3957 if (queue == NULL) 3958 return 0; 3959 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3960 if (conf == NULL || conf->tx_explicit != 0) 3961 return 0; 3962 queue_action = 1; 3963 action_n++; 3964 break; 3965 case RTE_FLOW_ACTION_TYPE_RSS: 3966 rss = actions->conf; 3967 if (rss == NULL || rss->queue_num == 0) 3968 return 0; 3969 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3970 if (conf == NULL || conf->tx_explicit != 0) 3971 return 0; 3972 queue_action = 1; 3973 action_n++; 3974 break; 3975 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3976 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3977 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3978 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3979 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3980 split++; 3981 action_n++; 3982 break; 3983 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3984 raw_encap = actions->conf; 3985 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3986 split++; 3987 action_n++; 3988 break; 3989 default: 3990 action_n++; 3991 break; 3992 } 3993 } 3994 if (split && queue_action) 3995 return action_n; 3996 return 0; 3997 } 3998 3999 /* Declare flow create/destroy prototype in advance. */ 4000 static uint32_t 4001 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 4002 const struct rte_flow_attr *attr, 4003 const struct rte_flow_item items[], 4004 const struct rte_flow_action actions[], 4005 bool external, struct rte_flow_error *error); 4006 4007 static void 4008 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 4009 uint32_t flow_idx); 4010 4011 int 4012 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused, 4013 struct mlx5_hlist_entry *entry, 4014 uint64_t key, void *cb_ctx __rte_unused) 4015 { 4016 struct mlx5_flow_mreg_copy_resource *mcp_res = 4017 container_of(entry, typeof(*mcp_res), hlist_ent); 4018 4019 return mcp_res->mark_id != key; 4020 } 4021 4022 struct mlx5_hlist_entry * 4023 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 4024 void *cb_ctx) 4025 { 4026 struct rte_eth_dev *dev = list->ctx; 4027 struct mlx5_priv *priv = dev->data->dev_private; 4028 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4029 struct mlx5_flow_mreg_copy_resource *mcp_res; 4030 struct rte_flow_error *error = ctx->error; 4031 uint32_t idx = 0; 4032 int ret; 4033 uint32_t mark_id = key; 4034 struct rte_flow_attr attr = { 4035 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4036 .ingress = 1, 4037 }; 4038 struct mlx5_rte_flow_item_tag tag_spec = { 4039 .data = mark_id, 4040 }; 4041 struct rte_flow_item items[] = { 4042 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4043 }; 4044 struct rte_flow_action_mark ftag = { 4045 .id = mark_id, 4046 }; 4047 struct mlx5_flow_action_copy_mreg cp_mreg = { 4048 .dst = REG_B, 4049 .src = REG_NON, 4050 }; 4051 struct rte_flow_action_jump jump = { 4052 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4053 }; 4054 struct rte_flow_action actions[] = { 4055 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4056 }; 4057 4058 /* Fill the register fileds in the flow. */ 4059 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4060 if (ret < 0) 4061 return NULL; 4062 tag_spec.id = ret; 4063 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4064 if (ret < 0) 4065 return NULL; 4066 cp_mreg.src = ret; 4067 /* Provide the full width of FLAG specific value. */ 4068 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4069 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4070 /* Build a new flow. */ 4071 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4072 items[0] = (struct rte_flow_item){ 4073 .type = (enum rte_flow_item_type) 4074 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4075 .spec = &tag_spec, 4076 }; 4077 items[1] = (struct rte_flow_item){ 4078 .type = RTE_FLOW_ITEM_TYPE_END, 4079 }; 4080 actions[0] = (struct rte_flow_action){ 4081 .type = (enum rte_flow_action_type) 4082 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4083 .conf = &ftag, 4084 }; 4085 actions[1] = (struct rte_flow_action){ 4086 .type = (enum rte_flow_action_type) 4087 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4088 .conf = &cp_mreg, 4089 }; 4090 actions[2] = (struct rte_flow_action){ 4091 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4092 .conf = &jump, 4093 }; 4094 actions[3] = (struct rte_flow_action){ 4095 .type = RTE_FLOW_ACTION_TYPE_END, 4096 }; 4097 } else { 4098 /* Default rule, wildcard match. */ 4099 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4100 items[0] = (struct rte_flow_item){ 4101 .type = RTE_FLOW_ITEM_TYPE_END, 4102 }; 4103 actions[0] = (struct rte_flow_action){ 4104 .type = (enum rte_flow_action_type) 4105 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4106 .conf = &cp_mreg, 4107 }; 4108 actions[1] = (struct rte_flow_action){ 4109 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4110 .conf = &jump, 4111 }; 4112 actions[2] = (struct rte_flow_action){ 4113 .type = RTE_FLOW_ACTION_TYPE_END, 4114 }; 4115 } 4116 /* Build a new entry. */ 4117 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4118 if (!mcp_res) { 4119 rte_errno = ENOMEM; 4120 return NULL; 4121 } 4122 mcp_res->idx = idx; 4123 mcp_res->mark_id = mark_id; 4124 /* 4125 * The copy Flows are not included in any list. There 4126 * ones are referenced from other Flows and can not 4127 * be applied, removed, deleted in ardbitrary order 4128 * by list traversing. 4129 */ 4130 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 4131 actions, false, error); 4132 if (!mcp_res->rix_flow) { 4133 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4134 return NULL; 4135 } 4136 return &mcp_res->hlist_ent; 4137 } 4138 4139 /** 4140 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4141 * 4142 * As mark_id is unique, if there's already a registered flow for the mark_id, 4143 * return by increasing the reference counter of the resource. Otherwise, create 4144 * the resource (mcp_res) and flow. 4145 * 4146 * Flow looks like, 4147 * - If ingress port is ANY and reg_c[1] is mark_id, 4148 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4149 * 4150 * For default flow (zero mark_id), flow is like, 4151 * - If ingress port is ANY, 4152 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4153 * 4154 * @param dev 4155 * Pointer to Ethernet device. 4156 * @param mark_id 4157 * ID of MARK action, zero means default flow for META. 4158 * @param[out] error 4159 * Perform verbose error reporting if not NULL. 4160 * 4161 * @return 4162 * Associated resource on success, NULL otherwise and rte_errno is set. 4163 */ 4164 static struct mlx5_flow_mreg_copy_resource * 4165 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4166 struct rte_flow_error *error) 4167 { 4168 struct mlx5_priv *priv = dev->data->dev_private; 4169 struct mlx5_hlist_entry *entry; 4170 struct mlx5_flow_cb_ctx ctx = { 4171 .dev = dev, 4172 .error = error, 4173 }; 4174 4175 /* Check if already registered. */ 4176 MLX5_ASSERT(priv->mreg_cp_tbl); 4177 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4178 if (!entry) 4179 return NULL; 4180 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4181 hlist_ent); 4182 } 4183 4184 void 4185 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 4186 { 4187 struct mlx5_flow_mreg_copy_resource *mcp_res = 4188 container_of(entry, typeof(*mcp_res), hlist_ent); 4189 struct rte_eth_dev *dev = list->ctx; 4190 struct mlx5_priv *priv = dev->data->dev_private; 4191 4192 MLX5_ASSERT(mcp_res->rix_flow); 4193 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 4194 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4195 } 4196 4197 /** 4198 * Release flow in RX_CP_TBL. 4199 * 4200 * @param dev 4201 * Pointer to Ethernet device. 4202 * @flow 4203 * Parent flow for wich copying is provided. 4204 */ 4205 static void 4206 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4207 struct rte_flow *flow) 4208 { 4209 struct mlx5_flow_mreg_copy_resource *mcp_res; 4210 struct mlx5_priv *priv = dev->data->dev_private; 4211 4212 if (!flow->rix_mreg_copy) 4213 return; 4214 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4215 flow->rix_mreg_copy); 4216 if (!mcp_res || !priv->mreg_cp_tbl) 4217 return; 4218 MLX5_ASSERT(mcp_res->rix_flow); 4219 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4220 flow->rix_mreg_copy = 0; 4221 } 4222 4223 /** 4224 * Remove the default copy action from RX_CP_TBL. 4225 * 4226 * This functions is called in the mlx5_dev_start(). No thread safe 4227 * is guaranteed. 4228 * 4229 * @param dev 4230 * Pointer to Ethernet device. 4231 */ 4232 static void 4233 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4234 { 4235 struct mlx5_hlist_entry *entry; 4236 struct mlx5_priv *priv = dev->data->dev_private; 4237 4238 /* Check if default flow is registered. */ 4239 if (!priv->mreg_cp_tbl) 4240 return; 4241 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 4242 MLX5_DEFAULT_COPY_ID, NULL); 4243 if (!entry) 4244 return; 4245 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4246 } 4247 4248 /** 4249 * Add the default copy action in in RX_CP_TBL. 4250 * 4251 * This functions is called in the mlx5_dev_start(). No thread safe 4252 * is guaranteed. 4253 * 4254 * @param dev 4255 * Pointer to Ethernet device. 4256 * @param[out] error 4257 * Perform verbose error reporting if not NULL. 4258 * 4259 * @return 4260 * 0 for success, negative value otherwise and rte_errno is set. 4261 */ 4262 static int 4263 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4264 struct rte_flow_error *error) 4265 { 4266 struct mlx5_priv *priv = dev->data->dev_private; 4267 struct mlx5_flow_mreg_copy_resource *mcp_res; 4268 4269 /* Check whether extensive metadata feature is engaged. */ 4270 if (!priv->config.dv_flow_en || 4271 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4272 !mlx5_flow_ext_mreg_supported(dev) || 4273 !priv->sh->dv_regc0_mask) 4274 return 0; 4275 /* 4276 * Add default mreg copy flow may be called multiple time, but 4277 * only be called once in stop. Avoid register it twice. 4278 */ 4279 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 4280 return 0; 4281 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 4282 if (!mcp_res) 4283 return -rte_errno; 4284 return 0; 4285 } 4286 4287 /** 4288 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4289 * 4290 * All the flow having Q/RSS action should be split by 4291 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4292 * performs the following, 4293 * - CQE->flow_tag := reg_c[1] (MARK) 4294 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4295 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4296 * but there should be a flow per each MARK ID set by MARK action. 4297 * 4298 * For the aforementioned reason, if there's a MARK action in flow's action 4299 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4300 * the MARK ID to CQE's flow_tag like, 4301 * - If reg_c[1] is mark_id, 4302 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4303 * 4304 * For SET_META action which stores value in reg_c[0], as the destination is 4305 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4306 * MARK ID means the default flow. The default flow looks like, 4307 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4308 * 4309 * @param dev 4310 * Pointer to Ethernet device. 4311 * @param flow 4312 * Pointer to flow structure. 4313 * @param[in] actions 4314 * Pointer to the list of actions. 4315 * @param[out] error 4316 * Perform verbose error reporting if not NULL. 4317 * 4318 * @return 4319 * 0 on success, negative value otherwise and rte_errno is set. 4320 */ 4321 static int 4322 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4323 struct rte_flow *flow, 4324 const struct rte_flow_action *actions, 4325 struct rte_flow_error *error) 4326 { 4327 struct mlx5_priv *priv = dev->data->dev_private; 4328 struct mlx5_dev_config *config = &priv->config; 4329 struct mlx5_flow_mreg_copy_resource *mcp_res; 4330 const struct rte_flow_action_mark *mark; 4331 4332 /* Check whether extensive metadata feature is engaged. */ 4333 if (!config->dv_flow_en || 4334 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4335 !mlx5_flow_ext_mreg_supported(dev) || 4336 !priv->sh->dv_regc0_mask) 4337 return 0; 4338 /* Find MARK action. */ 4339 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4340 switch (actions->type) { 4341 case RTE_FLOW_ACTION_TYPE_FLAG: 4342 mcp_res = flow_mreg_add_copy_action 4343 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4344 if (!mcp_res) 4345 return -rte_errno; 4346 flow->rix_mreg_copy = mcp_res->idx; 4347 return 0; 4348 case RTE_FLOW_ACTION_TYPE_MARK: 4349 mark = (const struct rte_flow_action_mark *) 4350 actions->conf; 4351 mcp_res = 4352 flow_mreg_add_copy_action(dev, mark->id, error); 4353 if (!mcp_res) 4354 return -rte_errno; 4355 flow->rix_mreg_copy = mcp_res->idx; 4356 return 0; 4357 default: 4358 break; 4359 } 4360 } 4361 return 0; 4362 } 4363 4364 #define MLX5_MAX_SPLIT_ACTIONS 24 4365 #define MLX5_MAX_SPLIT_ITEMS 24 4366 4367 /** 4368 * Split the hairpin flow. 4369 * Since HW can't support encap and push-vlan on Rx, we move these 4370 * actions to Tx. 4371 * If the count action is after the encap then we also 4372 * move the count action. in this case the count will also measure 4373 * the outer bytes. 4374 * 4375 * @param dev 4376 * Pointer to Ethernet device. 4377 * @param[in] actions 4378 * Associated actions (list terminated by the END action). 4379 * @param[out] actions_rx 4380 * Rx flow actions. 4381 * @param[out] actions_tx 4382 * Tx flow actions.. 4383 * @param[out] pattern_tx 4384 * The pattern items for the Tx flow. 4385 * @param[out] flow_id 4386 * The flow ID connected to this flow. 4387 * 4388 * @return 4389 * 0 on success. 4390 */ 4391 static int 4392 flow_hairpin_split(struct rte_eth_dev *dev, 4393 const struct rte_flow_action actions[], 4394 struct rte_flow_action actions_rx[], 4395 struct rte_flow_action actions_tx[], 4396 struct rte_flow_item pattern_tx[], 4397 uint32_t flow_id) 4398 { 4399 const struct rte_flow_action_raw_encap *raw_encap; 4400 const struct rte_flow_action_raw_decap *raw_decap; 4401 struct mlx5_rte_flow_action_set_tag *set_tag; 4402 struct rte_flow_action *tag_action; 4403 struct mlx5_rte_flow_item_tag *tag_item; 4404 struct rte_flow_item *item; 4405 char *addr; 4406 int encap = 0; 4407 4408 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4409 switch (actions->type) { 4410 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4411 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4412 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4413 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4414 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4415 rte_memcpy(actions_tx, actions, 4416 sizeof(struct rte_flow_action)); 4417 actions_tx++; 4418 break; 4419 case RTE_FLOW_ACTION_TYPE_COUNT: 4420 if (encap) { 4421 rte_memcpy(actions_tx, actions, 4422 sizeof(struct rte_flow_action)); 4423 actions_tx++; 4424 } else { 4425 rte_memcpy(actions_rx, actions, 4426 sizeof(struct rte_flow_action)); 4427 actions_rx++; 4428 } 4429 break; 4430 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4431 raw_encap = actions->conf; 4432 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4433 memcpy(actions_tx, actions, 4434 sizeof(struct rte_flow_action)); 4435 actions_tx++; 4436 encap = 1; 4437 } else { 4438 rte_memcpy(actions_rx, actions, 4439 sizeof(struct rte_flow_action)); 4440 actions_rx++; 4441 } 4442 break; 4443 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4444 raw_decap = actions->conf; 4445 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4446 memcpy(actions_tx, actions, 4447 sizeof(struct rte_flow_action)); 4448 actions_tx++; 4449 } else { 4450 rte_memcpy(actions_rx, actions, 4451 sizeof(struct rte_flow_action)); 4452 actions_rx++; 4453 } 4454 break; 4455 default: 4456 rte_memcpy(actions_rx, actions, 4457 sizeof(struct rte_flow_action)); 4458 actions_rx++; 4459 break; 4460 } 4461 } 4462 /* Add set meta action and end action for the Rx flow. */ 4463 tag_action = actions_rx; 4464 tag_action->type = (enum rte_flow_action_type) 4465 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4466 actions_rx++; 4467 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4468 actions_rx++; 4469 set_tag = (void *)actions_rx; 4470 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4471 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4472 .data = flow_id, 4473 }; 4474 MLX5_ASSERT(set_tag->id > REG_NON); 4475 tag_action->conf = set_tag; 4476 /* Create Tx item list. */ 4477 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4478 addr = (void *)&pattern_tx[2]; 4479 item = pattern_tx; 4480 item->type = (enum rte_flow_item_type) 4481 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4482 tag_item = (void *)addr; 4483 tag_item->data = flow_id; 4484 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4485 MLX5_ASSERT(set_tag->id > REG_NON); 4486 item->spec = tag_item; 4487 addr += sizeof(struct mlx5_rte_flow_item_tag); 4488 tag_item = (void *)addr; 4489 tag_item->data = UINT32_MAX; 4490 tag_item->id = UINT16_MAX; 4491 item->mask = tag_item; 4492 item->last = NULL; 4493 item++; 4494 item->type = RTE_FLOW_ITEM_TYPE_END; 4495 return 0; 4496 } 4497 4498 /** 4499 * The last stage of splitting chain, just creates the subflow 4500 * without any modification. 4501 * 4502 * @param[in] dev 4503 * Pointer to Ethernet device. 4504 * @param[in] flow 4505 * Parent flow structure pointer. 4506 * @param[in, out] sub_flow 4507 * Pointer to return the created subflow, may be NULL. 4508 * @param[in] attr 4509 * Flow rule attributes. 4510 * @param[in] items 4511 * Pattern specification (list terminated by the END pattern item). 4512 * @param[in] actions 4513 * Associated actions (list terminated by the END action). 4514 * @param[in] flow_split_info 4515 * Pointer to flow split info structure. 4516 * @param[out] error 4517 * Perform verbose error reporting if not NULL. 4518 * @return 4519 * 0 on success, negative value otherwise 4520 */ 4521 static int 4522 flow_create_split_inner(struct rte_eth_dev *dev, 4523 struct rte_flow *flow, 4524 struct mlx5_flow **sub_flow, 4525 const struct rte_flow_attr *attr, 4526 const struct rte_flow_item items[], 4527 const struct rte_flow_action actions[], 4528 struct mlx5_flow_split_info *flow_split_info, 4529 struct rte_flow_error *error) 4530 { 4531 struct mlx5_flow *dev_flow; 4532 4533 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4534 flow_split_info->flow_idx, error); 4535 if (!dev_flow) 4536 return -rte_errno; 4537 dev_flow->flow = flow; 4538 dev_flow->external = flow_split_info->external; 4539 dev_flow->skip_scale = flow_split_info->skip_scale; 4540 /* Subflow object was created, we must include one in the list. */ 4541 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4542 dev_flow->handle, next); 4543 /* 4544 * If dev_flow is as one of the suffix flow, some actions in suffix 4545 * flow may need some user defined item layer flags, and pass the 4546 * Metadate rxq mark flag to suffix flow as well. 4547 */ 4548 if (flow_split_info->prefix_layers) 4549 dev_flow->handle->layers = flow_split_info->prefix_layers; 4550 if (flow_split_info->prefix_mark) 4551 dev_flow->handle->mark = 1; 4552 if (sub_flow) 4553 *sub_flow = dev_flow; 4554 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4555 dev_flow->dv.table_id = flow_split_info->table_id; 4556 #endif 4557 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4558 } 4559 4560 /** 4561 * Get the sub policy of a meter. 4562 * 4563 * @param[in] dev 4564 * Pointer to Ethernet device. 4565 * @param[in] flow 4566 * Parent flow structure pointer. 4567 * @param[in] policy_id; 4568 * Meter Policy id. 4569 * @param[in] attr 4570 * Flow rule attributes. 4571 * @param[in] items 4572 * Pattern specification (list terminated by the END pattern item). 4573 * @param[out] error 4574 * Perform verbose error reporting if not NULL. 4575 * 4576 * @return 4577 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4578 */ 4579 static struct mlx5_flow_meter_sub_policy * 4580 get_meter_sub_policy(struct rte_eth_dev *dev, 4581 struct rte_flow *flow, 4582 uint32_t policy_id, 4583 const struct rte_flow_attr *attr, 4584 const struct rte_flow_item items[], 4585 struct rte_flow_error *error) 4586 { 4587 struct mlx5_flow_meter_policy *policy; 4588 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4589 4590 policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 4591 if (!policy) { 4592 rte_flow_error_set(error, EINVAL, 4593 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4594 "Failed to find Meter Policy."); 4595 goto exit; 4596 } 4597 if (policy->is_rss || 4598 (policy->is_queue && 4599 !policy->sub_policys[MLX5_MTR_DOMAIN_INGRESS][0]->rix_hrxq[0])) { 4600 struct mlx5_flow_workspace *wks = 4601 mlx5_flow_get_thread_workspace(); 4602 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4603 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4604 uint32_t i; 4605 4606 MLX5_ASSERT(wks); 4607 /** 4608 * This is a tmp dev_flow, 4609 * no need to register any matcher for it in translate. 4610 */ 4611 wks->skip_matcher_reg = 1; 4612 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4613 struct mlx5_flow dev_flow = {0}; 4614 struct mlx5_flow_handle dev_handle = { {0} }; 4615 4616 if (policy->is_rss) { 4617 const void *rss_act = 4618 policy->act_cnt[i].rss->conf; 4619 struct rte_flow_action rss_actions[2] = { 4620 [0] = { 4621 .type = RTE_FLOW_ACTION_TYPE_RSS, 4622 .conf = rss_act 4623 }, 4624 [1] = { 4625 .type = RTE_FLOW_ACTION_TYPE_END, 4626 .conf = NULL 4627 } 4628 }; 4629 4630 dev_flow.handle = &dev_handle; 4631 dev_flow.ingress = attr->ingress; 4632 dev_flow.flow = flow; 4633 dev_flow.external = 0; 4634 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4635 dev_flow.dv.transfer = attr->transfer; 4636 #endif 4637 /** 4638 * Translate RSS action to get rss hash fields. 4639 */ 4640 if (flow_drv_translate(dev, &dev_flow, attr, 4641 items, rss_actions, error)) 4642 goto exit; 4643 rss_desc_v[i] = wks->rss_desc; 4644 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4645 rss_desc_v[i].hash_fields = 4646 dev_flow.hash_fields; 4647 rss_desc_v[i].queue_num = 4648 rss_desc_v[i].hash_fields ? 4649 rss_desc_v[i].queue_num : 1; 4650 rss_desc_v[i].tunnel = 4651 !!(dev_flow.handle->layers & 4652 MLX5_FLOW_LAYER_TUNNEL); 4653 } else { 4654 /* This is queue action. */ 4655 rss_desc_v[i] = wks->rss_desc; 4656 rss_desc_v[i].key_len = 0; 4657 rss_desc_v[i].hash_fields = 0; 4658 rss_desc_v[i].queue = 4659 &policy->act_cnt[i].queue; 4660 rss_desc_v[i].queue_num = 1; 4661 } 4662 rss_desc[i] = &rss_desc_v[i]; 4663 } 4664 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4665 flow, policy, rss_desc); 4666 } else { 4667 enum mlx5_meter_domain mtr_domain = 4668 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4669 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4670 MLX5_MTR_DOMAIN_INGRESS; 4671 sub_policy = policy->sub_policys[mtr_domain][0]; 4672 } 4673 if (!sub_policy) { 4674 rte_flow_error_set(error, EINVAL, 4675 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4676 "Failed to get meter sub-policy."); 4677 goto exit; 4678 } 4679 exit: 4680 return sub_policy; 4681 } 4682 4683 /** 4684 * Split the meter flow. 4685 * 4686 * As meter flow will split to three sub flow, other than meter 4687 * action, the other actions make sense to only meter accepts 4688 * the packet. If it need to be dropped, no other additional 4689 * actions should be take. 4690 * 4691 * One kind of special action which decapsulates the L3 tunnel 4692 * header will be in the prefix sub flow, as not to take the 4693 * L3 tunnel header into account. 4694 * 4695 * @param[in] dev 4696 * Pointer to Ethernet device. 4697 * @param[in] flow 4698 * Parent flow structure pointer. 4699 * @param[in] fm 4700 * Pointer to flow meter structure. 4701 * @param[in] attr 4702 * Flow rule attributes. 4703 * @param[in] items 4704 * Pattern specification (list terminated by the END pattern item). 4705 * @param[out] sfx_items 4706 * Suffix flow match items (list terminated by the END pattern item). 4707 * @param[in] actions 4708 * Associated actions (list terminated by the END action). 4709 * @param[out] actions_sfx 4710 * Suffix flow actions. 4711 * @param[out] actions_pre 4712 * Prefix flow actions. 4713 * @param[out] mtr_flow_id 4714 * Pointer to meter flow id. 4715 * @param[out] error 4716 * Perform verbose error reporting if not NULL. 4717 * 4718 * @return 4719 * 0 on success, a negative errno value otherwise and rte_errno is set. 4720 */ 4721 static int 4722 flow_meter_split_prep(struct rte_eth_dev *dev, 4723 struct rte_flow *flow, 4724 struct mlx5_flow_meter_info *fm, 4725 const struct rte_flow_attr *attr, 4726 const struct rte_flow_item items[], 4727 struct rte_flow_item sfx_items[], 4728 const struct rte_flow_action actions[], 4729 struct rte_flow_action actions_sfx[], 4730 struct rte_flow_action actions_pre[], 4731 uint32_t *mtr_flow_id, 4732 struct rte_flow_error *error) 4733 { 4734 struct mlx5_priv *priv = dev->data->dev_private; 4735 struct rte_flow_action *tag_action = NULL; 4736 struct rte_flow_item *tag_item; 4737 struct mlx5_rte_flow_action_set_tag *set_tag; 4738 const struct rte_flow_action_raw_encap *raw_encap; 4739 const struct rte_flow_action_raw_decap *raw_decap; 4740 struct mlx5_rte_flow_item_tag *tag_item_spec; 4741 struct mlx5_rte_flow_item_tag *tag_item_mask; 4742 uint32_t tag_id = 0; 4743 struct rte_flow_item *vlan_item_dst = NULL; 4744 const struct rte_flow_item *vlan_item_src = NULL; 4745 struct rte_flow_action *hw_mtr_action; 4746 struct rte_flow_action *action_pre_head = NULL; 4747 int32_t flow_src_port = priv->representor_id; 4748 bool mtr_first; 4749 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 4750 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 4751 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 4752 uint32_t flow_id = 0; 4753 uint32_t flow_id_reversed = 0; 4754 uint8_t flow_id_bits = 0; 4755 int shift; 4756 4757 /* Prepare the suffix subflow items. */ 4758 tag_item = sfx_items++; 4759 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4760 struct mlx5_priv *port_priv; 4761 const struct rte_flow_item_port_id *pid_v; 4762 int item_type = items->type; 4763 4764 switch (item_type) { 4765 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4766 pid_v = items->spec; 4767 MLX5_ASSERT(pid_v); 4768 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 4769 if (!port_priv) 4770 return rte_flow_error_set(error, 4771 rte_errno, 4772 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 4773 pid_v, 4774 "Failed to get port info."); 4775 flow_src_port = port_priv->representor_id; 4776 memcpy(sfx_items, items, sizeof(*sfx_items)); 4777 sfx_items++; 4778 break; 4779 case RTE_FLOW_ITEM_TYPE_VLAN: 4780 /* Determine if copy vlan item below. */ 4781 vlan_item_src = items; 4782 vlan_item_dst = sfx_items++; 4783 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 4784 break; 4785 default: 4786 break; 4787 } 4788 } 4789 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4790 sfx_items++; 4791 mtr_first = priv->sh->meter_aso_en && 4792 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 4793 /* For ASO meter, meter must be before tag in TX direction. */ 4794 if (mtr_first) { 4795 action_pre_head = actions_pre++; 4796 /* Leave space for tag action. */ 4797 tag_action = actions_pre++; 4798 } 4799 /* Prepare the actions for prefix and suffix flow. */ 4800 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4801 struct rte_flow_action *action_cur = NULL; 4802 4803 switch (actions->type) { 4804 case RTE_FLOW_ACTION_TYPE_METER: 4805 if (mtr_first) { 4806 action_cur = action_pre_head; 4807 } else { 4808 /* Leave space for tag action. */ 4809 tag_action = actions_pre++; 4810 action_cur = actions_pre++; 4811 } 4812 break; 4813 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4814 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4815 action_cur = actions_pre++; 4816 break; 4817 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4818 raw_encap = actions->conf; 4819 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4820 action_cur = actions_pre++; 4821 break; 4822 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4823 raw_decap = actions->conf; 4824 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4825 action_cur = actions_pre++; 4826 break; 4827 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4828 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4829 if (vlan_item_dst && vlan_item_src) { 4830 memcpy(vlan_item_dst, vlan_item_src, 4831 sizeof(*vlan_item_dst)); 4832 /* 4833 * Convert to internal match item, it is used 4834 * for vlan push and set vid. 4835 */ 4836 vlan_item_dst->type = (enum rte_flow_item_type) 4837 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4838 } 4839 break; 4840 default: 4841 break; 4842 } 4843 if (!action_cur) 4844 action_cur = (fm->def_policy) ? 4845 actions_sfx++ : actions_pre++; 4846 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 4847 } 4848 /* Add end action to the actions. */ 4849 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4850 if (priv->sh->meter_aso_en) { 4851 /** 4852 * For ASO meter, need to add an extra jump action explicitly, 4853 * to jump from meter to policer table. 4854 */ 4855 struct mlx5_flow_meter_sub_policy *sub_policy; 4856 struct mlx5_flow_tbl_data_entry *tbl_data; 4857 4858 if (!fm->def_policy) { 4859 sub_policy = get_meter_sub_policy(dev, flow, 4860 fm->policy_id, attr, 4861 items, error); 4862 if (!sub_policy) 4863 return -rte_errno; 4864 } else { 4865 enum mlx5_meter_domain mtr_domain = 4866 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4867 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4868 MLX5_MTR_DOMAIN_INGRESS; 4869 4870 sub_policy = 4871 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 4872 } 4873 tbl_data = container_of(sub_policy->tbl_rsc, 4874 struct mlx5_flow_tbl_data_entry, tbl); 4875 hw_mtr_action = actions_pre++; 4876 hw_mtr_action->type = (enum rte_flow_action_type) 4877 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 4878 hw_mtr_action->conf = tbl_data->jump.action; 4879 } 4880 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4881 actions_pre++; 4882 if (!tag_action) 4883 return rte_flow_error_set(error, ENOMEM, 4884 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4885 "No tag action space."); 4886 if (!mtr_flow_id) { 4887 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 4888 goto exit; 4889 } 4890 /* Only default-policy Meter creates mtr flow id. */ 4891 if (fm->def_policy) { 4892 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 4893 if (!tag_id) 4894 return rte_flow_error_set(error, ENOMEM, 4895 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4896 "Failed to allocate meter flow id."); 4897 flow_id = tag_id - 1; 4898 flow_id_bits = (!flow_id) ? 1 : 4899 (MLX5_REG_BITS - __builtin_clz(flow_id)); 4900 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 4901 mtr_reg_bits) { 4902 mlx5_ipool_free(fm->flow_ipool, tag_id); 4903 return rte_flow_error_set(error, EINVAL, 4904 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4905 "Meter flow id exceeds max limit."); 4906 } 4907 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 4908 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 4909 } 4910 /* Build tag actions and items for meter_id/meter flow_id. */ 4911 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 4912 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4913 tag_item_mask = tag_item_spec + 1; 4914 /* Both flow_id and meter_id share the same register. */ 4915 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4916 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 4917 0, error), 4918 .offset = mtr_id_offset, 4919 .length = mtr_reg_bits, 4920 .data = flow->meter, 4921 }; 4922 /* 4923 * The color Reg bits used by flow_id are growing from 4924 * msb to lsb, so must do bit reverse for flow_id val in RegC. 4925 */ 4926 for (shift = 0; shift < flow_id_bits; shift++) 4927 flow_id_reversed = (flow_id_reversed << 1) | 4928 ((flow_id >> shift) & 0x1); 4929 set_tag->data |= 4930 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 4931 tag_item_spec->id = set_tag->id; 4932 tag_item_spec->data = set_tag->data << mtr_id_offset; 4933 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 4934 tag_action->type = (enum rte_flow_action_type) 4935 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4936 tag_action->conf = set_tag; 4937 tag_item->type = (enum rte_flow_item_type) 4938 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4939 tag_item->spec = tag_item_spec; 4940 tag_item->last = NULL; 4941 tag_item->mask = tag_item_mask; 4942 exit: 4943 if (mtr_flow_id) 4944 *mtr_flow_id = tag_id; 4945 return 0; 4946 } 4947 4948 /** 4949 * Split action list having QUEUE/RSS for metadata register copy. 4950 * 4951 * Once Q/RSS action is detected in user's action list, the flow action 4952 * should be split in order to copy metadata registers, which will happen in 4953 * RX_CP_TBL like, 4954 * - CQE->flow_tag := reg_c[1] (MARK) 4955 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4956 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4957 * This is because the last action of each flow must be a terminal action 4958 * (QUEUE, RSS or DROP). 4959 * 4960 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4961 * stored and kept in the mlx5_flow structure per each sub_flow. 4962 * 4963 * The Q/RSS action is replaced with, 4964 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4965 * And the following JUMP action is added at the end, 4966 * - JUMP, to RX_CP_TBL. 4967 * 4968 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4969 * flow_create_split_metadata() routine. The flow will look like, 4970 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4971 * 4972 * @param dev 4973 * Pointer to Ethernet device. 4974 * @param[out] split_actions 4975 * Pointer to store split actions to jump to CP_TBL. 4976 * @param[in] actions 4977 * Pointer to the list of original flow actions. 4978 * @param[in] qrss 4979 * Pointer to the Q/RSS action. 4980 * @param[in] actions_n 4981 * Number of original actions. 4982 * @param[out] error 4983 * Perform verbose error reporting if not NULL. 4984 * 4985 * @return 4986 * non-zero unique flow_id on success, otherwise 0 and 4987 * error/rte_error are set. 4988 */ 4989 static uint32_t 4990 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4991 struct rte_flow_action *split_actions, 4992 const struct rte_flow_action *actions, 4993 const struct rte_flow_action *qrss, 4994 int actions_n, struct rte_flow_error *error) 4995 { 4996 struct mlx5_priv *priv = dev->data->dev_private; 4997 struct mlx5_rte_flow_action_set_tag *set_tag; 4998 struct rte_flow_action_jump *jump; 4999 const int qrss_idx = qrss - actions; 5000 uint32_t flow_id = 0; 5001 int ret = 0; 5002 5003 /* 5004 * Given actions will be split 5005 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5006 * - Add jump to mreg CP_TBL. 5007 * As a result, there will be one more action. 5008 */ 5009 ++actions_n; 5010 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5011 set_tag = (void *)(split_actions + actions_n); 5012 /* 5013 * If tag action is not set to void(it means we are not the meter 5014 * suffix flow), add the tag action. Since meter suffix flow already 5015 * has the tag added. 5016 */ 5017 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 5018 /* 5019 * Allocate the new subflow ID. This one is unique within 5020 * device and not shared with representors. Otherwise, 5021 * we would have to resolve multi-thread access synch 5022 * issue. Each flow on the shared device is appended 5023 * with source vport identifier, so the resulting 5024 * flows will be unique in the shared (by master and 5025 * representors) domain even if they have coinciding 5026 * IDs. 5027 */ 5028 mlx5_ipool_malloc(priv->sh->ipool 5029 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5030 if (!flow_id) 5031 return rte_flow_error_set(error, ENOMEM, 5032 RTE_FLOW_ERROR_TYPE_ACTION, 5033 NULL, "can't allocate id " 5034 "for split Q/RSS subflow"); 5035 /* Internal SET_TAG action to set flow ID. */ 5036 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5037 .data = flow_id, 5038 }; 5039 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5040 if (ret < 0) 5041 return ret; 5042 set_tag->id = ret; 5043 /* Construct new actions array. */ 5044 /* Replace QUEUE/RSS action. */ 5045 split_actions[qrss_idx] = (struct rte_flow_action){ 5046 .type = (enum rte_flow_action_type) 5047 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5048 .conf = set_tag, 5049 }; 5050 } 5051 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5052 jump = (void *)(set_tag + 1); 5053 *jump = (struct rte_flow_action_jump){ 5054 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5055 }; 5056 split_actions[actions_n - 2] = (struct rte_flow_action){ 5057 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5058 .conf = jump, 5059 }; 5060 split_actions[actions_n - 1] = (struct rte_flow_action){ 5061 .type = RTE_FLOW_ACTION_TYPE_END, 5062 }; 5063 return flow_id; 5064 } 5065 5066 /** 5067 * Extend the given action list for Tx metadata copy. 5068 * 5069 * Copy the given action list to the ext_actions and add flow metadata register 5070 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5071 * 5072 * @param[out] ext_actions 5073 * Pointer to the extended action list. 5074 * @param[in] actions 5075 * Pointer to the list of actions. 5076 * @param[in] actions_n 5077 * Number of actions in the list. 5078 * @param[out] error 5079 * Perform verbose error reporting if not NULL. 5080 * @param[in] encap_idx 5081 * The encap action inndex. 5082 * 5083 * @return 5084 * 0 on success, negative value otherwise 5085 */ 5086 static int 5087 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5088 struct rte_flow_action *ext_actions, 5089 const struct rte_flow_action *actions, 5090 int actions_n, struct rte_flow_error *error, 5091 int encap_idx) 5092 { 5093 struct mlx5_flow_action_copy_mreg *cp_mreg = 5094 (struct mlx5_flow_action_copy_mreg *) 5095 (ext_actions + actions_n + 1); 5096 int ret; 5097 5098 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5099 if (ret < 0) 5100 return ret; 5101 cp_mreg->dst = ret; 5102 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5103 if (ret < 0) 5104 return ret; 5105 cp_mreg->src = ret; 5106 if (encap_idx != 0) 5107 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5108 if (encap_idx == actions_n - 1) { 5109 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5110 .type = (enum rte_flow_action_type) 5111 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5112 .conf = cp_mreg, 5113 }; 5114 ext_actions[actions_n] = (struct rte_flow_action){ 5115 .type = RTE_FLOW_ACTION_TYPE_END, 5116 }; 5117 } else { 5118 ext_actions[encap_idx] = (struct rte_flow_action){ 5119 .type = (enum rte_flow_action_type) 5120 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5121 .conf = cp_mreg, 5122 }; 5123 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5124 sizeof(*ext_actions) * (actions_n - encap_idx)); 5125 } 5126 return 0; 5127 } 5128 5129 /** 5130 * Check the match action from the action list. 5131 * 5132 * @param[in] actions 5133 * Pointer to the list of actions. 5134 * @param[in] attr 5135 * Flow rule attributes. 5136 * @param[in] action 5137 * The action to be check if exist. 5138 * @param[out] match_action_pos 5139 * Pointer to the position of the matched action if exists, otherwise is -1. 5140 * @param[out] qrss_action_pos 5141 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5142 * @param[out] modify_after_mirror 5143 * Pointer to the flag of modify action after FDB mirroring. 5144 * 5145 * @return 5146 * > 0 the total number of actions. 5147 * 0 if not found match action in action list. 5148 */ 5149 static int 5150 flow_check_match_action(const struct rte_flow_action actions[], 5151 const struct rte_flow_attr *attr, 5152 enum rte_flow_action_type action, 5153 int *match_action_pos, int *qrss_action_pos, 5154 int *modify_after_mirror) 5155 { 5156 const struct rte_flow_action_sample *sample; 5157 int actions_n = 0; 5158 uint32_t ratio = 0; 5159 int sub_type = 0; 5160 int flag = 0; 5161 int fdb_mirror = 0; 5162 5163 *match_action_pos = -1; 5164 *qrss_action_pos = -1; 5165 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5166 if (actions->type == action) { 5167 flag = 1; 5168 *match_action_pos = actions_n; 5169 } 5170 switch (actions->type) { 5171 case RTE_FLOW_ACTION_TYPE_QUEUE: 5172 case RTE_FLOW_ACTION_TYPE_RSS: 5173 *qrss_action_pos = actions_n; 5174 break; 5175 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5176 sample = actions->conf; 5177 ratio = sample->ratio; 5178 sub_type = ((const struct rte_flow_action *) 5179 (sample->actions))->type; 5180 if (ratio == 1 && attr->transfer) 5181 fdb_mirror = 1; 5182 break; 5183 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5184 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5185 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5186 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5187 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5188 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5189 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5190 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5191 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5192 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5193 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5194 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5195 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5196 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5197 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5198 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5199 case RTE_FLOW_ACTION_TYPE_FLAG: 5200 case RTE_FLOW_ACTION_TYPE_MARK: 5201 case RTE_FLOW_ACTION_TYPE_SET_META: 5202 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5203 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5204 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5205 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5206 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5207 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5208 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5209 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5210 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5211 case RTE_FLOW_ACTION_TYPE_METER: 5212 if (fdb_mirror) 5213 *modify_after_mirror = 1; 5214 break; 5215 default: 5216 break; 5217 } 5218 actions_n++; 5219 } 5220 if (flag && fdb_mirror && !*modify_after_mirror) { 5221 /* FDB mirroring uses the destination array to implement 5222 * instead of FLOW_SAMPLER object. 5223 */ 5224 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5225 flag = 0; 5226 } 5227 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5228 return flag ? actions_n + 1 : 0; 5229 } 5230 5231 #define SAMPLE_SUFFIX_ITEM 2 5232 5233 /** 5234 * Split the sample flow. 5235 * 5236 * As sample flow will split to two sub flow, sample flow with 5237 * sample action, the other actions will move to new suffix flow. 5238 * 5239 * Also add unique tag id with tag action in the sample flow, 5240 * the same tag id will be as match in the suffix flow. 5241 * 5242 * @param dev 5243 * Pointer to Ethernet device. 5244 * @param[in] add_tag 5245 * Add extra tag action flag. 5246 * @param[out] sfx_items 5247 * Suffix flow match items (list terminated by the END pattern item). 5248 * @param[in] actions 5249 * Associated actions (list terminated by the END action). 5250 * @param[out] actions_sfx 5251 * Suffix flow actions. 5252 * @param[out] actions_pre 5253 * Prefix flow actions. 5254 * @param[in] actions_n 5255 * The total number of actions. 5256 * @param[in] sample_action_pos 5257 * The sample action position. 5258 * @param[in] qrss_action_pos 5259 * The Queue/RSS action position. 5260 * @param[in] jump_table 5261 * Add extra jump action flag. 5262 * @param[out] error 5263 * Perform verbose error reporting if not NULL. 5264 * 5265 * @return 5266 * 0 on success, or unique flow_id, a negative errno value 5267 * otherwise and rte_errno is set. 5268 */ 5269 static int 5270 flow_sample_split_prep(struct rte_eth_dev *dev, 5271 int add_tag, 5272 struct rte_flow_item sfx_items[], 5273 const struct rte_flow_action actions[], 5274 struct rte_flow_action actions_sfx[], 5275 struct rte_flow_action actions_pre[], 5276 int actions_n, 5277 int sample_action_pos, 5278 int qrss_action_pos, 5279 int jump_table, 5280 struct rte_flow_error *error) 5281 { 5282 struct mlx5_priv *priv = dev->data->dev_private; 5283 struct mlx5_rte_flow_action_set_tag *set_tag; 5284 struct mlx5_rte_flow_item_tag *tag_spec; 5285 struct mlx5_rte_flow_item_tag *tag_mask; 5286 struct rte_flow_action_jump *jump_action; 5287 uint32_t tag_id = 0; 5288 int index; 5289 int append_index = 0; 5290 int ret; 5291 5292 if (sample_action_pos < 0) 5293 return rte_flow_error_set(error, EINVAL, 5294 RTE_FLOW_ERROR_TYPE_ACTION, 5295 NULL, "invalid position of sample " 5296 "action in list"); 5297 /* Prepare the actions for prefix and suffix flow. */ 5298 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5299 index = qrss_action_pos; 5300 /* Put the preceding the Queue/RSS action into prefix flow. */ 5301 if (index != 0) 5302 memcpy(actions_pre, actions, 5303 sizeof(struct rte_flow_action) * index); 5304 /* Put others preceding the sample action into prefix flow. */ 5305 if (sample_action_pos > index + 1) 5306 memcpy(actions_pre + index, actions + index + 1, 5307 sizeof(struct rte_flow_action) * 5308 (sample_action_pos - index - 1)); 5309 index = sample_action_pos - 1; 5310 /* Put Queue/RSS action into Suffix flow. */ 5311 memcpy(actions_sfx, actions + qrss_action_pos, 5312 sizeof(struct rte_flow_action)); 5313 actions_sfx++; 5314 } else { 5315 index = sample_action_pos; 5316 if (index != 0) 5317 memcpy(actions_pre, actions, 5318 sizeof(struct rte_flow_action) * index); 5319 } 5320 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5321 * For CX6DX and above, metadata registers Cx preserve their value, 5322 * add an extra tag action for NIC-RX and E-Switch Domain. 5323 */ 5324 if (add_tag) { 5325 /* Prepare the prefix tag action. */ 5326 append_index++; 5327 set_tag = (void *)(actions_pre + actions_n + append_index); 5328 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5329 if (ret < 0) 5330 return ret; 5331 mlx5_ipool_malloc(priv->sh->ipool 5332 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5333 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5334 .id = ret, 5335 .data = tag_id, 5336 }; 5337 /* Prepare the suffix subflow items. */ 5338 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5339 tag_spec->data = tag_id; 5340 tag_spec->id = set_tag->id; 5341 tag_mask = tag_spec + 1; 5342 tag_mask->data = UINT32_MAX; 5343 sfx_items[0] = (struct rte_flow_item){ 5344 .type = (enum rte_flow_item_type) 5345 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5346 .spec = tag_spec, 5347 .last = NULL, 5348 .mask = tag_mask, 5349 }; 5350 sfx_items[1] = (struct rte_flow_item){ 5351 .type = (enum rte_flow_item_type) 5352 RTE_FLOW_ITEM_TYPE_END, 5353 }; 5354 /* Prepare the tag action in prefix subflow. */ 5355 actions_pre[index++] = 5356 (struct rte_flow_action){ 5357 .type = (enum rte_flow_action_type) 5358 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5359 .conf = set_tag, 5360 }; 5361 } 5362 memcpy(actions_pre + index, actions + sample_action_pos, 5363 sizeof(struct rte_flow_action)); 5364 index += 1; 5365 /* For the modify action after the sample action in E-Switch mirroring, 5366 * Add the extra jump action in prefix subflow and jump into the next 5367 * table, then do the modify action in the new table. 5368 */ 5369 if (jump_table) { 5370 /* Prepare the prefix jump action. */ 5371 append_index++; 5372 jump_action = (void *)(actions_pre + actions_n + append_index); 5373 jump_action->group = jump_table; 5374 actions_pre[index++] = 5375 (struct rte_flow_action){ 5376 .type = (enum rte_flow_action_type) 5377 RTE_FLOW_ACTION_TYPE_JUMP, 5378 .conf = jump_action, 5379 }; 5380 } 5381 actions_pre[index] = (struct rte_flow_action){ 5382 .type = (enum rte_flow_action_type) 5383 RTE_FLOW_ACTION_TYPE_END, 5384 }; 5385 /* Put the actions after sample into Suffix flow. */ 5386 memcpy(actions_sfx, actions + sample_action_pos + 1, 5387 sizeof(struct rte_flow_action) * 5388 (actions_n - sample_action_pos - 1)); 5389 return tag_id; 5390 } 5391 5392 /** 5393 * The splitting for metadata feature. 5394 * 5395 * - Q/RSS action on NIC Rx should be split in order to pass by 5396 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5397 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5398 * 5399 * - All the actions on NIC Tx should have a mreg copy action to 5400 * copy reg_a from WQE to reg_c[0]. 5401 * 5402 * @param dev 5403 * Pointer to Ethernet device. 5404 * @param[in] flow 5405 * Parent flow structure pointer. 5406 * @param[in] attr 5407 * Flow rule attributes. 5408 * @param[in] items 5409 * Pattern specification (list terminated by the END pattern item). 5410 * @param[in] actions 5411 * Associated actions (list terminated by the END action). 5412 * @param[in] flow_split_info 5413 * Pointer to flow split info structure. 5414 * @param[out] error 5415 * Perform verbose error reporting if not NULL. 5416 * @return 5417 * 0 on success, negative value otherwise 5418 */ 5419 static int 5420 flow_create_split_metadata(struct rte_eth_dev *dev, 5421 struct rte_flow *flow, 5422 const struct rte_flow_attr *attr, 5423 const struct rte_flow_item items[], 5424 const struct rte_flow_action actions[], 5425 struct mlx5_flow_split_info *flow_split_info, 5426 struct rte_flow_error *error) 5427 { 5428 struct mlx5_priv *priv = dev->data->dev_private; 5429 struct mlx5_dev_config *config = &priv->config; 5430 const struct rte_flow_action *qrss = NULL; 5431 struct rte_flow_action *ext_actions = NULL; 5432 struct mlx5_flow *dev_flow = NULL; 5433 uint32_t qrss_id = 0; 5434 int mtr_sfx = 0; 5435 size_t act_size; 5436 int actions_n; 5437 int encap_idx; 5438 int ret; 5439 5440 /* Check whether extensive metadata feature is engaged. */ 5441 if (!config->dv_flow_en || 5442 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5443 !mlx5_flow_ext_mreg_supported(dev)) 5444 return flow_create_split_inner(dev, flow, NULL, attr, items, 5445 actions, flow_split_info, error); 5446 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5447 &encap_idx); 5448 if (qrss) { 5449 /* Exclude hairpin flows from splitting. */ 5450 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5451 const struct rte_flow_action_queue *queue; 5452 5453 queue = qrss->conf; 5454 if (mlx5_rxq_get_type(dev, queue->index) == 5455 MLX5_RXQ_TYPE_HAIRPIN) 5456 qrss = NULL; 5457 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5458 const struct rte_flow_action_rss *rss; 5459 5460 rss = qrss->conf; 5461 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5462 MLX5_RXQ_TYPE_HAIRPIN) 5463 qrss = NULL; 5464 } 5465 } 5466 if (qrss) { 5467 /* Check if it is in meter suffix table. */ 5468 mtr_sfx = attr->group == (attr->transfer ? 5469 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5470 MLX5_FLOW_TABLE_LEVEL_METER); 5471 /* 5472 * Q/RSS action on NIC Rx should be split in order to pass by 5473 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5474 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5475 */ 5476 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5477 sizeof(struct rte_flow_action_set_tag) + 5478 sizeof(struct rte_flow_action_jump); 5479 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5480 SOCKET_ID_ANY); 5481 if (!ext_actions) 5482 return rte_flow_error_set(error, ENOMEM, 5483 RTE_FLOW_ERROR_TYPE_ACTION, 5484 NULL, "no memory to split " 5485 "metadata flow"); 5486 /* 5487 * If we are the suffix flow of meter, tag already exist. 5488 * Set the tag action to void. 5489 */ 5490 if (mtr_sfx) 5491 ext_actions[qrss - actions].type = 5492 RTE_FLOW_ACTION_TYPE_VOID; 5493 else 5494 ext_actions[qrss - actions].type = 5495 (enum rte_flow_action_type) 5496 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5497 /* 5498 * Create the new actions list with removed Q/RSS action 5499 * and appended set tag and jump to register copy table 5500 * (RX_CP_TBL). We should preallocate unique tag ID here 5501 * in advance, because it is needed for set tag action. 5502 */ 5503 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5504 qrss, actions_n, error); 5505 if (!mtr_sfx && !qrss_id) { 5506 ret = -rte_errno; 5507 goto exit; 5508 } 5509 } else if (attr->egress && !attr->transfer) { 5510 /* 5511 * All the actions on NIC Tx should have a metadata register 5512 * copy action to copy reg_a from WQE to reg_c[meta] 5513 */ 5514 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5515 sizeof(struct mlx5_flow_action_copy_mreg); 5516 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5517 SOCKET_ID_ANY); 5518 if (!ext_actions) 5519 return rte_flow_error_set(error, ENOMEM, 5520 RTE_FLOW_ERROR_TYPE_ACTION, 5521 NULL, "no memory to split " 5522 "metadata flow"); 5523 /* Create the action list appended with copy register. */ 5524 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5525 actions_n, error, encap_idx); 5526 if (ret < 0) 5527 goto exit; 5528 } 5529 /* Add the unmodified original or prefix subflow. */ 5530 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5531 items, ext_actions ? ext_actions : 5532 actions, flow_split_info, error); 5533 if (ret < 0) 5534 goto exit; 5535 MLX5_ASSERT(dev_flow); 5536 if (qrss) { 5537 const struct rte_flow_attr q_attr = { 5538 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5539 .ingress = 1, 5540 }; 5541 /* Internal PMD action to set register. */ 5542 struct mlx5_rte_flow_item_tag q_tag_spec = { 5543 .data = qrss_id, 5544 .id = REG_NON, 5545 }; 5546 struct rte_flow_item q_items[] = { 5547 { 5548 .type = (enum rte_flow_item_type) 5549 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5550 .spec = &q_tag_spec, 5551 .last = NULL, 5552 .mask = NULL, 5553 }, 5554 { 5555 .type = RTE_FLOW_ITEM_TYPE_END, 5556 }, 5557 }; 5558 struct rte_flow_action q_actions[] = { 5559 { 5560 .type = qrss->type, 5561 .conf = qrss->conf, 5562 }, 5563 { 5564 .type = RTE_FLOW_ACTION_TYPE_END, 5565 }, 5566 }; 5567 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5568 5569 /* 5570 * Configure the tag item only if there is no meter subflow. 5571 * Since tag is already marked in the meter suffix subflow 5572 * we can just use the meter suffix items as is. 5573 */ 5574 if (qrss_id) { 5575 /* Not meter subflow. */ 5576 MLX5_ASSERT(!mtr_sfx); 5577 /* 5578 * Put unique id in prefix flow due to it is destroyed 5579 * after suffix flow and id will be freed after there 5580 * is no actual flows with this id and identifier 5581 * reallocation becomes possible (for example, for 5582 * other flows in other threads). 5583 */ 5584 dev_flow->handle->split_flow_id = qrss_id; 5585 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5586 error); 5587 if (ret < 0) 5588 goto exit; 5589 q_tag_spec.id = ret; 5590 } 5591 dev_flow = NULL; 5592 /* Add suffix subflow to execute Q/RSS. */ 5593 flow_split_info->prefix_layers = layers; 5594 flow_split_info->prefix_mark = 0; 5595 ret = flow_create_split_inner(dev, flow, &dev_flow, 5596 &q_attr, mtr_sfx ? items : 5597 q_items, q_actions, 5598 flow_split_info, error); 5599 if (ret < 0) 5600 goto exit; 5601 /* qrss ID should be freed if failed. */ 5602 qrss_id = 0; 5603 MLX5_ASSERT(dev_flow); 5604 } 5605 5606 exit: 5607 /* 5608 * We do not destroy the partially created sub_flows in case of error. 5609 * These ones are included into parent flow list and will be destroyed 5610 * by flow_drv_destroy. 5611 */ 5612 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5613 qrss_id); 5614 mlx5_free(ext_actions); 5615 return ret; 5616 } 5617 5618 /** 5619 * Create meter internal drop flow with the original pattern. 5620 * 5621 * @param dev 5622 * Pointer to Ethernet device. 5623 * @param[in] flow 5624 * Parent flow structure pointer. 5625 * @param[in] attr 5626 * Flow rule attributes. 5627 * @param[in] items 5628 * Pattern specification (list terminated by the END pattern item). 5629 * @param[in] flow_split_info 5630 * Pointer to flow split info structure. 5631 * @param[in] fm 5632 * Pointer to flow meter structure. 5633 * @param[out] error 5634 * Perform verbose error reporting if not NULL. 5635 * @return 5636 * 0 on success, negative value otherwise 5637 */ 5638 static uint32_t 5639 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5640 struct rte_flow *flow, 5641 const struct rte_flow_attr *attr, 5642 const struct rte_flow_item items[], 5643 struct mlx5_flow_split_info *flow_split_info, 5644 struct mlx5_flow_meter_info *fm, 5645 struct rte_flow_error *error) 5646 { 5647 struct mlx5_flow *dev_flow = NULL; 5648 struct rte_flow_attr drop_attr = *attr; 5649 struct rte_flow_action drop_actions[3]; 5650 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5651 5652 MLX5_ASSERT(fm->drop_cnt); 5653 drop_actions[0].type = 5654 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5655 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5656 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5657 drop_actions[1].conf = NULL; 5658 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5659 drop_actions[2].conf = NULL; 5660 drop_split_info.external = false; 5661 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5662 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5663 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5664 return flow_create_split_inner(dev, flow, &dev_flow, 5665 &drop_attr, items, drop_actions, 5666 &drop_split_info, error); 5667 } 5668 5669 /** 5670 * The splitting for meter feature. 5671 * 5672 * - The meter flow will be split to two flows as prefix and 5673 * suffix flow. The packets make sense only it pass the prefix 5674 * meter action. 5675 * 5676 * - Reg_C_5 is used for the packet to match betweend prefix and 5677 * suffix flow. 5678 * 5679 * @param dev 5680 * Pointer to Ethernet device. 5681 * @param[in] flow 5682 * Parent flow structure pointer. 5683 * @param[in] attr 5684 * Flow rule attributes. 5685 * @param[in] items 5686 * Pattern specification (list terminated by the END pattern item). 5687 * @param[in] actions 5688 * Associated actions (list terminated by the END action). 5689 * @param[in] flow_split_info 5690 * Pointer to flow split info structure. 5691 * @param[out] error 5692 * Perform verbose error reporting if not NULL. 5693 * @return 5694 * 0 on success, negative value otherwise 5695 */ 5696 static int 5697 flow_create_split_meter(struct rte_eth_dev *dev, 5698 struct rte_flow *flow, 5699 const struct rte_flow_attr *attr, 5700 const struct rte_flow_item items[], 5701 const struct rte_flow_action actions[], 5702 struct mlx5_flow_split_info *flow_split_info, 5703 struct rte_flow_error *error) 5704 { 5705 struct mlx5_priv *priv = dev->data->dev_private; 5706 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5707 struct rte_flow_action *sfx_actions = NULL; 5708 struct rte_flow_action *pre_actions = NULL; 5709 struct rte_flow_item *sfx_items = NULL; 5710 struct mlx5_flow *dev_flow = NULL; 5711 struct rte_flow_attr sfx_attr = *attr; 5712 struct mlx5_flow_meter_info *fm = NULL; 5713 uint8_t skip_scale_restore; 5714 bool has_mtr = false; 5715 bool has_modify = false; 5716 bool set_mtr_reg = true; 5717 uint32_t meter_id = 0; 5718 uint32_t mtr_idx = 0; 5719 uint32_t mtr_flow_id = 0; 5720 size_t act_size; 5721 size_t item_size; 5722 int actions_n = 0; 5723 int ret = 0; 5724 5725 if (priv->mtr_en) 5726 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 5727 &has_modify, &meter_id); 5728 if (has_mtr) { 5729 if (flow->meter) { 5730 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 5731 if (!fm) 5732 return rte_flow_error_set(error, EINVAL, 5733 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5734 NULL, "Meter not found."); 5735 } else { 5736 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 5737 if (!fm) 5738 return rte_flow_error_set(error, EINVAL, 5739 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5740 NULL, "Meter not found."); 5741 ret = mlx5_flow_meter_attach(priv, fm, 5742 &sfx_attr, error); 5743 if (ret) 5744 return -rte_errno; 5745 flow->meter = mtr_idx; 5746 } 5747 MLX5_ASSERT(wks); 5748 wks->fm = fm; 5749 /* 5750 * If it isn't default-policy Meter, and 5751 * 1. There's no action in flow to change 5752 * packet (modify/encap/decap etc.), OR 5753 * 2. No drop count needed for this meter. 5754 * no need to use regC to save meter id anymore. 5755 */ 5756 if (!fm->def_policy && (!has_modify || !fm->drop_cnt)) 5757 set_mtr_reg = false; 5758 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 5759 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 5760 sizeof(struct mlx5_rte_flow_action_set_tag); 5761 /* Suffix items: tag, vlan, port id, end. */ 5762 #define METER_SUFFIX_ITEM 4 5763 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5764 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5765 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5766 0, SOCKET_ID_ANY); 5767 if (!sfx_actions) 5768 return rte_flow_error_set(error, ENOMEM, 5769 RTE_FLOW_ERROR_TYPE_ACTION, 5770 NULL, "no memory to split " 5771 "meter flow"); 5772 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5773 act_size); 5774 /* There's no suffix flow for meter of non-default policy. */ 5775 if (!fm->def_policy) 5776 pre_actions = sfx_actions + 1; 5777 else 5778 pre_actions = sfx_actions + actions_n; 5779 ret = flow_meter_split_prep(dev, flow, fm, &sfx_attr, 5780 items, sfx_items, actions, 5781 sfx_actions, pre_actions, 5782 (set_mtr_reg ? &mtr_flow_id : NULL), 5783 error); 5784 if (ret) { 5785 ret = -rte_errno; 5786 goto exit; 5787 } 5788 /* Add the prefix subflow. */ 5789 flow_split_info->prefix_mark = 0; 5790 skip_scale_restore = flow_split_info->skip_scale; 5791 flow_split_info->skip_scale |= 5792 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5793 ret = flow_create_split_inner(dev, flow, &dev_flow, 5794 attr, items, pre_actions, 5795 flow_split_info, error); 5796 flow_split_info->skip_scale = skip_scale_restore; 5797 if (ret) { 5798 if (mtr_flow_id) 5799 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 5800 ret = -rte_errno; 5801 goto exit; 5802 } 5803 if (mtr_flow_id) { 5804 dev_flow->handle->split_flow_id = mtr_flow_id; 5805 dev_flow->handle->is_meter_flow_id = 1; 5806 } 5807 if (!fm->def_policy) { 5808 if (!set_mtr_reg && fm->drop_cnt) 5809 ret = 5810 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 5811 &sfx_attr, items, 5812 flow_split_info, 5813 fm, error); 5814 goto exit; 5815 } 5816 /* Setting the sfx group atrr. */ 5817 sfx_attr.group = sfx_attr.transfer ? 5818 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5819 MLX5_FLOW_TABLE_LEVEL_METER; 5820 flow_split_info->prefix_layers = 5821 flow_get_prefix_layer_flags(dev_flow); 5822 flow_split_info->prefix_mark = dev_flow->handle->mark; 5823 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 5824 } 5825 /* Add the prefix subflow. */ 5826 ret = flow_create_split_metadata(dev, flow, 5827 &sfx_attr, sfx_items ? 5828 sfx_items : items, 5829 sfx_actions ? sfx_actions : actions, 5830 flow_split_info, error); 5831 exit: 5832 if (sfx_actions) 5833 mlx5_free(sfx_actions); 5834 return ret; 5835 } 5836 5837 /** 5838 * The splitting for sample feature. 5839 * 5840 * Once Sample action is detected in the action list, the flow actions should 5841 * be split into prefix sub flow and suffix sub flow. 5842 * 5843 * The original items remain in the prefix sub flow, all actions preceding the 5844 * sample action and the sample action itself will be copied to the prefix 5845 * sub flow, the actions following the sample action will be copied to the 5846 * suffix sub flow, Queue action always be located in the suffix sub flow. 5847 * 5848 * In order to make the packet from prefix sub flow matches with suffix sub 5849 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5850 * flow uses tag item with the unique flow id. 5851 * 5852 * @param dev 5853 * Pointer to Ethernet device. 5854 * @param[in] flow 5855 * Parent flow structure pointer. 5856 * @param[in] attr 5857 * Flow rule attributes. 5858 * @param[in] items 5859 * Pattern specification (list terminated by the END pattern item). 5860 * @param[in] actions 5861 * Associated actions (list terminated by the END action). 5862 * @param[in] flow_split_info 5863 * Pointer to flow split info structure. 5864 * @param[out] error 5865 * Perform verbose error reporting if not NULL. 5866 * @return 5867 * 0 on success, negative value otherwise 5868 */ 5869 static int 5870 flow_create_split_sample(struct rte_eth_dev *dev, 5871 struct rte_flow *flow, 5872 const struct rte_flow_attr *attr, 5873 const struct rte_flow_item items[], 5874 const struct rte_flow_action actions[], 5875 struct mlx5_flow_split_info *flow_split_info, 5876 struct rte_flow_error *error) 5877 { 5878 struct mlx5_priv *priv = dev->data->dev_private; 5879 struct rte_flow_action *sfx_actions = NULL; 5880 struct rte_flow_action *pre_actions = NULL; 5881 struct rte_flow_item *sfx_items = NULL; 5882 struct mlx5_flow *dev_flow = NULL; 5883 struct rte_flow_attr sfx_attr = *attr; 5884 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5885 struct mlx5_flow_dv_sample_resource *sample_res; 5886 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5887 struct mlx5_flow_tbl_resource *sfx_tbl; 5888 #endif 5889 size_t act_size; 5890 size_t item_size; 5891 uint32_t fdb_tx = 0; 5892 int32_t tag_id = 0; 5893 int actions_n = 0; 5894 int sample_action_pos; 5895 int qrss_action_pos; 5896 int add_tag = 0; 5897 int modify_after_mirror = 0; 5898 uint16_t jump_table = 0; 5899 const uint32_t next_ft_step = 1; 5900 int ret = 0; 5901 5902 if (priv->sampler_en) 5903 actions_n = flow_check_match_action(actions, attr, 5904 RTE_FLOW_ACTION_TYPE_SAMPLE, 5905 &sample_action_pos, &qrss_action_pos, 5906 &modify_after_mirror); 5907 if (actions_n) { 5908 /* The prefix actions must includes sample, tag, end. */ 5909 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5910 + sizeof(struct mlx5_rte_flow_action_set_tag); 5911 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5912 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5913 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5914 item_size), 0, SOCKET_ID_ANY); 5915 if (!sfx_actions) 5916 return rte_flow_error_set(error, ENOMEM, 5917 RTE_FLOW_ERROR_TYPE_ACTION, 5918 NULL, "no memory to split " 5919 "sample flow"); 5920 /* The representor_id is UINT16_MAX for uplink. */ 5921 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 5922 /* 5923 * When reg_c_preserve is set, metadata registers Cx preserve 5924 * their value even through packet duplication. 5925 */ 5926 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 5927 if (add_tag) 5928 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5929 + act_size); 5930 if (modify_after_mirror) 5931 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 5932 next_ft_step; 5933 pre_actions = sfx_actions + actions_n; 5934 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 5935 actions, sfx_actions, 5936 pre_actions, actions_n, 5937 sample_action_pos, 5938 qrss_action_pos, jump_table, 5939 error); 5940 if (tag_id < 0 || (add_tag && !tag_id)) { 5941 ret = -rte_errno; 5942 goto exit; 5943 } 5944 if (modify_after_mirror) 5945 flow_split_info->skip_scale = 5946 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5947 /* Add the prefix subflow. */ 5948 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5949 items, pre_actions, 5950 flow_split_info, error); 5951 if (ret) { 5952 ret = -rte_errno; 5953 goto exit; 5954 } 5955 dev_flow->handle->split_flow_id = tag_id; 5956 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5957 if (!modify_after_mirror) { 5958 /* Set the sfx group attr. */ 5959 sample_res = (struct mlx5_flow_dv_sample_resource *) 5960 dev_flow->dv.sample_res; 5961 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5962 sample_res->normal_path_tbl; 5963 sfx_tbl_data = container_of(sfx_tbl, 5964 struct mlx5_flow_tbl_data_entry, 5965 tbl); 5966 sfx_attr.group = sfx_attr.transfer ? 5967 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 5968 } else { 5969 MLX5_ASSERT(attr->transfer); 5970 sfx_attr.group = jump_table; 5971 } 5972 flow_split_info->prefix_layers = 5973 flow_get_prefix_layer_flags(dev_flow); 5974 flow_split_info->prefix_mark = dev_flow->handle->mark; 5975 /* Suffix group level already be scaled with factor, set 5976 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 5977 * again in translation. 5978 */ 5979 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5980 #endif 5981 } 5982 /* Add the suffix subflow. */ 5983 ret = flow_create_split_meter(dev, flow, &sfx_attr, 5984 sfx_items ? sfx_items : items, 5985 sfx_actions ? sfx_actions : actions, 5986 flow_split_info, error); 5987 exit: 5988 if (sfx_actions) 5989 mlx5_free(sfx_actions); 5990 return ret; 5991 } 5992 5993 /** 5994 * Split the flow to subflow set. The splitters might be linked 5995 * in the chain, like this: 5996 * flow_create_split_outer() calls: 5997 * flow_create_split_meter() calls: 5998 * flow_create_split_metadata(meter_subflow_0) calls: 5999 * flow_create_split_inner(metadata_subflow_0) 6000 * flow_create_split_inner(metadata_subflow_1) 6001 * flow_create_split_inner(metadata_subflow_2) 6002 * flow_create_split_metadata(meter_subflow_1) calls: 6003 * flow_create_split_inner(metadata_subflow_0) 6004 * flow_create_split_inner(metadata_subflow_1) 6005 * flow_create_split_inner(metadata_subflow_2) 6006 * 6007 * This provide flexible way to add new levels of flow splitting. 6008 * The all of successfully created subflows are included to the 6009 * parent flow dev_flow list. 6010 * 6011 * @param dev 6012 * Pointer to Ethernet device. 6013 * @param[in] flow 6014 * Parent flow structure pointer. 6015 * @param[in] attr 6016 * Flow rule attributes. 6017 * @param[in] items 6018 * Pattern specification (list terminated by the END pattern item). 6019 * @param[in] actions 6020 * Associated actions (list terminated by the END action). 6021 * @param[in] flow_split_info 6022 * Pointer to flow split info structure. 6023 * @param[out] error 6024 * Perform verbose error reporting if not NULL. 6025 * @return 6026 * 0 on success, negative value otherwise 6027 */ 6028 static int 6029 flow_create_split_outer(struct rte_eth_dev *dev, 6030 struct rte_flow *flow, 6031 const struct rte_flow_attr *attr, 6032 const struct rte_flow_item items[], 6033 const struct rte_flow_action actions[], 6034 struct mlx5_flow_split_info *flow_split_info, 6035 struct rte_flow_error *error) 6036 { 6037 int ret; 6038 6039 ret = flow_create_split_sample(dev, flow, attr, items, 6040 actions, flow_split_info, error); 6041 MLX5_ASSERT(ret <= 0); 6042 return ret; 6043 } 6044 6045 static inline struct mlx5_flow_tunnel * 6046 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6047 { 6048 struct mlx5_flow_tunnel *tunnel; 6049 6050 #pragma GCC diagnostic push 6051 #pragma GCC diagnostic ignored "-Wcast-qual" 6052 tunnel = (typeof(tunnel))flow->tunnel; 6053 #pragma GCC diagnostic pop 6054 6055 return tunnel; 6056 } 6057 6058 /** 6059 * Adjust flow RSS workspace if needed. 6060 * 6061 * @param wks 6062 * Pointer to thread flow work space. 6063 * @param rss_desc 6064 * Pointer to RSS descriptor. 6065 * @param[in] nrssq_num 6066 * New RSS queue number. 6067 * 6068 * @return 6069 * 0 on success, -1 otherwise and rte_errno is set. 6070 */ 6071 static int 6072 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6073 struct mlx5_flow_rss_desc *rss_desc, 6074 uint32_t nrssq_num) 6075 { 6076 if (likely(nrssq_num <= wks->rssq_num)) 6077 return 0; 6078 rss_desc->queue = realloc(rss_desc->queue, 6079 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6080 if (!rss_desc->queue) { 6081 rte_errno = ENOMEM; 6082 return -1; 6083 } 6084 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6085 return 0; 6086 } 6087 6088 /** 6089 * Create a flow and add it to @p list. 6090 * 6091 * @param dev 6092 * Pointer to Ethernet device. 6093 * @param list 6094 * Pointer to a TAILQ flow list. If this parameter NULL, 6095 * no list insertion occurred, flow is just created, 6096 * this is caller's responsibility to track the 6097 * created flow. 6098 * @param[in] attr 6099 * Flow rule attributes. 6100 * @param[in] items 6101 * Pattern specification (list terminated by the END pattern item). 6102 * @param[in] actions 6103 * Associated actions (list terminated by the END action). 6104 * @param[in] external 6105 * This flow rule is created by request external to PMD. 6106 * @param[out] error 6107 * Perform verbose error reporting if not NULL. 6108 * 6109 * @return 6110 * A flow index on success, 0 otherwise and rte_errno is set. 6111 */ 6112 static uint32_t 6113 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 6114 const struct rte_flow_attr *attr, 6115 const struct rte_flow_item items[], 6116 const struct rte_flow_action original_actions[], 6117 bool external, struct rte_flow_error *error) 6118 { 6119 struct mlx5_priv *priv = dev->data->dev_private; 6120 struct rte_flow *flow = NULL; 6121 struct mlx5_flow *dev_flow; 6122 const struct rte_flow_action_rss *rss = NULL; 6123 struct mlx5_translated_action_handle 6124 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6125 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6126 union { 6127 struct mlx5_flow_expand_rss buf; 6128 uint8_t buffer[2048]; 6129 } expand_buffer; 6130 union { 6131 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6132 uint8_t buffer[2048]; 6133 } actions_rx; 6134 union { 6135 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6136 uint8_t buffer[2048]; 6137 } actions_hairpin_tx; 6138 union { 6139 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6140 uint8_t buffer[2048]; 6141 } items_tx; 6142 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6143 struct mlx5_flow_rss_desc *rss_desc; 6144 const struct rte_flow_action *p_actions_rx; 6145 uint32_t i; 6146 uint32_t idx = 0; 6147 int hairpin_flow; 6148 struct rte_flow_attr attr_tx = { .priority = 0 }; 6149 const struct rte_flow_action *actions; 6150 struct rte_flow_action *translated_actions = NULL; 6151 struct mlx5_flow_tunnel *tunnel; 6152 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6153 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6154 struct mlx5_flow_split_info flow_split_info = { 6155 .external = !!external, 6156 .skip_scale = 0, 6157 .flow_idx = 0, 6158 .prefix_mark = 0, 6159 .prefix_layers = 0, 6160 .table_id = 0 6161 }; 6162 int ret; 6163 6164 MLX5_ASSERT(wks); 6165 rss_desc = &wks->rss_desc; 6166 ret = flow_action_handles_translate(dev, original_actions, 6167 indir_actions, 6168 &indir_actions_n, 6169 &translated_actions, error); 6170 if (ret < 0) { 6171 MLX5_ASSERT(translated_actions == NULL); 6172 return 0; 6173 } 6174 actions = translated_actions ? translated_actions : original_actions; 6175 p_actions_rx = actions; 6176 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6177 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6178 external, hairpin_flow, error); 6179 if (ret < 0) 6180 goto error_before_hairpin_split; 6181 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 6182 if (!flow) { 6183 rte_errno = ENOMEM; 6184 goto error_before_hairpin_split; 6185 } 6186 if (hairpin_flow > 0) { 6187 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6188 rte_errno = EINVAL; 6189 goto error_before_hairpin_split; 6190 } 6191 flow_hairpin_split(dev, actions, actions_rx.actions, 6192 actions_hairpin_tx.actions, items_tx.items, 6193 idx); 6194 p_actions_rx = actions_rx.actions; 6195 } 6196 flow_split_info.flow_idx = idx; 6197 flow->drv_type = flow_get_drv_type(dev, attr); 6198 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6199 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6200 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6201 /* RSS Action only works on NIC RX domain */ 6202 if (attr->ingress && !attr->transfer) 6203 rss = flow_get_rss_action(dev, p_actions_rx); 6204 if (rss) { 6205 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6206 return 0; 6207 /* 6208 * The following information is required by 6209 * mlx5_flow_hashfields_adjust() in advance. 6210 */ 6211 rss_desc->level = rss->level; 6212 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 6213 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 6214 } 6215 flow->dev_handles = 0; 6216 if (rss && rss->types) { 6217 unsigned int graph_root; 6218 6219 graph_root = find_graph_root(items, rss->level); 6220 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6221 items, rss->types, 6222 mlx5_support_expansion, graph_root); 6223 MLX5_ASSERT(ret > 0 && 6224 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6225 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6226 for (i = 0; i < buf->entries; ++i) 6227 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6228 } 6229 } else { 6230 buf->entries = 1; 6231 buf->entry[0].pattern = (void *)(uintptr_t)items; 6232 } 6233 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6234 indir_actions_n); 6235 for (i = 0; i < buf->entries; ++i) { 6236 /* Initialize flow split data. */ 6237 flow_split_info.prefix_layers = 0; 6238 flow_split_info.prefix_mark = 0; 6239 flow_split_info.skip_scale = 0; 6240 /* 6241 * The splitter may create multiple dev_flows, 6242 * depending on configuration. In the simplest 6243 * case it just creates unmodified original flow. 6244 */ 6245 ret = flow_create_split_outer(dev, flow, attr, 6246 buf->entry[i].pattern, 6247 p_actions_rx, &flow_split_info, 6248 error); 6249 if (ret < 0) 6250 goto error; 6251 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6252 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6253 p_actions_rx, 6254 idx, 6255 wks->flows[0].tunnel, 6256 &default_miss_ctx, 6257 error); 6258 if (ret < 0) { 6259 mlx5_free(default_miss_ctx.queue); 6260 goto error; 6261 } 6262 } 6263 } 6264 /* Create the tx flow. */ 6265 if (hairpin_flow) { 6266 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6267 attr_tx.ingress = 0; 6268 attr_tx.egress = 1; 6269 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6270 actions_hairpin_tx.actions, 6271 idx, error); 6272 if (!dev_flow) 6273 goto error; 6274 dev_flow->flow = flow; 6275 dev_flow->external = 0; 6276 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6277 dev_flow->handle, next); 6278 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6279 items_tx.items, 6280 actions_hairpin_tx.actions, error); 6281 if (ret < 0) 6282 goto error; 6283 } 6284 /* 6285 * Update the metadata register copy table. If extensive 6286 * metadata feature is enabled and registers are supported 6287 * we might create the extra rte_flow for each unique 6288 * MARK/FLAG action ID. 6289 * 6290 * The table is updated for ingress Flows only, because 6291 * the egress Flows belong to the different device and 6292 * copy table should be updated in peer NIC Rx domain. 6293 */ 6294 if (attr->ingress && 6295 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6296 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6297 if (ret) 6298 goto error; 6299 } 6300 /* 6301 * If the flow is external (from application) OR device is started, 6302 * OR mreg discover, then apply immediately. 6303 */ 6304 if (external || dev->data->dev_started || 6305 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6306 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6307 ret = flow_drv_apply(dev, flow, error); 6308 if (ret < 0) 6309 goto error; 6310 } 6311 if (list) { 6312 rte_spinlock_lock(&priv->flow_list_lock); 6313 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 6314 flow, next); 6315 rte_spinlock_unlock(&priv->flow_list_lock); 6316 } 6317 flow_rxq_flags_set(dev, flow); 6318 rte_free(translated_actions); 6319 tunnel = flow_tunnel_from_rule(wks->flows); 6320 if (tunnel) { 6321 flow->tunnel = 1; 6322 flow->tunnel_id = tunnel->tunnel_id; 6323 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6324 mlx5_free(default_miss_ctx.queue); 6325 } 6326 mlx5_flow_pop_thread_workspace(); 6327 return idx; 6328 error: 6329 MLX5_ASSERT(flow); 6330 ret = rte_errno; /* Save rte_errno before cleanup. */ 6331 flow_mreg_del_copy_action(dev, flow); 6332 flow_drv_destroy(dev, flow); 6333 if (rss_desc->shared_rss) 6334 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6335 mlx5_ipool_get 6336 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6337 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6338 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 6339 rte_errno = ret; /* Restore rte_errno. */ 6340 ret = rte_errno; 6341 rte_errno = ret; 6342 mlx5_flow_pop_thread_workspace(); 6343 error_before_hairpin_split: 6344 rte_free(translated_actions); 6345 return 0; 6346 } 6347 6348 /** 6349 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6350 * incoming packets to table 1. 6351 * 6352 * Other flow rules, requested for group n, will be created in 6353 * e-switch table n+1. 6354 * Jump action to e-switch group n will be created to group n+1. 6355 * 6356 * Used when working in switchdev mode, to utilise advantages of table 1 6357 * and above. 6358 * 6359 * @param dev 6360 * Pointer to Ethernet device. 6361 * 6362 * @return 6363 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6364 */ 6365 struct rte_flow * 6366 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6367 { 6368 const struct rte_flow_attr attr = { 6369 .group = 0, 6370 .priority = 0, 6371 .ingress = 1, 6372 .egress = 0, 6373 .transfer = 1, 6374 }; 6375 const struct rte_flow_item pattern = { 6376 .type = RTE_FLOW_ITEM_TYPE_END, 6377 }; 6378 struct rte_flow_action_jump jump = { 6379 .group = 1, 6380 }; 6381 const struct rte_flow_action actions[] = { 6382 { 6383 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6384 .conf = &jump, 6385 }, 6386 { 6387 .type = RTE_FLOW_ACTION_TYPE_END, 6388 }, 6389 }; 6390 struct mlx5_priv *priv = dev->data->dev_private; 6391 struct rte_flow_error error; 6392 6393 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 6394 &attr, &pattern, 6395 actions, false, &error); 6396 } 6397 6398 /** 6399 * Validate a flow supported by the NIC. 6400 * 6401 * @see rte_flow_validate() 6402 * @see rte_flow_ops 6403 */ 6404 int 6405 mlx5_flow_validate(struct rte_eth_dev *dev, 6406 const struct rte_flow_attr *attr, 6407 const struct rte_flow_item items[], 6408 const struct rte_flow_action original_actions[], 6409 struct rte_flow_error *error) 6410 { 6411 int hairpin_flow; 6412 struct mlx5_translated_action_handle 6413 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6414 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6415 const struct rte_flow_action *actions; 6416 struct rte_flow_action *translated_actions = NULL; 6417 int ret = flow_action_handles_translate(dev, original_actions, 6418 indir_actions, 6419 &indir_actions_n, 6420 &translated_actions, error); 6421 6422 if (ret) 6423 return ret; 6424 actions = translated_actions ? translated_actions : original_actions; 6425 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6426 ret = flow_drv_validate(dev, attr, items, actions, 6427 true, hairpin_flow, error); 6428 rte_free(translated_actions); 6429 return ret; 6430 } 6431 6432 /** 6433 * Create a flow. 6434 * 6435 * @see rte_flow_create() 6436 * @see rte_flow_ops 6437 */ 6438 struct rte_flow * 6439 mlx5_flow_create(struct rte_eth_dev *dev, 6440 const struct rte_flow_attr *attr, 6441 const struct rte_flow_item items[], 6442 const struct rte_flow_action actions[], 6443 struct rte_flow_error *error) 6444 { 6445 struct mlx5_priv *priv = dev->data->dev_private; 6446 6447 /* 6448 * If the device is not started yet, it is not allowed to created a 6449 * flow from application. PMD default flows and traffic control flows 6450 * are not affected. 6451 */ 6452 if (unlikely(!dev->data->dev_started)) { 6453 DRV_LOG(DEBUG, "port %u is not started when " 6454 "inserting a flow", dev->data->port_id); 6455 rte_flow_error_set(error, ENODEV, 6456 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6457 NULL, 6458 "port not started"); 6459 return NULL; 6460 } 6461 6462 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 6463 attr, items, actions, true, error); 6464 } 6465 6466 /** 6467 * Destroy a flow in a list. 6468 * 6469 * @param dev 6470 * Pointer to Ethernet device. 6471 * @param list 6472 * Pointer to the Indexed flow list. If this parameter NULL, 6473 * there is no flow removal from the list. Be noted that as 6474 * flow is add to the indexed list, memory of the indexed 6475 * list points to maybe changed as flow destroyed. 6476 * @param[in] flow_idx 6477 * Index of flow to destroy. 6478 */ 6479 static void 6480 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 6481 uint32_t flow_idx) 6482 { 6483 struct mlx5_priv *priv = dev->data->dev_private; 6484 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6485 [MLX5_IPOOL_RTE_FLOW], flow_idx); 6486 6487 if (!flow) 6488 return; 6489 /* 6490 * Update RX queue flags only if port is started, otherwise it is 6491 * already clean. 6492 */ 6493 if (dev->data->dev_started) 6494 flow_rxq_flags_trim(dev, flow); 6495 flow_drv_destroy(dev, flow); 6496 if (list) { 6497 rte_spinlock_lock(&priv->flow_list_lock); 6498 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 6499 flow_idx, flow, next); 6500 rte_spinlock_unlock(&priv->flow_list_lock); 6501 } 6502 if (flow->tunnel) { 6503 struct mlx5_flow_tunnel *tunnel; 6504 6505 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6506 RTE_VERIFY(tunnel); 6507 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6508 mlx5_flow_tunnel_free(dev, tunnel); 6509 } 6510 flow_mreg_del_copy_action(dev, flow); 6511 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 6512 } 6513 6514 /** 6515 * Destroy all flows. 6516 * 6517 * @param dev 6518 * Pointer to Ethernet device. 6519 * @param list 6520 * Pointer to the Indexed flow list. 6521 * @param active 6522 * If flushing is called avtively. 6523 */ 6524 void 6525 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 6526 { 6527 uint32_t num_flushed = 0; 6528 6529 while (*list) { 6530 flow_list_destroy(dev, list, *list); 6531 num_flushed++; 6532 } 6533 if (active) { 6534 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6535 dev->data->port_id, num_flushed); 6536 } 6537 } 6538 6539 /** 6540 * Stop all default actions for flows. 6541 * 6542 * @param dev 6543 * Pointer to Ethernet device. 6544 */ 6545 void 6546 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6547 { 6548 flow_mreg_del_default_copy_action(dev); 6549 flow_rxq_flags_clear(dev); 6550 } 6551 6552 /** 6553 * Start all default actions for flows. 6554 * 6555 * @param dev 6556 * Pointer to Ethernet device. 6557 * @return 6558 * 0 on success, a negative errno value otherwise and rte_errno is set. 6559 */ 6560 int 6561 mlx5_flow_start_default(struct rte_eth_dev *dev) 6562 { 6563 struct rte_flow_error error; 6564 6565 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6566 return flow_mreg_add_default_copy_action(dev, &error); 6567 } 6568 6569 /** 6570 * Release key of thread specific flow workspace data. 6571 */ 6572 void 6573 flow_release_workspace(void *data) 6574 { 6575 struct mlx5_flow_workspace *wks = data; 6576 struct mlx5_flow_workspace *next; 6577 6578 while (wks) { 6579 next = wks->next; 6580 free(wks->rss_desc.queue); 6581 free(wks); 6582 wks = next; 6583 } 6584 } 6585 6586 /** 6587 * Get thread specific current flow workspace. 6588 * 6589 * @return pointer to thread specific flow workspace data, NULL on error. 6590 */ 6591 struct mlx5_flow_workspace* 6592 mlx5_flow_get_thread_workspace(void) 6593 { 6594 struct mlx5_flow_workspace *data; 6595 6596 data = mlx5_flow_os_get_specific_workspace(); 6597 MLX5_ASSERT(data && data->inuse); 6598 if (!data || !data->inuse) 6599 DRV_LOG(ERR, "flow workspace not initialized."); 6600 return data; 6601 } 6602 6603 /** 6604 * Allocate and init new flow workspace. 6605 * 6606 * @return pointer to flow workspace data, NULL on error. 6607 */ 6608 static struct mlx5_flow_workspace* 6609 flow_alloc_thread_workspace(void) 6610 { 6611 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6612 6613 if (!data) { 6614 DRV_LOG(ERR, "Failed to allocate flow workspace " 6615 "memory."); 6616 return NULL; 6617 } 6618 data->rss_desc.queue = calloc(1, 6619 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6620 if (!data->rss_desc.queue) 6621 goto err; 6622 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6623 return data; 6624 err: 6625 if (data->rss_desc.queue) 6626 free(data->rss_desc.queue); 6627 free(data); 6628 return NULL; 6629 } 6630 6631 /** 6632 * Get new thread specific flow workspace. 6633 * 6634 * If current workspace inuse, create new one and set as current. 6635 * 6636 * @return pointer to thread specific flow workspace data, NULL on error. 6637 */ 6638 static struct mlx5_flow_workspace* 6639 mlx5_flow_push_thread_workspace(void) 6640 { 6641 struct mlx5_flow_workspace *curr; 6642 struct mlx5_flow_workspace *data; 6643 6644 curr = mlx5_flow_os_get_specific_workspace(); 6645 if (!curr) { 6646 data = flow_alloc_thread_workspace(); 6647 if (!data) 6648 return NULL; 6649 } else if (!curr->inuse) { 6650 data = curr; 6651 } else if (curr->next) { 6652 data = curr->next; 6653 } else { 6654 data = flow_alloc_thread_workspace(); 6655 if (!data) 6656 return NULL; 6657 curr->next = data; 6658 data->prev = curr; 6659 } 6660 data->inuse = 1; 6661 data->flow_idx = 0; 6662 /* Set as current workspace */ 6663 if (mlx5_flow_os_set_specific_workspace(data)) 6664 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6665 return data; 6666 } 6667 6668 /** 6669 * Close current thread specific flow workspace. 6670 * 6671 * If previous workspace available, set it as current. 6672 * 6673 * @return pointer to thread specific flow workspace data, NULL on error. 6674 */ 6675 static void 6676 mlx5_flow_pop_thread_workspace(void) 6677 { 6678 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6679 6680 if (!data) 6681 return; 6682 if (!data->inuse) { 6683 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6684 return; 6685 } 6686 data->inuse = 0; 6687 if (!data->prev) 6688 return; 6689 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6690 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6691 } 6692 6693 /** 6694 * Verify the flow list is empty 6695 * 6696 * @param dev 6697 * Pointer to Ethernet device. 6698 * 6699 * @return the number of flows not released. 6700 */ 6701 int 6702 mlx5_flow_verify(struct rte_eth_dev *dev) 6703 { 6704 struct mlx5_priv *priv = dev->data->dev_private; 6705 struct rte_flow *flow; 6706 uint32_t idx; 6707 int ret = 0; 6708 6709 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 6710 flow, next) { 6711 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6712 dev->data->port_id, (void *)flow); 6713 ++ret; 6714 } 6715 return ret; 6716 } 6717 6718 /** 6719 * Enable default hairpin egress flow. 6720 * 6721 * @param dev 6722 * Pointer to Ethernet device. 6723 * @param queue 6724 * The queue index. 6725 * 6726 * @return 6727 * 0 on success, a negative errno value otherwise and rte_errno is set. 6728 */ 6729 int 6730 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6731 uint32_t queue) 6732 { 6733 struct mlx5_priv *priv = dev->data->dev_private; 6734 const struct rte_flow_attr attr = { 6735 .egress = 1, 6736 .priority = 0, 6737 }; 6738 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6739 .queue = queue, 6740 }; 6741 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6742 .queue = UINT32_MAX, 6743 }; 6744 struct rte_flow_item items[] = { 6745 { 6746 .type = (enum rte_flow_item_type) 6747 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6748 .spec = &queue_spec, 6749 .last = NULL, 6750 .mask = &queue_mask, 6751 }, 6752 { 6753 .type = RTE_FLOW_ITEM_TYPE_END, 6754 }, 6755 }; 6756 struct rte_flow_action_jump jump = { 6757 .group = MLX5_HAIRPIN_TX_TABLE, 6758 }; 6759 struct rte_flow_action actions[2]; 6760 uint32_t flow_idx; 6761 struct rte_flow_error error; 6762 6763 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6764 actions[0].conf = &jump; 6765 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6766 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6767 &attr, items, actions, false, &error); 6768 if (!flow_idx) { 6769 DRV_LOG(DEBUG, 6770 "Failed to create ctrl flow: rte_errno(%d)," 6771 " type(%d), message(%s)", 6772 rte_errno, error.type, 6773 error.message ? error.message : " (no stated reason)"); 6774 return -rte_errno; 6775 } 6776 return 0; 6777 } 6778 6779 /** 6780 * Enable a control flow configured from the control plane. 6781 * 6782 * @param dev 6783 * Pointer to Ethernet device. 6784 * @param eth_spec 6785 * An Ethernet flow spec to apply. 6786 * @param eth_mask 6787 * An Ethernet flow mask to apply. 6788 * @param vlan_spec 6789 * A VLAN flow spec to apply. 6790 * @param vlan_mask 6791 * A VLAN flow mask to apply. 6792 * 6793 * @return 6794 * 0 on success, a negative errno value otherwise and rte_errno is set. 6795 */ 6796 int 6797 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6798 struct rte_flow_item_eth *eth_spec, 6799 struct rte_flow_item_eth *eth_mask, 6800 struct rte_flow_item_vlan *vlan_spec, 6801 struct rte_flow_item_vlan *vlan_mask) 6802 { 6803 struct mlx5_priv *priv = dev->data->dev_private; 6804 const struct rte_flow_attr attr = { 6805 .ingress = 1, 6806 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6807 }; 6808 struct rte_flow_item items[] = { 6809 { 6810 .type = RTE_FLOW_ITEM_TYPE_ETH, 6811 .spec = eth_spec, 6812 .last = NULL, 6813 .mask = eth_mask, 6814 }, 6815 { 6816 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6817 RTE_FLOW_ITEM_TYPE_END, 6818 .spec = vlan_spec, 6819 .last = NULL, 6820 .mask = vlan_mask, 6821 }, 6822 { 6823 .type = RTE_FLOW_ITEM_TYPE_END, 6824 }, 6825 }; 6826 uint16_t queue[priv->reta_idx_n]; 6827 struct rte_flow_action_rss action_rss = { 6828 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6829 .level = 0, 6830 .types = priv->rss_conf.rss_hf, 6831 .key_len = priv->rss_conf.rss_key_len, 6832 .queue_num = priv->reta_idx_n, 6833 .key = priv->rss_conf.rss_key, 6834 .queue = queue, 6835 }; 6836 struct rte_flow_action actions[] = { 6837 { 6838 .type = RTE_FLOW_ACTION_TYPE_RSS, 6839 .conf = &action_rss, 6840 }, 6841 { 6842 .type = RTE_FLOW_ACTION_TYPE_END, 6843 }, 6844 }; 6845 uint32_t flow_idx; 6846 struct rte_flow_error error; 6847 unsigned int i; 6848 6849 if (!priv->reta_idx_n || !priv->rxqs_n) { 6850 return 0; 6851 } 6852 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6853 action_rss.types = 0; 6854 for (i = 0; i != priv->reta_idx_n; ++i) 6855 queue[i] = (*priv->reta_idx)[i]; 6856 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6857 &attr, items, actions, false, &error); 6858 if (!flow_idx) 6859 return -rte_errno; 6860 return 0; 6861 } 6862 6863 /** 6864 * Enable a flow control configured from the control plane. 6865 * 6866 * @param dev 6867 * Pointer to Ethernet device. 6868 * @param eth_spec 6869 * An Ethernet flow spec to apply. 6870 * @param eth_mask 6871 * An Ethernet flow mask to apply. 6872 * 6873 * @return 6874 * 0 on success, a negative errno value otherwise and rte_errno is set. 6875 */ 6876 int 6877 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6878 struct rte_flow_item_eth *eth_spec, 6879 struct rte_flow_item_eth *eth_mask) 6880 { 6881 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6882 } 6883 6884 /** 6885 * Create default miss flow rule matching lacp traffic 6886 * 6887 * @param dev 6888 * Pointer to Ethernet device. 6889 * @param eth_spec 6890 * An Ethernet flow spec to apply. 6891 * 6892 * @return 6893 * 0 on success, a negative errno value otherwise and rte_errno is set. 6894 */ 6895 int 6896 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6897 { 6898 struct mlx5_priv *priv = dev->data->dev_private; 6899 /* 6900 * The LACP matching is done by only using ether type since using 6901 * a multicast dst mac causes kernel to give low priority to this flow. 6902 */ 6903 static const struct rte_flow_item_eth lacp_spec = { 6904 .type = RTE_BE16(0x8809), 6905 }; 6906 static const struct rte_flow_item_eth lacp_mask = { 6907 .type = 0xffff, 6908 }; 6909 const struct rte_flow_attr attr = { 6910 .ingress = 1, 6911 }; 6912 struct rte_flow_item items[] = { 6913 { 6914 .type = RTE_FLOW_ITEM_TYPE_ETH, 6915 .spec = &lacp_spec, 6916 .mask = &lacp_mask, 6917 }, 6918 { 6919 .type = RTE_FLOW_ITEM_TYPE_END, 6920 }, 6921 }; 6922 struct rte_flow_action actions[] = { 6923 { 6924 .type = (enum rte_flow_action_type) 6925 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6926 }, 6927 { 6928 .type = RTE_FLOW_ACTION_TYPE_END, 6929 }, 6930 }; 6931 struct rte_flow_error error; 6932 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6933 &attr, items, actions, false, &error); 6934 6935 if (!flow_idx) 6936 return -rte_errno; 6937 return 0; 6938 } 6939 6940 /** 6941 * Destroy a flow. 6942 * 6943 * @see rte_flow_destroy() 6944 * @see rte_flow_ops 6945 */ 6946 int 6947 mlx5_flow_destroy(struct rte_eth_dev *dev, 6948 struct rte_flow *flow, 6949 struct rte_flow_error *error __rte_unused) 6950 { 6951 struct mlx5_priv *priv = dev->data->dev_private; 6952 6953 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6954 return 0; 6955 } 6956 6957 /** 6958 * Destroy all flows. 6959 * 6960 * @see rte_flow_flush() 6961 * @see rte_flow_ops 6962 */ 6963 int 6964 mlx5_flow_flush(struct rte_eth_dev *dev, 6965 struct rte_flow_error *error __rte_unused) 6966 { 6967 struct mlx5_priv *priv = dev->data->dev_private; 6968 6969 mlx5_flow_list_flush(dev, &priv->flows, false); 6970 return 0; 6971 } 6972 6973 /** 6974 * Isolated mode. 6975 * 6976 * @see rte_flow_isolate() 6977 * @see rte_flow_ops 6978 */ 6979 int 6980 mlx5_flow_isolate(struct rte_eth_dev *dev, 6981 int enable, 6982 struct rte_flow_error *error) 6983 { 6984 struct mlx5_priv *priv = dev->data->dev_private; 6985 6986 if (dev->data->dev_started) { 6987 rte_flow_error_set(error, EBUSY, 6988 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6989 NULL, 6990 "port must be stopped first"); 6991 return -rte_errno; 6992 } 6993 priv->isolated = !!enable; 6994 if (enable) 6995 dev->dev_ops = &mlx5_dev_ops_isolate; 6996 else 6997 dev->dev_ops = &mlx5_dev_ops; 6998 6999 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7000 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7001 7002 return 0; 7003 } 7004 7005 /** 7006 * Query a flow. 7007 * 7008 * @see rte_flow_query() 7009 * @see rte_flow_ops 7010 */ 7011 static int 7012 flow_drv_query(struct rte_eth_dev *dev, 7013 uint32_t flow_idx, 7014 const struct rte_flow_action *actions, 7015 void *data, 7016 struct rte_flow_error *error) 7017 { 7018 struct mlx5_priv *priv = dev->data->dev_private; 7019 const struct mlx5_flow_driver_ops *fops; 7020 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 7021 [MLX5_IPOOL_RTE_FLOW], 7022 flow_idx); 7023 enum mlx5_flow_drv_type ftype; 7024 7025 if (!flow) { 7026 return rte_flow_error_set(error, ENOENT, 7027 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7028 NULL, 7029 "invalid flow handle"); 7030 } 7031 ftype = flow->drv_type; 7032 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7033 fops = flow_get_drv_ops(ftype); 7034 7035 return fops->query(dev, flow, actions, data, error); 7036 } 7037 7038 /** 7039 * Query a flow. 7040 * 7041 * @see rte_flow_query() 7042 * @see rte_flow_ops 7043 */ 7044 int 7045 mlx5_flow_query(struct rte_eth_dev *dev, 7046 struct rte_flow *flow, 7047 const struct rte_flow_action *actions, 7048 void *data, 7049 struct rte_flow_error *error) 7050 { 7051 int ret; 7052 7053 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7054 error); 7055 if (ret < 0) 7056 return ret; 7057 return 0; 7058 } 7059 7060 /** 7061 * Get rte_flow callbacks. 7062 * 7063 * @param dev 7064 * Pointer to Ethernet device structure. 7065 * @param ops 7066 * Pointer to operation-specific structure. 7067 * 7068 * @return 0 7069 */ 7070 int 7071 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7072 const struct rte_flow_ops **ops) 7073 { 7074 *ops = &mlx5_flow_ops; 7075 return 0; 7076 } 7077 7078 /** 7079 * Validate meter policy actions. 7080 * Dispatcher for action type specific validation. 7081 * 7082 * @param[in] dev 7083 * Pointer to the Ethernet device structure. 7084 * @param[in] action 7085 * The meter policy action object to validate. 7086 * @param[in] attr 7087 * Attributes of flow to determine steering domain. 7088 * @param[out] is_rss 7089 * Is RSS or not. 7090 * @param[out] domain_bitmap 7091 * Domain bitmap. 7092 * @param[out] is_def_policy 7093 * Is default policy or not. 7094 * @param[out] error 7095 * Perform verbose error reporting if not NULL. Initialized in case of 7096 * error only. 7097 * 7098 * @return 7099 * 0 on success, otherwise negative errno value. 7100 */ 7101 int 7102 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7103 const struct rte_flow_action *actions[RTE_COLORS], 7104 struct rte_flow_attr *attr, 7105 bool *is_rss, 7106 uint8_t *domain_bitmap, 7107 bool *is_def_policy, 7108 struct rte_mtr_error *error) 7109 { 7110 const struct mlx5_flow_driver_ops *fops; 7111 7112 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7113 return fops->validate_mtr_acts(dev, actions, attr, 7114 is_rss, domain_bitmap, is_def_policy, error); 7115 } 7116 7117 /** 7118 * Destroy the meter table set. 7119 * 7120 * @param[in] dev 7121 * Pointer to Ethernet device. 7122 * @param[in] mtr_policy 7123 * Meter policy struct. 7124 */ 7125 void 7126 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7127 struct mlx5_flow_meter_policy *mtr_policy) 7128 { 7129 const struct mlx5_flow_driver_ops *fops; 7130 7131 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7132 fops->destroy_mtr_acts(dev, mtr_policy); 7133 } 7134 7135 /** 7136 * Create policy action, lock free, 7137 * (mutex should be acquired by caller). 7138 * Dispatcher for action type specific call. 7139 * 7140 * @param[in] dev 7141 * Pointer to the Ethernet device structure. 7142 * @param[in] mtr_policy 7143 * Meter policy struct. 7144 * @param[in] action 7145 * Action specification used to create meter actions. 7146 * @param[out] error 7147 * Perform verbose error reporting if not NULL. Initialized in case of 7148 * error only. 7149 * 7150 * @return 7151 * 0 on success, otherwise negative errno value. 7152 */ 7153 int 7154 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7155 struct mlx5_flow_meter_policy *mtr_policy, 7156 const struct rte_flow_action *actions[RTE_COLORS], 7157 struct rte_mtr_error *error) 7158 { 7159 const struct mlx5_flow_driver_ops *fops; 7160 7161 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7162 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7163 } 7164 7165 /** 7166 * Create policy rules, lock free, 7167 * (mutex should be acquired by caller). 7168 * Dispatcher for action type specific call. 7169 * 7170 * @param[in] dev 7171 * Pointer to the Ethernet device structure. 7172 * @param[in] mtr_policy 7173 * Meter policy struct. 7174 * 7175 * @return 7176 * 0 on success, -1 otherwise. 7177 */ 7178 int 7179 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7180 struct mlx5_flow_meter_policy *mtr_policy) 7181 { 7182 const struct mlx5_flow_driver_ops *fops; 7183 7184 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7185 return fops->create_policy_rules(dev, mtr_policy); 7186 } 7187 7188 /** 7189 * Destroy policy rules, lock free, 7190 * (mutex should be acquired by caller). 7191 * Dispatcher for action type specific call. 7192 * 7193 * @param[in] dev 7194 * Pointer to the Ethernet device structure. 7195 * @param[in] mtr_policy 7196 * Meter policy struct. 7197 */ 7198 void 7199 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7200 struct mlx5_flow_meter_policy *mtr_policy) 7201 { 7202 const struct mlx5_flow_driver_ops *fops; 7203 7204 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7205 fops->destroy_policy_rules(dev, mtr_policy); 7206 } 7207 7208 /** 7209 * Destroy the default policy table set. 7210 * 7211 * @param[in] dev 7212 * Pointer to Ethernet device. 7213 */ 7214 void 7215 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7216 { 7217 const struct mlx5_flow_driver_ops *fops; 7218 7219 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7220 fops->destroy_def_policy(dev); 7221 } 7222 7223 /** 7224 * Destroy the default policy table set. 7225 * 7226 * @param[in] dev 7227 * Pointer to Ethernet device. 7228 * 7229 * @return 7230 * 0 on success, -1 otherwise. 7231 */ 7232 int 7233 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7234 { 7235 const struct mlx5_flow_driver_ops *fops; 7236 7237 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7238 return fops->create_def_policy(dev); 7239 } 7240 7241 /** 7242 * Create the needed meter and suffix tables. 7243 * 7244 * @param[in] dev 7245 * Pointer to Ethernet device. 7246 * 7247 * @return 7248 * 0 on success, -1 otherwise. 7249 */ 7250 int 7251 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7252 struct mlx5_flow_meter_info *fm, 7253 uint32_t mtr_idx, 7254 uint8_t domain_bitmap) 7255 { 7256 const struct mlx5_flow_driver_ops *fops; 7257 7258 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7259 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7260 } 7261 7262 /** 7263 * Destroy the meter table set. 7264 * 7265 * @param[in] dev 7266 * Pointer to Ethernet device. 7267 * @param[in] tbl 7268 * Pointer to the meter table set. 7269 */ 7270 void 7271 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7272 struct mlx5_flow_meter_info *fm) 7273 { 7274 const struct mlx5_flow_driver_ops *fops; 7275 7276 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7277 fops->destroy_mtr_tbls(dev, fm); 7278 } 7279 7280 /** 7281 * Destroy the global meter drop table. 7282 * 7283 * @param[in] dev 7284 * Pointer to Ethernet device. 7285 */ 7286 void 7287 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7288 { 7289 const struct mlx5_flow_driver_ops *fops; 7290 7291 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7292 fops->destroy_mtr_drop_tbls(dev); 7293 } 7294 7295 /** 7296 * Destroy the sub policy table with RX queue. 7297 * 7298 * @param[in] dev 7299 * Pointer to Ethernet device. 7300 * @param[in] mtr_policy 7301 * Pointer to meter policy table. 7302 */ 7303 void 7304 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7305 struct mlx5_flow_meter_policy *mtr_policy) 7306 { 7307 const struct mlx5_flow_driver_ops *fops; 7308 7309 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7310 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7311 } 7312 7313 /** 7314 * Allocate the needed aso flow meter id. 7315 * 7316 * @param[in] dev 7317 * Pointer to Ethernet device. 7318 * 7319 * @return 7320 * Index to aso flow meter on success, NULL otherwise. 7321 */ 7322 uint32_t 7323 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7324 { 7325 const struct mlx5_flow_driver_ops *fops; 7326 7327 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7328 return fops->create_meter(dev); 7329 } 7330 7331 /** 7332 * Free the aso flow meter id. 7333 * 7334 * @param[in] dev 7335 * Pointer to Ethernet device. 7336 * @param[in] mtr_idx 7337 * Index to aso flow meter to be free. 7338 * 7339 * @return 7340 * 0 on success. 7341 */ 7342 void 7343 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7344 { 7345 const struct mlx5_flow_driver_ops *fops; 7346 7347 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7348 fops->free_meter(dev, mtr_idx); 7349 } 7350 7351 /** 7352 * Allocate a counter. 7353 * 7354 * @param[in] dev 7355 * Pointer to Ethernet device structure. 7356 * 7357 * @return 7358 * Index to allocated counter on success, 0 otherwise. 7359 */ 7360 uint32_t 7361 mlx5_counter_alloc(struct rte_eth_dev *dev) 7362 { 7363 const struct mlx5_flow_driver_ops *fops; 7364 struct rte_flow_attr attr = { .transfer = 0 }; 7365 7366 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7367 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7368 return fops->counter_alloc(dev); 7369 } 7370 DRV_LOG(ERR, 7371 "port %u counter allocate is not supported.", 7372 dev->data->port_id); 7373 return 0; 7374 } 7375 7376 /** 7377 * Free a counter. 7378 * 7379 * @param[in] dev 7380 * Pointer to Ethernet device structure. 7381 * @param[in] cnt 7382 * Index to counter to be free. 7383 */ 7384 void 7385 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7386 { 7387 const struct mlx5_flow_driver_ops *fops; 7388 struct rte_flow_attr attr = { .transfer = 0 }; 7389 7390 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7391 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7392 fops->counter_free(dev, cnt); 7393 return; 7394 } 7395 DRV_LOG(ERR, 7396 "port %u counter free is not supported.", 7397 dev->data->port_id); 7398 } 7399 7400 /** 7401 * Query counter statistics. 7402 * 7403 * @param[in] dev 7404 * Pointer to Ethernet device structure. 7405 * @param[in] cnt 7406 * Index to counter to query. 7407 * @param[in] clear 7408 * Set to clear counter statistics. 7409 * @param[out] pkts 7410 * The counter hits packets number to save. 7411 * @param[out] bytes 7412 * The counter hits bytes number to save. 7413 * 7414 * @return 7415 * 0 on success, a negative errno value otherwise. 7416 */ 7417 int 7418 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7419 bool clear, uint64_t *pkts, uint64_t *bytes) 7420 { 7421 const struct mlx5_flow_driver_ops *fops; 7422 struct rte_flow_attr attr = { .transfer = 0 }; 7423 7424 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7425 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7426 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7427 } 7428 DRV_LOG(ERR, 7429 "port %u counter query is not supported.", 7430 dev->data->port_id); 7431 return -ENOTSUP; 7432 } 7433 7434 /** 7435 * Allocate a new memory for the counter values wrapped by all the needed 7436 * management. 7437 * 7438 * @param[in] sh 7439 * Pointer to mlx5_dev_ctx_shared object. 7440 * 7441 * @return 7442 * 0 on success, a negative errno value otherwise. 7443 */ 7444 static int 7445 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7446 { 7447 struct mlx5_devx_mkey_attr mkey_attr; 7448 struct mlx5_counter_stats_mem_mng *mem_mng; 7449 volatile struct flow_counter_stats *raw_data; 7450 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7451 int size = (sizeof(struct flow_counter_stats) * 7452 MLX5_COUNTERS_PER_POOL + 7453 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7454 sizeof(struct mlx5_counter_stats_mem_mng); 7455 size_t pgsize = rte_mem_page_size(); 7456 uint8_t *mem; 7457 int i; 7458 7459 if (pgsize == (size_t)-1) { 7460 DRV_LOG(ERR, "Failed to get mem page size"); 7461 rte_errno = ENOMEM; 7462 return -ENOMEM; 7463 } 7464 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7465 if (!mem) { 7466 rte_errno = ENOMEM; 7467 return -ENOMEM; 7468 } 7469 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7470 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7471 mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size, 7472 IBV_ACCESS_LOCAL_WRITE); 7473 if (!mem_mng->umem) { 7474 rte_errno = errno; 7475 mlx5_free(mem); 7476 return -rte_errno; 7477 } 7478 memset(&mkey_attr, 0, sizeof(mkey_attr)); 7479 mkey_attr.addr = (uintptr_t)mem; 7480 mkey_attr.size = size; 7481 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7482 mkey_attr.pd = sh->pdn; 7483 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 7484 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 7485 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7486 if (!mem_mng->dm) { 7487 mlx5_os_umem_dereg(mem_mng->umem); 7488 rte_errno = errno; 7489 mlx5_free(mem); 7490 return -rte_errno; 7491 } 7492 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7493 raw_data = (volatile struct flow_counter_stats *)mem; 7494 for (i = 0; i < raws_n; ++i) { 7495 mem_mng->raws[i].mem_mng = mem_mng; 7496 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7497 } 7498 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7499 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7500 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7501 next); 7502 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7503 sh->cmng.mem_mng = mem_mng; 7504 return 0; 7505 } 7506 7507 /** 7508 * Set the statistic memory to the new counter pool. 7509 * 7510 * @param[in] sh 7511 * Pointer to mlx5_dev_ctx_shared object. 7512 * @param[in] pool 7513 * Pointer to the pool to set the statistic memory. 7514 * 7515 * @return 7516 * 0 on success, a negative errno value otherwise. 7517 */ 7518 static int 7519 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7520 struct mlx5_flow_counter_pool *pool) 7521 { 7522 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7523 /* Resize statistic memory once used out. */ 7524 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7525 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7526 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7527 return -1; 7528 } 7529 rte_spinlock_lock(&pool->sl); 7530 pool->raw = cmng->mem_mng->raws + pool->index % 7531 MLX5_CNT_CONTAINER_RESIZE; 7532 rte_spinlock_unlock(&pool->sl); 7533 pool->raw_hw = NULL; 7534 return 0; 7535 } 7536 7537 #define MLX5_POOL_QUERY_FREQ_US 1000000 7538 7539 /** 7540 * Set the periodic procedure for triggering asynchronous batch queries for all 7541 * the counter pools. 7542 * 7543 * @param[in] sh 7544 * Pointer to mlx5_dev_ctx_shared object. 7545 */ 7546 void 7547 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7548 { 7549 uint32_t pools_n, us; 7550 7551 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7552 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7553 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7554 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7555 sh->cmng.query_thread_on = 0; 7556 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7557 } else { 7558 sh->cmng.query_thread_on = 1; 7559 } 7560 } 7561 7562 /** 7563 * The periodic procedure for triggering asynchronous batch queries for all the 7564 * counter pools. This function is probably called by the host thread. 7565 * 7566 * @param[in] arg 7567 * The parameter for the alarm process. 7568 */ 7569 void 7570 mlx5_flow_query_alarm(void *arg) 7571 { 7572 struct mlx5_dev_ctx_shared *sh = arg; 7573 int ret; 7574 uint16_t pool_index = sh->cmng.pool_index; 7575 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7576 struct mlx5_flow_counter_pool *pool; 7577 uint16_t n_valid; 7578 7579 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7580 goto set_alarm; 7581 rte_spinlock_lock(&cmng->pool_update_sl); 7582 pool = cmng->pools[pool_index]; 7583 n_valid = cmng->n_valid; 7584 rte_spinlock_unlock(&cmng->pool_update_sl); 7585 /* Set the statistic memory to the new created pool. */ 7586 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7587 goto set_alarm; 7588 if (pool->raw_hw) 7589 /* There is a pool query in progress. */ 7590 goto set_alarm; 7591 pool->raw_hw = 7592 LIST_FIRST(&sh->cmng.free_stat_raws); 7593 if (!pool->raw_hw) 7594 /* No free counter statistics raw memory. */ 7595 goto set_alarm; 7596 /* 7597 * Identify the counters released between query trigger and query 7598 * handle more efficiently. The counter released in this gap period 7599 * should wait for a new round of query as the new arrived packets 7600 * will not be taken into account. 7601 */ 7602 pool->query_gen++; 7603 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7604 MLX5_COUNTERS_PER_POOL, 7605 NULL, NULL, 7606 pool->raw_hw->mem_mng->dm->id, 7607 (void *)(uintptr_t) 7608 pool->raw_hw->data, 7609 sh->devx_comp, 7610 (uint64_t)(uintptr_t)pool); 7611 if (ret) { 7612 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7613 " %d", pool->min_dcs->id); 7614 pool->raw_hw = NULL; 7615 goto set_alarm; 7616 } 7617 LIST_REMOVE(pool->raw_hw, next); 7618 sh->cmng.pending_queries++; 7619 pool_index++; 7620 if (pool_index >= n_valid) 7621 pool_index = 0; 7622 set_alarm: 7623 sh->cmng.pool_index = pool_index; 7624 mlx5_set_query_alarm(sh); 7625 } 7626 7627 /** 7628 * Check and callback event for new aged flow in the counter pool 7629 * 7630 * @param[in] sh 7631 * Pointer to mlx5_dev_ctx_shared object. 7632 * @param[in] pool 7633 * Pointer to Current counter pool. 7634 */ 7635 static void 7636 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7637 struct mlx5_flow_counter_pool *pool) 7638 { 7639 struct mlx5_priv *priv; 7640 struct mlx5_flow_counter *cnt; 7641 struct mlx5_age_info *age_info; 7642 struct mlx5_age_param *age_param; 7643 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7644 struct mlx5_counter_stats_raw *prev = pool->raw; 7645 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7646 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7647 uint16_t expected = AGE_CANDIDATE; 7648 uint32_t i; 7649 7650 pool->time_of_last_age_check = curr_time; 7651 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7652 cnt = MLX5_POOL_GET_CNT(pool, i); 7653 age_param = MLX5_CNT_TO_AGE(cnt); 7654 if (__atomic_load_n(&age_param->state, 7655 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7656 continue; 7657 if (cur->data[i].hits != prev->data[i].hits) { 7658 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7659 __ATOMIC_RELAXED); 7660 continue; 7661 } 7662 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7663 time_delta, 7664 __ATOMIC_RELAXED) <= age_param->timeout) 7665 continue; 7666 /** 7667 * Hold the lock first, or if between the 7668 * state AGE_TMOUT and tailq operation the 7669 * release happened, the release procedure 7670 * may delete a non-existent tailq node. 7671 */ 7672 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7673 age_info = GET_PORT_AGE_INFO(priv); 7674 rte_spinlock_lock(&age_info->aged_sl); 7675 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7676 AGE_TMOUT, false, 7677 __ATOMIC_RELAXED, 7678 __ATOMIC_RELAXED)) { 7679 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7680 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7681 } 7682 rte_spinlock_unlock(&age_info->aged_sl); 7683 } 7684 mlx5_age_event_prepare(sh); 7685 } 7686 7687 /** 7688 * Handler for the HW respond about ready values from an asynchronous batch 7689 * query. This function is probably called by the host thread. 7690 * 7691 * @param[in] sh 7692 * The pointer to the shared device context. 7693 * @param[in] async_id 7694 * The Devx async ID. 7695 * @param[in] status 7696 * The status of the completion. 7697 */ 7698 void 7699 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7700 uint64_t async_id, int status) 7701 { 7702 struct mlx5_flow_counter_pool *pool = 7703 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7704 struct mlx5_counter_stats_raw *raw_to_free; 7705 uint8_t query_gen = pool->query_gen ^ 1; 7706 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7707 enum mlx5_counter_type cnt_type = 7708 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7709 MLX5_COUNTER_TYPE_ORIGIN; 7710 7711 if (unlikely(status)) { 7712 raw_to_free = pool->raw_hw; 7713 } else { 7714 raw_to_free = pool->raw; 7715 if (pool->is_aged) 7716 mlx5_flow_aging_check(sh, pool); 7717 rte_spinlock_lock(&pool->sl); 7718 pool->raw = pool->raw_hw; 7719 rte_spinlock_unlock(&pool->sl); 7720 /* Be sure the new raw counters data is updated in memory. */ 7721 rte_io_wmb(); 7722 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7723 rte_spinlock_lock(&cmng->csl[cnt_type]); 7724 TAILQ_CONCAT(&cmng->counters[cnt_type], 7725 &pool->counters[query_gen], next); 7726 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7727 } 7728 } 7729 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7730 pool->raw_hw = NULL; 7731 sh->cmng.pending_queries--; 7732 } 7733 7734 static int 7735 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7736 const struct flow_grp_info *grp_info, 7737 struct rte_flow_error *error) 7738 { 7739 if (grp_info->transfer && grp_info->external && 7740 grp_info->fdb_def_rule) { 7741 if (group == UINT32_MAX) 7742 return rte_flow_error_set 7743 (error, EINVAL, 7744 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7745 NULL, 7746 "group index not supported"); 7747 *table = group + 1; 7748 } else { 7749 *table = group; 7750 } 7751 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7752 return 0; 7753 } 7754 7755 /** 7756 * Translate the rte_flow group index to HW table value. 7757 * 7758 * If tunnel offload is disabled, all group ids converted to flow table 7759 * id using the standard method. 7760 * If tunnel offload is enabled, group id can be converted using the 7761 * standard or tunnel conversion method. Group conversion method 7762 * selection depends on flags in `grp_info` parameter: 7763 * - Internal (grp_info.external == 0) groups conversion uses the 7764 * standard method. 7765 * - Group ids in JUMP action converted with the tunnel conversion. 7766 * - Group id in rule attribute conversion depends on a rule type and 7767 * group id value: 7768 * ** non zero group attributes converted with the tunnel method 7769 * ** zero group attribute in non-tunnel rule is converted using the 7770 * standard method - there's only one root table 7771 * ** zero group attribute in steer tunnel rule is converted with the 7772 * standard method - single root table 7773 * ** zero group attribute in match tunnel rule is a special OvS 7774 * case: that value is used for portability reasons. That group 7775 * id is converted with the tunnel conversion method. 7776 * 7777 * @param[in] dev 7778 * Port device 7779 * @param[in] tunnel 7780 * PMD tunnel offload object 7781 * @param[in] group 7782 * rte_flow group index value. 7783 * @param[out] table 7784 * HW table value. 7785 * @param[in] grp_info 7786 * flags used for conversion 7787 * @param[out] error 7788 * Pointer to error structure. 7789 * 7790 * @return 7791 * 0 on success, a negative errno value otherwise and rte_errno is set. 7792 */ 7793 int 7794 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7795 const struct mlx5_flow_tunnel *tunnel, 7796 uint32_t group, uint32_t *table, 7797 const struct flow_grp_info *grp_info, 7798 struct rte_flow_error *error) 7799 { 7800 int ret; 7801 bool standard_translation; 7802 7803 if (!grp_info->skip_scale && grp_info->external && 7804 group < MLX5_MAX_TABLES_EXTERNAL) 7805 group *= MLX5_FLOW_TABLE_FACTOR; 7806 if (is_tunnel_offload_active(dev)) { 7807 standard_translation = !grp_info->external || 7808 grp_info->std_tbl_fix; 7809 } else { 7810 standard_translation = true; 7811 } 7812 DRV_LOG(DEBUG, 7813 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 7814 dev->data->port_id, group, grp_info->transfer, 7815 grp_info->external, grp_info->fdb_def_rule, 7816 standard_translation ? "STANDARD" : "TUNNEL"); 7817 if (standard_translation) 7818 ret = flow_group_to_table(dev->data->port_id, group, table, 7819 grp_info, error); 7820 else 7821 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7822 table, error); 7823 7824 return ret; 7825 } 7826 7827 /** 7828 * Discover availability of metadata reg_c's. 7829 * 7830 * Iteratively use test flows to check availability. 7831 * 7832 * @param[in] dev 7833 * Pointer to the Ethernet device structure. 7834 * 7835 * @return 7836 * 0 on success, a negative errno value otherwise and rte_errno is set. 7837 */ 7838 int 7839 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7840 { 7841 struct mlx5_priv *priv = dev->data->dev_private; 7842 struct mlx5_dev_config *config = &priv->config; 7843 enum modify_reg idx; 7844 int n = 0; 7845 7846 /* reg_c[0] and reg_c[1] are reserved. */ 7847 config->flow_mreg_c[n++] = REG_C_0; 7848 config->flow_mreg_c[n++] = REG_C_1; 7849 /* Discover availability of other reg_c's. */ 7850 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7851 struct rte_flow_attr attr = { 7852 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7853 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7854 .ingress = 1, 7855 }; 7856 struct rte_flow_item items[] = { 7857 [0] = { 7858 .type = RTE_FLOW_ITEM_TYPE_END, 7859 }, 7860 }; 7861 struct rte_flow_action actions[] = { 7862 [0] = { 7863 .type = (enum rte_flow_action_type) 7864 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7865 .conf = &(struct mlx5_flow_action_copy_mreg){ 7866 .src = REG_C_1, 7867 .dst = idx, 7868 }, 7869 }, 7870 [1] = { 7871 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7872 .conf = &(struct rte_flow_action_jump){ 7873 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7874 }, 7875 }, 7876 [2] = { 7877 .type = RTE_FLOW_ACTION_TYPE_END, 7878 }, 7879 }; 7880 uint32_t flow_idx; 7881 struct rte_flow *flow; 7882 struct rte_flow_error error; 7883 7884 if (!config->dv_flow_en) 7885 break; 7886 /* Create internal flow, validation skips copy action. */ 7887 flow_idx = flow_list_create(dev, NULL, &attr, items, 7888 actions, false, &error); 7889 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 7890 flow_idx); 7891 if (!flow) 7892 continue; 7893 config->flow_mreg_c[n++] = idx; 7894 flow_list_destroy(dev, NULL, flow_idx); 7895 } 7896 for (; n < MLX5_MREG_C_NUM; ++n) 7897 config->flow_mreg_c[n] = REG_NON; 7898 return 0; 7899 } 7900 7901 int 7902 save_dump_file(const uint8_t *data, uint32_t size, 7903 uint32_t type, uint32_t id, void *arg, FILE *file) 7904 { 7905 char line[BUF_SIZE]; 7906 uint32_t out = 0; 7907 uint32_t k; 7908 uint32_t actions_num; 7909 struct rte_flow_query_count *count; 7910 7911 memset(line, 0, BUF_SIZE); 7912 switch (type) { 7913 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 7914 actions_num = *(uint32_t *)(arg); 7915 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,", 7916 type, id, actions_num); 7917 break; 7918 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 7919 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,", 7920 type, id); 7921 break; 7922 case DR_DUMP_REC_TYPE_PMD_COUNTER: 7923 count = (struct rte_flow_query_count *)arg; 7924 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type, 7925 id, count->hits, count->bytes); 7926 return 0; 7927 default: 7928 return -1; 7929 } 7930 7931 for (k = 0; k < size; k++) { 7932 /* Make sure we do not overrun the line buffer length. */ 7933 if (out >= BUF_SIZE - 4) { 7934 line[out] = '\0'; 7935 break; 7936 } 7937 out += snprintf(line + out, BUF_SIZE - out, "%02x", 7938 (data[k]) & 0xff); 7939 } 7940 fprintf(file, "%s\n", line); 7941 return 0; 7942 } 7943 7944 int 7945 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 7946 struct rte_flow_query_count *count, struct rte_flow_error *error) 7947 { 7948 struct rte_flow_action action[2]; 7949 enum mlx5_flow_drv_type ftype; 7950 const struct mlx5_flow_driver_ops *fops; 7951 7952 if (!flow) { 7953 return rte_flow_error_set(error, ENOENT, 7954 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7955 NULL, 7956 "invalid flow handle"); 7957 } 7958 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 7959 action[1].type = RTE_FLOW_ACTION_TYPE_END; 7960 if (flow->counter) { 7961 memset(count, 0, sizeof(struct rte_flow_query_count)); 7962 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 7963 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 7964 ftype < MLX5_FLOW_TYPE_MAX); 7965 fops = flow_get_drv_ops(ftype); 7966 return fops->query(dev, flow, action, count, error); 7967 } 7968 return -1; 7969 } 7970 7971 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7972 /** 7973 * Dump flow ipool data to file 7974 * 7975 * @param[in] dev 7976 * The pointer to Ethernet device. 7977 * @param[in] file 7978 * A pointer to a file for output. 7979 * @param[out] error 7980 * Perform verbose error reporting if not NULL. PMDs initialize this 7981 * structure in case of error only. 7982 * @return 7983 * 0 on success, a negative value otherwise. 7984 */ 7985 int 7986 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 7987 struct rte_flow *flow, FILE *file, 7988 struct rte_flow_error *error) 7989 { 7990 struct mlx5_priv *priv = dev->data->dev_private; 7991 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 7992 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 7993 uint32_t handle_idx; 7994 struct mlx5_flow_handle *dh; 7995 struct rte_flow_query_count count; 7996 uint32_t actions_num; 7997 const uint8_t *data; 7998 size_t size; 7999 uint32_t id; 8000 uint32_t type; 8001 8002 if (!flow) { 8003 return rte_flow_error_set(error, ENOENT, 8004 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8005 NULL, 8006 "invalid flow handle"); 8007 } 8008 handle_idx = flow->dev_handles; 8009 while (handle_idx) { 8010 dh = mlx5_ipool_get(priv->sh->ipool 8011 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8012 if (!dh) 8013 continue; 8014 handle_idx = dh->next.next; 8015 id = (uint32_t)(uintptr_t)dh->drv_flow; 8016 8017 /* query counter */ 8018 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8019 if (!mlx5_flow_query_counter(dev, flow, &count, error)) 8020 save_dump_file(NULL, 0, type, 8021 id, (void *)&count, file); 8022 8023 /* Get modify_hdr and encap_decap buf from ipools. */ 8024 encap_decap = NULL; 8025 modify_hdr = dh->dvh.modify_hdr; 8026 8027 if (dh->dvh.rix_encap_decap) { 8028 encap_decap = mlx5_ipool_get(priv->sh->ipool 8029 [MLX5_IPOOL_DECAP_ENCAP], 8030 dh->dvh.rix_encap_decap); 8031 } 8032 if (modify_hdr) { 8033 data = (const uint8_t *)modify_hdr->actions; 8034 size = (size_t)(modify_hdr->actions_num) * 8; 8035 actions_num = modify_hdr->actions_num; 8036 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8037 save_dump_file(data, size, type, id, 8038 (void *)(&actions_num), file); 8039 } 8040 if (encap_decap) { 8041 data = encap_decap->buf; 8042 size = encap_decap->size; 8043 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8044 save_dump_file(data, size, type, 8045 id, NULL, file); 8046 } 8047 } 8048 return 0; 8049 } 8050 #endif 8051 8052 /** 8053 * Dump flow raw hw data to file 8054 * 8055 * @param[in] dev 8056 * The pointer to Ethernet device. 8057 * @param[in] file 8058 * A pointer to a file for output. 8059 * @param[out] error 8060 * Perform verbose error reporting if not NULL. PMDs initialize this 8061 * structure in case of error only. 8062 * @return 8063 * 0 on success, a nagative value otherwise. 8064 */ 8065 int 8066 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8067 FILE *file, 8068 struct rte_flow_error *error __rte_unused) 8069 { 8070 struct mlx5_priv *priv = dev->data->dev_private; 8071 struct mlx5_dev_ctx_shared *sh = priv->sh; 8072 uint32_t handle_idx; 8073 int ret; 8074 struct mlx5_flow_handle *dh; 8075 struct rte_flow *flow; 8076 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8077 uint32_t idx; 8078 #endif 8079 8080 if (!priv->config.dv_flow_en) { 8081 if (fputs("device dv flow disabled\n", file) <= 0) 8082 return -errno; 8083 return -ENOTSUP; 8084 } 8085 8086 /* dump all */ 8087 if (!flow_idx) { 8088 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8089 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 8090 priv->flows, idx, flow, next) 8091 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8092 #endif 8093 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8094 sh->rx_domain, 8095 sh->tx_domain, file); 8096 } 8097 /* dump one */ 8098 flow = mlx5_ipool_get(priv->sh->ipool 8099 [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx); 8100 if (!flow) 8101 return -ENOENT; 8102 8103 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8104 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8105 #endif 8106 handle_idx = flow->dev_handles; 8107 while (handle_idx) { 8108 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8109 handle_idx); 8110 if (!dh) 8111 return -ENOENT; 8112 if (dh->drv_flow) { 8113 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8114 file); 8115 if (ret) 8116 return -ENOENT; 8117 } 8118 handle_idx = dh->next.next; 8119 } 8120 return 0; 8121 } 8122 8123 /** 8124 * Get aged-out flows. 8125 * 8126 * @param[in] dev 8127 * Pointer to the Ethernet device structure. 8128 * @param[in] context 8129 * The address of an array of pointers to the aged-out flows contexts. 8130 * @param[in] nb_countexts 8131 * The length of context array pointers. 8132 * @param[out] error 8133 * Perform verbose error reporting if not NULL. Initialized in case of 8134 * error only. 8135 * 8136 * @return 8137 * how many contexts get in success, otherwise negative errno value. 8138 * if nb_contexts is 0, return the amount of all aged contexts. 8139 * if nb_contexts is not 0 , return the amount of aged flows reported 8140 * in the context array. 8141 */ 8142 int 8143 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8144 uint32_t nb_contexts, struct rte_flow_error *error) 8145 { 8146 const struct mlx5_flow_driver_ops *fops; 8147 struct rte_flow_attr attr = { .transfer = 0 }; 8148 8149 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8150 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8151 return fops->get_aged_flows(dev, contexts, nb_contexts, 8152 error); 8153 } 8154 DRV_LOG(ERR, 8155 "port %u get aged flows is not supported.", 8156 dev->data->port_id); 8157 return -ENOTSUP; 8158 } 8159 8160 /* Wrapper for driver action_validate op callback */ 8161 static int 8162 flow_drv_action_validate(struct rte_eth_dev *dev, 8163 const struct rte_flow_indir_action_conf *conf, 8164 const struct rte_flow_action *action, 8165 const struct mlx5_flow_driver_ops *fops, 8166 struct rte_flow_error *error) 8167 { 8168 static const char err_msg[] = "indirect action validation unsupported"; 8169 8170 if (!fops->action_validate) { 8171 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8172 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8173 NULL, err_msg); 8174 return -rte_errno; 8175 } 8176 return fops->action_validate(dev, conf, action, error); 8177 } 8178 8179 /** 8180 * Destroys the shared action by handle. 8181 * 8182 * @param dev 8183 * Pointer to Ethernet device structure. 8184 * @param[in] handle 8185 * Handle for the indirect action object to be destroyed. 8186 * @param[out] error 8187 * Perform verbose error reporting if not NULL. PMDs initialize this 8188 * structure in case of error only. 8189 * 8190 * @return 8191 * 0 on success, a negative errno value otherwise and rte_errno is set. 8192 * 8193 * @note: wrapper for driver action_create op callback. 8194 */ 8195 static int 8196 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8197 struct rte_flow_action_handle *handle, 8198 struct rte_flow_error *error) 8199 { 8200 static const char err_msg[] = "indirect action destruction unsupported"; 8201 struct rte_flow_attr attr = { .transfer = 0 }; 8202 const struct mlx5_flow_driver_ops *fops = 8203 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8204 8205 if (!fops->action_destroy) { 8206 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8207 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8208 NULL, err_msg); 8209 return -rte_errno; 8210 } 8211 return fops->action_destroy(dev, handle, error); 8212 } 8213 8214 /* Wrapper for driver action_destroy op callback */ 8215 static int 8216 flow_drv_action_update(struct rte_eth_dev *dev, 8217 struct rte_flow_action_handle *handle, 8218 const void *update, 8219 const struct mlx5_flow_driver_ops *fops, 8220 struct rte_flow_error *error) 8221 { 8222 static const char err_msg[] = "indirect action update unsupported"; 8223 8224 if (!fops->action_update) { 8225 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8226 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8227 NULL, err_msg); 8228 return -rte_errno; 8229 } 8230 return fops->action_update(dev, handle, update, error); 8231 } 8232 8233 /* Wrapper for driver action_destroy op callback */ 8234 static int 8235 flow_drv_action_query(struct rte_eth_dev *dev, 8236 const struct rte_flow_action_handle *handle, 8237 void *data, 8238 const struct mlx5_flow_driver_ops *fops, 8239 struct rte_flow_error *error) 8240 { 8241 static const char err_msg[] = "indirect action query unsupported"; 8242 8243 if (!fops->action_query) { 8244 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8245 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8246 NULL, err_msg); 8247 return -rte_errno; 8248 } 8249 return fops->action_query(dev, handle, data, error); 8250 } 8251 8252 /** 8253 * Create indirect action for reuse in multiple flow rules. 8254 * 8255 * @param dev 8256 * Pointer to Ethernet device structure. 8257 * @param conf 8258 * Pointer to indirect action object configuration. 8259 * @param[in] action 8260 * Action configuration for indirect action object creation. 8261 * @param[out] error 8262 * Perform verbose error reporting if not NULL. PMDs initialize this 8263 * structure in case of error only. 8264 * @return 8265 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8266 */ 8267 static struct rte_flow_action_handle * 8268 mlx5_action_handle_create(struct rte_eth_dev *dev, 8269 const struct rte_flow_indir_action_conf *conf, 8270 const struct rte_flow_action *action, 8271 struct rte_flow_error *error) 8272 { 8273 static const char err_msg[] = "indirect action creation unsupported"; 8274 struct rte_flow_attr attr = { .transfer = 0 }; 8275 const struct mlx5_flow_driver_ops *fops = 8276 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8277 8278 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8279 return NULL; 8280 if (!fops->action_create) { 8281 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8282 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8283 NULL, err_msg); 8284 return NULL; 8285 } 8286 return fops->action_create(dev, conf, action, error); 8287 } 8288 8289 /** 8290 * Updates inplace the indirect action configuration pointed by *handle* 8291 * with the configuration provided as *update* argument. 8292 * The update of the indirect action configuration effects all flow rules 8293 * reusing the action via handle. 8294 * 8295 * @param dev 8296 * Pointer to Ethernet device structure. 8297 * @param[in] handle 8298 * Handle for the indirect action to be updated. 8299 * @param[in] update 8300 * Action specification used to modify the action pointed by handle. 8301 * *update* could be of same type with the action pointed by the *handle* 8302 * handle argument, or some other structures like a wrapper, depending on 8303 * the indirect action type. 8304 * @param[out] error 8305 * Perform verbose error reporting if not NULL. PMDs initialize this 8306 * structure in case of error only. 8307 * 8308 * @return 8309 * 0 on success, a negative errno value otherwise and rte_errno is set. 8310 */ 8311 static int 8312 mlx5_action_handle_update(struct rte_eth_dev *dev, 8313 struct rte_flow_action_handle *handle, 8314 const void *update, 8315 struct rte_flow_error *error) 8316 { 8317 struct rte_flow_attr attr = { .transfer = 0 }; 8318 const struct mlx5_flow_driver_ops *fops = 8319 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8320 int ret; 8321 8322 ret = flow_drv_action_validate(dev, NULL, 8323 (const struct rte_flow_action *)update, fops, error); 8324 if (ret) 8325 return ret; 8326 return flow_drv_action_update(dev, handle, update, fops, 8327 error); 8328 } 8329 8330 /** 8331 * Query the indirect action by handle. 8332 * 8333 * This function allows retrieving action-specific data such as counters. 8334 * Data is gathered by special action which may be present/referenced in 8335 * more than one flow rule definition. 8336 * 8337 * see @RTE_FLOW_ACTION_TYPE_COUNT 8338 * 8339 * @param dev 8340 * Pointer to Ethernet device structure. 8341 * @param[in] handle 8342 * Handle for the indirect action to query. 8343 * @param[in, out] data 8344 * Pointer to storage for the associated query data type. 8345 * @param[out] error 8346 * Perform verbose error reporting if not NULL. PMDs initialize this 8347 * structure in case of error only. 8348 * 8349 * @return 8350 * 0 on success, a negative errno value otherwise and rte_errno is set. 8351 */ 8352 static int 8353 mlx5_action_handle_query(struct rte_eth_dev *dev, 8354 const struct rte_flow_action_handle *handle, 8355 void *data, 8356 struct rte_flow_error *error) 8357 { 8358 struct rte_flow_attr attr = { .transfer = 0 }; 8359 const struct mlx5_flow_driver_ops *fops = 8360 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8361 8362 return flow_drv_action_query(dev, handle, data, fops, error); 8363 } 8364 8365 /** 8366 * Destroy all indirect actions (shared RSS). 8367 * 8368 * @param dev 8369 * Pointer to Ethernet device. 8370 * 8371 * @return 8372 * 0 on success, a negative errno value otherwise and rte_errno is set. 8373 */ 8374 int 8375 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8376 { 8377 struct rte_flow_error error; 8378 struct mlx5_priv *priv = dev->data->dev_private; 8379 struct mlx5_shared_action_rss *shared_rss; 8380 int ret = 0; 8381 uint32_t idx; 8382 8383 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8384 priv->rss_shared_actions, idx, shared_rss, next) { 8385 ret |= mlx5_action_handle_destroy(dev, 8386 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8387 } 8388 return ret; 8389 } 8390 8391 #ifndef HAVE_MLX5DV_DR 8392 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8393 #else 8394 #define MLX5_DOMAIN_SYNC_FLOW \ 8395 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8396 #endif 8397 8398 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8399 { 8400 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8401 const struct mlx5_flow_driver_ops *fops; 8402 int ret; 8403 struct rte_flow_attr attr = { .transfer = 0 }; 8404 8405 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8406 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8407 if (ret > 0) 8408 ret = -ret; 8409 return ret; 8410 } 8411 8412 const struct mlx5_flow_tunnel * 8413 mlx5_get_tof(const struct rte_flow_item *item, 8414 const struct rte_flow_action *action, 8415 enum mlx5_tof_rule_type *rule_type) 8416 { 8417 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 8418 if (item->type == (typeof(item->type)) 8419 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 8420 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 8421 return flow_items_to_tunnel(item); 8422 } 8423 } 8424 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 8425 if (action->type == (typeof(action->type)) 8426 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 8427 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 8428 return flow_actions_to_tunnel(action); 8429 } 8430 } 8431 return NULL; 8432 } 8433 8434 /** 8435 * tunnel offload functionalilty is defined for DV environment only 8436 */ 8437 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8438 __extension__ 8439 union tunnel_offload_mark { 8440 uint32_t val; 8441 struct { 8442 uint32_t app_reserve:8; 8443 uint32_t table_id:15; 8444 uint32_t transfer:1; 8445 uint32_t _unused_:8; 8446 }; 8447 }; 8448 8449 static bool 8450 mlx5_access_tunnel_offload_db 8451 (struct rte_eth_dev *dev, 8452 bool (*match)(struct rte_eth_dev *, 8453 struct mlx5_flow_tunnel *, const void *), 8454 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8455 void (*miss)(struct rte_eth_dev *, void *), 8456 void *ctx, bool lock_op); 8457 8458 static int 8459 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 8460 struct rte_flow *flow, 8461 const struct rte_flow_attr *attr, 8462 const struct rte_flow_action *app_actions, 8463 uint32_t flow_idx, 8464 const struct mlx5_flow_tunnel *tunnel, 8465 struct tunnel_default_miss_ctx *ctx, 8466 struct rte_flow_error *error) 8467 { 8468 struct mlx5_priv *priv = dev->data->dev_private; 8469 struct mlx5_flow *dev_flow; 8470 struct rte_flow_attr miss_attr = *attr; 8471 const struct rte_flow_item miss_items[2] = { 8472 { 8473 .type = RTE_FLOW_ITEM_TYPE_ETH, 8474 .spec = NULL, 8475 .last = NULL, 8476 .mask = NULL 8477 }, 8478 { 8479 .type = RTE_FLOW_ITEM_TYPE_END, 8480 .spec = NULL, 8481 .last = NULL, 8482 .mask = NULL 8483 } 8484 }; 8485 union tunnel_offload_mark mark_id; 8486 struct rte_flow_action_mark miss_mark; 8487 struct rte_flow_action miss_actions[3] = { 8488 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 8489 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 8490 }; 8491 const struct rte_flow_action_jump *jump_data; 8492 uint32_t i, flow_table = 0; /* prevent compilation warning */ 8493 struct flow_grp_info grp_info = { 8494 .external = 1, 8495 .transfer = attr->transfer, 8496 .fdb_def_rule = !!priv->fdb_def_rule, 8497 .std_tbl_fix = 0, 8498 }; 8499 int ret; 8500 8501 if (!attr->transfer) { 8502 uint32_t q_size; 8503 8504 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 8505 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 8506 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 8507 0, SOCKET_ID_ANY); 8508 if (!ctx->queue) 8509 return rte_flow_error_set 8510 (error, ENOMEM, 8511 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8512 NULL, "invalid default miss RSS"); 8513 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8514 ctx->action_rss.level = 0, 8515 ctx->action_rss.types = priv->rss_conf.rss_hf, 8516 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 8517 ctx->action_rss.queue_num = priv->reta_idx_n, 8518 ctx->action_rss.key = priv->rss_conf.rss_key, 8519 ctx->action_rss.queue = ctx->queue; 8520 if (!priv->reta_idx_n || !priv->rxqs_n) 8521 return rte_flow_error_set 8522 (error, EINVAL, 8523 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8524 NULL, "invalid port configuration"); 8525 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 8526 ctx->action_rss.types = 0; 8527 for (i = 0; i != priv->reta_idx_n; ++i) 8528 ctx->queue[i] = (*priv->reta_idx)[i]; 8529 } else { 8530 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 8531 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 8532 } 8533 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 8534 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 8535 jump_data = app_actions->conf; 8536 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 8537 miss_attr.group = jump_data->group; 8538 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 8539 &flow_table, &grp_info, error); 8540 if (ret) 8541 return rte_flow_error_set(error, EINVAL, 8542 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8543 NULL, "invalid tunnel id"); 8544 mark_id.app_reserve = 0; 8545 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 8546 mark_id.transfer = !!attr->transfer; 8547 mark_id._unused_ = 0; 8548 miss_mark.id = mark_id.val; 8549 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 8550 miss_items, miss_actions, flow_idx, error); 8551 if (!dev_flow) 8552 return -rte_errno; 8553 dev_flow->flow = flow; 8554 dev_flow->external = true; 8555 dev_flow->tunnel = tunnel; 8556 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 8557 /* Subflow object was created, we must include one in the list. */ 8558 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 8559 dev_flow->handle, next); 8560 DRV_LOG(DEBUG, 8561 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 8562 dev->data->port_id, tunnel->app_tunnel.type, 8563 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 8564 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 8565 miss_actions, error); 8566 if (!ret) 8567 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 8568 error); 8569 8570 return ret; 8571 } 8572 8573 static const struct mlx5_flow_tbl_data_entry * 8574 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 8575 { 8576 struct mlx5_priv *priv = dev->data->dev_private; 8577 struct mlx5_dev_ctx_shared *sh = priv->sh; 8578 struct mlx5_hlist_entry *he; 8579 union tunnel_offload_mark mbits = { .val = mark }; 8580 union mlx5_flow_tbl_key table_key = { 8581 { 8582 .level = tunnel_id_to_flow_tbl(mbits.table_id), 8583 .id = 0, 8584 .reserved = 0, 8585 .dummy = 0, 8586 .is_fdb = !!mbits.transfer, 8587 .is_egress = 0, 8588 } 8589 }; 8590 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 8591 return he ? 8592 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 8593 } 8594 8595 static void 8596 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 8597 struct mlx5_hlist_entry *entry) 8598 { 8599 struct mlx5_dev_ctx_shared *sh = list->ctx; 8600 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8601 8602 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8603 tunnel_flow_tbl_to_id(tte->flow_table)); 8604 mlx5_free(tte); 8605 } 8606 8607 static int 8608 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused, 8609 struct mlx5_hlist_entry *entry, 8610 uint64_t key, void *cb_ctx __rte_unused) 8611 { 8612 union tunnel_tbl_key tbl = { 8613 .val = key, 8614 }; 8615 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8616 8617 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 8618 } 8619 8620 static struct mlx5_hlist_entry * 8621 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key, 8622 void *ctx __rte_unused) 8623 { 8624 struct mlx5_dev_ctx_shared *sh = list->ctx; 8625 struct tunnel_tbl_entry *tte; 8626 union tunnel_tbl_key tbl = { 8627 .val = key, 8628 }; 8629 8630 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 8631 sizeof(*tte), 0, 8632 SOCKET_ID_ANY); 8633 if (!tte) 8634 goto err; 8635 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8636 &tte->flow_table); 8637 if (tte->flow_table >= MLX5_MAX_TABLES) { 8638 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 8639 tte->flow_table); 8640 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8641 tte->flow_table); 8642 goto err; 8643 } else if (!tte->flow_table) { 8644 goto err; 8645 } 8646 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 8647 tte->tunnel_id = tbl.tunnel_id; 8648 tte->group = tbl.group; 8649 return &tte->hash; 8650 err: 8651 if (tte) 8652 mlx5_free(tte); 8653 return NULL; 8654 } 8655 8656 static uint32_t 8657 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 8658 const struct mlx5_flow_tunnel *tunnel, 8659 uint32_t group, uint32_t *table, 8660 struct rte_flow_error *error) 8661 { 8662 struct mlx5_hlist_entry *he; 8663 struct tunnel_tbl_entry *tte; 8664 union tunnel_tbl_key key = { 8665 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 8666 .group = group 8667 }; 8668 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8669 struct mlx5_hlist *group_hash; 8670 8671 group_hash = tunnel ? tunnel->groups : thub->groups; 8672 he = mlx5_hlist_register(group_hash, key.val, NULL); 8673 if (!he) 8674 return rte_flow_error_set(error, EINVAL, 8675 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8676 NULL, 8677 "tunnel group index not supported"); 8678 tte = container_of(he, typeof(*tte), hash); 8679 *table = tte->flow_table; 8680 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 8681 dev->data->port_id, key.tunnel_id, group, *table); 8682 return 0; 8683 } 8684 8685 static void 8686 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8687 struct mlx5_flow_tunnel *tunnel) 8688 { 8689 struct mlx5_priv *priv = dev->data->dev_private; 8690 struct mlx5_indexed_pool *ipool; 8691 8692 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8693 dev->data->port_id, tunnel->tunnel_id); 8694 LIST_REMOVE(tunnel, chain); 8695 mlx5_hlist_destroy(tunnel->groups); 8696 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8697 mlx5_ipool_free(ipool, tunnel->tunnel_id); 8698 } 8699 8700 static bool 8701 mlx5_access_tunnel_offload_db 8702 (struct rte_eth_dev *dev, 8703 bool (*match)(struct rte_eth_dev *, 8704 struct mlx5_flow_tunnel *, const void *), 8705 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8706 void (*miss)(struct rte_eth_dev *, void *), 8707 void *ctx, bool lock_op) 8708 { 8709 bool verdict = false; 8710 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8711 struct mlx5_flow_tunnel *tunnel; 8712 8713 rte_spinlock_lock(&thub->sl); 8714 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 8715 verdict = match(dev, tunnel, (const void *)ctx); 8716 if (verdict) 8717 break; 8718 } 8719 if (!lock_op) 8720 rte_spinlock_unlock(&thub->sl); 8721 if (verdict && hit) 8722 hit(dev, tunnel, ctx); 8723 if (!verdict && miss) 8724 miss(dev, ctx); 8725 if (lock_op) 8726 rte_spinlock_unlock(&thub->sl); 8727 8728 return verdict; 8729 } 8730 8731 struct tunnel_db_find_tunnel_id_ctx { 8732 uint32_t tunnel_id; 8733 struct mlx5_flow_tunnel *tunnel; 8734 }; 8735 8736 static bool 8737 find_tunnel_id_match(struct rte_eth_dev *dev, 8738 struct mlx5_flow_tunnel *tunnel, const void *x) 8739 { 8740 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8741 8742 RTE_SET_USED(dev); 8743 return tunnel->tunnel_id == ctx->tunnel_id; 8744 } 8745 8746 static void 8747 find_tunnel_id_hit(struct rte_eth_dev *dev, 8748 struct mlx5_flow_tunnel *tunnel, void *x) 8749 { 8750 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8751 RTE_SET_USED(dev); 8752 ctx->tunnel = tunnel; 8753 } 8754 8755 static struct mlx5_flow_tunnel * 8756 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 8757 { 8758 struct tunnel_db_find_tunnel_id_ctx ctx = { 8759 .tunnel_id = id, 8760 }; 8761 8762 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 8763 find_tunnel_id_hit, NULL, &ctx, true); 8764 8765 return ctx.tunnel; 8766 } 8767 8768 static struct mlx5_flow_tunnel * 8769 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 8770 const struct rte_flow_tunnel *app_tunnel) 8771 { 8772 struct mlx5_priv *priv = dev->data->dev_private; 8773 struct mlx5_indexed_pool *ipool; 8774 struct mlx5_flow_tunnel *tunnel; 8775 uint32_t id; 8776 8777 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8778 tunnel = mlx5_ipool_zmalloc(ipool, &id); 8779 if (!tunnel) 8780 return NULL; 8781 if (id >= MLX5_MAX_TUNNELS) { 8782 mlx5_ipool_free(ipool, id); 8783 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 8784 return NULL; 8785 } 8786 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 8787 mlx5_flow_tunnel_grp2tbl_create_cb, 8788 mlx5_flow_tunnel_grp2tbl_match_cb, 8789 mlx5_flow_tunnel_grp2tbl_remove_cb); 8790 if (!tunnel->groups) { 8791 mlx5_ipool_free(ipool, id); 8792 return NULL; 8793 } 8794 tunnel->groups->ctx = priv->sh; 8795 /* initiate new PMD tunnel */ 8796 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8797 tunnel->tunnel_id = id; 8798 tunnel->action.type = (typeof(tunnel->action.type)) 8799 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8800 tunnel->action.conf = tunnel; 8801 tunnel->item.type = (typeof(tunnel->item.type)) 8802 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8803 tunnel->item.spec = tunnel; 8804 tunnel->item.last = NULL; 8805 tunnel->item.mask = NULL; 8806 8807 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8808 dev->data->port_id, tunnel->tunnel_id); 8809 8810 return tunnel; 8811 } 8812 8813 struct tunnel_db_get_tunnel_ctx { 8814 const struct rte_flow_tunnel *app_tunnel; 8815 struct mlx5_flow_tunnel *tunnel; 8816 }; 8817 8818 static bool get_tunnel_match(struct rte_eth_dev *dev, 8819 struct mlx5_flow_tunnel *tunnel, const void *x) 8820 { 8821 const struct tunnel_db_get_tunnel_ctx *ctx = x; 8822 8823 RTE_SET_USED(dev); 8824 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 8825 sizeof(*ctx->app_tunnel)); 8826 } 8827 8828 static void get_tunnel_hit(struct rte_eth_dev *dev, 8829 struct mlx5_flow_tunnel *tunnel, void *x) 8830 { 8831 /* called under tunnel spinlock protection */ 8832 struct tunnel_db_get_tunnel_ctx *ctx = x; 8833 8834 RTE_SET_USED(dev); 8835 tunnel->refctn++; 8836 ctx->tunnel = tunnel; 8837 } 8838 8839 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 8840 { 8841 /* called under tunnel spinlock protection */ 8842 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8843 struct tunnel_db_get_tunnel_ctx *ctx = x; 8844 8845 rte_spinlock_unlock(&thub->sl); 8846 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 8847 rte_spinlock_lock(&thub->sl); 8848 if (ctx->tunnel) { 8849 ctx->tunnel->refctn = 1; 8850 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 8851 } 8852 } 8853 8854 8855 static int 8856 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 8857 const struct rte_flow_tunnel *app_tunnel, 8858 struct mlx5_flow_tunnel **tunnel) 8859 { 8860 struct tunnel_db_get_tunnel_ctx ctx = { 8861 .app_tunnel = app_tunnel, 8862 }; 8863 8864 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 8865 get_tunnel_miss, &ctx, true); 8866 *tunnel = ctx.tunnel; 8867 return ctx.tunnel ? 0 : -ENOMEM; 8868 } 8869 8870 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 8871 { 8872 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 8873 8874 if (!thub) 8875 return; 8876 if (!LIST_EMPTY(&thub->tunnels)) 8877 DRV_LOG(WARNING, "port %u tunnels present", port_id); 8878 mlx5_hlist_destroy(thub->groups); 8879 mlx5_free(thub); 8880 } 8881 8882 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 8883 { 8884 int err; 8885 struct mlx5_flow_tunnel_hub *thub; 8886 8887 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 8888 0, SOCKET_ID_ANY); 8889 if (!thub) 8890 return -ENOMEM; 8891 LIST_INIT(&thub->tunnels); 8892 rte_spinlock_init(&thub->sl); 8893 thub->groups = mlx5_hlist_create("flow groups", 8894 rte_align32pow2(MLX5_MAX_TABLES), 0, 8895 0, mlx5_flow_tunnel_grp2tbl_create_cb, 8896 mlx5_flow_tunnel_grp2tbl_match_cb, 8897 mlx5_flow_tunnel_grp2tbl_remove_cb); 8898 if (!thub->groups) { 8899 err = -rte_errno; 8900 goto err; 8901 } 8902 thub->groups->ctx = sh; 8903 sh->tunnel_hub = thub; 8904 8905 return 0; 8906 8907 err: 8908 if (thub->groups) 8909 mlx5_hlist_destroy(thub->groups); 8910 if (thub) 8911 mlx5_free(thub); 8912 return err; 8913 } 8914 8915 static inline bool 8916 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 8917 struct rte_flow_tunnel *tunnel, 8918 const char *err_msg) 8919 { 8920 err_msg = NULL; 8921 if (!is_tunnel_offload_active(dev)) { 8922 err_msg = "tunnel offload was not activated"; 8923 goto out; 8924 } else if (!tunnel) { 8925 err_msg = "no application tunnel"; 8926 goto out; 8927 } 8928 8929 switch (tunnel->type) { 8930 default: 8931 err_msg = "unsupported tunnel type"; 8932 goto out; 8933 case RTE_FLOW_ITEM_TYPE_VXLAN: 8934 break; 8935 } 8936 8937 out: 8938 return !err_msg; 8939 } 8940 8941 static int 8942 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 8943 struct rte_flow_tunnel *app_tunnel, 8944 struct rte_flow_action **actions, 8945 uint32_t *num_of_actions, 8946 struct rte_flow_error *error) 8947 { 8948 int ret; 8949 struct mlx5_flow_tunnel *tunnel; 8950 const char *err_msg = NULL; 8951 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8952 8953 if (!verdict) 8954 return rte_flow_error_set(error, EINVAL, 8955 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8956 err_msg); 8957 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8958 if (ret < 0) { 8959 return rte_flow_error_set(error, ret, 8960 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8961 "failed to initialize pmd tunnel"); 8962 } 8963 *actions = &tunnel->action; 8964 *num_of_actions = 1; 8965 return 0; 8966 } 8967 8968 static int 8969 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 8970 struct rte_flow_tunnel *app_tunnel, 8971 struct rte_flow_item **items, 8972 uint32_t *num_of_items, 8973 struct rte_flow_error *error) 8974 { 8975 int ret; 8976 struct mlx5_flow_tunnel *tunnel; 8977 const char *err_msg = NULL; 8978 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8979 8980 if (!verdict) 8981 return rte_flow_error_set(error, EINVAL, 8982 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8983 err_msg); 8984 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8985 if (ret < 0) { 8986 return rte_flow_error_set(error, ret, 8987 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8988 "failed to initialize pmd tunnel"); 8989 } 8990 *items = &tunnel->item; 8991 *num_of_items = 1; 8992 return 0; 8993 } 8994 8995 struct tunnel_db_element_release_ctx { 8996 struct rte_flow_item *items; 8997 struct rte_flow_action *actions; 8998 uint32_t num_elements; 8999 struct rte_flow_error *error; 9000 int ret; 9001 }; 9002 9003 static bool 9004 tunnel_element_release_match(struct rte_eth_dev *dev, 9005 struct mlx5_flow_tunnel *tunnel, const void *x) 9006 { 9007 const struct tunnel_db_element_release_ctx *ctx = x; 9008 9009 RTE_SET_USED(dev); 9010 if (ctx->num_elements != 1) 9011 return false; 9012 else if (ctx->items) 9013 return ctx->items == &tunnel->item; 9014 else if (ctx->actions) 9015 return ctx->actions == &tunnel->action; 9016 9017 return false; 9018 } 9019 9020 static void 9021 tunnel_element_release_hit(struct rte_eth_dev *dev, 9022 struct mlx5_flow_tunnel *tunnel, void *x) 9023 { 9024 struct tunnel_db_element_release_ctx *ctx = x; 9025 ctx->ret = 0; 9026 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9027 mlx5_flow_tunnel_free(dev, tunnel); 9028 } 9029 9030 static void 9031 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9032 { 9033 struct tunnel_db_element_release_ctx *ctx = x; 9034 RTE_SET_USED(dev); 9035 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9036 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9037 "invalid argument"); 9038 } 9039 9040 static int 9041 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9042 struct rte_flow_item *pmd_items, 9043 uint32_t num_items, struct rte_flow_error *err) 9044 { 9045 struct tunnel_db_element_release_ctx ctx = { 9046 .items = pmd_items, 9047 .actions = NULL, 9048 .num_elements = num_items, 9049 .error = err, 9050 }; 9051 9052 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9053 tunnel_element_release_hit, 9054 tunnel_element_release_miss, &ctx, false); 9055 9056 return ctx.ret; 9057 } 9058 9059 static int 9060 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9061 struct rte_flow_action *pmd_actions, 9062 uint32_t num_actions, struct rte_flow_error *err) 9063 { 9064 struct tunnel_db_element_release_ctx ctx = { 9065 .items = NULL, 9066 .actions = pmd_actions, 9067 .num_elements = num_actions, 9068 .error = err, 9069 }; 9070 9071 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9072 tunnel_element_release_hit, 9073 tunnel_element_release_miss, &ctx, false); 9074 9075 return ctx.ret; 9076 } 9077 9078 static int 9079 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9080 struct rte_mbuf *m, 9081 struct rte_flow_restore_info *info, 9082 struct rte_flow_error *err) 9083 { 9084 uint64_t ol_flags = m->ol_flags; 9085 const struct mlx5_flow_tbl_data_entry *tble; 9086 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 9087 9088 if (!is_tunnel_offload_active(dev)) { 9089 info->flags = 0; 9090 return 0; 9091 } 9092 9093 if ((ol_flags & mask) != mask) 9094 goto err; 9095 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9096 if (!tble) { 9097 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9098 dev->data->port_id, m->hash.fdir.hi); 9099 goto err; 9100 } 9101 MLX5_ASSERT(tble->tunnel); 9102 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9103 info->group_id = tble->group_id; 9104 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9105 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9106 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9107 9108 return 0; 9109 9110 err: 9111 return rte_flow_error_set(err, EINVAL, 9112 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9113 "failed to get restore info"); 9114 } 9115 9116 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9117 static int 9118 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9119 __rte_unused struct rte_flow_tunnel *app_tunnel, 9120 __rte_unused struct rte_flow_action **actions, 9121 __rte_unused uint32_t *num_of_actions, 9122 __rte_unused struct rte_flow_error *error) 9123 { 9124 return -ENOTSUP; 9125 } 9126 9127 static int 9128 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9129 __rte_unused struct rte_flow_tunnel *app_tunnel, 9130 __rte_unused struct rte_flow_item **items, 9131 __rte_unused uint32_t *num_of_items, 9132 __rte_unused struct rte_flow_error *error) 9133 { 9134 return -ENOTSUP; 9135 } 9136 9137 static int 9138 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9139 __rte_unused struct rte_flow_item *pmd_items, 9140 __rte_unused uint32_t num_items, 9141 __rte_unused struct rte_flow_error *err) 9142 { 9143 return -ENOTSUP; 9144 } 9145 9146 static int 9147 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9148 __rte_unused struct rte_flow_action *pmd_action, 9149 __rte_unused uint32_t num_actions, 9150 __rte_unused struct rte_flow_error *err) 9151 { 9152 return -ENOTSUP; 9153 } 9154 9155 static int 9156 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9157 __rte_unused struct rte_mbuf *m, 9158 __rte_unused struct rte_flow_restore_info *i, 9159 __rte_unused struct rte_flow_error *err) 9160 { 9161 return -ENOTSUP; 9162 } 9163 9164 static int 9165 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9166 __rte_unused struct rte_flow *flow, 9167 __rte_unused const struct rte_flow_attr *attr, 9168 __rte_unused const struct rte_flow_action *actions, 9169 __rte_unused uint32_t flow_idx, 9170 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9171 __rte_unused struct tunnel_default_miss_ctx *ctx, 9172 __rte_unused struct rte_flow_error *error) 9173 { 9174 return -ENOTSUP; 9175 } 9176 9177 static struct mlx5_flow_tunnel * 9178 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9179 __rte_unused uint32_t id) 9180 { 9181 return NULL; 9182 } 9183 9184 static void 9185 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9186 __rte_unused struct mlx5_flow_tunnel *tunnel) 9187 { 9188 } 9189 9190 static uint32_t 9191 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9192 __rte_unused const struct mlx5_flow_tunnel *t, 9193 __rte_unused uint32_t group, 9194 __rte_unused uint32_t *table, 9195 struct rte_flow_error *error) 9196 { 9197 return rte_flow_error_set(error, ENOTSUP, 9198 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9199 "tunnel offload requires DV support"); 9200 } 9201 9202 void 9203 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9204 __rte_unused uint16_t port_id) 9205 { 9206 } 9207 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9208 9209 static void 9210 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9211 { 9212 int ret; 9213 struct rte_flow_error error; 9214 9215 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9216 char *item_name; 9217 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9218 sizeof(item_name), 9219 (void *)(uintptr_t)item->type, &error); 9220 if (ret > 0) 9221 printf("%s ", item_name); 9222 else 9223 printf("%d\n", (int)item->type); 9224 } 9225 printf("END\n"); 9226 } 9227