1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see ETH_RSS_* definitions). 102 */ 103 uint8_t optional; 104 /**< optional expand field. Default 0 to expand, 1 not go deeper. */ 105 }; 106 107 /** Object returned by mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_rss { 109 uint32_t entries; 110 /**< Number of entries @p patterns and @p priorities. */ 111 struct { 112 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 113 uint32_t priority; /**< Priority offset for each expansion. */ 114 } entry[]; 115 }; 116 117 static enum rte_flow_item_type 118 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 119 { 120 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 121 uint16_t ether_type = 0; 122 uint16_t ether_type_m; 123 uint8_t ip_next_proto = 0; 124 uint8_t ip_next_proto_m; 125 126 if (item == NULL || item->spec == NULL) 127 return ret; 128 switch (item->type) { 129 case RTE_FLOW_ITEM_TYPE_ETH: 130 if (item->mask) 131 ether_type_m = ((const struct rte_flow_item_eth *) 132 (item->mask))->type; 133 else 134 ether_type_m = rte_flow_item_eth_mask.type; 135 if (ether_type_m != RTE_BE16(0xFFFF)) 136 break; 137 ether_type = ((const struct rte_flow_item_eth *) 138 (item->spec))->type; 139 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 140 ret = RTE_FLOW_ITEM_TYPE_IPV4; 141 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 142 ret = RTE_FLOW_ITEM_TYPE_IPV6; 143 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 144 ret = RTE_FLOW_ITEM_TYPE_VLAN; 145 else 146 ret = RTE_FLOW_ITEM_TYPE_END; 147 break; 148 case RTE_FLOW_ITEM_TYPE_VLAN: 149 if (item->mask) 150 ether_type_m = ((const struct rte_flow_item_vlan *) 151 (item->mask))->inner_type; 152 else 153 ether_type_m = rte_flow_item_vlan_mask.inner_type; 154 if (ether_type_m != RTE_BE16(0xFFFF)) 155 break; 156 ether_type = ((const struct rte_flow_item_vlan *) 157 (item->spec))->inner_type; 158 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 159 ret = RTE_FLOW_ITEM_TYPE_IPV4; 160 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 161 ret = RTE_FLOW_ITEM_TYPE_IPV6; 162 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 163 ret = RTE_FLOW_ITEM_TYPE_VLAN; 164 else 165 ret = RTE_FLOW_ITEM_TYPE_END; 166 break; 167 case RTE_FLOW_ITEM_TYPE_IPV4: 168 if (item->mask) 169 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 170 (item->mask))->hdr.next_proto_id; 171 else 172 ip_next_proto_m = 173 rte_flow_item_ipv4_mask.hdr.next_proto_id; 174 if (ip_next_proto_m != 0xFF) 175 break; 176 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 177 (item->spec))->hdr.next_proto_id; 178 if (ip_next_proto == IPPROTO_UDP) 179 ret = RTE_FLOW_ITEM_TYPE_UDP; 180 else if (ip_next_proto == IPPROTO_TCP) 181 ret = RTE_FLOW_ITEM_TYPE_TCP; 182 else if (ip_next_proto == IPPROTO_IP) 183 ret = RTE_FLOW_ITEM_TYPE_IPV4; 184 else if (ip_next_proto == IPPROTO_IPV6) 185 ret = RTE_FLOW_ITEM_TYPE_IPV6; 186 else 187 ret = RTE_FLOW_ITEM_TYPE_END; 188 break; 189 case RTE_FLOW_ITEM_TYPE_IPV6: 190 if (item->mask) 191 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 192 (item->mask))->hdr.proto; 193 else 194 ip_next_proto_m = 195 rte_flow_item_ipv6_mask.hdr.proto; 196 if (ip_next_proto_m != 0xFF) 197 break; 198 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 199 (item->spec))->hdr.proto; 200 if (ip_next_proto == IPPROTO_UDP) 201 ret = RTE_FLOW_ITEM_TYPE_UDP; 202 else if (ip_next_proto == IPPROTO_TCP) 203 ret = RTE_FLOW_ITEM_TYPE_TCP; 204 else if (ip_next_proto == IPPROTO_IP) 205 ret = RTE_FLOW_ITEM_TYPE_IPV4; 206 else if (ip_next_proto == IPPROTO_IPV6) 207 ret = RTE_FLOW_ITEM_TYPE_IPV6; 208 else 209 ret = RTE_FLOW_ITEM_TYPE_END; 210 break; 211 default: 212 ret = RTE_FLOW_ITEM_TYPE_VOID; 213 break; 214 } 215 return ret; 216 } 217 218 #define MLX5_RSS_EXP_ELT_N 16 219 220 /** 221 * Expand RSS flows into several possible flows according to the RSS hash 222 * fields requested and the driver capabilities. 223 * 224 * @param[out] buf 225 * Buffer to store the result expansion. 226 * @param[in] size 227 * Buffer size in bytes. If 0, @p buf can be NULL. 228 * @param[in] pattern 229 * User flow pattern. 230 * @param[in] types 231 * RSS types to expand (see ETH_RSS_* definitions). 232 * @param[in] graph 233 * Input graph to expand @p pattern according to @p types. 234 * @param[in] graph_root_index 235 * Index of root node in @p graph, typically 0. 236 * 237 * @return 238 * A positive value representing the size of @p buf in bytes regardless of 239 * @p size on success, a negative errno value otherwise and rte_errno is 240 * set, the following errors are defined: 241 * 242 * -E2BIG: graph-depth @p graph is too deep. 243 */ 244 static int 245 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 246 const struct rte_flow_item *pattern, uint64_t types, 247 const struct mlx5_flow_expand_node graph[], 248 int graph_root_index) 249 { 250 const struct rte_flow_item *item; 251 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 252 const int *next_node; 253 const int *stack[MLX5_RSS_EXP_ELT_N]; 254 int stack_pos = 0; 255 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 256 unsigned int i; 257 size_t lsize; 258 size_t user_pattern_size = 0; 259 void *addr = NULL; 260 const struct mlx5_flow_expand_node *next = NULL; 261 struct rte_flow_item missed_item; 262 int missed = 0; 263 int elt = 0; 264 const struct rte_flow_item *last_item = NULL; 265 266 memset(&missed_item, 0, sizeof(missed_item)); 267 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 268 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 269 if (lsize <= size) { 270 buf->entry[0].priority = 0; 271 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 272 buf->entries = 0; 273 addr = buf->entry[0].pattern; 274 } 275 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 276 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 277 last_item = item; 278 for (i = 0; node->next && node->next[i]; ++i) { 279 next = &graph[node->next[i]]; 280 if (next->type == item->type) 281 break; 282 } 283 if (next) 284 node = next; 285 user_pattern_size += sizeof(*item); 286 } 287 user_pattern_size += sizeof(*item); /* Handle END item. */ 288 lsize += user_pattern_size; 289 /* Copy the user pattern in the first entry of the buffer. */ 290 if (lsize <= size) { 291 rte_memcpy(addr, pattern, user_pattern_size); 292 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 293 buf->entries = 1; 294 } 295 /* Start expanding. */ 296 memset(flow_items, 0, sizeof(flow_items)); 297 user_pattern_size -= sizeof(*item); 298 /* 299 * Check if the last valid item has spec set, need complete pattern, 300 * and the pattern can be used for expansion. 301 */ 302 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 303 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 304 /* Item type END indicates expansion is not required. */ 305 return lsize; 306 } 307 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 308 next = NULL; 309 missed = 1; 310 for (i = 0; node->next && node->next[i]; ++i) { 311 next = &graph[node->next[i]]; 312 if (next->type == missed_item.type) { 313 flow_items[0].type = missed_item.type; 314 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 315 break; 316 } 317 next = NULL; 318 } 319 } 320 if (next && missed) { 321 elt = 2; /* missed item + item end. */ 322 node = next; 323 lsize += elt * sizeof(*item) + user_pattern_size; 324 if ((node->rss_types & types) && lsize <= size) { 325 buf->entry[buf->entries].priority = 1; 326 buf->entry[buf->entries].pattern = addr; 327 buf->entries++; 328 rte_memcpy(addr, buf->entry[0].pattern, 329 user_pattern_size); 330 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 331 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 332 addr = (void *)(((uintptr_t)addr) + 333 elt * sizeof(*item)); 334 } 335 } 336 memset(flow_items, 0, sizeof(flow_items)); 337 next_node = node->next; 338 stack[stack_pos] = next_node; 339 node = next_node ? &graph[*next_node] : NULL; 340 while (node) { 341 flow_items[stack_pos].type = node->type; 342 if (node->rss_types & types) { 343 /* 344 * compute the number of items to copy from the 345 * expansion and copy it. 346 * When the stack_pos is 0, there are 1 element in it, 347 * plus the addition END item. 348 */ 349 elt = stack_pos + 2; 350 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 351 lsize += elt * sizeof(*item) + user_pattern_size; 352 if (lsize <= size) { 353 size_t n = elt * sizeof(*item); 354 355 buf->entry[buf->entries].priority = 356 stack_pos + 1 + missed; 357 buf->entry[buf->entries].pattern = addr; 358 buf->entries++; 359 rte_memcpy(addr, buf->entry[0].pattern, 360 user_pattern_size); 361 addr = (void *)(((uintptr_t)addr) + 362 user_pattern_size); 363 rte_memcpy(addr, &missed_item, 364 missed * sizeof(*item)); 365 addr = (void *)(((uintptr_t)addr) + 366 missed * sizeof(*item)); 367 rte_memcpy(addr, flow_items, n); 368 addr = (void *)(((uintptr_t)addr) + n); 369 } 370 } 371 /* Go deeper. */ 372 if (!node->optional && node->next) { 373 next_node = node->next; 374 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 375 rte_errno = E2BIG; 376 return -rte_errno; 377 } 378 stack[stack_pos] = next_node; 379 } else if (*(next_node + 1)) { 380 /* Follow up with the next possibility. */ 381 ++next_node; 382 } else { 383 /* Move to the next path. */ 384 if (stack_pos) 385 next_node = stack[--stack_pos]; 386 next_node++; 387 stack[stack_pos] = next_node; 388 } 389 node = *next_node ? &graph[*next_node] : NULL; 390 }; 391 return lsize; 392 } 393 394 enum mlx5_expansion { 395 MLX5_EXPANSION_ROOT, 396 MLX5_EXPANSION_ROOT_OUTER, 397 MLX5_EXPANSION_ROOT_ETH_VLAN, 398 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 399 MLX5_EXPANSION_OUTER_ETH, 400 MLX5_EXPANSION_OUTER_ETH_VLAN, 401 MLX5_EXPANSION_OUTER_VLAN, 402 MLX5_EXPANSION_OUTER_IPV4, 403 MLX5_EXPANSION_OUTER_IPV4_UDP, 404 MLX5_EXPANSION_OUTER_IPV4_TCP, 405 MLX5_EXPANSION_OUTER_IPV6, 406 MLX5_EXPANSION_OUTER_IPV6_UDP, 407 MLX5_EXPANSION_OUTER_IPV6_TCP, 408 MLX5_EXPANSION_VXLAN, 409 MLX5_EXPANSION_VXLAN_GPE, 410 MLX5_EXPANSION_GRE, 411 MLX5_EXPANSION_NVGRE, 412 MLX5_EXPANSION_GRE_KEY, 413 MLX5_EXPANSION_MPLS, 414 MLX5_EXPANSION_ETH, 415 MLX5_EXPANSION_ETH_VLAN, 416 MLX5_EXPANSION_VLAN, 417 MLX5_EXPANSION_IPV4, 418 MLX5_EXPANSION_IPV4_UDP, 419 MLX5_EXPANSION_IPV4_TCP, 420 MLX5_EXPANSION_IPV6, 421 MLX5_EXPANSION_IPV6_UDP, 422 MLX5_EXPANSION_IPV6_TCP, 423 }; 424 425 /** Supported expansion of items. */ 426 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 427 [MLX5_EXPANSION_ROOT] = { 428 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 429 MLX5_EXPANSION_IPV4, 430 MLX5_EXPANSION_IPV6), 431 .type = RTE_FLOW_ITEM_TYPE_END, 432 }, 433 [MLX5_EXPANSION_ROOT_OUTER] = { 434 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 435 MLX5_EXPANSION_OUTER_IPV4, 436 MLX5_EXPANSION_OUTER_IPV6), 437 .type = RTE_FLOW_ITEM_TYPE_END, 438 }, 439 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 440 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 441 .type = RTE_FLOW_ITEM_TYPE_END, 442 }, 443 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 444 .next = MLX5_FLOW_EXPAND_RSS_NEXT 445 (MLX5_EXPANSION_OUTER_ETH_VLAN), 446 .type = RTE_FLOW_ITEM_TYPE_END, 447 }, 448 [MLX5_EXPANSION_OUTER_ETH] = { 449 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 450 MLX5_EXPANSION_OUTER_IPV6, 451 MLX5_EXPANSION_MPLS), 452 .type = RTE_FLOW_ITEM_TYPE_ETH, 453 .rss_types = 0, 454 }, 455 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 456 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 457 .type = RTE_FLOW_ITEM_TYPE_ETH, 458 .rss_types = 0, 459 }, 460 [MLX5_EXPANSION_OUTER_VLAN] = { 461 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 462 MLX5_EXPANSION_OUTER_IPV6), 463 .type = RTE_FLOW_ITEM_TYPE_VLAN, 464 }, 465 [MLX5_EXPANSION_OUTER_IPV4] = { 466 .next = MLX5_FLOW_EXPAND_RSS_NEXT 467 (MLX5_EXPANSION_OUTER_IPV4_UDP, 468 MLX5_EXPANSION_OUTER_IPV4_TCP, 469 MLX5_EXPANSION_GRE, 470 MLX5_EXPANSION_NVGRE, 471 MLX5_EXPANSION_IPV4, 472 MLX5_EXPANSION_IPV6), 473 .type = RTE_FLOW_ITEM_TYPE_IPV4, 474 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 475 ETH_RSS_NONFRAG_IPV4_OTHER, 476 }, 477 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 478 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 479 MLX5_EXPANSION_VXLAN_GPE), 480 .type = RTE_FLOW_ITEM_TYPE_UDP, 481 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 482 }, 483 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 484 .type = RTE_FLOW_ITEM_TYPE_TCP, 485 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 486 }, 487 [MLX5_EXPANSION_OUTER_IPV6] = { 488 .next = MLX5_FLOW_EXPAND_RSS_NEXT 489 (MLX5_EXPANSION_OUTER_IPV6_UDP, 490 MLX5_EXPANSION_OUTER_IPV6_TCP, 491 MLX5_EXPANSION_IPV4, 492 MLX5_EXPANSION_IPV6, 493 MLX5_EXPANSION_GRE, 494 MLX5_EXPANSION_NVGRE), 495 .type = RTE_FLOW_ITEM_TYPE_IPV6, 496 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 497 ETH_RSS_NONFRAG_IPV6_OTHER, 498 }, 499 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 500 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 501 MLX5_EXPANSION_VXLAN_GPE), 502 .type = RTE_FLOW_ITEM_TYPE_UDP, 503 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 504 }, 505 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 506 .type = RTE_FLOW_ITEM_TYPE_TCP, 507 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 508 }, 509 [MLX5_EXPANSION_VXLAN] = { 510 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 511 MLX5_EXPANSION_IPV4, 512 MLX5_EXPANSION_IPV6), 513 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 514 }, 515 [MLX5_EXPANSION_VXLAN_GPE] = { 516 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 517 MLX5_EXPANSION_IPV4, 518 MLX5_EXPANSION_IPV6), 519 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 520 }, 521 [MLX5_EXPANSION_GRE] = { 522 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 523 MLX5_EXPANSION_IPV6, 524 MLX5_EXPANSION_GRE_KEY), 525 .type = RTE_FLOW_ITEM_TYPE_GRE, 526 }, 527 [MLX5_EXPANSION_GRE_KEY] = { 528 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 529 MLX5_EXPANSION_IPV6), 530 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 531 .optional = 1, 532 }, 533 [MLX5_EXPANSION_NVGRE] = { 534 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 535 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 536 }, 537 [MLX5_EXPANSION_MPLS] = { 538 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 539 MLX5_EXPANSION_IPV6), 540 .type = RTE_FLOW_ITEM_TYPE_MPLS, 541 }, 542 [MLX5_EXPANSION_ETH] = { 543 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 544 MLX5_EXPANSION_IPV6), 545 .type = RTE_FLOW_ITEM_TYPE_ETH, 546 }, 547 [MLX5_EXPANSION_ETH_VLAN] = { 548 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 549 .type = RTE_FLOW_ITEM_TYPE_ETH, 550 }, 551 [MLX5_EXPANSION_VLAN] = { 552 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 553 MLX5_EXPANSION_IPV6), 554 .type = RTE_FLOW_ITEM_TYPE_VLAN, 555 }, 556 [MLX5_EXPANSION_IPV4] = { 557 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 558 MLX5_EXPANSION_IPV4_TCP), 559 .type = RTE_FLOW_ITEM_TYPE_IPV4, 560 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 561 ETH_RSS_NONFRAG_IPV4_OTHER, 562 }, 563 [MLX5_EXPANSION_IPV4_UDP] = { 564 .type = RTE_FLOW_ITEM_TYPE_UDP, 565 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 566 }, 567 [MLX5_EXPANSION_IPV4_TCP] = { 568 .type = RTE_FLOW_ITEM_TYPE_TCP, 569 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 570 }, 571 [MLX5_EXPANSION_IPV6] = { 572 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 573 MLX5_EXPANSION_IPV6_TCP), 574 .type = RTE_FLOW_ITEM_TYPE_IPV6, 575 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 576 ETH_RSS_NONFRAG_IPV6_OTHER, 577 }, 578 [MLX5_EXPANSION_IPV6_UDP] = { 579 .type = RTE_FLOW_ITEM_TYPE_UDP, 580 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 581 }, 582 [MLX5_EXPANSION_IPV6_TCP] = { 583 .type = RTE_FLOW_ITEM_TYPE_TCP, 584 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 585 }, 586 }; 587 588 static struct rte_flow_action_handle * 589 mlx5_action_handle_create(struct rte_eth_dev *dev, 590 const struct rte_flow_indir_action_conf *conf, 591 const struct rte_flow_action *action, 592 struct rte_flow_error *error); 593 static int mlx5_action_handle_destroy 594 (struct rte_eth_dev *dev, 595 struct rte_flow_action_handle *handle, 596 struct rte_flow_error *error); 597 static int mlx5_action_handle_update 598 (struct rte_eth_dev *dev, 599 struct rte_flow_action_handle *handle, 600 const void *update, 601 struct rte_flow_error *error); 602 static int mlx5_action_handle_query 603 (struct rte_eth_dev *dev, 604 const struct rte_flow_action_handle *handle, 605 void *data, 606 struct rte_flow_error *error); 607 static int 608 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 609 struct rte_flow_tunnel *app_tunnel, 610 struct rte_flow_action **actions, 611 uint32_t *num_of_actions, 612 struct rte_flow_error *error); 613 static int 614 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 615 struct rte_flow_tunnel *app_tunnel, 616 struct rte_flow_item **items, 617 uint32_t *num_of_items, 618 struct rte_flow_error *error); 619 static int 620 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 621 struct rte_flow_item *pmd_items, 622 uint32_t num_items, struct rte_flow_error *err); 623 static int 624 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 625 struct rte_flow_action *pmd_actions, 626 uint32_t num_actions, 627 struct rte_flow_error *err); 628 static int 629 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 630 struct rte_mbuf *m, 631 struct rte_flow_restore_info *info, 632 struct rte_flow_error *err); 633 634 static const struct rte_flow_ops mlx5_flow_ops = { 635 .validate = mlx5_flow_validate, 636 .create = mlx5_flow_create, 637 .destroy = mlx5_flow_destroy, 638 .flush = mlx5_flow_flush, 639 .isolate = mlx5_flow_isolate, 640 .query = mlx5_flow_query, 641 .dev_dump = mlx5_flow_dev_dump, 642 .get_aged_flows = mlx5_flow_get_aged_flows, 643 .action_handle_create = mlx5_action_handle_create, 644 .action_handle_destroy = mlx5_action_handle_destroy, 645 .action_handle_update = mlx5_action_handle_update, 646 .action_handle_query = mlx5_action_handle_query, 647 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 648 .tunnel_match = mlx5_flow_tunnel_match, 649 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 650 .tunnel_item_release = mlx5_flow_tunnel_item_release, 651 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 652 }; 653 654 /* Tunnel information. */ 655 struct mlx5_flow_tunnel_info { 656 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 657 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 658 }; 659 660 static struct mlx5_flow_tunnel_info tunnels_info[] = { 661 { 662 .tunnel = MLX5_FLOW_LAYER_VXLAN, 663 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 664 }, 665 { 666 .tunnel = MLX5_FLOW_LAYER_GENEVE, 667 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 668 }, 669 { 670 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 671 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 672 }, 673 { 674 .tunnel = MLX5_FLOW_LAYER_GRE, 675 .ptype = RTE_PTYPE_TUNNEL_GRE, 676 }, 677 { 678 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 679 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 680 }, 681 { 682 .tunnel = MLX5_FLOW_LAYER_MPLS, 683 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 684 }, 685 { 686 .tunnel = MLX5_FLOW_LAYER_NVGRE, 687 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 688 }, 689 { 690 .tunnel = MLX5_FLOW_LAYER_IPIP, 691 .ptype = RTE_PTYPE_TUNNEL_IP, 692 }, 693 { 694 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 695 .ptype = RTE_PTYPE_TUNNEL_IP, 696 }, 697 { 698 .tunnel = MLX5_FLOW_LAYER_GTP, 699 .ptype = RTE_PTYPE_TUNNEL_GTPU, 700 }, 701 }; 702 703 704 705 /** 706 * Translate tag ID to register. 707 * 708 * @param[in] dev 709 * Pointer to the Ethernet device structure. 710 * @param[in] feature 711 * The feature that request the register. 712 * @param[in] id 713 * The request register ID. 714 * @param[out] error 715 * Error description in case of any. 716 * 717 * @return 718 * The request register on success, a negative errno 719 * value otherwise and rte_errno is set. 720 */ 721 int 722 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 723 enum mlx5_feature_name feature, 724 uint32_t id, 725 struct rte_flow_error *error) 726 { 727 struct mlx5_priv *priv = dev->data->dev_private; 728 struct mlx5_dev_config *config = &priv->config; 729 enum modify_reg start_reg; 730 bool skip_mtr_reg = false; 731 732 switch (feature) { 733 case MLX5_HAIRPIN_RX: 734 return REG_B; 735 case MLX5_HAIRPIN_TX: 736 return REG_A; 737 case MLX5_METADATA_RX: 738 switch (config->dv_xmeta_en) { 739 case MLX5_XMETA_MODE_LEGACY: 740 return REG_B; 741 case MLX5_XMETA_MODE_META16: 742 return REG_C_0; 743 case MLX5_XMETA_MODE_META32: 744 return REG_C_1; 745 } 746 break; 747 case MLX5_METADATA_TX: 748 return REG_A; 749 case MLX5_METADATA_FDB: 750 switch (config->dv_xmeta_en) { 751 case MLX5_XMETA_MODE_LEGACY: 752 return REG_NON; 753 case MLX5_XMETA_MODE_META16: 754 return REG_C_0; 755 case MLX5_XMETA_MODE_META32: 756 return REG_C_1; 757 } 758 break; 759 case MLX5_FLOW_MARK: 760 switch (config->dv_xmeta_en) { 761 case MLX5_XMETA_MODE_LEGACY: 762 return REG_NON; 763 case MLX5_XMETA_MODE_META16: 764 return REG_C_1; 765 case MLX5_XMETA_MODE_META32: 766 return REG_C_0; 767 } 768 break; 769 case MLX5_MTR_ID: 770 /* 771 * If meter color and meter id share one register, flow match 772 * should use the meter color register for match. 773 */ 774 if (priv->mtr_reg_share) 775 return priv->mtr_color_reg; 776 else 777 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 778 REG_C_3; 779 case MLX5_MTR_COLOR: 780 case MLX5_ASO_FLOW_HIT: 781 case MLX5_ASO_CONNTRACK: 782 /* All features use the same REG_C. */ 783 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 784 return priv->mtr_color_reg; 785 case MLX5_COPY_MARK: 786 /* 787 * Metadata COPY_MARK register using is in meter suffix sub 788 * flow while with meter. It's safe to share the same register. 789 */ 790 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 791 case MLX5_APP_TAG: 792 /* 793 * If meter is enable, it will engage the register for color 794 * match and flow match. If meter color match is not using the 795 * REG_C_2, need to skip the REG_C_x be used by meter color 796 * match. 797 * If meter is disable, free to use all available registers. 798 */ 799 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 800 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 801 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 802 if (id > (uint32_t)(REG_C_7 - start_reg)) 803 return rte_flow_error_set(error, EINVAL, 804 RTE_FLOW_ERROR_TYPE_ITEM, 805 NULL, "invalid tag id"); 806 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 807 return rte_flow_error_set(error, ENOTSUP, 808 RTE_FLOW_ERROR_TYPE_ITEM, 809 NULL, "unsupported tag id"); 810 /* 811 * This case means meter is using the REG_C_x great than 2. 812 * Take care not to conflict with meter color REG_C_x. 813 * If the available index REG_C_y >= REG_C_x, skip the 814 * color register. 815 */ 816 if (skip_mtr_reg && config->flow_mreg_c 817 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 818 if (id >= (uint32_t)(REG_C_7 - start_reg)) 819 return rte_flow_error_set(error, EINVAL, 820 RTE_FLOW_ERROR_TYPE_ITEM, 821 NULL, "invalid tag id"); 822 if (config->flow_mreg_c 823 [id + 1 + start_reg - REG_C_0] != REG_NON) 824 return config->flow_mreg_c 825 [id + 1 + start_reg - REG_C_0]; 826 return rte_flow_error_set(error, ENOTSUP, 827 RTE_FLOW_ERROR_TYPE_ITEM, 828 NULL, "unsupported tag id"); 829 } 830 return config->flow_mreg_c[id + start_reg - REG_C_0]; 831 } 832 MLX5_ASSERT(false); 833 return rte_flow_error_set(error, EINVAL, 834 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 835 NULL, "invalid feature name"); 836 } 837 838 /** 839 * Check extensive flow metadata register support. 840 * 841 * @param dev 842 * Pointer to rte_eth_dev structure. 843 * 844 * @return 845 * True if device supports extensive flow metadata register, otherwise false. 846 */ 847 bool 848 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 849 { 850 struct mlx5_priv *priv = dev->data->dev_private; 851 struct mlx5_dev_config *config = &priv->config; 852 853 /* 854 * Having available reg_c can be regarded inclusively as supporting 855 * extensive flow metadata register, which could mean, 856 * - metadata register copy action by modify header. 857 * - 16 modify header actions is supported. 858 * - reg_c's are preserved across different domain (FDB and NIC) on 859 * packet loopback by flow lookup miss. 860 */ 861 return config->flow_mreg_c[2] != REG_NON; 862 } 863 864 /** 865 * Get the lowest priority. 866 * 867 * @param[in] dev 868 * Pointer to the Ethernet device structure. 869 * @param[in] attributes 870 * Pointer to device flow rule attributes. 871 * 872 * @return 873 * The value of lowest priority of flow. 874 */ 875 uint32_t 876 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 877 const struct rte_flow_attr *attr) 878 { 879 struct mlx5_priv *priv = dev->data->dev_private; 880 881 if (!attr->group && !attr->transfer) 882 return priv->config.flow_prio - 2; 883 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 884 } 885 886 /** 887 * Calculate matcher priority of the flow. 888 * 889 * @param[in] dev 890 * Pointer to the Ethernet device structure. 891 * @param[in] attr 892 * Pointer to device flow rule attributes. 893 * @param[in] subpriority 894 * The priority based on the items. 895 * @return 896 * The matcher priority of the flow. 897 */ 898 uint16_t 899 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 900 const struct rte_flow_attr *attr, 901 uint32_t subpriority) 902 { 903 uint16_t priority = (uint16_t)attr->priority; 904 struct mlx5_priv *priv = dev->data->dev_private; 905 906 if (!attr->group && !attr->transfer) { 907 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 908 priority = priv->config.flow_prio - 1; 909 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 910 } 911 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 912 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 913 return priority * 3 + subpriority; 914 } 915 916 /** 917 * Verify the @p item specifications (spec, last, mask) are compatible with the 918 * NIC capabilities. 919 * 920 * @param[in] item 921 * Item specification. 922 * @param[in] mask 923 * @p item->mask or flow default bit-masks. 924 * @param[in] nic_mask 925 * Bit-masks covering supported fields by the NIC to compare with user mask. 926 * @param[in] size 927 * Bit-masks size in bytes. 928 * @param[in] range_accepted 929 * True if range of values is accepted for specific fields, false otherwise. 930 * @param[out] error 931 * Pointer to error structure. 932 * 933 * @return 934 * 0 on success, a negative errno value otherwise and rte_errno is set. 935 */ 936 int 937 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 938 const uint8_t *mask, 939 const uint8_t *nic_mask, 940 unsigned int size, 941 bool range_accepted, 942 struct rte_flow_error *error) 943 { 944 unsigned int i; 945 946 MLX5_ASSERT(nic_mask); 947 for (i = 0; i < size; ++i) 948 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 949 return rte_flow_error_set(error, ENOTSUP, 950 RTE_FLOW_ERROR_TYPE_ITEM, 951 item, 952 "mask enables non supported" 953 " bits"); 954 if (!item->spec && (item->mask || item->last)) 955 return rte_flow_error_set(error, EINVAL, 956 RTE_FLOW_ERROR_TYPE_ITEM, item, 957 "mask/last without a spec is not" 958 " supported"); 959 if (item->spec && item->last && !range_accepted) { 960 uint8_t spec[size]; 961 uint8_t last[size]; 962 unsigned int i; 963 int ret; 964 965 for (i = 0; i < size; ++i) { 966 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 967 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 968 } 969 ret = memcmp(spec, last, size); 970 if (ret != 0) 971 return rte_flow_error_set(error, EINVAL, 972 RTE_FLOW_ERROR_TYPE_ITEM, 973 item, 974 "range is not valid"); 975 } 976 return 0; 977 } 978 979 /** 980 * Adjust the hash fields according to the @p flow information. 981 * 982 * @param[in] dev_flow. 983 * Pointer to the mlx5_flow. 984 * @param[in] tunnel 985 * 1 when the hash field is for a tunnel item. 986 * @param[in] layer_types 987 * ETH_RSS_* types. 988 * @param[in] hash_fields 989 * Item hash fields. 990 * 991 * @return 992 * The hash fields that should be used. 993 */ 994 uint64_t 995 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 996 int tunnel __rte_unused, uint64_t layer_types, 997 uint64_t hash_fields) 998 { 999 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1000 int rss_request_inner = rss_desc->level >= 2; 1001 1002 /* Check RSS hash level for tunnel. */ 1003 if (tunnel && rss_request_inner) 1004 hash_fields |= IBV_RX_HASH_INNER; 1005 else if (tunnel || rss_request_inner) 1006 return 0; 1007 #endif 1008 /* Check if requested layer matches RSS hash fields. */ 1009 if (!(rss_desc->types & layer_types)) 1010 return 0; 1011 return hash_fields; 1012 } 1013 1014 /** 1015 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1016 * if several tunnel rules are used on this queue, the tunnel ptype will be 1017 * cleared. 1018 * 1019 * @param rxq_ctrl 1020 * Rx queue to update. 1021 */ 1022 static void 1023 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1024 { 1025 unsigned int i; 1026 uint32_t tunnel_ptype = 0; 1027 1028 /* Look up for the ptype to use. */ 1029 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1030 if (!rxq_ctrl->flow_tunnels_n[i]) 1031 continue; 1032 if (!tunnel_ptype) { 1033 tunnel_ptype = tunnels_info[i].ptype; 1034 } else { 1035 tunnel_ptype = 0; 1036 break; 1037 } 1038 } 1039 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1040 } 1041 1042 /** 1043 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1044 * flow. 1045 * 1046 * @param[in] dev 1047 * Pointer to the Ethernet device structure. 1048 * @param[in] dev_handle 1049 * Pointer to device flow handle structure. 1050 */ 1051 void 1052 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1053 struct mlx5_flow_handle *dev_handle) 1054 { 1055 struct mlx5_priv *priv = dev->data->dev_private; 1056 const int mark = dev_handle->mark; 1057 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1058 struct mlx5_ind_table_obj *ind_tbl = NULL; 1059 unsigned int i; 1060 1061 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1062 struct mlx5_hrxq *hrxq; 1063 1064 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1065 dev_handle->rix_hrxq); 1066 if (hrxq) 1067 ind_tbl = hrxq->ind_table; 1068 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1069 struct mlx5_shared_action_rss *shared_rss; 1070 1071 shared_rss = mlx5_ipool_get 1072 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1073 dev_handle->rix_srss); 1074 if (shared_rss) 1075 ind_tbl = shared_rss->ind_tbl; 1076 } 1077 if (!ind_tbl) 1078 return; 1079 for (i = 0; i != ind_tbl->queues_n; ++i) { 1080 int idx = ind_tbl->queues[i]; 1081 struct mlx5_rxq_ctrl *rxq_ctrl = 1082 container_of((*priv->rxqs)[idx], 1083 struct mlx5_rxq_ctrl, rxq); 1084 1085 /* 1086 * To support metadata register copy on Tx loopback, 1087 * this must be always enabled (metadata may arive 1088 * from other port - not from local flows only. 1089 */ 1090 if (priv->config.dv_flow_en && 1091 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1092 mlx5_flow_ext_mreg_supported(dev)) { 1093 rxq_ctrl->rxq.mark = 1; 1094 rxq_ctrl->flow_mark_n = 1; 1095 } else if (mark) { 1096 rxq_ctrl->rxq.mark = 1; 1097 rxq_ctrl->flow_mark_n++; 1098 } 1099 if (tunnel) { 1100 unsigned int j; 1101 1102 /* Increase the counter matching the flow. */ 1103 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1104 if ((tunnels_info[j].tunnel & 1105 dev_handle->layers) == 1106 tunnels_info[j].tunnel) { 1107 rxq_ctrl->flow_tunnels_n[j]++; 1108 break; 1109 } 1110 } 1111 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1112 } 1113 } 1114 } 1115 1116 /** 1117 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1118 * 1119 * @param[in] dev 1120 * Pointer to the Ethernet device structure. 1121 * @param[in] flow 1122 * Pointer to flow structure. 1123 */ 1124 static void 1125 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1126 { 1127 struct mlx5_priv *priv = dev->data->dev_private; 1128 uint32_t handle_idx; 1129 struct mlx5_flow_handle *dev_handle; 1130 1131 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1132 handle_idx, dev_handle, next) 1133 flow_drv_rxq_flags_set(dev, dev_handle); 1134 } 1135 1136 /** 1137 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1138 * device flow if no other flow uses it with the same kind of request. 1139 * 1140 * @param dev 1141 * Pointer to Ethernet device. 1142 * @param[in] dev_handle 1143 * Pointer to the device flow handle structure. 1144 */ 1145 static void 1146 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1147 struct mlx5_flow_handle *dev_handle) 1148 { 1149 struct mlx5_priv *priv = dev->data->dev_private; 1150 const int mark = dev_handle->mark; 1151 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1152 struct mlx5_ind_table_obj *ind_tbl = NULL; 1153 unsigned int i; 1154 1155 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1156 struct mlx5_hrxq *hrxq; 1157 1158 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1159 dev_handle->rix_hrxq); 1160 if (hrxq) 1161 ind_tbl = hrxq->ind_table; 1162 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1163 struct mlx5_shared_action_rss *shared_rss; 1164 1165 shared_rss = mlx5_ipool_get 1166 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1167 dev_handle->rix_srss); 1168 if (shared_rss) 1169 ind_tbl = shared_rss->ind_tbl; 1170 } 1171 if (!ind_tbl) 1172 return; 1173 MLX5_ASSERT(dev->data->dev_started); 1174 for (i = 0; i != ind_tbl->queues_n; ++i) { 1175 int idx = ind_tbl->queues[i]; 1176 struct mlx5_rxq_ctrl *rxq_ctrl = 1177 container_of((*priv->rxqs)[idx], 1178 struct mlx5_rxq_ctrl, rxq); 1179 1180 if (priv->config.dv_flow_en && 1181 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1182 mlx5_flow_ext_mreg_supported(dev)) { 1183 rxq_ctrl->rxq.mark = 1; 1184 rxq_ctrl->flow_mark_n = 1; 1185 } else if (mark) { 1186 rxq_ctrl->flow_mark_n--; 1187 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1188 } 1189 if (tunnel) { 1190 unsigned int j; 1191 1192 /* Decrease the counter matching the flow. */ 1193 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1194 if ((tunnels_info[j].tunnel & 1195 dev_handle->layers) == 1196 tunnels_info[j].tunnel) { 1197 rxq_ctrl->flow_tunnels_n[j]--; 1198 break; 1199 } 1200 } 1201 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1202 } 1203 } 1204 } 1205 1206 /** 1207 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1208 * @p flow if no other flow uses it with the same kind of request. 1209 * 1210 * @param dev 1211 * Pointer to Ethernet device. 1212 * @param[in] flow 1213 * Pointer to the flow. 1214 */ 1215 static void 1216 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1217 { 1218 struct mlx5_priv *priv = dev->data->dev_private; 1219 uint32_t handle_idx; 1220 struct mlx5_flow_handle *dev_handle; 1221 1222 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1223 handle_idx, dev_handle, next) 1224 flow_drv_rxq_flags_trim(dev, dev_handle); 1225 } 1226 1227 /** 1228 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1229 * 1230 * @param dev 1231 * Pointer to Ethernet device. 1232 */ 1233 static void 1234 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1235 { 1236 struct mlx5_priv *priv = dev->data->dev_private; 1237 unsigned int i; 1238 1239 for (i = 0; i != priv->rxqs_n; ++i) { 1240 struct mlx5_rxq_ctrl *rxq_ctrl; 1241 unsigned int j; 1242 1243 if (!(*priv->rxqs)[i]) 1244 continue; 1245 rxq_ctrl = container_of((*priv->rxqs)[i], 1246 struct mlx5_rxq_ctrl, rxq); 1247 rxq_ctrl->flow_mark_n = 0; 1248 rxq_ctrl->rxq.mark = 0; 1249 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1250 rxq_ctrl->flow_tunnels_n[j] = 0; 1251 rxq_ctrl->rxq.tunnel = 0; 1252 } 1253 } 1254 1255 /** 1256 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1257 * 1258 * @param[in] dev 1259 * Pointer to the Ethernet device structure. 1260 */ 1261 void 1262 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1263 { 1264 struct mlx5_priv *priv = dev->data->dev_private; 1265 struct mlx5_rxq_data *data; 1266 unsigned int i; 1267 1268 for (i = 0; i != priv->rxqs_n; ++i) { 1269 if (!(*priv->rxqs)[i]) 1270 continue; 1271 data = (*priv->rxqs)[i]; 1272 if (!rte_flow_dynf_metadata_avail()) { 1273 data->dynf_meta = 0; 1274 data->flow_meta_mask = 0; 1275 data->flow_meta_offset = -1; 1276 data->flow_meta_port_mask = 0; 1277 } else { 1278 data->dynf_meta = 1; 1279 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1280 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1281 data->flow_meta_port_mask = (uint32_t)~0; 1282 if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16) 1283 data->flow_meta_port_mask >>= 16; 1284 } 1285 } 1286 } 1287 1288 /* 1289 * return a pointer to the desired action in the list of actions. 1290 * 1291 * @param[in] actions 1292 * The list of actions to search the action in. 1293 * @param[in] action 1294 * The action to find. 1295 * 1296 * @return 1297 * Pointer to the action in the list, if found. NULL otherwise. 1298 */ 1299 const struct rte_flow_action * 1300 mlx5_flow_find_action(const struct rte_flow_action *actions, 1301 enum rte_flow_action_type action) 1302 { 1303 if (actions == NULL) 1304 return NULL; 1305 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1306 if (actions->type == action) 1307 return actions; 1308 return NULL; 1309 } 1310 1311 /* 1312 * Validate the flag action. 1313 * 1314 * @param[in] action_flags 1315 * Bit-fields that holds the actions detected until now. 1316 * @param[in] attr 1317 * Attributes of flow that includes this action. 1318 * @param[out] error 1319 * Pointer to error structure. 1320 * 1321 * @return 1322 * 0 on success, a negative errno value otherwise and rte_errno is set. 1323 */ 1324 int 1325 mlx5_flow_validate_action_flag(uint64_t action_flags, 1326 const struct rte_flow_attr *attr, 1327 struct rte_flow_error *error) 1328 { 1329 if (action_flags & MLX5_FLOW_ACTION_MARK) 1330 return rte_flow_error_set(error, EINVAL, 1331 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1332 "can't mark and flag in same flow"); 1333 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1334 return rte_flow_error_set(error, EINVAL, 1335 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1336 "can't have 2 flag" 1337 " actions in same flow"); 1338 if (attr->egress) 1339 return rte_flow_error_set(error, ENOTSUP, 1340 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1341 "flag action not supported for " 1342 "egress"); 1343 return 0; 1344 } 1345 1346 /* 1347 * Validate the mark action. 1348 * 1349 * @param[in] action 1350 * Pointer to the queue action. 1351 * @param[in] action_flags 1352 * Bit-fields that holds the actions detected until now. 1353 * @param[in] attr 1354 * Attributes of flow that includes this action. 1355 * @param[out] error 1356 * Pointer to error structure. 1357 * 1358 * @return 1359 * 0 on success, a negative errno value otherwise and rte_errno is set. 1360 */ 1361 int 1362 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1363 uint64_t action_flags, 1364 const struct rte_flow_attr *attr, 1365 struct rte_flow_error *error) 1366 { 1367 const struct rte_flow_action_mark *mark = action->conf; 1368 1369 if (!mark) 1370 return rte_flow_error_set(error, EINVAL, 1371 RTE_FLOW_ERROR_TYPE_ACTION, 1372 action, 1373 "configuration cannot be null"); 1374 if (mark->id >= MLX5_FLOW_MARK_MAX) 1375 return rte_flow_error_set(error, EINVAL, 1376 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1377 &mark->id, 1378 "mark id must in 0 <= id < " 1379 RTE_STR(MLX5_FLOW_MARK_MAX)); 1380 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1381 return rte_flow_error_set(error, EINVAL, 1382 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1383 "can't flag and mark in same flow"); 1384 if (action_flags & MLX5_FLOW_ACTION_MARK) 1385 return rte_flow_error_set(error, EINVAL, 1386 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1387 "can't have 2 mark actions in same" 1388 " flow"); 1389 if (attr->egress) 1390 return rte_flow_error_set(error, ENOTSUP, 1391 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1392 "mark action not supported for " 1393 "egress"); 1394 return 0; 1395 } 1396 1397 /* 1398 * Validate the drop action. 1399 * 1400 * @param[in] action_flags 1401 * Bit-fields that holds the actions detected until now. 1402 * @param[in] attr 1403 * Attributes of flow that includes this action. 1404 * @param[out] error 1405 * Pointer to error structure. 1406 * 1407 * @return 1408 * 0 on success, a negative errno value otherwise and rte_errno is set. 1409 */ 1410 int 1411 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1412 const struct rte_flow_attr *attr, 1413 struct rte_flow_error *error) 1414 { 1415 if (attr->egress) 1416 return rte_flow_error_set(error, ENOTSUP, 1417 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1418 "drop action not supported for " 1419 "egress"); 1420 return 0; 1421 } 1422 1423 /* 1424 * Validate the queue action. 1425 * 1426 * @param[in] action 1427 * Pointer to the queue action. 1428 * @param[in] action_flags 1429 * Bit-fields that holds the actions detected until now. 1430 * @param[in] dev 1431 * Pointer to the Ethernet device structure. 1432 * @param[in] attr 1433 * Attributes of flow that includes this action. 1434 * @param[out] error 1435 * Pointer to error structure. 1436 * 1437 * @return 1438 * 0 on success, a negative errno value otherwise and rte_errno is set. 1439 */ 1440 int 1441 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1442 uint64_t action_flags, 1443 struct rte_eth_dev *dev, 1444 const struct rte_flow_attr *attr, 1445 struct rte_flow_error *error) 1446 { 1447 struct mlx5_priv *priv = dev->data->dev_private; 1448 const struct rte_flow_action_queue *queue = action->conf; 1449 1450 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1451 return rte_flow_error_set(error, EINVAL, 1452 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1453 "can't have 2 fate actions in" 1454 " same flow"); 1455 if (!priv->rxqs_n) 1456 return rte_flow_error_set(error, EINVAL, 1457 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1458 NULL, "No Rx queues configured"); 1459 if (queue->index >= priv->rxqs_n) 1460 return rte_flow_error_set(error, EINVAL, 1461 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1462 &queue->index, 1463 "queue index out of range"); 1464 if (!(*priv->rxqs)[queue->index]) 1465 return rte_flow_error_set(error, EINVAL, 1466 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1467 &queue->index, 1468 "queue is not configured"); 1469 if (attr->egress) 1470 return rte_flow_error_set(error, ENOTSUP, 1471 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1472 "queue action not supported for " 1473 "egress"); 1474 return 0; 1475 } 1476 1477 /* 1478 * Validate the rss action. 1479 * 1480 * @param[in] dev 1481 * Pointer to the Ethernet device structure. 1482 * @param[in] action 1483 * Pointer to the queue action. 1484 * @param[out] error 1485 * Pointer to error structure. 1486 * 1487 * @return 1488 * 0 on success, a negative errno value otherwise and rte_errno is set. 1489 */ 1490 int 1491 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1492 const struct rte_flow_action *action, 1493 struct rte_flow_error *error) 1494 { 1495 struct mlx5_priv *priv = dev->data->dev_private; 1496 const struct rte_flow_action_rss *rss = action->conf; 1497 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1498 unsigned int i; 1499 1500 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1501 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1502 return rte_flow_error_set(error, ENOTSUP, 1503 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1504 &rss->func, 1505 "RSS hash function not supported"); 1506 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1507 if (rss->level > 2) 1508 #else 1509 if (rss->level > 1) 1510 #endif 1511 return rte_flow_error_set(error, ENOTSUP, 1512 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1513 &rss->level, 1514 "tunnel RSS is not supported"); 1515 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1516 if (rss->key_len == 0 && rss->key != NULL) 1517 return rte_flow_error_set(error, ENOTSUP, 1518 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1519 &rss->key_len, 1520 "RSS hash key length 0"); 1521 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1522 return rte_flow_error_set(error, ENOTSUP, 1523 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1524 &rss->key_len, 1525 "RSS hash key too small"); 1526 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1527 return rte_flow_error_set(error, ENOTSUP, 1528 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1529 &rss->key_len, 1530 "RSS hash key too large"); 1531 if (rss->queue_num > priv->config.ind_table_max_size) 1532 return rte_flow_error_set(error, ENOTSUP, 1533 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1534 &rss->queue_num, 1535 "number of queues too large"); 1536 if (rss->types & MLX5_RSS_HF_MASK) 1537 return rte_flow_error_set(error, ENOTSUP, 1538 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1539 &rss->types, 1540 "some RSS protocols are not" 1541 " supported"); 1542 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1543 !(rss->types & ETH_RSS_IP)) 1544 return rte_flow_error_set(error, EINVAL, 1545 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1546 "L3 partial RSS requested but L3 RSS" 1547 " type not specified"); 1548 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1549 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1550 return rte_flow_error_set(error, EINVAL, 1551 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1552 "L4 partial RSS requested but L4 RSS" 1553 " type not specified"); 1554 if (!priv->rxqs_n) 1555 return rte_flow_error_set(error, EINVAL, 1556 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1557 NULL, "No Rx queues configured"); 1558 if (!rss->queue_num) 1559 return rte_flow_error_set(error, EINVAL, 1560 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1561 NULL, "No queues configured"); 1562 for (i = 0; i != rss->queue_num; ++i) { 1563 struct mlx5_rxq_ctrl *rxq_ctrl; 1564 1565 if (rss->queue[i] >= priv->rxqs_n) 1566 return rte_flow_error_set 1567 (error, EINVAL, 1568 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1569 &rss->queue[i], "queue index out of range"); 1570 if (!(*priv->rxqs)[rss->queue[i]]) 1571 return rte_flow_error_set 1572 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1573 &rss->queue[i], "queue is not configured"); 1574 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1575 struct mlx5_rxq_ctrl, rxq); 1576 if (i == 0) 1577 rxq_type = rxq_ctrl->type; 1578 if (rxq_type != rxq_ctrl->type) 1579 return rte_flow_error_set 1580 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1581 &rss->queue[i], 1582 "combining hairpin and regular RSS queues is not supported"); 1583 } 1584 return 0; 1585 } 1586 1587 /* 1588 * Validate the rss action. 1589 * 1590 * @param[in] action 1591 * Pointer to the queue action. 1592 * @param[in] action_flags 1593 * Bit-fields that holds the actions detected until now. 1594 * @param[in] dev 1595 * Pointer to the Ethernet device structure. 1596 * @param[in] attr 1597 * Attributes of flow that includes this action. 1598 * @param[in] item_flags 1599 * Items that were detected. 1600 * @param[out] error 1601 * Pointer to error structure. 1602 * 1603 * @return 1604 * 0 on success, a negative errno value otherwise and rte_errno is set. 1605 */ 1606 int 1607 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1608 uint64_t action_flags, 1609 struct rte_eth_dev *dev, 1610 const struct rte_flow_attr *attr, 1611 uint64_t item_flags, 1612 struct rte_flow_error *error) 1613 { 1614 const struct rte_flow_action_rss *rss = action->conf; 1615 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1616 int ret; 1617 1618 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1619 return rte_flow_error_set(error, EINVAL, 1620 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1621 "can't have 2 fate actions" 1622 " in same flow"); 1623 ret = mlx5_validate_action_rss(dev, action, error); 1624 if (ret) 1625 return ret; 1626 if (attr->egress) 1627 return rte_flow_error_set(error, ENOTSUP, 1628 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1629 "rss action not supported for " 1630 "egress"); 1631 if (rss->level > 1 && !tunnel) 1632 return rte_flow_error_set(error, EINVAL, 1633 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1634 "inner RSS is not supported for " 1635 "non-tunnel flows"); 1636 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1637 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1638 return rte_flow_error_set(error, EINVAL, 1639 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1640 "RSS on eCPRI is not supported now"); 1641 } 1642 return 0; 1643 } 1644 1645 /* 1646 * Validate the default miss action. 1647 * 1648 * @param[in] action_flags 1649 * Bit-fields that holds the actions detected until now. 1650 * @param[out] error 1651 * Pointer to error structure. 1652 * 1653 * @return 1654 * 0 on success, a negative errno value otherwise and rte_errno is set. 1655 */ 1656 int 1657 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1658 const struct rte_flow_attr *attr, 1659 struct rte_flow_error *error) 1660 { 1661 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1662 return rte_flow_error_set(error, EINVAL, 1663 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1664 "can't have 2 fate actions in" 1665 " same flow"); 1666 if (attr->egress) 1667 return rte_flow_error_set(error, ENOTSUP, 1668 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1669 "default miss action not supported " 1670 "for egress"); 1671 if (attr->group) 1672 return rte_flow_error_set(error, ENOTSUP, 1673 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1674 "only group 0 is supported"); 1675 if (attr->transfer) 1676 return rte_flow_error_set(error, ENOTSUP, 1677 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1678 NULL, "transfer is not supported"); 1679 return 0; 1680 } 1681 1682 /* 1683 * Validate the count action. 1684 * 1685 * @param[in] dev 1686 * Pointer to the Ethernet device structure. 1687 * @param[in] attr 1688 * Attributes of flow that includes this action. 1689 * @param[out] error 1690 * Pointer to error structure. 1691 * 1692 * @return 1693 * 0 on success, a negative errno value otherwise and rte_errno is set. 1694 */ 1695 int 1696 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1697 const struct rte_flow_attr *attr, 1698 struct rte_flow_error *error) 1699 { 1700 if (attr->egress) 1701 return rte_flow_error_set(error, ENOTSUP, 1702 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1703 "count action not supported for " 1704 "egress"); 1705 return 0; 1706 } 1707 1708 /* 1709 * Validate the ASO CT action. 1710 * 1711 * @param[in] dev 1712 * Pointer to the Ethernet device structure. 1713 * @param[in] conntrack 1714 * Pointer to the CT action profile. 1715 * @param[out] error 1716 * Pointer to error structure. 1717 * 1718 * @return 1719 * 0 on success, a negative errno value otherwise and rte_errno is set. 1720 */ 1721 int 1722 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1723 const struct rte_flow_action_conntrack *conntrack, 1724 struct rte_flow_error *error) 1725 { 1726 RTE_SET_USED(dev); 1727 1728 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1729 return rte_flow_error_set(error, EINVAL, 1730 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1731 "Invalid CT state"); 1732 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1733 return rte_flow_error_set(error, EINVAL, 1734 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1735 "Invalid last TCP packet flag"); 1736 return 0; 1737 } 1738 1739 /** 1740 * Verify the @p attributes will be correctly understood by the NIC and store 1741 * them in the @p flow if everything is correct. 1742 * 1743 * @param[in] dev 1744 * Pointer to the Ethernet device structure. 1745 * @param[in] attributes 1746 * Pointer to flow attributes 1747 * @param[out] error 1748 * Pointer to error structure. 1749 * 1750 * @return 1751 * 0 on success, a negative errno value otherwise and rte_errno is set. 1752 */ 1753 int 1754 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1755 const struct rte_flow_attr *attributes, 1756 struct rte_flow_error *error) 1757 { 1758 struct mlx5_priv *priv = dev->data->dev_private; 1759 uint32_t priority_max = priv->config.flow_prio - 1; 1760 1761 if (attributes->group) 1762 return rte_flow_error_set(error, ENOTSUP, 1763 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1764 NULL, "groups is not supported"); 1765 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1766 attributes->priority >= priority_max) 1767 return rte_flow_error_set(error, ENOTSUP, 1768 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1769 NULL, "priority out of range"); 1770 if (attributes->egress) 1771 return rte_flow_error_set(error, ENOTSUP, 1772 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1773 "egress is not supported"); 1774 if (attributes->transfer && !priv->config.dv_esw_en) 1775 return rte_flow_error_set(error, ENOTSUP, 1776 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1777 NULL, "transfer is not supported"); 1778 if (!attributes->ingress) 1779 return rte_flow_error_set(error, EINVAL, 1780 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1781 NULL, 1782 "ingress attribute is mandatory"); 1783 return 0; 1784 } 1785 1786 /** 1787 * Validate ICMP6 item. 1788 * 1789 * @param[in] item 1790 * Item specification. 1791 * @param[in] item_flags 1792 * Bit-fields that holds the items detected until now. 1793 * @param[in] ext_vlan_sup 1794 * Whether extended VLAN features are supported or not. 1795 * @param[out] error 1796 * Pointer to error structure. 1797 * 1798 * @return 1799 * 0 on success, a negative errno value otherwise and rte_errno is set. 1800 */ 1801 int 1802 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1803 uint64_t item_flags, 1804 uint8_t target_protocol, 1805 struct rte_flow_error *error) 1806 { 1807 const struct rte_flow_item_icmp6 *mask = item->mask; 1808 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1809 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1810 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1811 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1812 MLX5_FLOW_LAYER_OUTER_L4; 1813 int ret; 1814 1815 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1816 return rte_flow_error_set(error, EINVAL, 1817 RTE_FLOW_ERROR_TYPE_ITEM, item, 1818 "protocol filtering not compatible" 1819 " with ICMP6 layer"); 1820 if (!(item_flags & l3m)) 1821 return rte_flow_error_set(error, EINVAL, 1822 RTE_FLOW_ERROR_TYPE_ITEM, item, 1823 "IPv6 is mandatory to filter on" 1824 " ICMP6"); 1825 if (item_flags & l4m) 1826 return rte_flow_error_set(error, EINVAL, 1827 RTE_FLOW_ERROR_TYPE_ITEM, item, 1828 "multiple L4 layers not supported"); 1829 if (!mask) 1830 mask = &rte_flow_item_icmp6_mask; 1831 ret = mlx5_flow_item_acceptable 1832 (item, (const uint8_t *)mask, 1833 (const uint8_t *)&rte_flow_item_icmp6_mask, 1834 sizeof(struct rte_flow_item_icmp6), 1835 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1836 if (ret < 0) 1837 return ret; 1838 return 0; 1839 } 1840 1841 /** 1842 * Validate ICMP item. 1843 * 1844 * @param[in] item 1845 * Item specification. 1846 * @param[in] item_flags 1847 * Bit-fields that holds the items detected until now. 1848 * @param[out] error 1849 * Pointer to error structure. 1850 * 1851 * @return 1852 * 0 on success, a negative errno value otherwise and rte_errno is set. 1853 */ 1854 int 1855 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1856 uint64_t item_flags, 1857 uint8_t target_protocol, 1858 struct rte_flow_error *error) 1859 { 1860 const struct rte_flow_item_icmp *mask = item->mask; 1861 const struct rte_flow_item_icmp nic_mask = { 1862 .hdr.icmp_type = 0xff, 1863 .hdr.icmp_code = 0xff, 1864 .hdr.icmp_ident = RTE_BE16(0xffff), 1865 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1866 }; 1867 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1868 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1869 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1870 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1871 MLX5_FLOW_LAYER_OUTER_L4; 1872 int ret; 1873 1874 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1875 return rte_flow_error_set(error, EINVAL, 1876 RTE_FLOW_ERROR_TYPE_ITEM, item, 1877 "protocol filtering not compatible" 1878 " with ICMP layer"); 1879 if (!(item_flags & l3m)) 1880 return rte_flow_error_set(error, EINVAL, 1881 RTE_FLOW_ERROR_TYPE_ITEM, item, 1882 "IPv4 is mandatory to filter" 1883 " on ICMP"); 1884 if (item_flags & l4m) 1885 return rte_flow_error_set(error, EINVAL, 1886 RTE_FLOW_ERROR_TYPE_ITEM, item, 1887 "multiple L4 layers not supported"); 1888 if (!mask) 1889 mask = &nic_mask; 1890 ret = mlx5_flow_item_acceptable 1891 (item, (const uint8_t *)mask, 1892 (const uint8_t *)&nic_mask, 1893 sizeof(struct rte_flow_item_icmp), 1894 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1895 if (ret < 0) 1896 return ret; 1897 return 0; 1898 } 1899 1900 /** 1901 * Validate Ethernet item. 1902 * 1903 * @param[in] item 1904 * Item specification. 1905 * @param[in] item_flags 1906 * Bit-fields that holds the items detected until now. 1907 * @param[out] error 1908 * Pointer to error structure. 1909 * 1910 * @return 1911 * 0 on success, a negative errno value otherwise and rte_errno is set. 1912 */ 1913 int 1914 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1915 uint64_t item_flags, bool ext_vlan_sup, 1916 struct rte_flow_error *error) 1917 { 1918 const struct rte_flow_item_eth *mask = item->mask; 1919 const struct rte_flow_item_eth nic_mask = { 1920 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1921 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1922 .type = RTE_BE16(0xffff), 1923 .has_vlan = ext_vlan_sup ? 1 : 0, 1924 }; 1925 int ret; 1926 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1927 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1928 MLX5_FLOW_LAYER_OUTER_L2; 1929 1930 if (item_flags & ethm) 1931 return rte_flow_error_set(error, ENOTSUP, 1932 RTE_FLOW_ERROR_TYPE_ITEM, item, 1933 "multiple L2 layers not supported"); 1934 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1935 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1936 return rte_flow_error_set(error, EINVAL, 1937 RTE_FLOW_ERROR_TYPE_ITEM, item, 1938 "L2 layer should not follow " 1939 "L3 layers"); 1940 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1941 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1942 return rte_flow_error_set(error, EINVAL, 1943 RTE_FLOW_ERROR_TYPE_ITEM, item, 1944 "L2 layer should not follow VLAN"); 1945 if (!mask) 1946 mask = &rte_flow_item_eth_mask; 1947 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1948 (const uint8_t *)&nic_mask, 1949 sizeof(struct rte_flow_item_eth), 1950 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1951 return ret; 1952 } 1953 1954 /** 1955 * Validate VLAN item. 1956 * 1957 * @param[in] item 1958 * Item specification. 1959 * @param[in] item_flags 1960 * Bit-fields that holds the items detected until now. 1961 * @param[in] dev 1962 * Ethernet device flow is being created on. 1963 * @param[out] error 1964 * Pointer to error structure. 1965 * 1966 * @return 1967 * 0 on success, a negative errno value otherwise and rte_errno is set. 1968 */ 1969 int 1970 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1971 uint64_t item_flags, 1972 struct rte_eth_dev *dev, 1973 struct rte_flow_error *error) 1974 { 1975 const struct rte_flow_item_vlan *spec = item->spec; 1976 const struct rte_flow_item_vlan *mask = item->mask; 1977 const struct rte_flow_item_vlan nic_mask = { 1978 .tci = RTE_BE16(UINT16_MAX), 1979 .inner_type = RTE_BE16(UINT16_MAX), 1980 }; 1981 uint16_t vlan_tag = 0; 1982 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1983 int ret; 1984 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1985 MLX5_FLOW_LAYER_INNER_L4) : 1986 (MLX5_FLOW_LAYER_OUTER_L3 | 1987 MLX5_FLOW_LAYER_OUTER_L4); 1988 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1989 MLX5_FLOW_LAYER_OUTER_VLAN; 1990 1991 if (item_flags & vlanm) 1992 return rte_flow_error_set(error, EINVAL, 1993 RTE_FLOW_ERROR_TYPE_ITEM, item, 1994 "multiple VLAN layers not supported"); 1995 else if ((item_flags & l34m) != 0) 1996 return rte_flow_error_set(error, EINVAL, 1997 RTE_FLOW_ERROR_TYPE_ITEM, item, 1998 "VLAN cannot follow L3/L4 layer"); 1999 if (!mask) 2000 mask = &rte_flow_item_vlan_mask; 2001 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2002 (const uint8_t *)&nic_mask, 2003 sizeof(struct rte_flow_item_vlan), 2004 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2005 if (ret) 2006 return ret; 2007 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2008 struct mlx5_priv *priv = dev->data->dev_private; 2009 2010 if (priv->vmwa_context) { 2011 /* 2012 * Non-NULL context means we have a virtual machine 2013 * and SR-IOV enabled, we have to create VLAN interface 2014 * to make hypervisor to setup E-Switch vport 2015 * context correctly. We avoid creating the multiple 2016 * VLAN interfaces, so we cannot support VLAN tag mask. 2017 */ 2018 return rte_flow_error_set(error, EINVAL, 2019 RTE_FLOW_ERROR_TYPE_ITEM, 2020 item, 2021 "VLAN tag mask is not" 2022 " supported in virtual" 2023 " environment"); 2024 } 2025 } 2026 if (spec) { 2027 vlan_tag = spec->tci; 2028 vlan_tag &= mask->tci; 2029 } 2030 /* 2031 * From verbs perspective an empty VLAN is equivalent 2032 * to a packet without VLAN layer. 2033 */ 2034 if (!vlan_tag) 2035 return rte_flow_error_set(error, EINVAL, 2036 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2037 item->spec, 2038 "VLAN cannot be empty"); 2039 return 0; 2040 } 2041 2042 /** 2043 * Validate IPV4 item. 2044 * 2045 * @param[in] item 2046 * Item specification. 2047 * @param[in] item_flags 2048 * Bit-fields that holds the items detected until now. 2049 * @param[in] last_item 2050 * Previous validated item in the pattern items. 2051 * @param[in] ether_type 2052 * Type in the ethernet layer header (including dot1q). 2053 * @param[in] acc_mask 2054 * Acceptable mask, if NULL default internal default mask 2055 * will be used to check whether item fields are supported. 2056 * @param[in] range_accepted 2057 * True if range of values is accepted for specific fields, false otherwise. 2058 * @param[out] error 2059 * Pointer to error structure. 2060 * 2061 * @return 2062 * 0 on success, a negative errno value otherwise and rte_errno is set. 2063 */ 2064 int 2065 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2066 uint64_t item_flags, 2067 uint64_t last_item, 2068 uint16_t ether_type, 2069 const struct rte_flow_item_ipv4 *acc_mask, 2070 bool range_accepted, 2071 struct rte_flow_error *error) 2072 { 2073 const struct rte_flow_item_ipv4 *mask = item->mask; 2074 const struct rte_flow_item_ipv4 *spec = item->spec; 2075 const struct rte_flow_item_ipv4 nic_mask = { 2076 .hdr = { 2077 .src_addr = RTE_BE32(0xffffffff), 2078 .dst_addr = RTE_BE32(0xffffffff), 2079 .type_of_service = 0xff, 2080 .next_proto_id = 0xff, 2081 }, 2082 }; 2083 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2084 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2085 MLX5_FLOW_LAYER_OUTER_L3; 2086 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2087 MLX5_FLOW_LAYER_OUTER_L4; 2088 int ret; 2089 uint8_t next_proto = 0xFF; 2090 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2091 MLX5_FLOW_LAYER_OUTER_VLAN | 2092 MLX5_FLOW_LAYER_INNER_VLAN); 2093 2094 if ((last_item & l2_vlan) && ether_type && 2095 ether_type != RTE_ETHER_TYPE_IPV4) 2096 return rte_flow_error_set(error, EINVAL, 2097 RTE_FLOW_ERROR_TYPE_ITEM, item, 2098 "IPv4 cannot follow L2/VLAN layer " 2099 "which ether type is not IPv4"); 2100 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2101 if (mask && spec) 2102 next_proto = mask->hdr.next_proto_id & 2103 spec->hdr.next_proto_id; 2104 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2105 return rte_flow_error_set(error, EINVAL, 2106 RTE_FLOW_ERROR_TYPE_ITEM, 2107 item, 2108 "multiple tunnel " 2109 "not supported"); 2110 } 2111 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2112 return rte_flow_error_set(error, EINVAL, 2113 RTE_FLOW_ERROR_TYPE_ITEM, item, 2114 "wrong tunnel type - IPv6 specified " 2115 "but IPv4 item provided"); 2116 if (item_flags & l3m) 2117 return rte_flow_error_set(error, ENOTSUP, 2118 RTE_FLOW_ERROR_TYPE_ITEM, item, 2119 "multiple L3 layers not supported"); 2120 else if (item_flags & l4m) 2121 return rte_flow_error_set(error, EINVAL, 2122 RTE_FLOW_ERROR_TYPE_ITEM, item, 2123 "L3 cannot follow an L4 layer."); 2124 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2125 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2126 return rte_flow_error_set(error, EINVAL, 2127 RTE_FLOW_ERROR_TYPE_ITEM, item, 2128 "L3 cannot follow an NVGRE layer."); 2129 if (!mask) 2130 mask = &rte_flow_item_ipv4_mask; 2131 else if (mask->hdr.next_proto_id != 0 && 2132 mask->hdr.next_proto_id != 0xff) 2133 return rte_flow_error_set(error, EINVAL, 2134 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2135 "partial mask is not supported" 2136 " for protocol"); 2137 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2138 acc_mask ? (const uint8_t *)acc_mask 2139 : (const uint8_t *)&nic_mask, 2140 sizeof(struct rte_flow_item_ipv4), 2141 range_accepted, error); 2142 if (ret < 0) 2143 return ret; 2144 return 0; 2145 } 2146 2147 /** 2148 * Validate IPV6 item. 2149 * 2150 * @param[in] item 2151 * Item specification. 2152 * @param[in] item_flags 2153 * Bit-fields that holds the items detected until now. 2154 * @param[in] last_item 2155 * Previous validated item in the pattern items. 2156 * @param[in] ether_type 2157 * Type in the ethernet layer header (including dot1q). 2158 * @param[in] acc_mask 2159 * Acceptable mask, if NULL default internal default mask 2160 * will be used to check whether item fields are supported. 2161 * @param[out] error 2162 * Pointer to error structure. 2163 * 2164 * @return 2165 * 0 on success, a negative errno value otherwise and rte_errno is set. 2166 */ 2167 int 2168 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2169 uint64_t item_flags, 2170 uint64_t last_item, 2171 uint16_t ether_type, 2172 const struct rte_flow_item_ipv6 *acc_mask, 2173 struct rte_flow_error *error) 2174 { 2175 const struct rte_flow_item_ipv6 *mask = item->mask; 2176 const struct rte_flow_item_ipv6 *spec = item->spec; 2177 const struct rte_flow_item_ipv6 nic_mask = { 2178 .hdr = { 2179 .src_addr = 2180 "\xff\xff\xff\xff\xff\xff\xff\xff" 2181 "\xff\xff\xff\xff\xff\xff\xff\xff", 2182 .dst_addr = 2183 "\xff\xff\xff\xff\xff\xff\xff\xff" 2184 "\xff\xff\xff\xff\xff\xff\xff\xff", 2185 .vtc_flow = RTE_BE32(0xffffffff), 2186 .proto = 0xff, 2187 }, 2188 }; 2189 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2190 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2191 MLX5_FLOW_LAYER_OUTER_L3; 2192 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2193 MLX5_FLOW_LAYER_OUTER_L4; 2194 int ret; 2195 uint8_t next_proto = 0xFF; 2196 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2197 MLX5_FLOW_LAYER_OUTER_VLAN | 2198 MLX5_FLOW_LAYER_INNER_VLAN); 2199 2200 if ((last_item & l2_vlan) && ether_type && 2201 ether_type != RTE_ETHER_TYPE_IPV6) 2202 return rte_flow_error_set(error, EINVAL, 2203 RTE_FLOW_ERROR_TYPE_ITEM, item, 2204 "IPv6 cannot follow L2/VLAN layer " 2205 "which ether type is not IPv6"); 2206 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2207 next_proto = spec->hdr.proto; 2208 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 2209 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2210 return rte_flow_error_set(error, EINVAL, 2211 RTE_FLOW_ERROR_TYPE_ITEM, 2212 item, 2213 "multiple tunnel " 2214 "not supported"); 2215 } 2216 if (next_proto == IPPROTO_HOPOPTS || 2217 next_proto == IPPROTO_ROUTING || 2218 next_proto == IPPROTO_FRAGMENT || 2219 next_proto == IPPROTO_ESP || 2220 next_proto == IPPROTO_AH || 2221 next_proto == IPPROTO_DSTOPTS) 2222 return rte_flow_error_set(error, EINVAL, 2223 RTE_FLOW_ERROR_TYPE_ITEM, item, 2224 "IPv6 proto (next header) should " 2225 "not be set as extension header"); 2226 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2227 return rte_flow_error_set(error, EINVAL, 2228 RTE_FLOW_ERROR_TYPE_ITEM, item, 2229 "wrong tunnel type - IPv4 specified " 2230 "but IPv6 item provided"); 2231 if (item_flags & l3m) 2232 return rte_flow_error_set(error, ENOTSUP, 2233 RTE_FLOW_ERROR_TYPE_ITEM, item, 2234 "multiple L3 layers not supported"); 2235 else if (item_flags & l4m) 2236 return rte_flow_error_set(error, EINVAL, 2237 RTE_FLOW_ERROR_TYPE_ITEM, item, 2238 "L3 cannot follow an L4 layer."); 2239 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2240 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2241 return rte_flow_error_set(error, EINVAL, 2242 RTE_FLOW_ERROR_TYPE_ITEM, item, 2243 "L3 cannot follow an NVGRE layer."); 2244 if (!mask) 2245 mask = &rte_flow_item_ipv6_mask; 2246 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2247 acc_mask ? (const uint8_t *)acc_mask 2248 : (const uint8_t *)&nic_mask, 2249 sizeof(struct rte_flow_item_ipv6), 2250 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2251 if (ret < 0) 2252 return ret; 2253 return 0; 2254 } 2255 2256 /** 2257 * Validate UDP item. 2258 * 2259 * @param[in] item 2260 * Item specification. 2261 * @param[in] item_flags 2262 * Bit-fields that holds the items detected until now. 2263 * @param[in] target_protocol 2264 * The next protocol in the previous item. 2265 * @param[in] flow_mask 2266 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2267 * @param[out] error 2268 * Pointer to error structure. 2269 * 2270 * @return 2271 * 0 on success, a negative errno value otherwise and rte_errno is set. 2272 */ 2273 int 2274 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2275 uint64_t item_flags, 2276 uint8_t target_protocol, 2277 struct rte_flow_error *error) 2278 { 2279 const struct rte_flow_item_udp *mask = item->mask; 2280 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2281 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2282 MLX5_FLOW_LAYER_OUTER_L3; 2283 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2284 MLX5_FLOW_LAYER_OUTER_L4; 2285 int ret; 2286 2287 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2288 return rte_flow_error_set(error, EINVAL, 2289 RTE_FLOW_ERROR_TYPE_ITEM, item, 2290 "protocol filtering not compatible" 2291 " with UDP layer"); 2292 if (!(item_flags & l3m)) 2293 return rte_flow_error_set(error, EINVAL, 2294 RTE_FLOW_ERROR_TYPE_ITEM, item, 2295 "L3 is mandatory to filter on L4"); 2296 if (item_flags & l4m) 2297 return rte_flow_error_set(error, EINVAL, 2298 RTE_FLOW_ERROR_TYPE_ITEM, item, 2299 "multiple L4 layers not supported"); 2300 if (!mask) 2301 mask = &rte_flow_item_udp_mask; 2302 ret = mlx5_flow_item_acceptable 2303 (item, (const uint8_t *)mask, 2304 (const uint8_t *)&rte_flow_item_udp_mask, 2305 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2306 error); 2307 if (ret < 0) 2308 return ret; 2309 return 0; 2310 } 2311 2312 /** 2313 * Validate TCP item. 2314 * 2315 * @param[in] item 2316 * Item specification. 2317 * @param[in] item_flags 2318 * Bit-fields that holds the items detected until now. 2319 * @param[in] target_protocol 2320 * The next protocol in the previous item. 2321 * @param[out] error 2322 * Pointer to error structure. 2323 * 2324 * @return 2325 * 0 on success, a negative errno value otherwise and rte_errno is set. 2326 */ 2327 int 2328 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2329 uint64_t item_flags, 2330 uint8_t target_protocol, 2331 const struct rte_flow_item_tcp *flow_mask, 2332 struct rte_flow_error *error) 2333 { 2334 const struct rte_flow_item_tcp *mask = item->mask; 2335 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2336 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2337 MLX5_FLOW_LAYER_OUTER_L3; 2338 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2339 MLX5_FLOW_LAYER_OUTER_L4; 2340 int ret; 2341 2342 MLX5_ASSERT(flow_mask); 2343 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2344 return rte_flow_error_set(error, EINVAL, 2345 RTE_FLOW_ERROR_TYPE_ITEM, item, 2346 "protocol filtering not compatible" 2347 " with TCP layer"); 2348 if (!(item_flags & l3m)) 2349 return rte_flow_error_set(error, EINVAL, 2350 RTE_FLOW_ERROR_TYPE_ITEM, item, 2351 "L3 is mandatory to filter on L4"); 2352 if (item_flags & l4m) 2353 return rte_flow_error_set(error, EINVAL, 2354 RTE_FLOW_ERROR_TYPE_ITEM, item, 2355 "multiple L4 layers not supported"); 2356 if (!mask) 2357 mask = &rte_flow_item_tcp_mask; 2358 ret = mlx5_flow_item_acceptable 2359 (item, (const uint8_t *)mask, 2360 (const uint8_t *)flow_mask, 2361 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2362 error); 2363 if (ret < 0) 2364 return ret; 2365 return 0; 2366 } 2367 2368 /** 2369 * Validate VXLAN item. 2370 * 2371 * @param[in] item 2372 * Item specification. 2373 * @param[in] item_flags 2374 * Bit-fields that holds the items detected until now. 2375 * @param[in] target_protocol 2376 * The next protocol in the previous item. 2377 * @param[out] error 2378 * Pointer to error structure. 2379 * 2380 * @return 2381 * 0 on success, a negative errno value otherwise and rte_errno is set. 2382 */ 2383 int 2384 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2385 uint64_t item_flags, 2386 struct rte_flow_error *error) 2387 { 2388 const struct rte_flow_item_vxlan *spec = item->spec; 2389 const struct rte_flow_item_vxlan *mask = item->mask; 2390 int ret; 2391 union vni { 2392 uint32_t vlan_id; 2393 uint8_t vni[4]; 2394 } id = { .vlan_id = 0, }; 2395 2396 2397 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2398 return rte_flow_error_set(error, ENOTSUP, 2399 RTE_FLOW_ERROR_TYPE_ITEM, item, 2400 "multiple tunnel layers not" 2401 " supported"); 2402 /* 2403 * Verify only UDPv4 is present as defined in 2404 * https://tools.ietf.org/html/rfc7348 2405 */ 2406 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2407 return rte_flow_error_set(error, EINVAL, 2408 RTE_FLOW_ERROR_TYPE_ITEM, item, 2409 "no outer UDP layer found"); 2410 if (!mask) 2411 mask = &rte_flow_item_vxlan_mask; 2412 ret = mlx5_flow_item_acceptable 2413 (item, (const uint8_t *)mask, 2414 (const uint8_t *)&rte_flow_item_vxlan_mask, 2415 sizeof(struct rte_flow_item_vxlan), 2416 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2417 if (ret < 0) 2418 return ret; 2419 if (spec) { 2420 memcpy(&id.vni[1], spec->vni, 3); 2421 memcpy(&id.vni[1], mask->vni, 3); 2422 } 2423 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2424 return rte_flow_error_set(error, ENOTSUP, 2425 RTE_FLOW_ERROR_TYPE_ITEM, item, 2426 "VXLAN tunnel must be fully defined"); 2427 return 0; 2428 } 2429 2430 /** 2431 * Validate VXLAN_GPE item. 2432 * 2433 * @param[in] item 2434 * Item specification. 2435 * @param[in] item_flags 2436 * Bit-fields that holds the items detected until now. 2437 * @param[in] priv 2438 * Pointer to the private data structure. 2439 * @param[in] target_protocol 2440 * The next protocol in the previous item. 2441 * @param[out] error 2442 * Pointer to error structure. 2443 * 2444 * @return 2445 * 0 on success, a negative errno value otherwise and rte_errno is set. 2446 */ 2447 int 2448 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2449 uint64_t item_flags, 2450 struct rte_eth_dev *dev, 2451 struct rte_flow_error *error) 2452 { 2453 struct mlx5_priv *priv = dev->data->dev_private; 2454 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2455 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2456 int ret; 2457 union vni { 2458 uint32_t vlan_id; 2459 uint8_t vni[4]; 2460 } id = { .vlan_id = 0, }; 2461 2462 if (!priv->config.l3_vxlan_en) 2463 return rte_flow_error_set(error, ENOTSUP, 2464 RTE_FLOW_ERROR_TYPE_ITEM, item, 2465 "L3 VXLAN is not enabled by device" 2466 " parameter and/or not configured in" 2467 " firmware"); 2468 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2469 return rte_flow_error_set(error, ENOTSUP, 2470 RTE_FLOW_ERROR_TYPE_ITEM, item, 2471 "multiple tunnel layers not" 2472 " supported"); 2473 /* 2474 * Verify only UDPv4 is present as defined in 2475 * https://tools.ietf.org/html/rfc7348 2476 */ 2477 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2478 return rte_flow_error_set(error, EINVAL, 2479 RTE_FLOW_ERROR_TYPE_ITEM, item, 2480 "no outer UDP layer found"); 2481 if (!mask) 2482 mask = &rte_flow_item_vxlan_gpe_mask; 2483 ret = mlx5_flow_item_acceptable 2484 (item, (const uint8_t *)mask, 2485 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2486 sizeof(struct rte_flow_item_vxlan_gpe), 2487 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2488 if (ret < 0) 2489 return ret; 2490 if (spec) { 2491 if (spec->protocol) 2492 return rte_flow_error_set(error, ENOTSUP, 2493 RTE_FLOW_ERROR_TYPE_ITEM, 2494 item, 2495 "VxLAN-GPE protocol" 2496 " not supported"); 2497 memcpy(&id.vni[1], spec->vni, 3); 2498 memcpy(&id.vni[1], mask->vni, 3); 2499 } 2500 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2501 return rte_flow_error_set(error, ENOTSUP, 2502 RTE_FLOW_ERROR_TYPE_ITEM, item, 2503 "VXLAN-GPE tunnel must be fully" 2504 " defined"); 2505 return 0; 2506 } 2507 /** 2508 * Validate GRE Key item. 2509 * 2510 * @param[in] item 2511 * Item specification. 2512 * @param[in] item_flags 2513 * Bit flags to mark detected items. 2514 * @param[in] gre_item 2515 * Pointer to gre_item 2516 * @param[out] error 2517 * Pointer to error structure. 2518 * 2519 * @return 2520 * 0 on success, a negative errno value otherwise and rte_errno is set. 2521 */ 2522 int 2523 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2524 uint64_t item_flags, 2525 const struct rte_flow_item *gre_item, 2526 struct rte_flow_error *error) 2527 { 2528 const rte_be32_t *mask = item->mask; 2529 int ret = 0; 2530 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2531 const struct rte_flow_item_gre *gre_spec; 2532 const struct rte_flow_item_gre *gre_mask; 2533 2534 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2535 return rte_flow_error_set(error, ENOTSUP, 2536 RTE_FLOW_ERROR_TYPE_ITEM, item, 2537 "Multiple GRE key not support"); 2538 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2539 return rte_flow_error_set(error, ENOTSUP, 2540 RTE_FLOW_ERROR_TYPE_ITEM, item, 2541 "No preceding GRE header"); 2542 if (item_flags & MLX5_FLOW_LAYER_INNER) 2543 return rte_flow_error_set(error, ENOTSUP, 2544 RTE_FLOW_ERROR_TYPE_ITEM, item, 2545 "GRE key following a wrong item"); 2546 gre_mask = gre_item->mask; 2547 if (!gre_mask) 2548 gre_mask = &rte_flow_item_gre_mask; 2549 gre_spec = gre_item->spec; 2550 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2551 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2552 return rte_flow_error_set(error, EINVAL, 2553 RTE_FLOW_ERROR_TYPE_ITEM, item, 2554 "Key bit must be on"); 2555 2556 if (!mask) 2557 mask = &gre_key_default_mask; 2558 ret = mlx5_flow_item_acceptable 2559 (item, (const uint8_t *)mask, 2560 (const uint8_t *)&gre_key_default_mask, 2561 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2562 return ret; 2563 } 2564 2565 /** 2566 * Validate GRE item. 2567 * 2568 * @param[in] item 2569 * Item specification. 2570 * @param[in] item_flags 2571 * Bit flags to mark detected items. 2572 * @param[in] target_protocol 2573 * The next protocol in the previous item. 2574 * @param[out] error 2575 * Pointer to error structure. 2576 * 2577 * @return 2578 * 0 on success, a negative errno value otherwise and rte_errno is set. 2579 */ 2580 int 2581 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2582 uint64_t item_flags, 2583 uint8_t target_protocol, 2584 struct rte_flow_error *error) 2585 { 2586 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2587 const struct rte_flow_item_gre *mask = item->mask; 2588 int ret; 2589 const struct rte_flow_item_gre nic_mask = { 2590 .c_rsvd0_ver = RTE_BE16(0xB000), 2591 .protocol = RTE_BE16(UINT16_MAX), 2592 }; 2593 2594 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2595 return rte_flow_error_set(error, EINVAL, 2596 RTE_FLOW_ERROR_TYPE_ITEM, item, 2597 "protocol filtering not compatible" 2598 " with this GRE layer"); 2599 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2600 return rte_flow_error_set(error, ENOTSUP, 2601 RTE_FLOW_ERROR_TYPE_ITEM, item, 2602 "multiple tunnel layers not" 2603 " supported"); 2604 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2605 return rte_flow_error_set(error, ENOTSUP, 2606 RTE_FLOW_ERROR_TYPE_ITEM, item, 2607 "L3 Layer is missing"); 2608 if (!mask) 2609 mask = &rte_flow_item_gre_mask; 2610 ret = mlx5_flow_item_acceptable 2611 (item, (const uint8_t *)mask, 2612 (const uint8_t *)&nic_mask, 2613 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2614 error); 2615 if (ret < 0) 2616 return ret; 2617 #ifndef HAVE_MLX5DV_DR 2618 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2619 if (spec && (spec->protocol & mask->protocol)) 2620 return rte_flow_error_set(error, ENOTSUP, 2621 RTE_FLOW_ERROR_TYPE_ITEM, item, 2622 "without MPLS support the" 2623 " specification cannot be used for" 2624 " filtering"); 2625 #endif 2626 #endif 2627 return 0; 2628 } 2629 2630 /** 2631 * Validate Geneve item. 2632 * 2633 * @param[in] item 2634 * Item specification. 2635 * @param[in] itemFlags 2636 * Bit-fields that holds the items detected until now. 2637 * @param[in] enPriv 2638 * Pointer to the private data structure. 2639 * @param[out] error 2640 * Pointer to error structure. 2641 * 2642 * @return 2643 * 0 on success, a negative errno value otherwise and rte_errno is set. 2644 */ 2645 2646 int 2647 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2648 uint64_t item_flags, 2649 struct rte_eth_dev *dev, 2650 struct rte_flow_error *error) 2651 { 2652 struct mlx5_priv *priv = dev->data->dev_private; 2653 const struct rte_flow_item_geneve *spec = item->spec; 2654 const struct rte_flow_item_geneve *mask = item->mask; 2655 int ret; 2656 uint16_t gbhdr; 2657 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2658 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2659 const struct rte_flow_item_geneve nic_mask = { 2660 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2661 .vni = "\xff\xff\xff", 2662 .protocol = RTE_BE16(UINT16_MAX), 2663 }; 2664 2665 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2666 return rte_flow_error_set(error, ENOTSUP, 2667 RTE_FLOW_ERROR_TYPE_ITEM, item, 2668 "L3 Geneve is not enabled by device" 2669 " parameter and/or not configured in" 2670 " firmware"); 2671 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2672 return rte_flow_error_set(error, ENOTSUP, 2673 RTE_FLOW_ERROR_TYPE_ITEM, item, 2674 "multiple tunnel layers not" 2675 " supported"); 2676 /* 2677 * Verify only UDPv4 is present as defined in 2678 * https://tools.ietf.org/html/rfc7348 2679 */ 2680 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2681 return rte_flow_error_set(error, EINVAL, 2682 RTE_FLOW_ERROR_TYPE_ITEM, item, 2683 "no outer UDP layer found"); 2684 if (!mask) 2685 mask = &rte_flow_item_geneve_mask; 2686 ret = mlx5_flow_item_acceptable 2687 (item, (const uint8_t *)mask, 2688 (const uint8_t *)&nic_mask, 2689 sizeof(struct rte_flow_item_geneve), 2690 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2691 if (ret) 2692 return ret; 2693 if (spec) { 2694 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2695 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2696 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2697 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2698 return rte_flow_error_set(error, ENOTSUP, 2699 RTE_FLOW_ERROR_TYPE_ITEM, 2700 item, 2701 "Geneve protocol unsupported" 2702 " fields are being used"); 2703 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2704 return rte_flow_error_set 2705 (error, ENOTSUP, 2706 RTE_FLOW_ERROR_TYPE_ITEM, 2707 item, 2708 "Unsupported Geneve options length"); 2709 } 2710 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2711 return rte_flow_error_set 2712 (error, ENOTSUP, 2713 RTE_FLOW_ERROR_TYPE_ITEM, item, 2714 "Geneve tunnel must be fully defined"); 2715 return 0; 2716 } 2717 2718 /** 2719 * Validate Geneve TLV option item. 2720 * 2721 * @param[in] item 2722 * Item specification. 2723 * @param[in] last_item 2724 * Previous validated item in the pattern items. 2725 * @param[in] geneve_item 2726 * Previous GENEVE item specification. 2727 * @param[in] dev 2728 * Pointer to the rte_eth_dev structure. 2729 * @param[out] error 2730 * Pointer to error structure. 2731 * 2732 * @return 2733 * 0 on success, a negative errno value otherwise and rte_errno is set. 2734 */ 2735 int 2736 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2737 uint64_t last_item, 2738 const struct rte_flow_item *geneve_item, 2739 struct rte_eth_dev *dev, 2740 struct rte_flow_error *error) 2741 { 2742 struct mlx5_priv *priv = dev->data->dev_private; 2743 struct mlx5_dev_ctx_shared *sh = priv->sh; 2744 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2745 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2746 uint8_t data_max_supported = 2747 hca_attr->max_geneve_tlv_option_data_len * 4; 2748 struct mlx5_dev_config *config = &priv->config; 2749 const struct rte_flow_item_geneve *geneve_spec; 2750 const struct rte_flow_item_geneve *geneve_mask; 2751 const struct rte_flow_item_geneve_opt *spec = item->spec; 2752 const struct rte_flow_item_geneve_opt *mask = item->mask; 2753 unsigned int i; 2754 unsigned int data_len; 2755 uint8_t tlv_option_len; 2756 uint16_t optlen_m, optlen_v; 2757 const struct rte_flow_item_geneve_opt full_mask = { 2758 .option_class = RTE_BE16(0xffff), 2759 .option_type = 0xff, 2760 .option_len = 0x1f, 2761 }; 2762 2763 if (!mask) 2764 mask = &rte_flow_item_geneve_opt_mask; 2765 if (!spec) 2766 return rte_flow_error_set 2767 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2768 "Geneve TLV opt class/type/length must be specified"); 2769 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2770 return rte_flow_error_set 2771 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2772 "Geneve TLV opt length exceeeds the limit (31)"); 2773 /* Check if class type and length masks are full. */ 2774 if (full_mask.option_class != mask->option_class || 2775 full_mask.option_type != mask->option_type || 2776 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2777 return rte_flow_error_set 2778 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2779 "Geneve TLV opt class/type/length masks must be full"); 2780 /* Check if length is supported */ 2781 if ((uint32_t)spec->option_len > 2782 config->hca_attr.max_geneve_tlv_option_data_len) 2783 return rte_flow_error_set 2784 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2785 "Geneve TLV opt length not supported"); 2786 if (config->hca_attr.max_geneve_tlv_options > 1) 2787 DRV_LOG(DEBUG, 2788 "max_geneve_tlv_options supports more than 1 option"); 2789 /* Check GENEVE item preceding. */ 2790 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2791 return rte_flow_error_set 2792 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2793 "Geneve opt item must be preceded with Geneve item"); 2794 geneve_spec = geneve_item->spec; 2795 geneve_mask = geneve_item->mask ? geneve_item->mask : 2796 &rte_flow_item_geneve_mask; 2797 /* Check if GENEVE TLV option size doesn't exceed option length */ 2798 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2799 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2800 tlv_option_len = spec->option_len & mask->option_len; 2801 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2802 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2803 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2804 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2805 if ((optlen_v & optlen_m) <= tlv_option_len) 2806 return rte_flow_error_set 2807 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2808 "GENEVE TLV option length exceeds optlen"); 2809 } 2810 /* Check if length is 0 or data is 0. */ 2811 if (spec->data == NULL || spec->option_len == 0) 2812 return rte_flow_error_set 2813 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2814 "Geneve TLV opt with zero data/length not supported"); 2815 /* Check not all data & mask are 0. */ 2816 data_len = spec->option_len * 4; 2817 if (mask->data == NULL) { 2818 for (i = 0; i < data_len; i++) 2819 if (spec->data[i]) 2820 break; 2821 if (i == data_len) 2822 return rte_flow_error_set(error, ENOTSUP, 2823 RTE_FLOW_ERROR_TYPE_ITEM, item, 2824 "Can't match on Geneve option data 0"); 2825 } else { 2826 for (i = 0; i < data_len; i++) 2827 if (spec->data[i] & mask->data[i]) 2828 break; 2829 if (i == data_len) 2830 return rte_flow_error_set(error, ENOTSUP, 2831 RTE_FLOW_ERROR_TYPE_ITEM, item, 2832 "Can't match on Geneve option data and mask 0"); 2833 /* Check data mask supported. */ 2834 for (i = data_max_supported; i < data_len ; i++) 2835 if (mask->data[i]) 2836 return rte_flow_error_set(error, ENOTSUP, 2837 RTE_FLOW_ERROR_TYPE_ITEM, item, 2838 "Data mask is of unsupported size"); 2839 } 2840 /* Check GENEVE option is supported in NIC. */ 2841 if (!config->hca_attr.geneve_tlv_opt) 2842 return rte_flow_error_set 2843 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2844 "Geneve TLV opt not supported"); 2845 /* Check if we already have geneve option with different type/class. */ 2846 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2847 geneve_opt_resource = sh->geneve_tlv_option_resource; 2848 if (geneve_opt_resource != NULL) 2849 if (geneve_opt_resource->option_class != spec->option_class || 2850 geneve_opt_resource->option_type != spec->option_type || 2851 geneve_opt_resource->length != spec->option_len) { 2852 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2853 return rte_flow_error_set(error, ENOTSUP, 2854 RTE_FLOW_ERROR_TYPE_ITEM, item, 2855 "Only one Geneve TLV option supported"); 2856 } 2857 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2858 return 0; 2859 } 2860 2861 /** 2862 * Validate MPLS item. 2863 * 2864 * @param[in] dev 2865 * Pointer to the rte_eth_dev structure. 2866 * @param[in] item 2867 * Item specification. 2868 * @param[in] item_flags 2869 * Bit-fields that holds the items detected until now. 2870 * @param[in] prev_layer 2871 * The protocol layer indicated in previous item. 2872 * @param[out] error 2873 * Pointer to error structure. 2874 * 2875 * @return 2876 * 0 on success, a negative errno value otherwise and rte_errno is set. 2877 */ 2878 int 2879 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2880 const struct rte_flow_item *item __rte_unused, 2881 uint64_t item_flags __rte_unused, 2882 uint64_t prev_layer __rte_unused, 2883 struct rte_flow_error *error) 2884 { 2885 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2886 const struct rte_flow_item_mpls *mask = item->mask; 2887 struct mlx5_priv *priv = dev->data->dev_private; 2888 int ret; 2889 2890 if (!priv->config.mpls_en) 2891 return rte_flow_error_set(error, ENOTSUP, 2892 RTE_FLOW_ERROR_TYPE_ITEM, item, 2893 "MPLS not supported or" 2894 " disabled in firmware" 2895 " configuration."); 2896 /* MPLS over IP, UDP, GRE is allowed */ 2897 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2898 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2899 MLX5_FLOW_LAYER_GRE | 2900 MLX5_FLOW_LAYER_GRE_KEY))) 2901 return rte_flow_error_set(error, EINVAL, 2902 RTE_FLOW_ERROR_TYPE_ITEM, item, 2903 "protocol filtering not compatible" 2904 " with MPLS layer"); 2905 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2906 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2907 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2908 return rte_flow_error_set(error, ENOTSUP, 2909 RTE_FLOW_ERROR_TYPE_ITEM, item, 2910 "multiple tunnel layers not" 2911 " supported"); 2912 if (!mask) 2913 mask = &rte_flow_item_mpls_mask; 2914 ret = mlx5_flow_item_acceptable 2915 (item, (const uint8_t *)mask, 2916 (const uint8_t *)&rte_flow_item_mpls_mask, 2917 sizeof(struct rte_flow_item_mpls), 2918 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2919 if (ret < 0) 2920 return ret; 2921 return 0; 2922 #else 2923 return rte_flow_error_set(error, ENOTSUP, 2924 RTE_FLOW_ERROR_TYPE_ITEM, item, 2925 "MPLS is not supported by Verbs, please" 2926 " update."); 2927 #endif 2928 } 2929 2930 /** 2931 * Validate NVGRE item. 2932 * 2933 * @param[in] item 2934 * Item specification. 2935 * @param[in] item_flags 2936 * Bit flags to mark detected items. 2937 * @param[in] target_protocol 2938 * The next protocol in the previous item. 2939 * @param[out] error 2940 * Pointer to error structure. 2941 * 2942 * @return 2943 * 0 on success, a negative errno value otherwise and rte_errno is set. 2944 */ 2945 int 2946 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2947 uint64_t item_flags, 2948 uint8_t target_protocol, 2949 struct rte_flow_error *error) 2950 { 2951 const struct rte_flow_item_nvgre *mask = item->mask; 2952 int ret; 2953 2954 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2955 return rte_flow_error_set(error, EINVAL, 2956 RTE_FLOW_ERROR_TYPE_ITEM, item, 2957 "protocol filtering not compatible" 2958 " with this GRE layer"); 2959 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2960 return rte_flow_error_set(error, ENOTSUP, 2961 RTE_FLOW_ERROR_TYPE_ITEM, item, 2962 "multiple tunnel layers not" 2963 " supported"); 2964 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2965 return rte_flow_error_set(error, ENOTSUP, 2966 RTE_FLOW_ERROR_TYPE_ITEM, item, 2967 "L3 Layer is missing"); 2968 if (!mask) 2969 mask = &rte_flow_item_nvgre_mask; 2970 ret = mlx5_flow_item_acceptable 2971 (item, (const uint8_t *)mask, 2972 (const uint8_t *)&rte_flow_item_nvgre_mask, 2973 sizeof(struct rte_flow_item_nvgre), 2974 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2975 if (ret < 0) 2976 return ret; 2977 return 0; 2978 } 2979 2980 /** 2981 * Validate eCPRI item. 2982 * 2983 * @param[in] item 2984 * Item specification. 2985 * @param[in] item_flags 2986 * Bit-fields that holds the items detected until now. 2987 * @param[in] last_item 2988 * Previous validated item in the pattern items. 2989 * @param[in] ether_type 2990 * Type in the ethernet layer header (including dot1q). 2991 * @param[in] acc_mask 2992 * Acceptable mask, if NULL default internal default mask 2993 * will be used to check whether item fields are supported. 2994 * @param[out] error 2995 * Pointer to error structure. 2996 * 2997 * @return 2998 * 0 on success, a negative errno value otherwise and rte_errno is set. 2999 */ 3000 int 3001 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3002 uint64_t item_flags, 3003 uint64_t last_item, 3004 uint16_t ether_type, 3005 const struct rte_flow_item_ecpri *acc_mask, 3006 struct rte_flow_error *error) 3007 { 3008 const struct rte_flow_item_ecpri *mask = item->mask; 3009 const struct rte_flow_item_ecpri nic_mask = { 3010 .hdr = { 3011 .common = { 3012 .u32 = 3013 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3014 .type = 0xFF, 3015 }).u32), 3016 }, 3017 .dummy[0] = 0xFFFFFFFF, 3018 }, 3019 }; 3020 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3021 MLX5_FLOW_LAYER_OUTER_VLAN); 3022 struct rte_flow_item_ecpri mask_lo; 3023 3024 if (!(last_item & outer_l2_vlan) && 3025 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3026 return rte_flow_error_set(error, EINVAL, 3027 RTE_FLOW_ERROR_TYPE_ITEM, item, 3028 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3029 if ((last_item & outer_l2_vlan) && ether_type && 3030 ether_type != RTE_ETHER_TYPE_ECPRI) 3031 return rte_flow_error_set(error, EINVAL, 3032 RTE_FLOW_ERROR_TYPE_ITEM, item, 3033 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3034 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3035 return rte_flow_error_set(error, EINVAL, 3036 RTE_FLOW_ERROR_TYPE_ITEM, item, 3037 "eCPRI with tunnel is not supported right now"); 3038 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3039 return rte_flow_error_set(error, ENOTSUP, 3040 RTE_FLOW_ERROR_TYPE_ITEM, item, 3041 "multiple L3 layers not supported"); 3042 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3043 return rte_flow_error_set(error, EINVAL, 3044 RTE_FLOW_ERROR_TYPE_ITEM, item, 3045 "eCPRI cannot coexist with a TCP layer"); 3046 /* In specification, eCPRI could be over UDP layer. */ 3047 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3048 return rte_flow_error_set(error, EINVAL, 3049 RTE_FLOW_ERROR_TYPE_ITEM, item, 3050 "eCPRI over UDP layer is not yet supported right now"); 3051 /* Mask for type field in common header could be zero. */ 3052 if (!mask) 3053 mask = &rte_flow_item_ecpri_mask; 3054 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3055 /* Input mask is in big-endian format. */ 3056 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3057 return rte_flow_error_set(error, EINVAL, 3058 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3059 "partial mask is not supported for protocol"); 3060 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3061 return rte_flow_error_set(error, EINVAL, 3062 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3063 "message header mask must be after a type mask"); 3064 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3065 acc_mask ? (const uint8_t *)acc_mask 3066 : (const uint8_t *)&nic_mask, 3067 sizeof(struct rte_flow_item_ecpri), 3068 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3069 } 3070 3071 /** 3072 * Release resource related QUEUE/RSS action split. 3073 * 3074 * @param dev 3075 * Pointer to Ethernet device. 3076 * @param flow 3077 * Flow to release id's from. 3078 */ 3079 static void 3080 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 3081 struct rte_flow *flow) 3082 { 3083 struct mlx5_priv *priv = dev->data->dev_private; 3084 uint32_t handle_idx; 3085 struct mlx5_flow_handle *dev_handle; 3086 3087 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 3088 handle_idx, dev_handle, next) 3089 if (dev_handle->split_flow_id && 3090 !dev_handle->is_meter_flow_id) 3091 mlx5_ipool_free(priv->sh->ipool 3092 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 3093 dev_handle->split_flow_id); 3094 } 3095 3096 static int 3097 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3098 const struct rte_flow_attr *attr __rte_unused, 3099 const struct rte_flow_item items[] __rte_unused, 3100 const struct rte_flow_action actions[] __rte_unused, 3101 bool external __rte_unused, 3102 int hairpin __rte_unused, 3103 struct rte_flow_error *error) 3104 { 3105 return rte_flow_error_set(error, ENOTSUP, 3106 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3107 } 3108 3109 static struct mlx5_flow * 3110 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3111 const struct rte_flow_attr *attr __rte_unused, 3112 const struct rte_flow_item items[] __rte_unused, 3113 const struct rte_flow_action actions[] __rte_unused, 3114 struct rte_flow_error *error) 3115 { 3116 rte_flow_error_set(error, ENOTSUP, 3117 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3118 return NULL; 3119 } 3120 3121 static int 3122 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3123 struct mlx5_flow *dev_flow __rte_unused, 3124 const struct rte_flow_attr *attr __rte_unused, 3125 const struct rte_flow_item items[] __rte_unused, 3126 const struct rte_flow_action actions[] __rte_unused, 3127 struct rte_flow_error *error) 3128 { 3129 return rte_flow_error_set(error, ENOTSUP, 3130 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3131 } 3132 3133 static int 3134 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3135 struct rte_flow *flow __rte_unused, 3136 struct rte_flow_error *error) 3137 { 3138 return rte_flow_error_set(error, ENOTSUP, 3139 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3140 } 3141 3142 static void 3143 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3144 struct rte_flow *flow __rte_unused) 3145 { 3146 } 3147 3148 static void 3149 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3150 struct rte_flow *flow __rte_unused) 3151 { 3152 } 3153 3154 static int 3155 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3156 struct rte_flow *flow __rte_unused, 3157 const struct rte_flow_action *actions __rte_unused, 3158 void *data __rte_unused, 3159 struct rte_flow_error *error) 3160 { 3161 return rte_flow_error_set(error, ENOTSUP, 3162 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3163 } 3164 3165 static int 3166 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3167 uint32_t domains __rte_unused, 3168 uint32_t flags __rte_unused) 3169 { 3170 return 0; 3171 } 3172 3173 /* Void driver to protect from null pointer reference. */ 3174 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3175 .validate = flow_null_validate, 3176 .prepare = flow_null_prepare, 3177 .translate = flow_null_translate, 3178 .apply = flow_null_apply, 3179 .remove = flow_null_remove, 3180 .destroy = flow_null_destroy, 3181 .query = flow_null_query, 3182 .sync_domain = flow_null_sync_domain, 3183 }; 3184 3185 /** 3186 * Select flow driver type according to flow attributes and device 3187 * configuration. 3188 * 3189 * @param[in] dev 3190 * Pointer to the dev structure. 3191 * @param[in] attr 3192 * Pointer to the flow attributes. 3193 * 3194 * @return 3195 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3196 */ 3197 static enum mlx5_flow_drv_type 3198 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3199 { 3200 struct mlx5_priv *priv = dev->data->dev_private; 3201 /* The OS can determine first a specific flow type (DV, VERBS) */ 3202 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3203 3204 if (type != MLX5_FLOW_TYPE_MAX) 3205 return type; 3206 /* If no OS specific type - continue with DV/VERBS selection */ 3207 if (attr->transfer && priv->config.dv_esw_en) 3208 type = MLX5_FLOW_TYPE_DV; 3209 if (!attr->transfer) 3210 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3211 MLX5_FLOW_TYPE_VERBS; 3212 return type; 3213 } 3214 3215 #define flow_get_drv_ops(type) flow_drv_ops[type] 3216 3217 /** 3218 * Flow driver validation API. This abstracts calling driver specific functions. 3219 * The type of flow driver is determined according to flow attributes. 3220 * 3221 * @param[in] dev 3222 * Pointer to the dev structure. 3223 * @param[in] attr 3224 * Pointer to the flow attributes. 3225 * @param[in] items 3226 * Pointer to the list of items. 3227 * @param[in] actions 3228 * Pointer to the list of actions. 3229 * @param[in] external 3230 * This flow rule is created by request external to PMD. 3231 * @param[in] hairpin 3232 * Number of hairpin TX actions, 0 means classic flow. 3233 * @param[out] error 3234 * Pointer to the error structure. 3235 * 3236 * @return 3237 * 0 on success, a negative errno value otherwise and rte_errno is set. 3238 */ 3239 static inline int 3240 flow_drv_validate(struct rte_eth_dev *dev, 3241 const struct rte_flow_attr *attr, 3242 const struct rte_flow_item items[], 3243 const struct rte_flow_action actions[], 3244 bool external, int hairpin, struct rte_flow_error *error) 3245 { 3246 const struct mlx5_flow_driver_ops *fops; 3247 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3248 3249 fops = flow_get_drv_ops(type); 3250 return fops->validate(dev, attr, items, actions, external, 3251 hairpin, error); 3252 } 3253 3254 /** 3255 * Flow driver preparation API. This abstracts calling driver specific 3256 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3257 * calculates the size of memory required for device flow, allocates the memory, 3258 * initializes the device flow and returns the pointer. 3259 * 3260 * @note 3261 * This function initializes device flow structure such as dv or verbs in 3262 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3263 * rest. For example, adding returning device flow to flow->dev_flow list and 3264 * setting backward reference to the flow should be done out of this function. 3265 * layers field is not filled either. 3266 * 3267 * @param[in] dev 3268 * Pointer to the dev structure. 3269 * @param[in] attr 3270 * Pointer to the flow attributes. 3271 * @param[in] items 3272 * Pointer to the list of items. 3273 * @param[in] actions 3274 * Pointer to the list of actions. 3275 * @param[in] flow_idx 3276 * This memory pool index to the flow. 3277 * @param[out] error 3278 * Pointer to the error structure. 3279 * 3280 * @return 3281 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3282 */ 3283 static inline struct mlx5_flow * 3284 flow_drv_prepare(struct rte_eth_dev *dev, 3285 const struct rte_flow *flow, 3286 const struct rte_flow_attr *attr, 3287 const struct rte_flow_item items[], 3288 const struct rte_flow_action actions[], 3289 uint32_t flow_idx, 3290 struct rte_flow_error *error) 3291 { 3292 const struct mlx5_flow_driver_ops *fops; 3293 enum mlx5_flow_drv_type type = flow->drv_type; 3294 struct mlx5_flow *mlx5_flow = NULL; 3295 3296 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3297 fops = flow_get_drv_ops(type); 3298 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3299 if (mlx5_flow) 3300 mlx5_flow->flow_idx = flow_idx; 3301 return mlx5_flow; 3302 } 3303 3304 /** 3305 * Flow driver translation API. This abstracts calling driver specific 3306 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3307 * translates a generic flow into a driver flow. flow_drv_prepare() must 3308 * precede. 3309 * 3310 * @note 3311 * dev_flow->layers could be filled as a result of parsing during translation 3312 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3313 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3314 * flow->actions could be overwritten even though all the expanded dev_flows 3315 * have the same actions. 3316 * 3317 * @param[in] dev 3318 * Pointer to the rte dev structure. 3319 * @param[in, out] dev_flow 3320 * Pointer to the mlx5 flow. 3321 * @param[in] attr 3322 * Pointer to the flow attributes. 3323 * @param[in] items 3324 * Pointer to the list of items. 3325 * @param[in] actions 3326 * Pointer to the list of actions. 3327 * @param[out] error 3328 * Pointer to the error structure. 3329 * 3330 * @return 3331 * 0 on success, a negative errno value otherwise and rte_errno is set. 3332 */ 3333 static inline int 3334 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3335 const struct rte_flow_attr *attr, 3336 const struct rte_flow_item items[], 3337 const struct rte_flow_action actions[], 3338 struct rte_flow_error *error) 3339 { 3340 const struct mlx5_flow_driver_ops *fops; 3341 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3342 3343 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3344 fops = flow_get_drv_ops(type); 3345 return fops->translate(dev, dev_flow, attr, items, actions, error); 3346 } 3347 3348 /** 3349 * Flow driver apply API. This abstracts calling driver specific functions. 3350 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3351 * translated driver flows on to device. flow_drv_translate() must precede. 3352 * 3353 * @param[in] dev 3354 * Pointer to Ethernet device structure. 3355 * @param[in, out] flow 3356 * Pointer to flow structure. 3357 * @param[out] error 3358 * Pointer to error structure. 3359 * 3360 * @return 3361 * 0 on success, a negative errno value otherwise and rte_errno is set. 3362 */ 3363 static inline int 3364 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3365 struct rte_flow_error *error) 3366 { 3367 const struct mlx5_flow_driver_ops *fops; 3368 enum mlx5_flow_drv_type type = flow->drv_type; 3369 3370 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3371 fops = flow_get_drv_ops(type); 3372 return fops->apply(dev, flow, error); 3373 } 3374 3375 /** 3376 * Flow driver destroy API. This abstracts calling driver specific functions. 3377 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3378 * on device and releases resources of the flow. 3379 * 3380 * @param[in] dev 3381 * Pointer to Ethernet device. 3382 * @param[in, out] flow 3383 * Pointer to flow structure. 3384 */ 3385 static inline void 3386 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3387 { 3388 const struct mlx5_flow_driver_ops *fops; 3389 enum mlx5_flow_drv_type type = flow->drv_type; 3390 3391 flow_mreg_split_qrss_release(dev, flow); 3392 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3393 fops = flow_get_drv_ops(type); 3394 fops->destroy(dev, flow); 3395 } 3396 3397 /** 3398 * Flow driver find RSS policy tbl API. This abstracts calling driver 3399 * specific functions. Parent flow (rte_flow) should have driver 3400 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3401 * 3402 * @param[in] dev 3403 * Pointer to Ethernet device. 3404 * @param[in, out] flow 3405 * Pointer to flow structure. 3406 * @param[in] policy 3407 * Pointer to meter policy table. 3408 * @param[in] rss_desc 3409 * Pointer to rss_desc 3410 */ 3411 static struct mlx5_flow_meter_sub_policy * 3412 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3413 struct rte_flow *flow, 3414 struct mlx5_flow_meter_policy *policy, 3415 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3416 { 3417 const struct mlx5_flow_driver_ops *fops; 3418 enum mlx5_flow_drv_type type = flow->drv_type; 3419 3420 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3421 fops = flow_get_drv_ops(type); 3422 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3423 } 3424 3425 /** 3426 * Get RSS action from the action list. 3427 * 3428 * @param[in] dev 3429 * Pointer to Ethernet device. 3430 * @param[in] actions 3431 * Pointer to the list of actions. 3432 * @param[in] flow 3433 * Parent flow structure pointer. 3434 * 3435 * @return 3436 * Pointer to the RSS action if exist, else return NULL. 3437 */ 3438 static const struct rte_flow_action_rss* 3439 flow_get_rss_action(struct rte_eth_dev *dev, 3440 const struct rte_flow_action actions[]) 3441 { 3442 struct mlx5_priv *priv = dev->data->dev_private; 3443 const struct rte_flow_action_rss *rss = NULL; 3444 3445 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3446 switch (actions->type) { 3447 case RTE_FLOW_ACTION_TYPE_RSS: 3448 rss = actions->conf; 3449 break; 3450 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3451 { 3452 const struct rte_flow_action_sample *sample = 3453 actions->conf; 3454 const struct rte_flow_action *act = sample->actions; 3455 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3456 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3457 rss = act->conf; 3458 break; 3459 } 3460 case RTE_FLOW_ACTION_TYPE_METER: 3461 { 3462 uint32_t mtr_idx; 3463 struct mlx5_flow_meter_info *fm; 3464 struct mlx5_flow_meter_policy *policy; 3465 const struct rte_flow_action_meter *mtr = actions->conf; 3466 3467 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3468 if (fm) { 3469 policy = mlx5_flow_meter_policy_find(dev, 3470 fm->policy_id, NULL); 3471 if (policy && policy->is_rss) 3472 rss = 3473 policy->act_cnt[RTE_COLOR_GREEN].rss->conf; 3474 } 3475 break; 3476 } 3477 default: 3478 break; 3479 } 3480 } 3481 return rss; 3482 } 3483 3484 /** 3485 * Get ASO age action by index. 3486 * 3487 * @param[in] dev 3488 * Pointer to the Ethernet device structure. 3489 * @param[in] age_idx 3490 * Index to the ASO age action. 3491 * 3492 * @return 3493 * The specified ASO age action. 3494 */ 3495 struct mlx5_aso_age_action* 3496 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3497 { 3498 uint16_t pool_idx = age_idx & UINT16_MAX; 3499 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3500 struct mlx5_priv *priv = dev->data->dev_private; 3501 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3502 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3503 3504 return &pool->actions[offset - 1]; 3505 } 3506 3507 /* maps indirect action to translated direct in some actions array */ 3508 struct mlx5_translated_action_handle { 3509 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3510 int index; /**< Index in related array of rte_flow_action. */ 3511 }; 3512 3513 /** 3514 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3515 * direct action if translation possible. 3516 * This functionality used to run same execution path for both direct and 3517 * indirect actions on flow create. All necessary preparations for indirect 3518 * action handling should be performed on *handle* actions list returned 3519 * from this call. 3520 * 3521 * @param[in] dev 3522 * Pointer to Ethernet device. 3523 * @param[in] actions 3524 * List of actions to translate. 3525 * @param[out] handle 3526 * List to store translated indirect action object handles. 3527 * @param[in, out] indir_n 3528 * Size of *handle* array. On return should be updated with number of 3529 * indirect actions retrieved from the *actions* list. 3530 * @param[out] translated_actions 3531 * List of actions where all indirect actions were translated to direct 3532 * if possible. NULL if no translation took place. 3533 * @param[out] error 3534 * Pointer to the error structure. 3535 * 3536 * @return 3537 * 0 on success, a negative errno value otherwise and rte_errno is set. 3538 */ 3539 static int 3540 flow_action_handles_translate(struct rte_eth_dev *dev, 3541 const struct rte_flow_action actions[], 3542 struct mlx5_translated_action_handle *handle, 3543 int *indir_n, 3544 struct rte_flow_action **translated_actions, 3545 struct rte_flow_error *error) 3546 { 3547 struct mlx5_priv *priv = dev->data->dev_private; 3548 struct rte_flow_action *translated = NULL; 3549 size_t actions_size; 3550 int n; 3551 int copied_n = 0; 3552 struct mlx5_translated_action_handle *handle_end = NULL; 3553 3554 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3555 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3556 continue; 3557 if (copied_n == *indir_n) { 3558 return rte_flow_error_set 3559 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3560 NULL, "too many shared actions"); 3561 } 3562 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3563 sizeof(actions[n].conf)); 3564 handle[copied_n].index = n; 3565 copied_n++; 3566 } 3567 n++; 3568 *indir_n = copied_n; 3569 if (!copied_n) 3570 return 0; 3571 actions_size = sizeof(struct rte_flow_action) * n; 3572 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3573 if (!translated) { 3574 rte_errno = ENOMEM; 3575 return -ENOMEM; 3576 } 3577 memcpy(translated, actions, actions_size); 3578 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3579 struct mlx5_shared_action_rss *shared_rss; 3580 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3581 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3582 uint32_t idx = act_idx & 3583 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3584 3585 switch (type) { 3586 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3587 shared_rss = mlx5_ipool_get 3588 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3589 translated[handle->index].type = 3590 RTE_FLOW_ACTION_TYPE_RSS; 3591 translated[handle->index].conf = 3592 &shared_rss->origin; 3593 break; 3594 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3595 translated[handle->index].type = 3596 (enum rte_flow_action_type) 3597 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3598 translated[handle->index].conf = (void *)(uintptr_t)idx; 3599 break; 3600 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3601 if (priv->sh->flow_hit_aso_en) { 3602 translated[handle->index].type = 3603 (enum rte_flow_action_type) 3604 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3605 translated[handle->index].conf = 3606 (void *)(uintptr_t)idx; 3607 break; 3608 } 3609 /* Fall-through */ 3610 case MLX5_INDIRECT_ACTION_TYPE_CT: 3611 if (priv->sh->ct_aso_en) { 3612 translated[handle->index].type = 3613 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3614 translated[handle->index].conf = 3615 (void *)(uintptr_t)idx; 3616 break; 3617 } 3618 /* Fall-through */ 3619 default: 3620 mlx5_free(translated); 3621 return rte_flow_error_set 3622 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3623 NULL, "invalid indirect action type"); 3624 } 3625 } 3626 *translated_actions = translated; 3627 return 0; 3628 } 3629 3630 /** 3631 * Get Shared RSS action from the action list. 3632 * 3633 * @param[in] dev 3634 * Pointer to Ethernet device. 3635 * @param[in] shared 3636 * Pointer to the list of actions. 3637 * @param[in] shared_n 3638 * Actions list length. 3639 * 3640 * @return 3641 * The MLX5 RSS action ID if exists, otherwise return 0. 3642 */ 3643 static uint32_t 3644 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3645 struct mlx5_translated_action_handle *handle, 3646 int shared_n) 3647 { 3648 struct mlx5_translated_action_handle *handle_end; 3649 struct mlx5_priv *priv = dev->data->dev_private; 3650 struct mlx5_shared_action_rss *shared_rss; 3651 3652 3653 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3654 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3655 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3656 uint32_t idx = act_idx & 3657 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3658 switch (type) { 3659 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3660 shared_rss = mlx5_ipool_get 3661 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3662 idx); 3663 __atomic_add_fetch(&shared_rss->refcnt, 1, 3664 __ATOMIC_RELAXED); 3665 return idx; 3666 default: 3667 break; 3668 } 3669 } 3670 return 0; 3671 } 3672 3673 static unsigned int 3674 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3675 { 3676 const struct rte_flow_item *item; 3677 unsigned int has_vlan = 0; 3678 3679 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3680 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3681 has_vlan = 1; 3682 break; 3683 } 3684 } 3685 if (has_vlan) 3686 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3687 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3688 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3689 MLX5_EXPANSION_ROOT_OUTER; 3690 } 3691 3692 /** 3693 * Get layer flags from the prefix flow. 3694 * 3695 * Some flows may be split to several subflows, the prefix subflow gets the 3696 * match items and the suffix sub flow gets the actions. 3697 * Some actions need the user defined match item flags to get the detail for 3698 * the action. 3699 * This function helps the suffix flow to get the item layer flags from prefix 3700 * subflow. 3701 * 3702 * @param[in] dev_flow 3703 * Pointer the created preifx subflow. 3704 * 3705 * @return 3706 * The layers get from prefix subflow. 3707 */ 3708 static inline uint64_t 3709 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3710 { 3711 uint64_t layers = 0; 3712 3713 /* 3714 * Layers bits could be localization, but usually the compiler will 3715 * help to do the optimization work for source code. 3716 * If no decap actions, use the layers directly. 3717 */ 3718 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3719 return dev_flow->handle->layers; 3720 /* Convert L3 layers with decap action. */ 3721 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3722 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3723 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3724 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3725 /* Convert L4 layers with decap action. */ 3726 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3727 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3728 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3729 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3730 return layers; 3731 } 3732 3733 /** 3734 * Get metadata split action information. 3735 * 3736 * @param[in] actions 3737 * Pointer to the list of actions. 3738 * @param[out] qrss 3739 * Pointer to the return pointer. 3740 * @param[out] qrss_type 3741 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3742 * if no QUEUE/RSS is found. 3743 * @param[out] encap_idx 3744 * Pointer to the index of the encap action if exists, otherwise the last 3745 * action index. 3746 * 3747 * @return 3748 * Total number of actions. 3749 */ 3750 static int 3751 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3752 const struct rte_flow_action **qrss, 3753 int *encap_idx) 3754 { 3755 const struct rte_flow_action_raw_encap *raw_encap; 3756 int actions_n = 0; 3757 int raw_decap_idx = -1; 3758 3759 *encap_idx = -1; 3760 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3761 switch (actions->type) { 3762 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3763 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3764 *encap_idx = actions_n; 3765 break; 3766 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3767 raw_decap_idx = actions_n; 3768 break; 3769 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3770 raw_encap = actions->conf; 3771 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3772 *encap_idx = raw_decap_idx != -1 ? 3773 raw_decap_idx : actions_n; 3774 break; 3775 case RTE_FLOW_ACTION_TYPE_QUEUE: 3776 case RTE_FLOW_ACTION_TYPE_RSS: 3777 *qrss = actions; 3778 break; 3779 default: 3780 break; 3781 } 3782 actions_n++; 3783 } 3784 if (*encap_idx == -1) 3785 *encap_idx = actions_n; 3786 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3787 return actions_n + 1; 3788 } 3789 3790 /** 3791 * Check if the action will change packet. 3792 * 3793 * @param dev 3794 * Pointer to Ethernet device. 3795 * @param[in] type 3796 * action type. 3797 * 3798 * @return 3799 * true if action will change packet, false otherwise. 3800 */ 3801 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 3802 enum rte_flow_action_type type) 3803 { 3804 struct mlx5_priv *priv = dev->data->dev_private; 3805 3806 switch (type) { 3807 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3808 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3809 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3810 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3811 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3812 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3813 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3814 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3815 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3816 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3817 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3818 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3819 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3820 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3821 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 3822 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 3823 case RTE_FLOW_ACTION_TYPE_SET_META: 3824 case RTE_FLOW_ACTION_TYPE_SET_TAG: 3825 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 3826 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3827 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3828 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3829 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3830 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3831 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3832 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3833 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3834 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3835 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 3836 return true; 3837 case RTE_FLOW_ACTION_TYPE_FLAG: 3838 case RTE_FLOW_ACTION_TYPE_MARK: 3839 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 3840 return true; 3841 else 3842 return false; 3843 default: 3844 return false; 3845 } 3846 } 3847 3848 /** 3849 * Check meter action from the action list. 3850 * 3851 * @param dev 3852 * Pointer to Ethernet device. 3853 * @param[in] actions 3854 * Pointer to the list of actions. 3855 * @param[out] has_mtr 3856 * Pointer to the meter exist flag. 3857 * @param[out] has_modify 3858 * Pointer to the flag showing there's packet change action. 3859 * @param[out] meter_id 3860 * Pointer to the meter id. 3861 * 3862 * @return 3863 * Total number of actions. 3864 */ 3865 static int 3866 flow_check_meter_action(struct rte_eth_dev *dev, 3867 const struct rte_flow_action actions[], 3868 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 3869 { 3870 const struct rte_flow_action_meter *mtr = NULL; 3871 int actions_n = 0; 3872 3873 MLX5_ASSERT(has_mtr); 3874 *has_mtr = false; 3875 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3876 switch (actions->type) { 3877 case RTE_FLOW_ACTION_TYPE_METER: 3878 mtr = actions->conf; 3879 *meter_id = mtr->mtr_id; 3880 *has_mtr = true; 3881 break; 3882 default: 3883 break; 3884 } 3885 if (!*has_mtr) 3886 *has_modify |= flow_check_modify_action_type(dev, 3887 actions->type); 3888 actions_n++; 3889 } 3890 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3891 return actions_n + 1; 3892 } 3893 3894 /** 3895 * Check if the flow should be split due to hairpin. 3896 * The reason for the split is that in current HW we can't 3897 * support encap and push-vlan on Rx, so if a flow contains 3898 * these actions we move it to Tx. 3899 * 3900 * @param dev 3901 * Pointer to Ethernet device. 3902 * @param[in] attr 3903 * Flow rule attributes. 3904 * @param[in] actions 3905 * Associated actions (list terminated by the END action). 3906 * 3907 * @return 3908 * > 0 the number of actions and the flow should be split, 3909 * 0 when no split required. 3910 */ 3911 static int 3912 flow_check_hairpin_split(struct rte_eth_dev *dev, 3913 const struct rte_flow_attr *attr, 3914 const struct rte_flow_action actions[]) 3915 { 3916 int queue_action = 0; 3917 int action_n = 0; 3918 int split = 0; 3919 const struct rte_flow_action_queue *queue; 3920 const struct rte_flow_action_rss *rss; 3921 const struct rte_flow_action_raw_encap *raw_encap; 3922 const struct rte_eth_hairpin_conf *conf; 3923 3924 if (!attr->ingress) 3925 return 0; 3926 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3927 switch (actions->type) { 3928 case RTE_FLOW_ACTION_TYPE_QUEUE: 3929 queue = actions->conf; 3930 if (queue == NULL) 3931 return 0; 3932 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3933 if (conf == NULL || conf->tx_explicit != 0) 3934 return 0; 3935 queue_action = 1; 3936 action_n++; 3937 break; 3938 case RTE_FLOW_ACTION_TYPE_RSS: 3939 rss = actions->conf; 3940 if (rss == NULL || rss->queue_num == 0) 3941 return 0; 3942 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3943 if (conf == NULL || conf->tx_explicit != 0) 3944 return 0; 3945 queue_action = 1; 3946 action_n++; 3947 break; 3948 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3949 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3950 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3951 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3952 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3953 split++; 3954 action_n++; 3955 break; 3956 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3957 raw_encap = actions->conf; 3958 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3959 split++; 3960 action_n++; 3961 break; 3962 default: 3963 action_n++; 3964 break; 3965 } 3966 } 3967 if (split && queue_action) 3968 return action_n; 3969 return 0; 3970 } 3971 3972 /* Declare flow create/destroy prototype in advance. */ 3973 static uint32_t 3974 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3975 const struct rte_flow_attr *attr, 3976 const struct rte_flow_item items[], 3977 const struct rte_flow_action actions[], 3978 bool external, struct rte_flow_error *error); 3979 3980 static void 3981 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3982 uint32_t flow_idx); 3983 3984 int 3985 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused, 3986 struct mlx5_hlist_entry *entry, 3987 uint64_t key, void *cb_ctx __rte_unused) 3988 { 3989 struct mlx5_flow_mreg_copy_resource *mcp_res = 3990 container_of(entry, typeof(*mcp_res), hlist_ent); 3991 3992 return mcp_res->mark_id != key; 3993 } 3994 3995 struct mlx5_hlist_entry * 3996 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 3997 void *cb_ctx) 3998 { 3999 struct rte_eth_dev *dev = list->ctx; 4000 struct mlx5_priv *priv = dev->data->dev_private; 4001 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4002 struct mlx5_flow_mreg_copy_resource *mcp_res; 4003 struct rte_flow_error *error = ctx->error; 4004 uint32_t idx = 0; 4005 int ret; 4006 uint32_t mark_id = key; 4007 struct rte_flow_attr attr = { 4008 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4009 .ingress = 1, 4010 }; 4011 struct mlx5_rte_flow_item_tag tag_spec = { 4012 .data = mark_id, 4013 }; 4014 struct rte_flow_item items[] = { 4015 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4016 }; 4017 struct rte_flow_action_mark ftag = { 4018 .id = mark_id, 4019 }; 4020 struct mlx5_flow_action_copy_mreg cp_mreg = { 4021 .dst = REG_B, 4022 .src = REG_NON, 4023 }; 4024 struct rte_flow_action_jump jump = { 4025 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4026 }; 4027 struct rte_flow_action actions[] = { 4028 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4029 }; 4030 4031 /* Fill the register fileds in the flow. */ 4032 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4033 if (ret < 0) 4034 return NULL; 4035 tag_spec.id = ret; 4036 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4037 if (ret < 0) 4038 return NULL; 4039 cp_mreg.src = ret; 4040 /* Provide the full width of FLAG specific value. */ 4041 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4042 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4043 /* Build a new flow. */ 4044 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4045 items[0] = (struct rte_flow_item){ 4046 .type = (enum rte_flow_item_type) 4047 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4048 .spec = &tag_spec, 4049 }; 4050 items[1] = (struct rte_flow_item){ 4051 .type = RTE_FLOW_ITEM_TYPE_END, 4052 }; 4053 actions[0] = (struct rte_flow_action){ 4054 .type = (enum rte_flow_action_type) 4055 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4056 .conf = &ftag, 4057 }; 4058 actions[1] = (struct rte_flow_action){ 4059 .type = (enum rte_flow_action_type) 4060 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4061 .conf = &cp_mreg, 4062 }; 4063 actions[2] = (struct rte_flow_action){ 4064 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4065 .conf = &jump, 4066 }; 4067 actions[3] = (struct rte_flow_action){ 4068 .type = RTE_FLOW_ACTION_TYPE_END, 4069 }; 4070 } else { 4071 /* Default rule, wildcard match. */ 4072 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4073 items[0] = (struct rte_flow_item){ 4074 .type = RTE_FLOW_ITEM_TYPE_END, 4075 }; 4076 actions[0] = (struct rte_flow_action){ 4077 .type = (enum rte_flow_action_type) 4078 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4079 .conf = &cp_mreg, 4080 }; 4081 actions[1] = (struct rte_flow_action){ 4082 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4083 .conf = &jump, 4084 }; 4085 actions[2] = (struct rte_flow_action){ 4086 .type = RTE_FLOW_ACTION_TYPE_END, 4087 }; 4088 } 4089 /* Build a new entry. */ 4090 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4091 if (!mcp_res) { 4092 rte_errno = ENOMEM; 4093 return NULL; 4094 } 4095 mcp_res->idx = idx; 4096 mcp_res->mark_id = mark_id; 4097 /* 4098 * The copy Flows are not included in any list. There 4099 * ones are referenced from other Flows and can not 4100 * be applied, removed, deleted in ardbitrary order 4101 * by list traversing. 4102 */ 4103 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 4104 actions, false, error); 4105 if (!mcp_res->rix_flow) { 4106 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4107 return NULL; 4108 } 4109 return &mcp_res->hlist_ent; 4110 } 4111 4112 /** 4113 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4114 * 4115 * As mark_id is unique, if there's already a registered flow for the mark_id, 4116 * return by increasing the reference counter of the resource. Otherwise, create 4117 * the resource (mcp_res) and flow. 4118 * 4119 * Flow looks like, 4120 * - If ingress port is ANY and reg_c[1] is mark_id, 4121 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4122 * 4123 * For default flow (zero mark_id), flow is like, 4124 * - If ingress port is ANY, 4125 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4126 * 4127 * @param dev 4128 * Pointer to Ethernet device. 4129 * @param mark_id 4130 * ID of MARK action, zero means default flow for META. 4131 * @param[out] error 4132 * Perform verbose error reporting if not NULL. 4133 * 4134 * @return 4135 * Associated resource on success, NULL otherwise and rte_errno is set. 4136 */ 4137 static struct mlx5_flow_mreg_copy_resource * 4138 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4139 struct rte_flow_error *error) 4140 { 4141 struct mlx5_priv *priv = dev->data->dev_private; 4142 struct mlx5_hlist_entry *entry; 4143 struct mlx5_flow_cb_ctx ctx = { 4144 .dev = dev, 4145 .error = error, 4146 }; 4147 4148 /* Check if already registered. */ 4149 MLX5_ASSERT(priv->mreg_cp_tbl); 4150 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4151 if (!entry) 4152 return NULL; 4153 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4154 hlist_ent); 4155 } 4156 4157 void 4158 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 4159 { 4160 struct mlx5_flow_mreg_copy_resource *mcp_res = 4161 container_of(entry, typeof(*mcp_res), hlist_ent); 4162 struct rte_eth_dev *dev = list->ctx; 4163 struct mlx5_priv *priv = dev->data->dev_private; 4164 4165 MLX5_ASSERT(mcp_res->rix_flow); 4166 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 4167 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4168 } 4169 4170 /** 4171 * Release flow in RX_CP_TBL. 4172 * 4173 * @param dev 4174 * Pointer to Ethernet device. 4175 * @flow 4176 * Parent flow for wich copying is provided. 4177 */ 4178 static void 4179 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4180 struct rte_flow *flow) 4181 { 4182 struct mlx5_flow_mreg_copy_resource *mcp_res; 4183 struct mlx5_priv *priv = dev->data->dev_private; 4184 4185 if (!flow->rix_mreg_copy) 4186 return; 4187 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4188 flow->rix_mreg_copy); 4189 if (!mcp_res || !priv->mreg_cp_tbl) 4190 return; 4191 MLX5_ASSERT(mcp_res->rix_flow); 4192 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4193 flow->rix_mreg_copy = 0; 4194 } 4195 4196 /** 4197 * Remove the default copy action from RX_CP_TBL. 4198 * 4199 * This functions is called in the mlx5_dev_start(). No thread safe 4200 * is guaranteed. 4201 * 4202 * @param dev 4203 * Pointer to Ethernet device. 4204 */ 4205 static void 4206 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4207 { 4208 struct mlx5_hlist_entry *entry; 4209 struct mlx5_priv *priv = dev->data->dev_private; 4210 4211 /* Check if default flow is registered. */ 4212 if (!priv->mreg_cp_tbl) 4213 return; 4214 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 4215 MLX5_DEFAULT_COPY_ID, NULL); 4216 if (!entry) 4217 return; 4218 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4219 } 4220 4221 /** 4222 * Add the default copy action in in RX_CP_TBL. 4223 * 4224 * This functions is called in the mlx5_dev_start(). No thread safe 4225 * is guaranteed. 4226 * 4227 * @param dev 4228 * Pointer to Ethernet device. 4229 * @param[out] error 4230 * Perform verbose error reporting if not NULL. 4231 * 4232 * @return 4233 * 0 for success, negative value otherwise and rte_errno is set. 4234 */ 4235 static int 4236 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4237 struct rte_flow_error *error) 4238 { 4239 struct mlx5_priv *priv = dev->data->dev_private; 4240 struct mlx5_flow_mreg_copy_resource *mcp_res; 4241 4242 /* Check whether extensive metadata feature is engaged. */ 4243 if (!priv->config.dv_flow_en || 4244 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4245 !mlx5_flow_ext_mreg_supported(dev) || 4246 !priv->sh->dv_regc0_mask) 4247 return 0; 4248 /* 4249 * Add default mreg copy flow may be called multiple time, but 4250 * only be called once in stop. Avoid register it twice. 4251 */ 4252 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 4253 return 0; 4254 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 4255 if (!mcp_res) 4256 return -rte_errno; 4257 return 0; 4258 } 4259 4260 /** 4261 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4262 * 4263 * All the flow having Q/RSS action should be split by 4264 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4265 * performs the following, 4266 * - CQE->flow_tag := reg_c[1] (MARK) 4267 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4268 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4269 * but there should be a flow per each MARK ID set by MARK action. 4270 * 4271 * For the aforementioned reason, if there's a MARK action in flow's action 4272 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4273 * the MARK ID to CQE's flow_tag like, 4274 * - If reg_c[1] is mark_id, 4275 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4276 * 4277 * For SET_META action which stores value in reg_c[0], as the destination is 4278 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4279 * MARK ID means the default flow. The default flow looks like, 4280 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4281 * 4282 * @param dev 4283 * Pointer to Ethernet device. 4284 * @param flow 4285 * Pointer to flow structure. 4286 * @param[in] actions 4287 * Pointer to the list of actions. 4288 * @param[out] error 4289 * Perform verbose error reporting if not NULL. 4290 * 4291 * @return 4292 * 0 on success, negative value otherwise and rte_errno is set. 4293 */ 4294 static int 4295 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4296 struct rte_flow *flow, 4297 const struct rte_flow_action *actions, 4298 struct rte_flow_error *error) 4299 { 4300 struct mlx5_priv *priv = dev->data->dev_private; 4301 struct mlx5_dev_config *config = &priv->config; 4302 struct mlx5_flow_mreg_copy_resource *mcp_res; 4303 const struct rte_flow_action_mark *mark; 4304 4305 /* Check whether extensive metadata feature is engaged. */ 4306 if (!config->dv_flow_en || 4307 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4308 !mlx5_flow_ext_mreg_supported(dev) || 4309 !priv->sh->dv_regc0_mask) 4310 return 0; 4311 /* Find MARK action. */ 4312 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4313 switch (actions->type) { 4314 case RTE_FLOW_ACTION_TYPE_FLAG: 4315 mcp_res = flow_mreg_add_copy_action 4316 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4317 if (!mcp_res) 4318 return -rte_errno; 4319 flow->rix_mreg_copy = mcp_res->idx; 4320 return 0; 4321 case RTE_FLOW_ACTION_TYPE_MARK: 4322 mark = (const struct rte_flow_action_mark *) 4323 actions->conf; 4324 mcp_res = 4325 flow_mreg_add_copy_action(dev, mark->id, error); 4326 if (!mcp_res) 4327 return -rte_errno; 4328 flow->rix_mreg_copy = mcp_res->idx; 4329 return 0; 4330 default: 4331 break; 4332 } 4333 } 4334 return 0; 4335 } 4336 4337 #define MLX5_MAX_SPLIT_ACTIONS 24 4338 #define MLX5_MAX_SPLIT_ITEMS 24 4339 4340 /** 4341 * Split the hairpin flow. 4342 * Since HW can't support encap and push-vlan on Rx, we move these 4343 * actions to Tx. 4344 * If the count action is after the encap then we also 4345 * move the count action. in this case the count will also measure 4346 * the outer bytes. 4347 * 4348 * @param dev 4349 * Pointer to Ethernet device. 4350 * @param[in] actions 4351 * Associated actions (list terminated by the END action). 4352 * @param[out] actions_rx 4353 * Rx flow actions. 4354 * @param[out] actions_tx 4355 * Tx flow actions.. 4356 * @param[out] pattern_tx 4357 * The pattern items for the Tx flow. 4358 * @param[out] flow_id 4359 * The flow ID connected to this flow. 4360 * 4361 * @return 4362 * 0 on success. 4363 */ 4364 static int 4365 flow_hairpin_split(struct rte_eth_dev *dev, 4366 const struct rte_flow_action actions[], 4367 struct rte_flow_action actions_rx[], 4368 struct rte_flow_action actions_tx[], 4369 struct rte_flow_item pattern_tx[], 4370 uint32_t flow_id) 4371 { 4372 const struct rte_flow_action_raw_encap *raw_encap; 4373 const struct rte_flow_action_raw_decap *raw_decap; 4374 struct mlx5_rte_flow_action_set_tag *set_tag; 4375 struct rte_flow_action *tag_action; 4376 struct mlx5_rte_flow_item_tag *tag_item; 4377 struct rte_flow_item *item; 4378 char *addr; 4379 int encap = 0; 4380 4381 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4382 switch (actions->type) { 4383 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4384 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4385 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4386 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4387 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4388 rte_memcpy(actions_tx, actions, 4389 sizeof(struct rte_flow_action)); 4390 actions_tx++; 4391 break; 4392 case RTE_FLOW_ACTION_TYPE_COUNT: 4393 if (encap) { 4394 rte_memcpy(actions_tx, actions, 4395 sizeof(struct rte_flow_action)); 4396 actions_tx++; 4397 } else { 4398 rte_memcpy(actions_rx, actions, 4399 sizeof(struct rte_flow_action)); 4400 actions_rx++; 4401 } 4402 break; 4403 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4404 raw_encap = actions->conf; 4405 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4406 memcpy(actions_tx, actions, 4407 sizeof(struct rte_flow_action)); 4408 actions_tx++; 4409 encap = 1; 4410 } else { 4411 rte_memcpy(actions_rx, actions, 4412 sizeof(struct rte_flow_action)); 4413 actions_rx++; 4414 } 4415 break; 4416 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4417 raw_decap = actions->conf; 4418 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4419 memcpy(actions_tx, actions, 4420 sizeof(struct rte_flow_action)); 4421 actions_tx++; 4422 } else { 4423 rte_memcpy(actions_rx, actions, 4424 sizeof(struct rte_flow_action)); 4425 actions_rx++; 4426 } 4427 break; 4428 default: 4429 rte_memcpy(actions_rx, actions, 4430 sizeof(struct rte_flow_action)); 4431 actions_rx++; 4432 break; 4433 } 4434 } 4435 /* Add set meta action and end action for the Rx flow. */ 4436 tag_action = actions_rx; 4437 tag_action->type = (enum rte_flow_action_type) 4438 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4439 actions_rx++; 4440 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4441 actions_rx++; 4442 set_tag = (void *)actions_rx; 4443 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4444 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4445 .data = flow_id, 4446 }; 4447 MLX5_ASSERT(set_tag->id > REG_NON); 4448 tag_action->conf = set_tag; 4449 /* Create Tx item list. */ 4450 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4451 addr = (void *)&pattern_tx[2]; 4452 item = pattern_tx; 4453 item->type = (enum rte_flow_item_type) 4454 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4455 tag_item = (void *)addr; 4456 tag_item->data = flow_id; 4457 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4458 MLX5_ASSERT(set_tag->id > REG_NON); 4459 item->spec = tag_item; 4460 addr += sizeof(struct mlx5_rte_flow_item_tag); 4461 tag_item = (void *)addr; 4462 tag_item->data = UINT32_MAX; 4463 tag_item->id = UINT16_MAX; 4464 item->mask = tag_item; 4465 item->last = NULL; 4466 item++; 4467 item->type = RTE_FLOW_ITEM_TYPE_END; 4468 return 0; 4469 } 4470 4471 /** 4472 * The last stage of splitting chain, just creates the subflow 4473 * without any modification. 4474 * 4475 * @param[in] dev 4476 * Pointer to Ethernet device. 4477 * @param[in] flow 4478 * Parent flow structure pointer. 4479 * @param[in, out] sub_flow 4480 * Pointer to return the created subflow, may be NULL. 4481 * @param[in] attr 4482 * Flow rule attributes. 4483 * @param[in] items 4484 * Pattern specification (list terminated by the END pattern item). 4485 * @param[in] actions 4486 * Associated actions (list terminated by the END action). 4487 * @param[in] flow_split_info 4488 * Pointer to flow split info structure. 4489 * @param[out] error 4490 * Perform verbose error reporting if not NULL. 4491 * @return 4492 * 0 on success, negative value otherwise 4493 */ 4494 static int 4495 flow_create_split_inner(struct rte_eth_dev *dev, 4496 struct rte_flow *flow, 4497 struct mlx5_flow **sub_flow, 4498 const struct rte_flow_attr *attr, 4499 const struct rte_flow_item items[], 4500 const struct rte_flow_action actions[], 4501 struct mlx5_flow_split_info *flow_split_info, 4502 struct rte_flow_error *error) 4503 { 4504 struct mlx5_flow *dev_flow; 4505 4506 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4507 flow_split_info->flow_idx, error); 4508 if (!dev_flow) 4509 return -rte_errno; 4510 dev_flow->flow = flow; 4511 dev_flow->external = flow_split_info->external; 4512 dev_flow->skip_scale = flow_split_info->skip_scale; 4513 /* Subflow object was created, we must include one in the list. */ 4514 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4515 dev_flow->handle, next); 4516 /* 4517 * If dev_flow is as one of the suffix flow, some actions in suffix 4518 * flow may need some user defined item layer flags, and pass the 4519 * Metadate rxq mark flag to suffix flow as well. 4520 */ 4521 if (flow_split_info->prefix_layers) 4522 dev_flow->handle->layers = flow_split_info->prefix_layers; 4523 if (flow_split_info->prefix_mark) 4524 dev_flow->handle->mark = 1; 4525 if (sub_flow) 4526 *sub_flow = dev_flow; 4527 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4528 dev_flow->dv.table_id = flow_split_info->table_id; 4529 #endif 4530 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4531 } 4532 4533 /** 4534 * Get the sub policy of a meter. 4535 * 4536 * @param[in] dev 4537 * Pointer to Ethernet device. 4538 * @param[in] flow 4539 * Parent flow structure pointer. 4540 * @param[in] policy_id; 4541 * Meter Policy id. 4542 * @param[in] attr 4543 * Flow rule attributes. 4544 * @param[in] items 4545 * Pattern specification (list terminated by the END pattern item). 4546 * @param[out] error 4547 * Perform verbose error reporting if not NULL. 4548 * 4549 * @return 4550 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4551 */ 4552 static struct mlx5_flow_meter_sub_policy * 4553 get_meter_sub_policy(struct rte_eth_dev *dev, 4554 struct rte_flow *flow, 4555 uint32_t policy_id, 4556 const struct rte_flow_attr *attr, 4557 const struct rte_flow_item items[], 4558 struct rte_flow_error *error) 4559 { 4560 struct mlx5_flow_meter_policy *policy; 4561 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4562 4563 policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 4564 if (!policy) { 4565 rte_flow_error_set(error, EINVAL, 4566 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4567 "Failed to find Meter Policy."); 4568 goto exit; 4569 } 4570 if (policy->is_rss) { 4571 struct mlx5_flow_workspace *wks = 4572 mlx5_flow_get_thread_workspace(); 4573 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4574 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4575 uint32_t i; 4576 4577 MLX5_ASSERT(wks); 4578 /** 4579 * This is a tmp dev_flow, 4580 * no need to register any matcher for it in translate. 4581 */ 4582 wks->skip_matcher_reg = 1; 4583 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4584 struct mlx5_flow dev_flow = {0}; 4585 struct mlx5_flow_handle dev_handle = { {0} }; 4586 const void *rss_act = policy->act_cnt[i].rss->conf; 4587 struct rte_flow_action rss_actions[2] = { 4588 [0] = { 4589 .type = RTE_FLOW_ACTION_TYPE_RSS, 4590 .conf = rss_act 4591 }, 4592 [1] = { 4593 .type = RTE_FLOW_ACTION_TYPE_END, 4594 .conf = NULL 4595 } 4596 }; 4597 4598 dev_flow.handle = &dev_handle; 4599 dev_flow.ingress = attr->ingress; 4600 dev_flow.flow = flow; 4601 dev_flow.external = 0; 4602 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4603 dev_flow.dv.transfer = attr->transfer; 4604 #endif 4605 /* Translate RSS action to get rss hash fields. */ 4606 if (flow_drv_translate(dev, &dev_flow, attr, 4607 items, rss_actions, error)) 4608 goto exit; 4609 rss_desc_v[i] = wks->rss_desc; 4610 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4611 rss_desc_v[i].hash_fields = dev_flow.hash_fields; 4612 rss_desc_v[i].queue_num = rss_desc_v[i].hash_fields ? 4613 rss_desc_v[i].queue_num : 1; 4614 rss_desc[i] = &rss_desc_v[i]; 4615 } 4616 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4617 flow, policy, rss_desc); 4618 } else { 4619 enum mlx5_meter_domain mtr_domain = 4620 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4621 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4622 MLX5_MTR_DOMAIN_INGRESS; 4623 sub_policy = policy->sub_policys[mtr_domain][0]; 4624 } 4625 if (!sub_policy) { 4626 rte_flow_error_set(error, EINVAL, 4627 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4628 "Failed to get meter sub-policy."); 4629 goto exit; 4630 } 4631 exit: 4632 return sub_policy; 4633 } 4634 4635 /** 4636 * Split the meter flow. 4637 * 4638 * As meter flow will split to three sub flow, other than meter 4639 * action, the other actions make sense to only meter accepts 4640 * the packet. If it need to be dropped, no other additional 4641 * actions should be take. 4642 * 4643 * One kind of special action which decapsulates the L3 tunnel 4644 * header will be in the prefix sub flow, as not to take the 4645 * L3 tunnel header into account. 4646 * 4647 * @param[in] dev 4648 * Pointer to Ethernet device. 4649 * @param[in] flow 4650 * Parent flow structure pointer. 4651 * @param[in] fm 4652 * Pointer to flow meter structure. 4653 * @param[in] attr 4654 * Flow rule attributes. 4655 * @param[in] items 4656 * Pattern specification (list terminated by the END pattern item). 4657 * @param[out] sfx_items 4658 * Suffix flow match items (list terminated by the END pattern item). 4659 * @param[in] actions 4660 * Associated actions (list terminated by the END action). 4661 * @param[out] actions_sfx 4662 * Suffix flow actions. 4663 * @param[out] actions_pre 4664 * Prefix flow actions. 4665 * @param[out] mtr_flow_id 4666 * Pointer to meter flow id. 4667 * @param[out] error 4668 * Perform verbose error reporting if not NULL. 4669 * 4670 * @return 4671 * 0 on success, a negative errno value otherwise and rte_errno is set. 4672 */ 4673 static int 4674 flow_meter_split_prep(struct rte_eth_dev *dev, 4675 struct rte_flow *flow, 4676 struct mlx5_flow_meter_info *fm, 4677 const struct rte_flow_attr *attr, 4678 const struct rte_flow_item items[], 4679 struct rte_flow_item sfx_items[], 4680 const struct rte_flow_action actions[], 4681 struct rte_flow_action actions_sfx[], 4682 struct rte_flow_action actions_pre[], 4683 uint32_t *mtr_flow_id, 4684 struct rte_flow_error *error) 4685 { 4686 struct mlx5_priv *priv = dev->data->dev_private; 4687 struct rte_flow_action *tag_action = NULL; 4688 struct rte_flow_item *tag_item; 4689 struct mlx5_rte_flow_action_set_tag *set_tag; 4690 const struct rte_flow_action_raw_encap *raw_encap; 4691 const struct rte_flow_action_raw_decap *raw_decap; 4692 struct mlx5_rte_flow_item_tag *tag_item_spec; 4693 struct mlx5_rte_flow_item_tag *tag_item_mask; 4694 uint32_t tag_id = 0; 4695 bool copy_vlan = false; 4696 struct rte_flow_action *hw_mtr_action; 4697 struct rte_flow_action *action_pre_head = NULL; 4698 bool mtr_first = priv->sh->meter_aso_en && 4699 (attr->egress || 4700 (attr->transfer && priv->representor_id != UINT16_MAX)); 4701 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 4702 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 4703 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 4704 uint32_t flow_id = 0; 4705 uint32_t flow_id_reversed = 0; 4706 uint8_t flow_id_bits = 0; 4707 int shift; 4708 4709 /* For ASO meter, meter must be before tag in TX direction. */ 4710 if (mtr_first) { 4711 action_pre_head = actions_pre++; 4712 /* Leave space for tag action. */ 4713 tag_action = actions_pre++; 4714 } 4715 /* Prepare the actions for prefix and suffix flow. */ 4716 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4717 struct rte_flow_action *action_cur = NULL; 4718 4719 switch (actions->type) { 4720 case RTE_FLOW_ACTION_TYPE_METER: 4721 if (mtr_first) { 4722 action_cur = action_pre_head; 4723 } else { 4724 /* Leave space for tag action. */ 4725 tag_action = actions_pre++; 4726 action_cur = actions_pre++; 4727 } 4728 break; 4729 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4730 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4731 action_cur = actions_pre++; 4732 break; 4733 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4734 raw_encap = actions->conf; 4735 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4736 action_cur = actions_pre++; 4737 break; 4738 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4739 raw_decap = actions->conf; 4740 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4741 action_cur = actions_pre++; 4742 break; 4743 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4744 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4745 copy_vlan = true; 4746 break; 4747 default: 4748 break; 4749 } 4750 if (!action_cur) 4751 action_cur = (fm->def_policy) ? 4752 actions_sfx++ : actions_pre++; 4753 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 4754 } 4755 /* Add end action to the actions. */ 4756 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4757 if (priv->sh->meter_aso_en) { 4758 /** 4759 * For ASO meter, need to add an extra jump action explicitly, 4760 * to jump from meter to policer table. 4761 */ 4762 struct mlx5_flow_meter_sub_policy *sub_policy; 4763 struct mlx5_flow_tbl_data_entry *tbl_data; 4764 4765 if (!fm->def_policy) { 4766 sub_policy = get_meter_sub_policy(dev, flow, 4767 fm->policy_id, attr, 4768 items, error); 4769 if (!sub_policy) 4770 return -rte_errno; 4771 } else { 4772 enum mlx5_meter_domain mtr_domain = 4773 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4774 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4775 MLX5_MTR_DOMAIN_INGRESS; 4776 4777 sub_policy = 4778 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 4779 } 4780 tbl_data = container_of(sub_policy->tbl_rsc, 4781 struct mlx5_flow_tbl_data_entry, tbl); 4782 hw_mtr_action = actions_pre++; 4783 hw_mtr_action->type = (enum rte_flow_action_type) 4784 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 4785 hw_mtr_action->conf = tbl_data->jump.action; 4786 } 4787 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4788 actions_pre++; 4789 if (!tag_action) 4790 return rte_flow_error_set(error, ENOMEM, 4791 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4792 "No tag action space."); 4793 if (!mtr_flow_id) { 4794 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 4795 goto exit; 4796 } 4797 /* Only default-policy Meter creates mtr flow id. */ 4798 if (fm->def_policy) { 4799 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 4800 if (!tag_id) 4801 return rte_flow_error_set(error, ENOMEM, 4802 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4803 "Failed to allocate meter flow id."); 4804 flow_id = tag_id - 1; 4805 flow_id_bits = (!flow_id) ? 1 : 4806 (MLX5_REG_BITS - __builtin_clz(flow_id)); 4807 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 4808 mtr_reg_bits) { 4809 mlx5_ipool_free(fm->flow_ipool, tag_id); 4810 return rte_flow_error_set(error, EINVAL, 4811 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4812 "Meter flow id exceeds max limit."); 4813 } 4814 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 4815 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 4816 } 4817 /* Prepare the suffix subflow items. */ 4818 tag_item = sfx_items++; 4819 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4820 int item_type = items->type; 4821 4822 switch (item_type) { 4823 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4824 memcpy(sfx_items, items, sizeof(*sfx_items)); 4825 sfx_items++; 4826 break; 4827 case RTE_FLOW_ITEM_TYPE_VLAN: 4828 if (copy_vlan) { 4829 memcpy(sfx_items, items, sizeof(*sfx_items)); 4830 /* 4831 * Convert to internal match item, it is used 4832 * for vlan push and set vid. 4833 */ 4834 sfx_items->type = (enum rte_flow_item_type) 4835 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4836 sfx_items++; 4837 } 4838 break; 4839 default: 4840 break; 4841 } 4842 } 4843 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4844 sfx_items++; 4845 /* Build tag actions and items for meter_id/meter flow_id. */ 4846 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 4847 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4848 tag_item_mask = tag_item_spec + 1; 4849 /* Both flow_id and meter_id share the same register. */ 4850 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4851 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 4852 0, error), 4853 .offset = mtr_id_offset, 4854 .length = mtr_reg_bits, 4855 .data = flow->meter, 4856 }; 4857 /* 4858 * The color Reg bits used by flow_id are growing from 4859 * msb to lsb, so must do bit reverse for flow_id val in RegC. 4860 */ 4861 for (shift = 0; shift < flow_id_bits; shift++) 4862 flow_id_reversed = (flow_id_reversed << 1) | 4863 ((flow_id >> shift) & 0x1); 4864 set_tag->data |= 4865 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 4866 tag_item_spec->id = set_tag->id; 4867 tag_item_spec->data = set_tag->data << mtr_id_offset; 4868 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 4869 tag_action->type = (enum rte_flow_action_type) 4870 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4871 tag_action->conf = set_tag; 4872 tag_item->type = (enum rte_flow_item_type) 4873 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4874 tag_item->spec = tag_item_spec; 4875 tag_item->last = NULL; 4876 tag_item->mask = tag_item_mask; 4877 exit: 4878 if (mtr_flow_id) 4879 *mtr_flow_id = tag_id; 4880 return 0; 4881 } 4882 4883 /** 4884 * Split action list having QUEUE/RSS for metadata register copy. 4885 * 4886 * Once Q/RSS action is detected in user's action list, the flow action 4887 * should be split in order to copy metadata registers, which will happen in 4888 * RX_CP_TBL like, 4889 * - CQE->flow_tag := reg_c[1] (MARK) 4890 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4891 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4892 * This is because the last action of each flow must be a terminal action 4893 * (QUEUE, RSS or DROP). 4894 * 4895 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4896 * stored and kept in the mlx5_flow structure per each sub_flow. 4897 * 4898 * The Q/RSS action is replaced with, 4899 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4900 * And the following JUMP action is added at the end, 4901 * - JUMP, to RX_CP_TBL. 4902 * 4903 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4904 * flow_create_split_metadata() routine. The flow will look like, 4905 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4906 * 4907 * @param dev 4908 * Pointer to Ethernet device. 4909 * @param[out] split_actions 4910 * Pointer to store split actions to jump to CP_TBL. 4911 * @param[in] actions 4912 * Pointer to the list of original flow actions. 4913 * @param[in] qrss 4914 * Pointer to the Q/RSS action. 4915 * @param[in] actions_n 4916 * Number of original actions. 4917 * @param[out] error 4918 * Perform verbose error reporting if not NULL. 4919 * 4920 * @return 4921 * non-zero unique flow_id on success, otherwise 0 and 4922 * error/rte_error are set. 4923 */ 4924 static uint32_t 4925 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4926 struct rte_flow_action *split_actions, 4927 const struct rte_flow_action *actions, 4928 const struct rte_flow_action *qrss, 4929 int actions_n, struct rte_flow_error *error) 4930 { 4931 struct mlx5_priv *priv = dev->data->dev_private; 4932 struct mlx5_rte_flow_action_set_tag *set_tag; 4933 struct rte_flow_action_jump *jump; 4934 const int qrss_idx = qrss - actions; 4935 uint32_t flow_id = 0; 4936 int ret = 0; 4937 4938 /* 4939 * Given actions will be split 4940 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4941 * - Add jump to mreg CP_TBL. 4942 * As a result, there will be one more action. 4943 */ 4944 ++actions_n; 4945 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4946 set_tag = (void *)(split_actions + actions_n); 4947 /* 4948 * If tag action is not set to void(it means we are not the meter 4949 * suffix flow), add the tag action. Since meter suffix flow already 4950 * has the tag added. 4951 */ 4952 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4953 /* 4954 * Allocate the new subflow ID. This one is unique within 4955 * device and not shared with representors. Otherwise, 4956 * we would have to resolve multi-thread access synch 4957 * issue. Each flow on the shared device is appended 4958 * with source vport identifier, so the resulting 4959 * flows will be unique in the shared (by master and 4960 * representors) domain even if they have coinciding 4961 * IDs. 4962 */ 4963 mlx5_ipool_malloc(priv->sh->ipool 4964 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 4965 if (!flow_id) 4966 return rte_flow_error_set(error, ENOMEM, 4967 RTE_FLOW_ERROR_TYPE_ACTION, 4968 NULL, "can't allocate id " 4969 "for split Q/RSS subflow"); 4970 /* Internal SET_TAG action to set flow ID. */ 4971 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4972 .data = flow_id, 4973 }; 4974 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4975 if (ret < 0) 4976 return ret; 4977 set_tag->id = ret; 4978 /* Construct new actions array. */ 4979 /* Replace QUEUE/RSS action. */ 4980 split_actions[qrss_idx] = (struct rte_flow_action){ 4981 .type = (enum rte_flow_action_type) 4982 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4983 .conf = set_tag, 4984 }; 4985 } 4986 /* JUMP action to jump to mreg copy table (CP_TBL). */ 4987 jump = (void *)(set_tag + 1); 4988 *jump = (struct rte_flow_action_jump){ 4989 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4990 }; 4991 split_actions[actions_n - 2] = (struct rte_flow_action){ 4992 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4993 .conf = jump, 4994 }; 4995 split_actions[actions_n - 1] = (struct rte_flow_action){ 4996 .type = RTE_FLOW_ACTION_TYPE_END, 4997 }; 4998 return flow_id; 4999 } 5000 5001 /** 5002 * Extend the given action list for Tx metadata copy. 5003 * 5004 * Copy the given action list to the ext_actions and add flow metadata register 5005 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5006 * 5007 * @param[out] ext_actions 5008 * Pointer to the extended action list. 5009 * @param[in] actions 5010 * Pointer to the list of actions. 5011 * @param[in] actions_n 5012 * Number of actions in the list. 5013 * @param[out] error 5014 * Perform verbose error reporting if not NULL. 5015 * @param[in] encap_idx 5016 * The encap action inndex. 5017 * 5018 * @return 5019 * 0 on success, negative value otherwise 5020 */ 5021 static int 5022 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5023 struct rte_flow_action *ext_actions, 5024 const struct rte_flow_action *actions, 5025 int actions_n, struct rte_flow_error *error, 5026 int encap_idx) 5027 { 5028 struct mlx5_flow_action_copy_mreg *cp_mreg = 5029 (struct mlx5_flow_action_copy_mreg *) 5030 (ext_actions + actions_n + 1); 5031 int ret; 5032 5033 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5034 if (ret < 0) 5035 return ret; 5036 cp_mreg->dst = ret; 5037 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5038 if (ret < 0) 5039 return ret; 5040 cp_mreg->src = ret; 5041 if (encap_idx != 0) 5042 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5043 if (encap_idx == actions_n - 1) { 5044 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5045 .type = (enum rte_flow_action_type) 5046 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5047 .conf = cp_mreg, 5048 }; 5049 ext_actions[actions_n] = (struct rte_flow_action){ 5050 .type = RTE_FLOW_ACTION_TYPE_END, 5051 }; 5052 } else { 5053 ext_actions[encap_idx] = (struct rte_flow_action){ 5054 .type = (enum rte_flow_action_type) 5055 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5056 .conf = cp_mreg, 5057 }; 5058 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5059 sizeof(*ext_actions) * (actions_n - encap_idx)); 5060 } 5061 return 0; 5062 } 5063 5064 /** 5065 * Check the match action from the action list. 5066 * 5067 * @param[in] actions 5068 * Pointer to the list of actions. 5069 * @param[in] attr 5070 * Flow rule attributes. 5071 * @param[in] action 5072 * The action to be check if exist. 5073 * @param[out] match_action_pos 5074 * Pointer to the position of the matched action if exists, otherwise is -1. 5075 * @param[out] qrss_action_pos 5076 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5077 * @param[out] modify_after_mirror 5078 * Pointer to the flag of modify action after FDB mirroring. 5079 * 5080 * @return 5081 * > 0 the total number of actions. 5082 * 0 if not found match action in action list. 5083 */ 5084 static int 5085 flow_check_match_action(const struct rte_flow_action actions[], 5086 const struct rte_flow_attr *attr, 5087 enum rte_flow_action_type action, 5088 int *match_action_pos, int *qrss_action_pos, 5089 int *modify_after_mirror) 5090 { 5091 const struct rte_flow_action_sample *sample; 5092 int actions_n = 0; 5093 uint32_t ratio = 0; 5094 int sub_type = 0; 5095 int flag = 0; 5096 int fdb_mirror = 0; 5097 5098 *match_action_pos = -1; 5099 *qrss_action_pos = -1; 5100 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5101 if (actions->type == action) { 5102 flag = 1; 5103 *match_action_pos = actions_n; 5104 } 5105 switch (actions->type) { 5106 case RTE_FLOW_ACTION_TYPE_QUEUE: 5107 case RTE_FLOW_ACTION_TYPE_RSS: 5108 *qrss_action_pos = actions_n; 5109 break; 5110 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5111 sample = actions->conf; 5112 ratio = sample->ratio; 5113 sub_type = ((const struct rte_flow_action *) 5114 (sample->actions))->type; 5115 if (ratio == 1 && attr->transfer) 5116 fdb_mirror = 1; 5117 break; 5118 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5119 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5120 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5121 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5122 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5123 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5124 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5125 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5126 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5127 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5128 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5129 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5130 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5131 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5132 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5133 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5134 case RTE_FLOW_ACTION_TYPE_FLAG: 5135 case RTE_FLOW_ACTION_TYPE_MARK: 5136 case RTE_FLOW_ACTION_TYPE_SET_META: 5137 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5138 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5139 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5140 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5141 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5142 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5143 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5144 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5145 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5146 case RTE_FLOW_ACTION_TYPE_METER: 5147 if (fdb_mirror) 5148 *modify_after_mirror = 1; 5149 break; 5150 default: 5151 break; 5152 } 5153 actions_n++; 5154 } 5155 if (flag && fdb_mirror && !*modify_after_mirror) { 5156 /* FDB mirroring uses the destination array to implement 5157 * instead of FLOW_SAMPLER object. 5158 */ 5159 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5160 flag = 0; 5161 } 5162 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5163 return flag ? actions_n + 1 : 0; 5164 } 5165 5166 #define SAMPLE_SUFFIX_ITEM 2 5167 5168 /** 5169 * Split the sample flow. 5170 * 5171 * As sample flow will split to two sub flow, sample flow with 5172 * sample action, the other actions will move to new suffix flow. 5173 * 5174 * Also add unique tag id with tag action in the sample flow, 5175 * the same tag id will be as match in the suffix flow. 5176 * 5177 * @param dev 5178 * Pointer to Ethernet device. 5179 * @param[in] add_tag 5180 * Add extra tag action flag. 5181 * @param[out] sfx_items 5182 * Suffix flow match items (list terminated by the END pattern item). 5183 * @param[in] actions 5184 * Associated actions (list terminated by the END action). 5185 * @param[out] actions_sfx 5186 * Suffix flow actions. 5187 * @param[out] actions_pre 5188 * Prefix flow actions. 5189 * @param[in] actions_n 5190 * The total number of actions. 5191 * @param[in] sample_action_pos 5192 * The sample action position. 5193 * @param[in] qrss_action_pos 5194 * The Queue/RSS action position. 5195 * @param[in] jump_table 5196 * Add extra jump action flag. 5197 * @param[out] error 5198 * Perform verbose error reporting if not NULL. 5199 * 5200 * @return 5201 * 0 on success, or unique flow_id, a negative errno value 5202 * otherwise and rte_errno is set. 5203 */ 5204 static int 5205 flow_sample_split_prep(struct rte_eth_dev *dev, 5206 int add_tag, 5207 struct rte_flow_item sfx_items[], 5208 const struct rte_flow_action actions[], 5209 struct rte_flow_action actions_sfx[], 5210 struct rte_flow_action actions_pre[], 5211 int actions_n, 5212 int sample_action_pos, 5213 int qrss_action_pos, 5214 int jump_table, 5215 struct rte_flow_error *error) 5216 { 5217 struct mlx5_priv *priv = dev->data->dev_private; 5218 struct mlx5_rte_flow_action_set_tag *set_tag; 5219 struct mlx5_rte_flow_item_tag *tag_spec; 5220 struct mlx5_rte_flow_item_tag *tag_mask; 5221 struct rte_flow_action_jump *jump_action; 5222 uint32_t tag_id = 0; 5223 int index; 5224 int append_index = 0; 5225 int ret; 5226 5227 if (sample_action_pos < 0) 5228 return rte_flow_error_set(error, EINVAL, 5229 RTE_FLOW_ERROR_TYPE_ACTION, 5230 NULL, "invalid position of sample " 5231 "action in list"); 5232 /* Prepare the actions for prefix and suffix flow. */ 5233 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5234 index = qrss_action_pos; 5235 /* Put the preceding the Queue/RSS action into prefix flow. */ 5236 if (index != 0) 5237 memcpy(actions_pre, actions, 5238 sizeof(struct rte_flow_action) * index); 5239 /* Put others preceding the sample action into prefix flow. */ 5240 if (sample_action_pos > index + 1) 5241 memcpy(actions_pre + index, actions + index + 1, 5242 sizeof(struct rte_flow_action) * 5243 (sample_action_pos - index - 1)); 5244 index = sample_action_pos - 1; 5245 /* Put Queue/RSS action into Suffix flow. */ 5246 memcpy(actions_sfx, actions + qrss_action_pos, 5247 sizeof(struct rte_flow_action)); 5248 actions_sfx++; 5249 } else { 5250 index = sample_action_pos; 5251 if (index != 0) 5252 memcpy(actions_pre, actions, 5253 sizeof(struct rte_flow_action) * index); 5254 } 5255 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5256 * For CX6DX and above, metadata registers Cx preserve their value, 5257 * add an extra tag action for NIC-RX and E-Switch Domain. 5258 */ 5259 if (add_tag) { 5260 /* Prepare the prefix tag action. */ 5261 append_index++; 5262 set_tag = (void *)(actions_pre + actions_n + append_index); 5263 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5264 if (ret < 0) 5265 return ret; 5266 mlx5_ipool_malloc(priv->sh->ipool 5267 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5268 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5269 .id = ret, 5270 .data = tag_id, 5271 }; 5272 /* Prepare the suffix subflow items. */ 5273 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5274 tag_spec->data = tag_id; 5275 tag_spec->id = set_tag->id; 5276 tag_mask = tag_spec + 1; 5277 tag_mask->data = UINT32_MAX; 5278 sfx_items[0] = (struct rte_flow_item){ 5279 .type = (enum rte_flow_item_type) 5280 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5281 .spec = tag_spec, 5282 .last = NULL, 5283 .mask = tag_mask, 5284 }; 5285 sfx_items[1] = (struct rte_flow_item){ 5286 .type = (enum rte_flow_item_type) 5287 RTE_FLOW_ITEM_TYPE_END, 5288 }; 5289 /* Prepare the tag action in prefix subflow. */ 5290 actions_pre[index++] = 5291 (struct rte_flow_action){ 5292 .type = (enum rte_flow_action_type) 5293 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5294 .conf = set_tag, 5295 }; 5296 } 5297 memcpy(actions_pre + index, actions + sample_action_pos, 5298 sizeof(struct rte_flow_action)); 5299 index += 1; 5300 /* For the modify action after the sample action in E-Switch mirroring, 5301 * Add the extra jump action in prefix subflow and jump into the next 5302 * table, then do the modify action in the new table. 5303 */ 5304 if (jump_table) { 5305 /* Prepare the prefix jump action. */ 5306 append_index++; 5307 jump_action = (void *)(actions_pre + actions_n + append_index); 5308 jump_action->group = jump_table; 5309 actions_pre[index++] = 5310 (struct rte_flow_action){ 5311 .type = (enum rte_flow_action_type) 5312 RTE_FLOW_ACTION_TYPE_JUMP, 5313 .conf = jump_action, 5314 }; 5315 } 5316 actions_pre[index] = (struct rte_flow_action){ 5317 .type = (enum rte_flow_action_type) 5318 RTE_FLOW_ACTION_TYPE_END, 5319 }; 5320 /* Put the actions after sample into Suffix flow. */ 5321 memcpy(actions_sfx, actions + sample_action_pos + 1, 5322 sizeof(struct rte_flow_action) * 5323 (actions_n - sample_action_pos - 1)); 5324 return tag_id; 5325 } 5326 5327 /** 5328 * The splitting for metadata feature. 5329 * 5330 * - Q/RSS action on NIC Rx should be split in order to pass by 5331 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5332 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5333 * 5334 * - All the actions on NIC Tx should have a mreg copy action to 5335 * copy reg_a from WQE to reg_c[0]. 5336 * 5337 * @param dev 5338 * Pointer to Ethernet device. 5339 * @param[in] flow 5340 * Parent flow structure pointer. 5341 * @param[in] attr 5342 * Flow rule attributes. 5343 * @param[in] items 5344 * Pattern specification (list terminated by the END pattern item). 5345 * @param[in] actions 5346 * Associated actions (list terminated by the END action). 5347 * @param[in] flow_split_info 5348 * Pointer to flow split info structure. 5349 * @param[out] error 5350 * Perform verbose error reporting if not NULL. 5351 * @return 5352 * 0 on success, negative value otherwise 5353 */ 5354 static int 5355 flow_create_split_metadata(struct rte_eth_dev *dev, 5356 struct rte_flow *flow, 5357 const struct rte_flow_attr *attr, 5358 const struct rte_flow_item items[], 5359 const struct rte_flow_action actions[], 5360 struct mlx5_flow_split_info *flow_split_info, 5361 struct rte_flow_error *error) 5362 { 5363 struct mlx5_priv *priv = dev->data->dev_private; 5364 struct mlx5_dev_config *config = &priv->config; 5365 const struct rte_flow_action *qrss = NULL; 5366 struct rte_flow_action *ext_actions = NULL; 5367 struct mlx5_flow *dev_flow = NULL; 5368 uint32_t qrss_id = 0; 5369 int mtr_sfx = 0; 5370 size_t act_size; 5371 int actions_n; 5372 int encap_idx; 5373 int ret; 5374 5375 /* Check whether extensive metadata feature is engaged. */ 5376 if (!config->dv_flow_en || 5377 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5378 !mlx5_flow_ext_mreg_supported(dev)) 5379 return flow_create_split_inner(dev, flow, NULL, attr, items, 5380 actions, flow_split_info, error); 5381 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5382 &encap_idx); 5383 if (qrss) { 5384 /* Exclude hairpin flows from splitting. */ 5385 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5386 const struct rte_flow_action_queue *queue; 5387 5388 queue = qrss->conf; 5389 if (mlx5_rxq_get_type(dev, queue->index) == 5390 MLX5_RXQ_TYPE_HAIRPIN) 5391 qrss = NULL; 5392 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5393 const struct rte_flow_action_rss *rss; 5394 5395 rss = qrss->conf; 5396 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5397 MLX5_RXQ_TYPE_HAIRPIN) 5398 qrss = NULL; 5399 } 5400 } 5401 if (qrss) { 5402 /* Check if it is in meter suffix table. */ 5403 mtr_sfx = attr->group == (attr->transfer ? 5404 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5405 MLX5_FLOW_TABLE_LEVEL_METER); 5406 /* 5407 * Q/RSS action on NIC Rx should be split in order to pass by 5408 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5409 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5410 */ 5411 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5412 sizeof(struct rte_flow_action_set_tag) + 5413 sizeof(struct rte_flow_action_jump); 5414 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5415 SOCKET_ID_ANY); 5416 if (!ext_actions) 5417 return rte_flow_error_set(error, ENOMEM, 5418 RTE_FLOW_ERROR_TYPE_ACTION, 5419 NULL, "no memory to split " 5420 "metadata flow"); 5421 /* 5422 * If we are the suffix flow of meter, tag already exist. 5423 * Set the tag action to void. 5424 */ 5425 if (mtr_sfx) 5426 ext_actions[qrss - actions].type = 5427 RTE_FLOW_ACTION_TYPE_VOID; 5428 else 5429 ext_actions[qrss - actions].type = 5430 (enum rte_flow_action_type) 5431 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5432 /* 5433 * Create the new actions list with removed Q/RSS action 5434 * and appended set tag and jump to register copy table 5435 * (RX_CP_TBL). We should preallocate unique tag ID here 5436 * in advance, because it is needed for set tag action. 5437 */ 5438 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5439 qrss, actions_n, error); 5440 if (!mtr_sfx && !qrss_id) { 5441 ret = -rte_errno; 5442 goto exit; 5443 } 5444 } else if (attr->egress && !attr->transfer) { 5445 /* 5446 * All the actions on NIC Tx should have a metadata register 5447 * copy action to copy reg_a from WQE to reg_c[meta] 5448 */ 5449 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5450 sizeof(struct mlx5_flow_action_copy_mreg); 5451 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5452 SOCKET_ID_ANY); 5453 if (!ext_actions) 5454 return rte_flow_error_set(error, ENOMEM, 5455 RTE_FLOW_ERROR_TYPE_ACTION, 5456 NULL, "no memory to split " 5457 "metadata flow"); 5458 /* Create the action list appended with copy register. */ 5459 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5460 actions_n, error, encap_idx); 5461 if (ret < 0) 5462 goto exit; 5463 } 5464 /* Add the unmodified original or prefix subflow. */ 5465 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5466 items, ext_actions ? ext_actions : 5467 actions, flow_split_info, error); 5468 if (ret < 0) 5469 goto exit; 5470 MLX5_ASSERT(dev_flow); 5471 if (qrss) { 5472 const struct rte_flow_attr q_attr = { 5473 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5474 .ingress = 1, 5475 }; 5476 /* Internal PMD action to set register. */ 5477 struct mlx5_rte_flow_item_tag q_tag_spec = { 5478 .data = qrss_id, 5479 .id = REG_NON, 5480 }; 5481 struct rte_flow_item q_items[] = { 5482 { 5483 .type = (enum rte_flow_item_type) 5484 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5485 .spec = &q_tag_spec, 5486 .last = NULL, 5487 .mask = NULL, 5488 }, 5489 { 5490 .type = RTE_FLOW_ITEM_TYPE_END, 5491 }, 5492 }; 5493 struct rte_flow_action q_actions[] = { 5494 { 5495 .type = qrss->type, 5496 .conf = qrss->conf, 5497 }, 5498 { 5499 .type = RTE_FLOW_ACTION_TYPE_END, 5500 }, 5501 }; 5502 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5503 5504 /* 5505 * Configure the tag item only if there is no meter subflow. 5506 * Since tag is already marked in the meter suffix subflow 5507 * we can just use the meter suffix items as is. 5508 */ 5509 if (qrss_id) { 5510 /* Not meter subflow. */ 5511 MLX5_ASSERT(!mtr_sfx); 5512 /* 5513 * Put unique id in prefix flow due to it is destroyed 5514 * after suffix flow and id will be freed after there 5515 * is no actual flows with this id and identifier 5516 * reallocation becomes possible (for example, for 5517 * other flows in other threads). 5518 */ 5519 dev_flow->handle->split_flow_id = qrss_id; 5520 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5521 error); 5522 if (ret < 0) 5523 goto exit; 5524 q_tag_spec.id = ret; 5525 } 5526 dev_flow = NULL; 5527 /* Add suffix subflow to execute Q/RSS. */ 5528 flow_split_info->prefix_layers = layers; 5529 flow_split_info->prefix_mark = 0; 5530 ret = flow_create_split_inner(dev, flow, &dev_flow, 5531 &q_attr, mtr_sfx ? items : 5532 q_items, q_actions, 5533 flow_split_info, error); 5534 if (ret < 0) 5535 goto exit; 5536 /* qrss ID should be freed if failed. */ 5537 qrss_id = 0; 5538 MLX5_ASSERT(dev_flow); 5539 } 5540 5541 exit: 5542 /* 5543 * We do not destroy the partially created sub_flows in case of error. 5544 * These ones are included into parent flow list and will be destroyed 5545 * by flow_drv_destroy. 5546 */ 5547 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5548 qrss_id); 5549 mlx5_free(ext_actions); 5550 return ret; 5551 } 5552 5553 /** 5554 * Create meter internal drop flow with the original pattern. 5555 * 5556 * @param dev 5557 * Pointer to Ethernet device. 5558 * @param[in] flow 5559 * Parent flow structure pointer. 5560 * @param[in] attr 5561 * Flow rule attributes. 5562 * @param[in] items 5563 * Pattern specification (list terminated by the END pattern item). 5564 * @param[in] flow_split_info 5565 * Pointer to flow split info structure. 5566 * @param[in] fm 5567 * Pointer to flow meter structure. 5568 * @param[out] error 5569 * Perform verbose error reporting if not NULL. 5570 * @return 5571 * 0 on success, negative value otherwise 5572 */ 5573 static uint32_t 5574 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5575 struct rte_flow *flow, 5576 const struct rte_flow_attr *attr, 5577 const struct rte_flow_item items[], 5578 struct mlx5_flow_split_info *flow_split_info, 5579 struct mlx5_flow_meter_info *fm, 5580 struct rte_flow_error *error) 5581 { 5582 struct mlx5_flow *dev_flow = NULL; 5583 struct rte_flow_attr drop_attr = *attr; 5584 struct rte_flow_action drop_actions[3]; 5585 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5586 5587 MLX5_ASSERT(fm->drop_cnt); 5588 drop_actions[0].type = 5589 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5590 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5591 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5592 drop_actions[1].conf = NULL; 5593 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5594 drop_actions[2].conf = NULL; 5595 drop_split_info.external = false; 5596 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5597 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5598 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5599 return flow_create_split_inner(dev, flow, &dev_flow, 5600 &drop_attr, items, drop_actions, 5601 &drop_split_info, error); 5602 } 5603 5604 /** 5605 * The splitting for meter feature. 5606 * 5607 * - The meter flow will be split to two flows as prefix and 5608 * suffix flow. The packets make sense only it pass the prefix 5609 * meter action. 5610 * 5611 * - Reg_C_5 is used for the packet to match betweend prefix and 5612 * suffix flow. 5613 * 5614 * @param dev 5615 * Pointer to Ethernet device. 5616 * @param[in] flow 5617 * Parent flow structure pointer. 5618 * @param[in] attr 5619 * Flow rule attributes. 5620 * @param[in] items 5621 * Pattern specification (list terminated by the END pattern item). 5622 * @param[in] actions 5623 * Associated actions (list terminated by the END action). 5624 * @param[in] flow_split_info 5625 * Pointer to flow split info structure. 5626 * @param[out] error 5627 * Perform verbose error reporting if not NULL. 5628 * @return 5629 * 0 on success, negative value otherwise 5630 */ 5631 static int 5632 flow_create_split_meter(struct rte_eth_dev *dev, 5633 struct rte_flow *flow, 5634 const struct rte_flow_attr *attr, 5635 const struct rte_flow_item items[], 5636 const struct rte_flow_action actions[], 5637 struct mlx5_flow_split_info *flow_split_info, 5638 struct rte_flow_error *error) 5639 { 5640 struct mlx5_priv *priv = dev->data->dev_private; 5641 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5642 struct rte_flow_action *sfx_actions = NULL; 5643 struct rte_flow_action *pre_actions = NULL; 5644 struct rte_flow_item *sfx_items = NULL; 5645 struct mlx5_flow *dev_flow = NULL; 5646 struct rte_flow_attr sfx_attr = *attr; 5647 struct mlx5_flow_meter_info *fm = NULL; 5648 uint8_t skip_scale_restore; 5649 bool has_mtr = false; 5650 bool has_modify = false; 5651 bool set_mtr_reg = true; 5652 uint32_t meter_id = 0; 5653 uint32_t mtr_idx = 0; 5654 uint32_t mtr_flow_id = 0; 5655 size_t act_size; 5656 size_t item_size; 5657 int actions_n = 0; 5658 int ret = 0; 5659 5660 if (priv->mtr_en) 5661 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 5662 &has_modify, &meter_id); 5663 if (has_mtr) { 5664 if (flow->meter) { 5665 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 5666 if (!fm) 5667 return rte_flow_error_set(error, EINVAL, 5668 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5669 NULL, "Meter not found."); 5670 } else { 5671 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 5672 if (!fm) 5673 return rte_flow_error_set(error, EINVAL, 5674 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5675 NULL, "Meter not found."); 5676 ret = mlx5_flow_meter_attach(priv, fm, 5677 &sfx_attr, error); 5678 if (ret) 5679 return -rte_errno; 5680 flow->meter = mtr_idx; 5681 } 5682 MLX5_ASSERT(wks); 5683 wks->fm = fm; 5684 /* 5685 * If it isn't default-policy Meter, and 5686 * 1. There's no action in flow to change 5687 * packet (modify/encap/decap etc.), OR 5688 * 2. No drop count needed for this meter. 5689 * no need to use regC to save meter id anymore. 5690 */ 5691 if (!fm->def_policy && (!has_modify || !fm->drop_cnt)) 5692 set_mtr_reg = false; 5693 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 5694 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 5695 sizeof(struct mlx5_rte_flow_action_set_tag); 5696 /* Suffix items: tag, vlan, port id, end. */ 5697 #define METER_SUFFIX_ITEM 4 5698 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5699 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5700 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5701 0, SOCKET_ID_ANY); 5702 if (!sfx_actions) 5703 return rte_flow_error_set(error, ENOMEM, 5704 RTE_FLOW_ERROR_TYPE_ACTION, 5705 NULL, "no memory to split " 5706 "meter flow"); 5707 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5708 act_size); 5709 /* There's no suffix flow for meter of non-default policy. */ 5710 if (!fm->def_policy) 5711 pre_actions = sfx_actions + 1; 5712 else 5713 pre_actions = sfx_actions + actions_n; 5714 ret = flow_meter_split_prep(dev, flow, fm, &sfx_attr, 5715 items, sfx_items, actions, 5716 sfx_actions, pre_actions, 5717 (set_mtr_reg ? &mtr_flow_id : NULL), 5718 error); 5719 if (ret) { 5720 ret = -rte_errno; 5721 goto exit; 5722 } 5723 /* Add the prefix subflow. */ 5724 flow_split_info->prefix_mark = 0; 5725 skip_scale_restore = flow_split_info->skip_scale; 5726 flow_split_info->skip_scale |= 5727 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5728 ret = flow_create_split_inner(dev, flow, &dev_flow, 5729 attr, items, pre_actions, 5730 flow_split_info, error); 5731 flow_split_info->skip_scale = skip_scale_restore; 5732 if (ret) { 5733 if (mtr_flow_id) 5734 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 5735 ret = -rte_errno; 5736 goto exit; 5737 } 5738 if (mtr_flow_id) { 5739 dev_flow->handle->split_flow_id = mtr_flow_id; 5740 dev_flow->handle->is_meter_flow_id = 1; 5741 } 5742 if (!fm->def_policy) { 5743 if (!set_mtr_reg && fm->drop_cnt) 5744 ret = 5745 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 5746 &sfx_attr, items, 5747 flow_split_info, 5748 fm, error); 5749 goto exit; 5750 } 5751 /* Setting the sfx group atrr. */ 5752 sfx_attr.group = sfx_attr.transfer ? 5753 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5754 MLX5_FLOW_TABLE_LEVEL_METER; 5755 flow_split_info->prefix_layers = 5756 flow_get_prefix_layer_flags(dev_flow); 5757 flow_split_info->prefix_mark = dev_flow->handle->mark; 5758 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 5759 } 5760 /* Add the prefix subflow. */ 5761 ret = flow_create_split_metadata(dev, flow, 5762 &sfx_attr, sfx_items ? 5763 sfx_items : items, 5764 sfx_actions ? sfx_actions : actions, 5765 flow_split_info, error); 5766 exit: 5767 if (sfx_actions) 5768 mlx5_free(sfx_actions); 5769 return ret; 5770 } 5771 5772 /** 5773 * The splitting for sample feature. 5774 * 5775 * Once Sample action is detected in the action list, the flow actions should 5776 * be split into prefix sub flow and suffix sub flow. 5777 * 5778 * The original items remain in the prefix sub flow, all actions preceding the 5779 * sample action and the sample action itself will be copied to the prefix 5780 * sub flow, the actions following the sample action will be copied to the 5781 * suffix sub flow, Queue action always be located in the suffix sub flow. 5782 * 5783 * In order to make the packet from prefix sub flow matches with suffix sub 5784 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5785 * flow uses tag item with the unique flow id. 5786 * 5787 * @param dev 5788 * Pointer to Ethernet device. 5789 * @param[in] flow 5790 * Parent flow structure pointer. 5791 * @param[in] attr 5792 * Flow rule attributes. 5793 * @param[in] items 5794 * Pattern specification (list terminated by the END pattern item). 5795 * @param[in] actions 5796 * Associated actions (list terminated by the END action). 5797 * @param[in] flow_split_info 5798 * Pointer to flow split info structure. 5799 * @param[out] error 5800 * Perform verbose error reporting if not NULL. 5801 * @return 5802 * 0 on success, negative value otherwise 5803 */ 5804 static int 5805 flow_create_split_sample(struct rte_eth_dev *dev, 5806 struct rte_flow *flow, 5807 const struct rte_flow_attr *attr, 5808 const struct rte_flow_item items[], 5809 const struct rte_flow_action actions[], 5810 struct mlx5_flow_split_info *flow_split_info, 5811 struct rte_flow_error *error) 5812 { 5813 struct mlx5_priv *priv = dev->data->dev_private; 5814 struct rte_flow_action *sfx_actions = NULL; 5815 struct rte_flow_action *pre_actions = NULL; 5816 struct rte_flow_item *sfx_items = NULL; 5817 struct mlx5_flow *dev_flow = NULL; 5818 struct rte_flow_attr sfx_attr = *attr; 5819 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5820 struct mlx5_flow_dv_sample_resource *sample_res; 5821 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5822 struct mlx5_flow_tbl_resource *sfx_tbl; 5823 #endif 5824 size_t act_size; 5825 size_t item_size; 5826 uint32_t fdb_tx = 0; 5827 int32_t tag_id = 0; 5828 int actions_n = 0; 5829 int sample_action_pos; 5830 int qrss_action_pos; 5831 int add_tag = 0; 5832 int modify_after_mirror = 0; 5833 uint16_t jump_table = 0; 5834 const uint32_t next_ft_step = 1; 5835 int ret = 0; 5836 5837 if (priv->sampler_en) 5838 actions_n = flow_check_match_action(actions, attr, 5839 RTE_FLOW_ACTION_TYPE_SAMPLE, 5840 &sample_action_pos, &qrss_action_pos, 5841 &modify_after_mirror); 5842 if (actions_n) { 5843 /* The prefix actions must includes sample, tag, end. */ 5844 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5845 + sizeof(struct mlx5_rte_flow_action_set_tag); 5846 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5847 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5848 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5849 item_size), 0, SOCKET_ID_ANY); 5850 if (!sfx_actions) 5851 return rte_flow_error_set(error, ENOMEM, 5852 RTE_FLOW_ERROR_TYPE_ACTION, 5853 NULL, "no memory to split " 5854 "sample flow"); 5855 /* The representor_id is -1 for uplink. */ 5856 fdb_tx = (attr->transfer && priv->representor_id != -1); 5857 /* 5858 * When reg_c_preserve is set, metadata registers Cx preserve 5859 * their value even through packet duplication. 5860 */ 5861 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 5862 if (add_tag) 5863 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5864 + act_size); 5865 if (modify_after_mirror) 5866 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 5867 next_ft_step; 5868 pre_actions = sfx_actions + actions_n; 5869 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 5870 actions, sfx_actions, 5871 pre_actions, actions_n, 5872 sample_action_pos, 5873 qrss_action_pos, jump_table, 5874 error); 5875 if (tag_id < 0 || (add_tag && !tag_id)) { 5876 ret = -rte_errno; 5877 goto exit; 5878 } 5879 if (modify_after_mirror) 5880 flow_split_info->skip_scale = 5881 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5882 /* Add the prefix subflow. */ 5883 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5884 items, pre_actions, 5885 flow_split_info, error); 5886 if (ret) { 5887 ret = -rte_errno; 5888 goto exit; 5889 } 5890 dev_flow->handle->split_flow_id = tag_id; 5891 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5892 if (!modify_after_mirror) { 5893 /* Set the sfx group attr. */ 5894 sample_res = (struct mlx5_flow_dv_sample_resource *) 5895 dev_flow->dv.sample_res; 5896 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5897 sample_res->normal_path_tbl; 5898 sfx_tbl_data = container_of(sfx_tbl, 5899 struct mlx5_flow_tbl_data_entry, 5900 tbl); 5901 sfx_attr.group = sfx_attr.transfer ? 5902 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 5903 } else { 5904 MLX5_ASSERT(attr->transfer); 5905 sfx_attr.group = jump_table; 5906 } 5907 flow_split_info->prefix_layers = 5908 flow_get_prefix_layer_flags(dev_flow); 5909 flow_split_info->prefix_mark = dev_flow->handle->mark; 5910 /* Suffix group level already be scaled with factor, set 5911 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 5912 * again in translation. 5913 */ 5914 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5915 #endif 5916 } 5917 /* Add the suffix subflow. */ 5918 ret = flow_create_split_meter(dev, flow, &sfx_attr, 5919 sfx_items ? sfx_items : items, 5920 sfx_actions ? sfx_actions : actions, 5921 flow_split_info, error); 5922 exit: 5923 if (sfx_actions) 5924 mlx5_free(sfx_actions); 5925 return ret; 5926 } 5927 5928 /** 5929 * Split the flow to subflow set. The splitters might be linked 5930 * in the chain, like this: 5931 * flow_create_split_outer() calls: 5932 * flow_create_split_meter() calls: 5933 * flow_create_split_metadata(meter_subflow_0) calls: 5934 * flow_create_split_inner(metadata_subflow_0) 5935 * flow_create_split_inner(metadata_subflow_1) 5936 * flow_create_split_inner(metadata_subflow_2) 5937 * flow_create_split_metadata(meter_subflow_1) calls: 5938 * flow_create_split_inner(metadata_subflow_0) 5939 * flow_create_split_inner(metadata_subflow_1) 5940 * flow_create_split_inner(metadata_subflow_2) 5941 * 5942 * This provide flexible way to add new levels of flow splitting. 5943 * The all of successfully created subflows are included to the 5944 * parent flow dev_flow list. 5945 * 5946 * @param dev 5947 * Pointer to Ethernet device. 5948 * @param[in] flow 5949 * Parent flow structure pointer. 5950 * @param[in] attr 5951 * Flow rule attributes. 5952 * @param[in] items 5953 * Pattern specification (list terminated by the END pattern item). 5954 * @param[in] actions 5955 * Associated actions (list terminated by the END action). 5956 * @param[in] flow_split_info 5957 * Pointer to flow split info structure. 5958 * @param[out] error 5959 * Perform verbose error reporting if not NULL. 5960 * @return 5961 * 0 on success, negative value otherwise 5962 */ 5963 static int 5964 flow_create_split_outer(struct rte_eth_dev *dev, 5965 struct rte_flow *flow, 5966 const struct rte_flow_attr *attr, 5967 const struct rte_flow_item items[], 5968 const struct rte_flow_action actions[], 5969 struct mlx5_flow_split_info *flow_split_info, 5970 struct rte_flow_error *error) 5971 { 5972 int ret; 5973 5974 ret = flow_create_split_sample(dev, flow, attr, items, 5975 actions, flow_split_info, error); 5976 MLX5_ASSERT(ret <= 0); 5977 return ret; 5978 } 5979 5980 static inline struct mlx5_flow_tunnel * 5981 flow_tunnel_from_rule(const struct mlx5_flow *flow) 5982 { 5983 struct mlx5_flow_tunnel *tunnel; 5984 5985 #pragma GCC diagnostic push 5986 #pragma GCC diagnostic ignored "-Wcast-qual" 5987 tunnel = (typeof(tunnel))flow->tunnel; 5988 #pragma GCC diagnostic pop 5989 5990 return tunnel; 5991 } 5992 5993 /** 5994 * Adjust flow RSS workspace if needed. 5995 * 5996 * @param wks 5997 * Pointer to thread flow work space. 5998 * @param rss_desc 5999 * Pointer to RSS descriptor. 6000 * @param[in] nrssq_num 6001 * New RSS queue number. 6002 * 6003 * @return 6004 * 0 on success, -1 otherwise and rte_errno is set. 6005 */ 6006 static int 6007 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6008 struct mlx5_flow_rss_desc *rss_desc, 6009 uint32_t nrssq_num) 6010 { 6011 if (likely(nrssq_num <= wks->rssq_num)) 6012 return 0; 6013 rss_desc->queue = realloc(rss_desc->queue, 6014 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6015 if (!rss_desc->queue) { 6016 rte_errno = ENOMEM; 6017 return -1; 6018 } 6019 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6020 return 0; 6021 } 6022 6023 /** 6024 * Create a flow and add it to @p list. 6025 * 6026 * @param dev 6027 * Pointer to Ethernet device. 6028 * @param list 6029 * Pointer to a TAILQ flow list. If this parameter NULL, 6030 * no list insertion occurred, flow is just created, 6031 * this is caller's responsibility to track the 6032 * created flow. 6033 * @param[in] attr 6034 * Flow rule attributes. 6035 * @param[in] items 6036 * Pattern specification (list terminated by the END pattern item). 6037 * @param[in] actions 6038 * Associated actions (list terminated by the END action). 6039 * @param[in] external 6040 * This flow rule is created by request external to PMD. 6041 * @param[out] error 6042 * Perform verbose error reporting if not NULL. 6043 * 6044 * @return 6045 * A flow index on success, 0 otherwise and rte_errno is set. 6046 */ 6047 static uint32_t 6048 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 6049 const struct rte_flow_attr *attr, 6050 const struct rte_flow_item items[], 6051 const struct rte_flow_action original_actions[], 6052 bool external, struct rte_flow_error *error) 6053 { 6054 struct mlx5_priv *priv = dev->data->dev_private; 6055 struct rte_flow *flow = NULL; 6056 struct mlx5_flow *dev_flow; 6057 const struct rte_flow_action_rss *rss = NULL; 6058 struct mlx5_translated_action_handle 6059 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6060 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6061 union { 6062 struct mlx5_flow_expand_rss buf; 6063 uint8_t buffer[2048]; 6064 } expand_buffer; 6065 union { 6066 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6067 uint8_t buffer[2048]; 6068 } actions_rx; 6069 union { 6070 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6071 uint8_t buffer[2048]; 6072 } actions_hairpin_tx; 6073 union { 6074 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6075 uint8_t buffer[2048]; 6076 } items_tx; 6077 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6078 struct mlx5_flow_rss_desc *rss_desc; 6079 const struct rte_flow_action *p_actions_rx; 6080 uint32_t i; 6081 uint32_t idx = 0; 6082 int hairpin_flow; 6083 struct rte_flow_attr attr_tx = { .priority = 0 }; 6084 const struct rte_flow_action *actions; 6085 struct rte_flow_action *translated_actions = NULL; 6086 struct mlx5_flow_tunnel *tunnel; 6087 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6088 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6089 struct mlx5_flow_split_info flow_split_info = { 6090 .external = !!external, 6091 .skip_scale = 0, 6092 .flow_idx = 0, 6093 .prefix_mark = 0, 6094 .prefix_layers = 0, 6095 .table_id = 0 6096 }; 6097 int ret; 6098 6099 MLX5_ASSERT(wks); 6100 rss_desc = &wks->rss_desc; 6101 ret = flow_action_handles_translate(dev, original_actions, 6102 indir_actions, 6103 &indir_actions_n, 6104 &translated_actions, error); 6105 if (ret < 0) { 6106 MLX5_ASSERT(translated_actions == NULL); 6107 return 0; 6108 } 6109 actions = translated_actions ? translated_actions : original_actions; 6110 p_actions_rx = actions; 6111 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6112 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6113 external, hairpin_flow, error); 6114 if (ret < 0) 6115 goto error_before_hairpin_split; 6116 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 6117 if (!flow) { 6118 rte_errno = ENOMEM; 6119 goto error_before_hairpin_split; 6120 } 6121 if (hairpin_flow > 0) { 6122 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6123 rte_errno = EINVAL; 6124 goto error_before_hairpin_split; 6125 } 6126 flow_hairpin_split(dev, actions, actions_rx.actions, 6127 actions_hairpin_tx.actions, items_tx.items, 6128 idx); 6129 p_actions_rx = actions_rx.actions; 6130 } 6131 flow_split_info.flow_idx = idx; 6132 flow->drv_type = flow_get_drv_type(dev, attr); 6133 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6134 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6135 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6136 /* RSS Action only works on NIC RX domain */ 6137 if (attr->ingress && !attr->transfer) 6138 rss = flow_get_rss_action(dev, p_actions_rx); 6139 if (rss) { 6140 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6141 return 0; 6142 /* 6143 * The following information is required by 6144 * mlx5_flow_hashfields_adjust() in advance. 6145 */ 6146 rss_desc->level = rss->level; 6147 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 6148 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 6149 } 6150 flow->dev_handles = 0; 6151 if (rss && rss->types) { 6152 unsigned int graph_root; 6153 6154 graph_root = find_graph_root(items, rss->level); 6155 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6156 items, rss->types, 6157 mlx5_support_expansion, graph_root); 6158 MLX5_ASSERT(ret > 0 && 6159 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6160 } else { 6161 buf->entries = 1; 6162 buf->entry[0].pattern = (void *)(uintptr_t)items; 6163 } 6164 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6165 indir_actions_n); 6166 for (i = 0; i < buf->entries; ++i) { 6167 /* Initialize flow split data. */ 6168 flow_split_info.prefix_layers = 0; 6169 flow_split_info.prefix_mark = 0; 6170 flow_split_info.skip_scale = 0; 6171 /* 6172 * The splitter may create multiple dev_flows, 6173 * depending on configuration. In the simplest 6174 * case it just creates unmodified original flow. 6175 */ 6176 ret = flow_create_split_outer(dev, flow, attr, 6177 buf->entry[i].pattern, 6178 p_actions_rx, &flow_split_info, 6179 error); 6180 if (ret < 0) 6181 goto error; 6182 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6183 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6184 p_actions_rx, 6185 idx, 6186 wks->flows[0].tunnel, 6187 &default_miss_ctx, 6188 error); 6189 if (ret < 0) { 6190 mlx5_free(default_miss_ctx.queue); 6191 goto error; 6192 } 6193 } 6194 } 6195 /* Create the tx flow. */ 6196 if (hairpin_flow) { 6197 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6198 attr_tx.ingress = 0; 6199 attr_tx.egress = 1; 6200 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6201 actions_hairpin_tx.actions, 6202 idx, error); 6203 if (!dev_flow) 6204 goto error; 6205 dev_flow->flow = flow; 6206 dev_flow->external = 0; 6207 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6208 dev_flow->handle, next); 6209 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6210 items_tx.items, 6211 actions_hairpin_tx.actions, error); 6212 if (ret < 0) 6213 goto error; 6214 } 6215 /* 6216 * Update the metadata register copy table. If extensive 6217 * metadata feature is enabled and registers are supported 6218 * we might create the extra rte_flow for each unique 6219 * MARK/FLAG action ID. 6220 * 6221 * The table is updated for ingress Flows only, because 6222 * the egress Flows belong to the different device and 6223 * copy table should be updated in peer NIC Rx domain. 6224 */ 6225 if (attr->ingress && 6226 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6227 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6228 if (ret) 6229 goto error; 6230 } 6231 /* 6232 * If the flow is external (from application) OR device is started, 6233 * OR mreg discover, then apply immediately. 6234 */ 6235 if (external || dev->data->dev_started || 6236 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6237 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6238 ret = flow_drv_apply(dev, flow, error); 6239 if (ret < 0) 6240 goto error; 6241 } 6242 if (list) { 6243 rte_spinlock_lock(&priv->flow_list_lock); 6244 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 6245 flow, next); 6246 rte_spinlock_unlock(&priv->flow_list_lock); 6247 } 6248 flow_rxq_flags_set(dev, flow); 6249 rte_free(translated_actions); 6250 tunnel = flow_tunnel_from_rule(wks->flows); 6251 if (tunnel) { 6252 flow->tunnel = 1; 6253 flow->tunnel_id = tunnel->tunnel_id; 6254 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6255 mlx5_free(default_miss_ctx.queue); 6256 } 6257 mlx5_flow_pop_thread_workspace(); 6258 return idx; 6259 error: 6260 MLX5_ASSERT(flow); 6261 ret = rte_errno; /* Save rte_errno before cleanup. */ 6262 flow_mreg_del_copy_action(dev, flow); 6263 flow_drv_destroy(dev, flow); 6264 if (rss_desc->shared_rss) 6265 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6266 mlx5_ipool_get 6267 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6268 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6269 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 6270 rte_errno = ret; /* Restore rte_errno. */ 6271 ret = rte_errno; 6272 rte_errno = ret; 6273 mlx5_flow_pop_thread_workspace(); 6274 error_before_hairpin_split: 6275 rte_free(translated_actions); 6276 return 0; 6277 } 6278 6279 /** 6280 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6281 * incoming packets to table 1. 6282 * 6283 * Other flow rules, requested for group n, will be created in 6284 * e-switch table n+1. 6285 * Jump action to e-switch group n will be created to group n+1. 6286 * 6287 * Used when working in switchdev mode, to utilise advantages of table 1 6288 * and above. 6289 * 6290 * @param dev 6291 * Pointer to Ethernet device. 6292 * 6293 * @return 6294 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6295 */ 6296 struct rte_flow * 6297 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6298 { 6299 const struct rte_flow_attr attr = { 6300 .group = 0, 6301 .priority = 0, 6302 .ingress = 1, 6303 .egress = 0, 6304 .transfer = 1, 6305 }; 6306 const struct rte_flow_item pattern = { 6307 .type = RTE_FLOW_ITEM_TYPE_END, 6308 }; 6309 struct rte_flow_action_jump jump = { 6310 .group = 1, 6311 }; 6312 const struct rte_flow_action actions[] = { 6313 { 6314 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6315 .conf = &jump, 6316 }, 6317 { 6318 .type = RTE_FLOW_ACTION_TYPE_END, 6319 }, 6320 }; 6321 struct mlx5_priv *priv = dev->data->dev_private; 6322 struct rte_flow_error error; 6323 6324 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 6325 &attr, &pattern, 6326 actions, false, &error); 6327 } 6328 6329 /** 6330 * Validate a flow supported by the NIC. 6331 * 6332 * @see rte_flow_validate() 6333 * @see rte_flow_ops 6334 */ 6335 int 6336 mlx5_flow_validate(struct rte_eth_dev *dev, 6337 const struct rte_flow_attr *attr, 6338 const struct rte_flow_item items[], 6339 const struct rte_flow_action original_actions[], 6340 struct rte_flow_error *error) 6341 { 6342 int hairpin_flow; 6343 struct mlx5_translated_action_handle 6344 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6345 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6346 const struct rte_flow_action *actions; 6347 struct rte_flow_action *translated_actions = NULL; 6348 int ret = flow_action_handles_translate(dev, original_actions, 6349 indir_actions, 6350 &indir_actions_n, 6351 &translated_actions, error); 6352 6353 if (ret) 6354 return ret; 6355 actions = translated_actions ? translated_actions : original_actions; 6356 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6357 ret = flow_drv_validate(dev, attr, items, actions, 6358 true, hairpin_flow, error); 6359 rte_free(translated_actions); 6360 return ret; 6361 } 6362 6363 /** 6364 * Create a flow. 6365 * 6366 * @see rte_flow_create() 6367 * @see rte_flow_ops 6368 */ 6369 struct rte_flow * 6370 mlx5_flow_create(struct rte_eth_dev *dev, 6371 const struct rte_flow_attr *attr, 6372 const struct rte_flow_item items[], 6373 const struct rte_flow_action actions[], 6374 struct rte_flow_error *error) 6375 { 6376 struct mlx5_priv *priv = dev->data->dev_private; 6377 6378 /* 6379 * If the device is not started yet, it is not allowed to created a 6380 * flow from application. PMD default flows and traffic control flows 6381 * are not affected. 6382 */ 6383 if (unlikely(!dev->data->dev_started)) { 6384 DRV_LOG(DEBUG, "port %u is not started when " 6385 "inserting a flow", dev->data->port_id); 6386 rte_flow_error_set(error, ENODEV, 6387 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6388 NULL, 6389 "port not started"); 6390 return NULL; 6391 } 6392 6393 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 6394 attr, items, actions, true, error); 6395 } 6396 6397 /** 6398 * Destroy a flow in a list. 6399 * 6400 * @param dev 6401 * Pointer to Ethernet device. 6402 * @param list 6403 * Pointer to the Indexed flow list. If this parameter NULL, 6404 * there is no flow removal from the list. Be noted that as 6405 * flow is add to the indexed list, memory of the indexed 6406 * list points to maybe changed as flow destroyed. 6407 * @param[in] flow_idx 6408 * Index of flow to destroy. 6409 */ 6410 static void 6411 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 6412 uint32_t flow_idx) 6413 { 6414 struct mlx5_priv *priv = dev->data->dev_private; 6415 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6416 [MLX5_IPOOL_RTE_FLOW], flow_idx); 6417 6418 if (!flow) 6419 return; 6420 /* 6421 * Update RX queue flags only if port is started, otherwise it is 6422 * already clean. 6423 */ 6424 if (dev->data->dev_started) 6425 flow_rxq_flags_trim(dev, flow); 6426 flow_drv_destroy(dev, flow); 6427 if (list) { 6428 rte_spinlock_lock(&priv->flow_list_lock); 6429 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 6430 flow_idx, flow, next); 6431 rte_spinlock_unlock(&priv->flow_list_lock); 6432 } 6433 if (flow->tunnel) { 6434 struct mlx5_flow_tunnel *tunnel; 6435 6436 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6437 RTE_VERIFY(tunnel); 6438 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6439 mlx5_flow_tunnel_free(dev, tunnel); 6440 } 6441 flow_mreg_del_copy_action(dev, flow); 6442 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 6443 } 6444 6445 /** 6446 * Destroy all flows. 6447 * 6448 * @param dev 6449 * Pointer to Ethernet device. 6450 * @param list 6451 * Pointer to the Indexed flow list. 6452 * @param active 6453 * If flushing is called avtively. 6454 */ 6455 void 6456 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 6457 { 6458 uint32_t num_flushed = 0; 6459 6460 while (*list) { 6461 flow_list_destroy(dev, list, *list); 6462 num_flushed++; 6463 } 6464 if (active) { 6465 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6466 dev->data->port_id, num_flushed); 6467 } 6468 } 6469 6470 /** 6471 * Stop all default actions for flows. 6472 * 6473 * @param dev 6474 * Pointer to Ethernet device. 6475 */ 6476 void 6477 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6478 { 6479 flow_mreg_del_default_copy_action(dev); 6480 flow_rxq_flags_clear(dev); 6481 } 6482 6483 /** 6484 * Start all default actions for flows. 6485 * 6486 * @param dev 6487 * Pointer to Ethernet device. 6488 * @return 6489 * 0 on success, a negative errno value otherwise and rte_errno is set. 6490 */ 6491 int 6492 mlx5_flow_start_default(struct rte_eth_dev *dev) 6493 { 6494 struct rte_flow_error error; 6495 6496 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6497 return flow_mreg_add_default_copy_action(dev, &error); 6498 } 6499 6500 /** 6501 * Release key of thread specific flow workspace data. 6502 */ 6503 void 6504 flow_release_workspace(void *data) 6505 { 6506 struct mlx5_flow_workspace *wks = data; 6507 struct mlx5_flow_workspace *next; 6508 6509 while (wks) { 6510 next = wks->next; 6511 free(wks->rss_desc.queue); 6512 free(wks); 6513 wks = next; 6514 } 6515 } 6516 6517 /** 6518 * Get thread specific current flow workspace. 6519 * 6520 * @return pointer to thread specific flow workspace data, NULL on error. 6521 */ 6522 struct mlx5_flow_workspace* 6523 mlx5_flow_get_thread_workspace(void) 6524 { 6525 struct mlx5_flow_workspace *data; 6526 6527 data = mlx5_flow_os_get_specific_workspace(); 6528 MLX5_ASSERT(data && data->inuse); 6529 if (!data || !data->inuse) 6530 DRV_LOG(ERR, "flow workspace not initialized."); 6531 return data; 6532 } 6533 6534 /** 6535 * Allocate and init new flow workspace. 6536 * 6537 * @return pointer to flow workspace data, NULL on error. 6538 */ 6539 static struct mlx5_flow_workspace* 6540 flow_alloc_thread_workspace(void) 6541 { 6542 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6543 6544 if (!data) { 6545 DRV_LOG(ERR, "Failed to allocate flow workspace " 6546 "memory."); 6547 return NULL; 6548 } 6549 data->rss_desc.queue = calloc(1, 6550 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6551 if (!data->rss_desc.queue) 6552 goto err; 6553 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6554 return data; 6555 err: 6556 if (data->rss_desc.queue) 6557 free(data->rss_desc.queue); 6558 free(data); 6559 return NULL; 6560 } 6561 6562 /** 6563 * Get new thread specific flow workspace. 6564 * 6565 * If current workspace inuse, create new one and set as current. 6566 * 6567 * @return pointer to thread specific flow workspace data, NULL on error. 6568 */ 6569 static struct mlx5_flow_workspace* 6570 mlx5_flow_push_thread_workspace(void) 6571 { 6572 struct mlx5_flow_workspace *curr; 6573 struct mlx5_flow_workspace *data; 6574 6575 curr = mlx5_flow_os_get_specific_workspace(); 6576 if (!curr) { 6577 data = flow_alloc_thread_workspace(); 6578 if (!data) 6579 return NULL; 6580 } else if (!curr->inuse) { 6581 data = curr; 6582 } else if (curr->next) { 6583 data = curr->next; 6584 } else { 6585 data = flow_alloc_thread_workspace(); 6586 if (!data) 6587 return NULL; 6588 curr->next = data; 6589 data->prev = curr; 6590 } 6591 data->inuse = 1; 6592 data->flow_idx = 0; 6593 /* Set as current workspace */ 6594 if (mlx5_flow_os_set_specific_workspace(data)) 6595 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6596 return data; 6597 } 6598 6599 /** 6600 * Close current thread specific flow workspace. 6601 * 6602 * If previous workspace available, set it as current. 6603 * 6604 * @return pointer to thread specific flow workspace data, NULL on error. 6605 */ 6606 static void 6607 mlx5_flow_pop_thread_workspace(void) 6608 { 6609 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6610 6611 if (!data) 6612 return; 6613 if (!data->inuse) { 6614 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6615 return; 6616 } 6617 data->inuse = 0; 6618 if (!data->prev) 6619 return; 6620 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6621 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6622 } 6623 6624 /** 6625 * Verify the flow list is empty 6626 * 6627 * @param dev 6628 * Pointer to Ethernet device. 6629 * 6630 * @return the number of flows not released. 6631 */ 6632 int 6633 mlx5_flow_verify(struct rte_eth_dev *dev) 6634 { 6635 struct mlx5_priv *priv = dev->data->dev_private; 6636 struct rte_flow *flow; 6637 uint32_t idx; 6638 int ret = 0; 6639 6640 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 6641 flow, next) { 6642 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6643 dev->data->port_id, (void *)flow); 6644 ++ret; 6645 } 6646 return ret; 6647 } 6648 6649 /** 6650 * Enable default hairpin egress flow. 6651 * 6652 * @param dev 6653 * Pointer to Ethernet device. 6654 * @param queue 6655 * The queue index. 6656 * 6657 * @return 6658 * 0 on success, a negative errno value otherwise and rte_errno is set. 6659 */ 6660 int 6661 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6662 uint32_t queue) 6663 { 6664 struct mlx5_priv *priv = dev->data->dev_private; 6665 const struct rte_flow_attr attr = { 6666 .egress = 1, 6667 .priority = 0, 6668 }; 6669 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6670 .queue = queue, 6671 }; 6672 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6673 .queue = UINT32_MAX, 6674 }; 6675 struct rte_flow_item items[] = { 6676 { 6677 .type = (enum rte_flow_item_type) 6678 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6679 .spec = &queue_spec, 6680 .last = NULL, 6681 .mask = &queue_mask, 6682 }, 6683 { 6684 .type = RTE_FLOW_ITEM_TYPE_END, 6685 }, 6686 }; 6687 struct rte_flow_action_jump jump = { 6688 .group = MLX5_HAIRPIN_TX_TABLE, 6689 }; 6690 struct rte_flow_action actions[2]; 6691 uint32_t flow_idx; 6692 struct rte_flow_error error; 6693 6694 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6695 actions[0].conf = &jump; 6696 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6697 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6698 &attr, items, actions, false, &error); 6699 if (!flow_idx) { 6700 DRV_LOG(DEBUG, 6701 "Failed to create ctrl flow: rte_errno(%d)," 6702 " type(%d), message(%s)", 6703 rte_errno, error.type, 6704 error.message ? error.message : " (no stated reason)"); 6705 return -rte_errno; 6706 } 6707 return 0; 6708 } 6709 6710 /** 6711 * Enable a control flow configured from the control plane. 6712 * 6713 * @param dev 6714 * Pointer to Ethernet device. 6715 * @param eth_spec 6716 * An Ethernet flow spec to apply. 6717 * @param eth_mask 6718 * An Ethernet flow mask to apply. 6719 * @param vlan_spec 6720 * A VLAN flow spec to apply. 6721 * @param vlan_mask 6722 * A VLAN flow mask to apply. 6723 * 6724 * @return 6725 * 0 on success, a negative errno value otherwise and rte_errno is set. 6726 */ 6727 int 6728 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6729 struct rte_flow_item_eth *eth_spec, 6730 struct rte_flow_item_eth *eth_mask, 6731 struct rte_flow_item_vlan *vlan_spec, 6732 struct rte_flow_item_vlan *vlan_mask) 6733 { 6734 struct mlx5_priv *priv = dev->data->dev_private; 6735 const struct rte_flow_attr attr = { 6736 .ingress = 1, 6737 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6738 }; 6739 struct rte_flow_item items[] = { 6740 { 6741 .type = RTE_FLOW_ITEM_TYPE_ETH, 6742 .spec = eth_spec, 6743 .last = NULL, 6744 .mask = eth_mask, 6745 }, 6746 { 6747 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6748 RTE_FLOW_ITEM_TYPE_END, 6749 .spec = vlan_spec, 6750 .last = NULL, 6751 .mask = vlan_mask, 6752 }, 6753 { 6754 .type = RTE_FLOW_ITEM_TYPE_END, 6755 }, 6756 }; 6757 uint16_t queue[priv->reta_idx_n]; 6758 struct rte_flow_action_rss action_rss = { 6759 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6760 .level = 0, 6761 .types = priv->rss_conf.rss_hf, 6762 .key_len = priv->rss_conf.rss_key_len, 6763 .queue_num = priv->reta_idx_n, 6764 .key = priv->rss_conf.rss_key, 6765 .queue = queue, 6766 }; 6767 struct rte_flow_action actions[] = { 6768 { 6769 .type = RTE_FLOW_ACTION_TYPE_RSS, 6770 .conf = &action_rss, 6771 }, 6772 { 6773 .type = RTE_FLOW_ACTION_TYPE_END, 6774 }, 6775 }; 6776 uint32_t flow_idx; 6777 struct rte_flow_error error; 6778 unsigned int i; 6779 6780 if (!priv->reta_idx_n || !priv->rxqs_n) { 6781 return 0; 6782 } 6783 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6784 action_rss.types = 0; 6785 for (i = 0; i != priv->reta_idx_n; ++i) 6786 queue[i] = (*priv->reta_idx)[i]; 6787 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6788 &attr, items, actions, false, &error); 6789 if (!flow_idx) 6790 return -rte_errno; 6791 return 0; 6792 } 6793 6794 /** 6795 * Enable a flow control configured from the control plane. 6796 * 6797 * @param dev 6798 * Pointer to Ethernet device. 6799 * @param eth_spec 6800 * An Ethernet flow spec to apply. 6801 * @param eth_mask 6802 * An Ethernet flow mask to apply. 6803 * 6804 * @return 6805 * 0 on success, a negative errno value otherwise and rte_errno is set. 6806 */ 6807 int 6808 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6809 struct rte_flow_item_eth *eth_spec, 6810 struct rte_flow_item_eth *eth_mask) 6811 { 6812 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6813 } 6814 6815 /** 6816 * Create default miss flow rule matching lacp traffic 6817 * 6818 * @param dev 6819 * Pointer to Ethernet device. 6820 * @param eth_spec 6821 * An Ethernet flow spec to apply. 6822 * 6823 * @return 6824 * 0 on success, a negative errno value otherwise and rte_errno is set. 6825 */ 6826 int 6827 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6828 { 6829 struct mlx5_priv *priv = dev->data->dev_private; 6830 /* 6831 * The LACP matching is done by only using ether type since using 6832 * a multicast dst mac causes kernel to give low priority to this flow. 6833 */ 6834 static const struct rte_flow_item_eth lacp_spec = { 6835 .type = RTE_BE16(0x8809), 6836 }; 6837 static const struct rte_flow_item_eth lacp_mask = { 6838 .type = 0xffff, 6839 }; 6840 const struct rte_flow_attr attr = { 6841 .ingress = 1, 6842 }; 6843 struct rte_flow_item items[] = { 6844 { 6845 .type = RTE_FLOW_ITEM_TYPE_ETH, 6846 .spec = &lacp_spec, 6847 .mask = &lacp_mask, 6848 }, 6849 { 6850 .type = RTE_FLOW_ITEM_TYPE_END, 6851 }, 6852 }; 6853 struct rte_flow_action actions[] = { 6854 { 6855 .type = (enum rte_flow_action_type) 6856 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6857 }, 6858 { 6859 .type = RTE_FLOW_ACTION_TYPE_END, 6860 }, 6861 }; 6862 struct rte_flow_error error; 6863 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6864 &attr, items, actions, false, &error); 6865 6866 if (!flow_idx) 6867 return -rte_errno; 6868 return 0; 6869 } 6870 6871 /** 6872 * Destroy a flow. 6873 * 6874 * @see rte_flow_destroy() 6875 * @see rte_flow_ops 6876 */ 6877 int 6878 mlx5_flow_destroy(struct rte_eth_dev *dev, 6879 struct rte_flow *flow, 6880 struct rte_flow_error *error __rte_unused) 6881 { 6882 struct mlx5_priv *priv = dev->data->dev_private; 6883 6884 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6885 return 0; 6886 } 6887 6888 /** 6889 * Destroy all flows. 6890 * 6891 * @see rte_flow_flush() 6892 * @see rte_flow_ops 6893 */ 6894 int 6895 mlx5_flow_flush(struct rte_eth_dev *dev, 6896 struct rte_flow_error *error __rte_unused) 6897 { 6898 struct mlx5_priv *priv = dev->data->dev_private; 6899 6900 mlx5_flow_list_flush(dev, &priv->flows, false); 6901 return 0; 6902 } 6903 6904 /** 6905 * Isolated mode. 6906 * 6907 * @see rte_flow_isolate() 6908 * @see rte_flow_ops 6909 */ 6910 int 6911 mlx5_flow_isolate(struct rte_eth_dev *dev, 6912 int enable, 6913 struct rte_flow_error *error) 6914 { 6915 struct mlx5_priv *priv = dev->data->dev_private; 6916 6917 if (dev->data->dev_started) { 6918 rte_flow_error_set(error, EBUSY, 6919 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6920 NULL, 6921 "port must be stopped first"); 6922 return -rte_errno; 6923 } 6924 priv->isolated = !!enable; 6925 if (enable) 6926 dev->dev_ops = &mlx5_dev_ops_isolate; 6927 else 6928 dev->dev_ops = &mlx5_dev_ops; 6929 6930 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 6931 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 6932 6933 return 0; 6934 } 6935 6936 /** 6937 * Query a flow. 6938 * 6939 * @see rte_flow_query() 6940 * @see rte_flow_ops 6941 */ 6942 static int 6943 flow_drv_query(struct rte_eth_dev *dev, 6944 uint32_t flow_idx, 6945 const struct rte_flow_action *actions, 6946 void *data, 6947 struct rte_flow_error *error) 6948 { 6949 struct mlx5_priv *priv = dev->data->dev_private; 6950 const struct mlx5_flow_driver_ops *fops; 6951 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6952 [MLX5_IPOOL_RTE_FLOW], 6953 flow_idx); 6954 enum mlx5_flow_drv_type ftype; 6955 6956 if (!flow) { 6957 return rte_flow_error_set(error, ENOENT, 6958 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6959 NULL, 6960 "invalid flow handle"); 6961 } 6962 ftype = flow->drv_type; 6963 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 6964 fops = flow_get_drv_ops(ftype); 6965 6966 return fops->query(dev, flow, actions, data, error); 6967 } 6968 6969 /** 6970 * Query a flow. 6971 * 6972 * @see rte_flow_query() 6973 * @see rte_flow_ops 6974 */ 6975 int 6976 mlx5_flow_query(struct rte_eth_dev *dev, 6977 struct rte_flow *flow, 6978 const struct rte_flow_action *actions, 6979 void *data, 6980 struct rte_flow_error *error) 6981 { 6982 int ret; 6983 6984 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 6985 error); 6986 if (ret < 0) 6987 return ret; 6988 return 0; 6989 } 6990 6991 /** 6992 * Get rte_flow callbacks. 6993 * 6994 * @param dev 6995 * Pointer to Ethernet device structure. 6996 * @param ops 6997 * Pointer to operation-specific structure. 6998 * 6999 * @return 0 7000 */ 7001 int 7002 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7003 const struct rte_flow_ops **ops) 7004 { 7005 *ops = &mlx5_flow_ops; 7006 return 0; 7007 } 7008 7009 /** 7010 * Validate meter policy actions. 7011 * Dispatcher for action type specific validation. 7012 * 7013 * @param[in] dev 7014 * Pointer to the Ethernet device structure. 7015 * @param[in] action 7016 * The meter policy action object to validate. 7017 * @param[in] attr 7018 * Attributes of flow to determine steering domain. 7019 * @param[out] is_rss 7020 * Is RSS or not. 7021 * @param[out] domain_bitmap 7022 * Domain bitmap. 7023 * @param[out] is_def_policy 7024 * Is default policy or not. 7025 * @param[out] error 7026 * Perform verbose error reporting if not NULL. Initialized in case of 7027 * error only. 7028 * 7029 * @return 7030 * 0 on success, otherwise negative errno value. 7031 */ 7032 int 7033 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7034 const struct rte_flow_action *actions[RTE_COLORS], 7035 struct rte_flow_attr *attr, 7036 bool *is_rss, 7037 uint8_t *domain_bitmap, 7038 bool *is_def_policy, 7039 struct rte_mtr_error *error) 7040 { 7041 const struct mlx5_flow_driver_ops *fops; 7042 7043 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7044 return fops->validate_mtr_acts(dev, actions, attr, 7045 is_rss, domain_bitmap, is_def_policy, error); 7046 } 7047 7048 /** 7049 * Destroy the meter table set. 7050 * 7051 * @param[in] dev 7052 * Pointer to Ethernet device. 7053 * @param[in] mtr_policy 7054 * Meter policy struct. 7055 */ 7056 void 7057 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7058 struct mlx5_flow_meter_policy *mtr_policy) 7059 { 7060 const struct mlx5_flow_driver_ops *fops; 7061 7062 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7063 fops->destroy_mtr_acts(dev, mtr_policy); 7064 } 7065 7066 /** 7067 * Create policy action, lock free, 7068 * (mutex should be acquired by caller). 7069 * Dispatcher for action type specific call. 7070 * 7071 * @param[in] dev 7072 * Pointer to the Ethernet device structure. 7073 * @param[in] mtr_policy 7074 * Meter policy struct. 7075 * @param[in] action 7076 * Action specification used to create meter actions. 7077 * @param[out] error 7078 * Perform verbose error reporting if not NULL. Initialized in case of 7079 * error only. 7080 * 7081 * @return 7082 * 0 on success, otherwise negative errno value. 7083 */ 7084 int 7085 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7086 struct mlx5_flow_meter_policy *mtr_policy, 7087 const struct rte_flow_action *actions[RTE_COLORS], 7088 struct rte_mtr_error *error) 7089 { 7090 const struct mlx5_flow_driver_ops *fops; 7091 7092 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7093 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7094 } 7095 7096 /** 7097 * Create policy rules, lock free, 7098 * (mutex should be acquired by caller). 7099 * Dispatcher for action type specific call. 7100 * 7101 * @param[in] dev 7102 * Pointer to the Ethernet device structure. 7103 * @param[in] mtr_policy 7104 * Meter policy struct. 7105 * 7106 * @return 7107 * 0 on success, -1 otherwise. 7108 */ 7109 int 7110 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7111 struct mlx5_flow_meter_policy *mtr_policy) 7112 { 7113 const struct mlx5_flow_driver_ops *fops; 7114 7115 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7116 return fops->create_policy_rules(dev, mtr_policy); 7117 } 7118 7119 /** 7120 * Destroy policy rules, lock free, 7121 * (mutex should be acquired by caller). 7122 * Dispatcher for action type specific call. 7123 * 7124 * @param[in] dev 7125 * Pointer to the Ethernet device structure. 7126 * @param[in] mtr_policy 7127 * Meter policy struct. 7128 */ 7129 void 7130 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7131 struct mlx5_flow_meter_policy *mtr_policy) 7132 { 7133 const struct mlx5_flow_driver_ops *fops; 7134 7135 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7136 fops->destroy_policy_rules(dev, mtr_policy); 7137 } 7138 7139 /** 7140 * Destroy the default policy table set. 7141 * 7142 * @param[in] dev 7143 * Pointer to Ethernet device. 7144 */ 7145 void 7146 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7147 { 7148 const struct mlx5_flow_driver_ops *fops; 7149 7150 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7151 fops->destroy_def_policy(dev); 7152 } 7153 7154 /** 7155 * Destroy the default policy table set. 7156 * 7157 * @param[in] dev 7158 * Pointer to Ethernet device. 7159 * 7160 * @return 7161 * 0 on success, -1 otherwise. 7162 */ 7163 int 7164 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7165 { 7166 const struct mlx5_flow_driver_ops *fops; 7167 7168 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7169 return fops->create_def_policy(dev); 7170 } 7171 7172 /** 7173 * Create the needed meter and suffix tables. 7174 * 7175 * @param[in] dev 7176 * Pointer to Ethernet device. 7177 * 7178 * @return 7179 * 0 on success, -1 otherwise. 7180 */ 7181 int 7182 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7183 struct mlx5_flow_meter_info *fm, 7184 uint32_t mtr_idx, 7185 uint8_t domain_bitmap) 7186 { 7187 const struct mlx5_flow_driver_ops *fops; 7188 7189 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7190 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7191 } 7192 7193 /** 7194 * Destroy the meter table set. 7195 * 7196 * @param[in] dev 7197 * Pointer to Ethernet device. 7198 * @param[in] tbl 7199 * Pointer to the meter table set. 7200 */ 7201 void 7202 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7203 struct mlx5_flow_meter_info *fm) 7204 { 7205 const struct mlx5_flow_driver_ops *fops; 7206 7207 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7208 fops->destroy_mtr_tbls(dev, fm); 7209 } 7210 7211 /** 7212 * Destroy the global meter drop table. 7213 * 7214 * @param[in] dev 7215 * Pointer to Ethernet device. 7216 */ 7217 void 7218 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7219 { 7220 const struct mlx5_flow_driver_ops *fops; 7221 7222 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7223 fops->destroy_mtr_drop_tbls(dev); 7224 } 7225 7226 /** 7227 * Allocate the needed aso flow meter id. 7228 * 7229 * @param[in] dev 7230 * Pointer to Ethernet device. 7231 * 7232 * @return 7233 * Index to aso flow meter on success, NULL otherwise. 7234 */ 7235 uint32_t 7236 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7237 { 7238 const struct mlx5_flow_driver_ops *fops; 7239 7240 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7241 return fops->create_meter(dev); 7242 } 7243 7244 /** 7245 * Free the aso flow meter id. 7246 * 7247 * @param[in] dev 7248 * Pointer to Ethernet device. 7249 * @param[in] mtr_idx 7250 * Index to aso flow meter to be free. 7251 * 7252 * @return 7253 * 0 on success. 7254 */ 7255 void 7256 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7257 { 7258 const struct mlx5_flow_driver_ops *fops; 7259 7260 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7261 fops->free_meter(dev, mtr_idx); 7262 } 7263 7264 /** 7265 * Allocate a counter. 7266 * 7267 * @param[in] dev 7268 * Pointer to Ethernet device structure. 7269 * 7270 * @return 7271 * Index to allocated counter on success, 0 otherwise. 7272 */ 7273 uint32_t 7274 mlx5_counter_alloc(struct rte_eth_dev *dev) 7275 { 7276 const struct mlx5_flow_driver_ops *fops; 7277 struct rte_flow_attr attr = { .transfer = 0 }; 7278 7279 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7280 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7281 return fops->counter_alloc(dev); 7282 } 7283 DRV_LOG(ERR, 7284 "port %u counter allocate is not supported.", 7285 dev->data->port_id); 7286 return 0; 7287 } 7288 7289 /** 7290 * Free a counter. 7291 * 7292 * @param[in] dev 7293 * Pointer to Ethernet device structure. 7294 * @param[in] cnt 7295 * Index to counter to be free. 7296 */ 7297 void 7298 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7299 { 7300 const struct mlx5_flow_driver_ops *fops; 7301 struct rte_flow_attr attr = { .transfer = 0 }; 7302 7303 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7304 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7305 fops->counter_free(dev, cnt); 7306 return; 7307 } 7308 DRV_LOG(ERR, 7309 "port %u counter free is not supported.", 7310 dev->data->port_id); 7311 } 7312 7313 /** 7314 * Query counter statistics. 7315 * 7316 * @param[in] dev 7317 * Pointer to Ethernet device structure. 7318 * @param[in] cnt 7319 * Index to counter to query. 7320 * @param[in] clear 7321 * Set to clear counter statistics. 7322 * @param[out] pkts 7323 * The counter hits packets number to save. 7324 * @param[out] bytes 7325 * The counter hits bytes number to save. 7326 * 7327 * @return 7328 * 0 on success, a negative errno value otherwise. 7329 */ 7330 int 7331 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7332 bool clear, uint64_t *pkts, uint64_t *bytes) 7333 { 7334 const struct mlx5_flow_driver_ops *fops; 7335 struct rte_flow_attr attr = { .transfer = 0 }; 7336 7337 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7338 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7339 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7340 } 7341 DRV_LOG(ERR, 7342 "port %u counter query is not supported.", 7343 dev->data->port_id); 7344 return -ENOTSUP; 7345 } 7346 7347 /** 7348 * Allocate a new memory for the counter values wrapped by all the needed 7349 * management. 7350 * 7351 * @param[in] sh 7352 * Pointer to mlx5_dev_ctx_shared object. 7353 * 7354 * @return 7355 * 0 on success, a negative errno value otherwise. 7356 */ 7357 static int 7358 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7359 { 7360 struct mlx5_devx_mkey_attr mkey_attr; 7361 struct mlx5_counter_stats_mem_mng *mem_mng; 7362 volatile struct flow_counter_stats *raw_data; 7363 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7364 int size = (sizeof(struct flow_counter_stats) * 7365 MLX5_COUNTERS_PER_POOL + 7366 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7367 sizeof(struct mlx5_counter_stats_mem_mng); 7368 size_t pgsize = rte_mem_page_size(); 7369 uint8_t *mem; 7370 int i; 7371 7372 if (pgsize == (size_t)-1) { 7373 DRV_LOG(ERR, "Failed to get mem page size"); 7374 rte_errno = ENOMEM; 7375 return -ENOMEM; 7376 } 7377 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7378 if (!mem) { 7379 rte_errno = ENOMEM; 7380 return -ENOMEM; 7381 } 7382 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7383 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7384 mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size, 7385 IBV_ACCESS_LOCAL_WRITE); 7386 if (!mem_mng->umem) { 7387 rte_errno = errno; 7388 mlx5_free(mem); 7389 return -rte_errno; 7390 } 7391 memset(&mkey_attr, 0, sizeof(mkey_attr)); 7392 mkey_attr.addr = (uintptr_t)mem; 7393 mkey_attr.size = size; 7394 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7395 mkey_attr.pd = sh->pdn; 7396 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 7397 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 7398 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7399 if (!mem_mng->dm) { 7400 mlx5_os_umem_dereg(mem_mng->umem); 7401 rte_errno = errno; 7402 mlx5_free(mem); 7403 return -rte_errno; 7404 } 7405 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7406 raw_data = (volatile struct flow_counter_stats *)mem; 7407 for (i = 0; i < raws_n; ++i) { 7408 mem_mng->raws[i].mem_mng = mem_mng; 7409 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7410 } 7411 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7412 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7413 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7414 next); 7415 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7416 sh->cmng.mem_mng = mem_mng; 7417 return 0; 7418 } 7419 7420 /** 7421 * Set the statistic memory to the new counter pool. 7422 * 7423 * @param[in] sh 7424 * Pointer to mlx5_dev_ctx_shared object. 7425 * @param[in] pool 7426 * Pointer to the pool to set the statistic memory. 7427 * 7428 * @return 7429 * 0 on success, a negative errno value otherwise. 7430 */ 7431 static int 7432 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7433 struct mlx5_flow_counter_pool *pool) 7434 { 7435 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7436 /* Resize statistic memory once used out. */ 7437 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7438 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7439 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7440 return -1; 7441 } 7442 rte_spinlock_lock(&pool->sl); 7443 pool->raw = cmng->mem_mng->raws + pool->index % 7444 MLX5_CNT_CONTAINER_RESIZE; 7445 rte_spinlock_unlock(&pool->sl); 7446 pool->raw_hw = NULL; 7447 return 0; 7448 } 7449 7450 #define MLX5_POOL_QUERY_FREQ_US 1000000 7451 7452 /** 7453 * Set the periodic procedure for triggering asynchronous batch queries for all 7454 * the counter pools. 7455 * 7456 * @param[in] sh 7457 * Pointer to mlx5_dev_ctx_shared object. 7458 */ 7459 void 7460 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7461 { 7462 uint32_t pools_n, us; 7463 7464 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7465 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7466 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7467 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7468 sh->cmng.query_thread_on = 0; 7469 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7470 } else { 7471 sh->cmng.query_thread_on = 1; 7472 } 7473 } 7474 7475 /** 7476 * The periodic procedure for triggering asynchronous batch queries for all the 7477 * counter pools. This function is probably called by the host thread. 7478 * 7479 * @param[in] arg 7480 * The parameter for the alarm process. 7481 */ 7482 void 7483 mlx5_flow_query_alarm(void *arg) 7484 { 7485 struct mlx5_dev_ctx_shared *sh = arg; 7486 int ret; 7487 uint16_t pool_index = sh->cmng.pool_index; 7488 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7489 struct mlx5_flow_counter_pool *pool; 7490 uint16_t n_valid; 7491 7492 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7493 goto set_alarm; 7494 rte_spinlock_lock(&cmng->pool_update_sl); 7495 pool = cmng->pools[pool_index]; 7496 n_valid = cmng->n_valid; 7497 rte_spinlock_unlock(&cmng->pool_update_sl); 7498 /* Set the statistic memory to the new created pool. */ 7499 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7500 goto set_alarm; 7501 if (pool->raw_hw) 7502 /* There is a pool query in progress. */ 7503 goto set_alarm; 7504 pool->raw_hw = 7505 LIST_FIRST(&sh->cmng.free_stat_raws); 7506 if (!pool->raw_hw) 7507 /* No free counter statistics raw memory. */ 7508 goto set_alarm; 7509 /* 7510 * Identify the counters released between query trigger and query 7511 * handle more efficiently. The counter released in this gap period 7512 * should wait for a new round of query as the new arrived packets 7513 * will not be taken into account. 7514 */ 7515 pool->query_gen++; 7516 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7517 MLX5_COUNTERS_PER_POOL, 7518 NULL, NULL, 7519 pool->raw_hw->mem_mng->dm->id, 7520 (void *)(uintptr_t) 7521 pool->raw_hw->data, 7522 sh->devx_comp, 7523 (uint64_t)(uintptr_t)pool); 7524 if (ret) { 7525 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7526 " %d", pool->min_dcs->id); 7527 pool->raw_hw = NULL; 7528 goto set_alarm; 7529 } 7530 LIST_REMOVE(pool->raw_hw, next); 7531 sh->cmng.pending_queries++; 7532 pool_index++; 7533 if (pool_index >= n_valid) 7534 pool_index = 0; 7535 set_alarm: 7536 sh->cmng.pool_index = pool_index; 7537 mlx5_set_query_alarm(sh); 7538 } 7539 7540 /** 7541 * Check and callback event for new aged flow in the counter pool 7542 * 7543 * @param[in] sh 7544 * Pointer to mlx5_dev_ctx_shared object. 7545 * @param[in] pool 7546 * Pointer to Current counter pool. 7547 */ 7548 static void 7549 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7550 struct mlx5_flow_counter_pool *pool) 7551 { 7552 struct mlx5_priv *priv; 7553 struct mlx5_flow_counter *cnt; 7554 struct mlx5_age_info *age_info; 7555 struct mlx5_age_param *age_param; 7556 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7557 struct mlx5_counter_stats_raw *prev = pool->raw; 7558 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7559 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7560 uint16_t expected = AGE_CANDIDATE; 7561 uint32_t i; 7562 7563 pool->time_of_last_age_check = curr_time; 7564 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7565 cnt = MLX5_POOL_GET_CNT(pool, i); 7566 age_param = MLX5_CNT_TO_AGE(cnt); 7567 if (__atomic_load_n(&age_param->state, 7568 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7569 continue; 7570 if (cur->data[i].hits != prev->data[i].hits) { 7571 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7572 __ATOMIC_RELAXED); 7573 continue; 7574 } 7575 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7576 time_delta, 7577 __ATOMIC_RELAXED) <= age_param->timeout) 7578 continue; 7579 /** 7580 * Hold the lock first, or if between the 7581 * state AGE_TMOUT and tailq operation the 7582 * release happened, the release procedure 7583 * may delete a non-existent tailq node. 7584 */ 7585 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7586 age_info = GET_PORT_AGE_INFO(priv); 7587 rte_spinlock_lock(&age_info->aged_sl); 7588 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7589 AGE_TMOUT, false, 7590 __ATOMIC_RELAXED, 7591 __ATOMIC_RELAXED)) { 7592 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7593 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7594 } 7595 rte_spinlock_unlock(&age_info->aged_sl); 7596 } 7597 mlx5_age_event_prepare(sh); 7598 } 7599 7600 /** 7601 * Handler for the HW respond about ready values from an asynchronous batch 7602 * query. This function is probably called by the host thread. 7603 * 7604 * @param[in] sh 7605 * The pointer to the shared device context. 7606 * @param[in] async_id 7607 * The Devx async ID. 7608 * @param[in] status 7609 * The status of the completion. 7610 */ 7611 void 7612 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7613 uint64_t async_id, int status) 7614 { 7615 struct mlx5_flow_counter_pool *pool = 7616 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7617 struct mlx5_counter_stats_raw *raw_to_free; 7618 uint8_t query_gen = pool->query_gen ^ 1; 7619 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7620 enum mlx5_counter_type cnt_type = 7621 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7622 MLX5_COUNTER_TYPE_ORIGIN; 7623 7624 if (unlikely(status)) { 7625 raw_to_free = pool->raw_hw; 7626 } else { 7627 raw_to_free = pool->raw; 7628 if (pool->is_aged) 7629 mlx5_flow_aging_check(sh, pool); 7630 rte_spinlock_lock(&pool->sl); 7631 pool->raw = pool->raw_hw; 7632 rte_spinlock_unlock(&pool->sl); 7633 /* Be sure the new raw counters data is updated in memory. */ 7634 rte_io_wmb(); 7635 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7636 rte_spinlock_lock(&cmng->csl[cnt_type]); 7637 TAILQ_CONCAT(&cmng->counters[cnt_type], 7638 &pool->counters[query_gen], next); 7639 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7640 } 7641 } 7642 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7643 pool->raw_hw = NULL; 7644 sh->cmng.pending_queries--; 7645 } 7646 7647 static int 7648 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7649 const struct flow_grp_info *grp_info, 7650 struct rte_flow_error *error) 7651 { 7652 if (grp_info->transfer && grp_info->external && 7653 grp_info->fdb_def_rule) { 7654 if (group == UINT32_MAX) 7655 return rte_flow_error_set 7656 (error, EINVAL, 7657 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7658 NULL, 7659 "group index not supported"); 7660 *table = group + 1; 7661 } else { 7662 *table = group; 7663 } 7664 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7665 return 0; 7666 } 7667 7668 /** 7669 * Translate the rte_flow group index to HW table value. 7670 * 7671 * If tunnel offload is disabled, all group ids converted to flow table 7672 * id using the standard method. 7673 * If tunnel offload is enabled, group id can be converted using the 7674 * standard or tunnel conversion method. Group conversion method 7675 * selection depends on flags in `grp_info` parameter: 7676 * - Internal (grp_info.external == 0) groups conversion uses the 7677 * standard method. 7678 * - Group ids in JUMP action converted with the tunnel conversion. 7679 * - Group id in rule attribute conversion depends on a rule type and 7680 * group id value: 7681 * ** non zero group attributes converted with the tunnel method 7682 * ** zero group attribute in non-tunnel rule is converted using the 7683 * standard method - there's only one root table 7684 * ** zero group attribute in steer tunnel rule is converted with the 7685 * standard method - single root table 7686 * ** zero group attribute in match tunnel rule is a special OvS 7687 * case: that value is used for portability reasons. That group 7688 * id is converted with the tunnel conversion method. 7689 * 7690 * @param[in] dev 7691 * Port device 7692 * @param[in] tunnel 7693 * PMD tunnel offload object 7694 * @param[in] group 7695 * rte_flow group index value. 7696 * @param[out] table 7697 * HW table value. 7698 * @param[in] grp_info 7699 * flags used for conversion 7700 * @param[out] error 7701 * Pointer to error structure. 7702 * 7703 * @return 7704 * 0 on success, a negative errno value otherwise and rte_errno is set. 7705 */ 7706 int 7707 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7708 const struct mlx5_flow_tunnel *tunnel, 7709 uint32_t group, uint32_t *table, 7710 const struct flow_grp_info *grp_info, 7711 struct rte_flow_error *error) 7712 { 7713 int ret; 7714 bool standard_translation; 7715 7716 if (!grp_info->skip_scale && grp_info->external && 7717 group < MLX5_MAX_TABLES_EXTERNAL) 7718 group *= MLX5_FLOW_TABLE_FACTOR; 7719 if (is_tunnel_offload_active(dev)) { 7720 standard_translation = !grp_info->external || 7721 grp_info->std_tbl_fix; 7722 } else { 7723 standard_translation = true; 7724 } 7725 DRV_LOG(DEBUG, 7726 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 7727 dev->data->port_id, group, grp_info->transfer, 7728 grp_info->external, grp_info->fdb_def_rule, 7729 standard_translation ? "STANDARD" : "TUNNEL"); 7730 if (standard_translation) 7731 ret = flow_group_to_table(dev->data->port_id, group, table, 7732 grp_info, error); 7733 else 7734 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7735 table, error); 7736 7737 return ret; 7738 } 7739 7740 /** 7741 * Discover availability of metadata reg_c's. 7742 * 7743 * Iteratively use test flows to check availability. 7744 * 7745 * @param[in] dev 7746 * Pointer to the Ethernet device structure. 7747 * 7748 * @return 7749 * 0 on success, a negative errno value otherwise and rte_errno is set. 7750 */ 7751 int 7752 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7753 { 7754 struct mlx5_priv *priv = dev->data->dev_private; 7755 struct mlx5_dev_config *config = &priv->config; 7756 enum modify_reg idx; 7757 int n = 0; 7758 7759 /* reg_c[0] and reg_c[1] are reserved. */ 7760 config->flow_mreg_c[n++] = REG_C_0; 7761 config->flow_mreg_c[n++] = REG_C_1; 7762 /* Discover availability of other reg_c's. */ 7763 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7764 struct rte_flow_attr attr = { 7765 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7766 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7767 .ingress = 1, 7768 }; 7769 struct rte_flow_item items[] = { 7770 [0] = { 7771 .type = RTE_FLOW_ITEM_TYPE_END, 7772 }, 7773 }; 7774 struct rte_flow_action actions[] = { 7775 [0] = { 7776 .type = (enum rte_flow_action_type) 7777 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7778 .conf = &(struct mlx5_flow_action_copy_mreg){ 7779 .src = REG_C_1, 7780 .dst = idx, 7781 }, 7782 }, 7783 [1] = { 7784 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7785 .conf = &(struct rte_flow_action_jump){ 7786 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7787 }, 7788 }, 7789 [2] = { 7790 .type = RTE_FLOW_ACTION_TYPE_END, 7791 }, 7792 }; 7793 uint32_t flow_idx; 7794 struct rte_flow *flow; 7795 struct rte_flow_error error; 7796 7797 if (!config->dv_flow_en) 7798 break; 7799 /* Create internal flow, validation skips copy action. */ 7800 flow_idx = flow_list_create(dev, NULL, &attr, items, 7801 actions, false, &error); 7802 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 7803 flow_idx); 7804 if (!flow) 7805 continue; 7806 config->flow_mreg_c[n++] = idx; 7807 flow_list_destroy(dev, NULL, flow_idx); 7808 } 7809 for (; n < MLX5_MREG_C_NUM; ++n) 7810 config->flow_mreg_c[n] = REG_NON; 7811 return 0; 7812 } 7813 7814 /** 7815 * Dump flow raw hw data to file 7816 * 7817 * @param[in] dev 7818 * The pointer to Ethernet device. 7819 * @param[in] file 7820 * A pointer to a file for output. 7821 * @param[out] error 7822 * Perform verbose error reporting if not NULL. PMDs initialize this 7823 * structure in case of error only. 7824 * @return 7825 * 0 on success, a nagative value otherwise. 7826 */ 7827 int 7828 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 7829 FILE *file, 7830 struct rte_flow_error *error __rte_unused) 7831 { 7832 struct mlx5_priv *priv = dev->data->dev_private; 7833 struct mlx5_dev_ctx_shared *sh = priv->sh; 7834 uint32_t handle_idx; 7835 int ret; 7836 struct mlx5_flow_handle *dh; 7837 struct rte_flow *flow; 7838 7839 if (!priv->config.dv_flow_en) { 7840 if (fputs("device dv flow disabled\n", file) <= 0) 7841 return -errno; 7842 return -ENOTSUP; 7843 } 7844 7845 /* dump all */ 7846 if (!flow_idx) 7847 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 7848 sh->rx_domain, 7849 sh->tx_domain, file); 7850 /* dump one */ 7851 flow = mlx5_ipool_get(priv->sh->ipool 7852 [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx); 7853 if (!flow) 7854 return -ENOENT; 7855 7856 handle_idx = flow->dev_handles; 7857 while (handle_idx) { 7858 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 7859 handle_idx); 7860 if (!dh) 7861 return -ENOENT; 7862 if (dh->drv_flow) { 7863 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 7864 file); 7865 if (ret) 7866 return -ENOENT; 7867 } 7868 handle_idx = dh->next.next; 7869 } 7870 return 0; 7871 } 7872 7873 /** 7874 * Get aged-out flows. 7875 * 7876 * @param[in] dev 7877 * Pointer to the Ethernet device structure. 7878 * @param[in] context 7879 * The address of an array of pointers to the aged-out flows contexts. 7880 * @param[in] nb_countexts 7881 * The length of context array pointers. 7882 * @param[out] error 7883 * Perform verbose error reporting if not NULL. Initialized in case of 7884 * error only. 7885 * 7886 * @return 7887 * how many contexts get in success, otherwise negative errno value. 7888 * if nb_contexts is 0, return the amount of all aged contexts. 7889 * if nb_contexts is not 0 , return the amount of aged flows reported 7890 * in the context array. 7891 */ 7892 int 7893 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 7894 uint32_t nb_contexts, struct rte_flow_error *error) 7895 { 7896 const struct mlx5_flow_driver_ops *fops; 7897 struct rte_flow_attr attr = { .transfer = 0 }; 7898 7899 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7900 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7901 return fops->get_aged_flows(dev, contexts, nb_contexts, 7902 error); 7903 } 7904 DRV_LOG(ERR, 7905 "port %u get aged flows is not supported.", 7906 dev->data->port_id); 7907 return -ENOTSUP; 7908 } 7909 7910 /* Wrapper for driver action_validate op callback */ 7911 static int 7912 flow_drv_action_validate(struct rte_eth_dev *dev, 7913 const struct rte_flow_indir_action_conf *conf, 7914 const struct rte_flow_action *action, 7915 const struct mlx5_flow_driver_ops *fops, 7916 struct rte_flow_error *error) 7917 { 7918 static const char err_msg[] = "indirect action validation unsupported"; 7919 7920 if (!fops->action_validate) { 7921 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7922 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7923 NULL, err_msg); 7924 return -rte_errno; 7925 } 7926 return fops->action_validate(dev, conf, action, error); 7927 } 7928 7929 /** 7930 * Destroys the shared action by handle. 7931 * 7932 * @param dev 7933 * Pointer to Ethernet device structure. 7934 * @param[in] handle 7935 * Handle for the indirect action object to be destroyed. 7936 * @param[out] error 7937 * Perform verbose error reporting if not NULL. PMDs initialize this 7938 * structure in case of error only. 7939 * 7940 * @return 7941 * 0 on success, a negative errno value otherwise and rte_errno is set. 7942 * 7943 * @note: wrapper for driver action_create op callback. 7944 */ 7945 static int 7946 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 7947 struct rte_flow_action_handle *handle, 7948 struct rte_flow_error *error) 7949 { 7950 static const char err_msg[] = "indirect action destruction unsupported"; 7951 struct rte_flow_attr attr = { .transfer = 0 }; 7952 const struct mlx5_flow_driver_ops *fops = 7953 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7954 7955 if (!fops->action_destroy) { 7956 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7957 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7958 NULL, err_msg); 7959 return -rte_errno; 7960 } 7961 return fops->action_destroy(dev, handle, error); 7962 } 7963 7964 /* Wrapper for driver action_destroy op callback */ 7965 static int 7966 flow_drv_action_update(struct rte_eth_dev *dev, 7967 struct rte_flow_action_handle *handle, 7968 const void *update, 7969 const struct mlx5_flow_driver_ops *fops, 7970 struct rte_flow_error *error) 7971 { 7972 static const char err_msg[] = "indirect action update unsupported"; 7973 7974 if (!fops->action_update) { 7975 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7976 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7977 NULL, err_msg); 7978 return -rte_errno; 7979 } 7980 return fops->action_update(dev, handle, update, error); 7981 } 7982 7983 /* Wrapper for driver action_destroy op callback */ 7984 static int 7985 flow_drv_action_query(struct rte_eth_dev *dev, 7986 const struct rte_flow_action_handle *handle, 7987 void *data, 7988 const struct mlx5_flow_driver_ops *fops, 7989 struct rte_flow_error *error) 7990 { 7991 static const char err_msg[] = "indirect action query unsupported"; 7992 7993 if (!fops->action_query) { 7994 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7995 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7996 NULL, err_msg); 7997 return -rte_errno; 7998 } 7999 return fops->action_query(dev, handle, data, error); 8000 } 8001 8002 /** 8003 * Create indirect action for reuse in multiple flow rules. 8004 * 8005 * @param dev 8006 * Pointer to Ethernet device structure. 8007 * @param conf 8008 * Pointer to indirect action object configuration. 8009 * @param[in] action 8010 * Action configuration for indirect action object creation. 8011 * @param[out] error 8012 * Perform verbose error reporting if not NULL. PMDs initialize this 8013 * structure in case of error only. 8014 * @return 8015 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8016 */ 8017 static struct rte_flow_action_handle * 8018 mlx5_action_handle_create(struct rte_eth_dev *dev, 8019 const struct rte_flow_indir_action_conf *conf, 8020 const struct rte_flow_action *action, 8021 struct rte_flow_error *error) 8022 { 8023 static const char err_msg[] = "indirect action creation unsupported"; 8024 struct rte_flow_attr attr = { .transfer = 0 }; 8025 const struct mlx5_flow_driver_ops *fops = 8026 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8027 8028 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8029 return NULL; 8030 if (!fops->action_create) { 8031 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8032 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8033 NULL, err_msg); 8034 return NULL; 8035 } 8036 return fops->action_create(dev, conf, action, error); 8037 } 8038 8039 /** 8040 * Updates inplace the indirect action configuration pointed by *handle* 8041 * with the configuration provided as *update* argument. 8042 * The update of the indirect action configuration effects all flow rules 8043 * reusing the action via handle. 8044 * 8045 * @param dev 8046 * Pointer to Ethernet device structure. 8047 * @param[in] handle 8048 * Handle for the indirect action to be updated. 8049 * @param[in] update 8050 * Action specification used to modify the action pointed by handle. 8051 * *update* could be of same type with the action pointed by the *handle* 8052 * handle argument, or some other structures like a wrapper, depending on 8053 * the indirect action type. 8054 * @param[out] error 8055 * Perform verbose error reporting if not NULL. PMDs initialize this 8056 * structure in case of error only. 8057 * 8058 * @return 8059 * 0 on success, a negative errno value otherwise and rte_errno is set. 8060 */ 8061 static int 8062 mlx5_action_handle_update(struct rte_eth_dev *dev, 8063 struct rte_flow_action_handle *handle, 8064 const void *update, 8065 struct rte_flow_error *error) 8066 { 8067 struct rte_flow_attr attr = { .transfer = 0 }; 8068 const struct mlx5_flow_driver_ops *fops = 8069 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8070 int ret; 8071 8072 ret = flow_drv_action_validate(dev, NULL, 8073 (const struct rte_flow_action *)update, fops, error); 8074 if (ret) 8075 return ret; 8076 return flow_drv_action_update(dev, handle, update, fops, 8077 error); 8078 } 8079 8080 /** 8081 * Query the indirect action by handle. 8082 * 8083 * This function allows retrieving action-specific data such as counters. 8084 * Data is gathered by special action which may be present/referenced in 8085 * more than one flow rule definition. 8086 * 8087 * see @RTE_FLOW_ACTION_TYPE_COUNT 8088 * 8089 * @param dev 8090 * Pointer to Ethernet device structure. 8091 * @param[in] handle 8092 * Handle for the indirect action to query. 8093 * @param[in, out] data 8094 * Pointer to storage for the associated query data type. 8095 * @param[out] error 8096 * Perform verbose error reporting if not NULL. PMDs initialize this 8097 * structure in case of error only. 8098 * 8099 * @return 8100 * 0 on success, a negative errno value otherwise and rte_errno is set. 8101 */ 8102 static int 8103 mlx5_action_handle_query(struct rte_eth_dev *dev, 8104 const struct rte_flow_action_handle *handle, 8105 void *data, 8106 struct rte_flow_error *error) 8107 { 8108 struct rte_flow_attr attr = { .transfer = 0 }; 8109 const struct mlx5_flow_driver_ops *fops = 8110 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8111 8112 return flow_drv_action_query(dev, handle, data, fops, error); 8113 } 8114 8115 /** 8116 * Destroy all indirect actions (shared RSS). 8117 * 8118 * @param dev 8119 * Pointer to Ethernet device. 8120 * 8121 * @return 8122 * 0 on success, a negative errno value otherwise and rte_errno is set. 8123 */ 8124 int 8125 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8126 { 8127 struct rte_flow_error error; 8128 struct mlx5_priv *priv = dev->data->dev_private; 8129 struct mlx5_shared_action_rss *shared_rss; 8130 int ret = 0; 8131 uint32_t idx; 8132 8133 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8134 priv->rss_shared_actions, idx, shared_rss, next) { 8135 ret |= mlx5_action_handle_destroy(dev, 8136 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8137 } 8138 return ret; 8139 } 8140 8141 #ifndef HAVE_MLX5DV_DR 8142 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8143 #else 8144 #define MLX5_DOMAIN_SYNC_FLOW \ 8145 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8146 #endif 8147 8148 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8149 { 8150 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8151 const struct mlx5_flow_driver_ops *fops; 8152 int ret; 8153 struct rte_flow_attr attr = { .transfer = 0 }; 8154 8155 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8156 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8157 if (ret > 0) 8158 ret = -ret; 8159 return ret; 8160 } 8161 8162 const struct mlx5_flow_tunnel * 8163 mlx5_get_tof(const struct rte_flow_item *item, 8164 const struct rte_flow_action *action, 8165 enum mlx5_tof_rule_type *rule_type) 8166 { 8167 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 8168 if (item->type == (typeof(item->type)) 8169 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 8170 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 8171 return flow_items_to_tunnel(item); 8172 } 8173 } 8174 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 8175 if (action->type == (typeof(action->type)) 8176 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 8177 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 8178 return flow_actions_to_tunnel(action); 8179 } 8180 } 8181 return NULL; 8182 } 8183 8184 /** 8185 * tunnel offload functionalilty is defined for DV environment only 8186 */ 8187 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8188 __extension__ 8189 union tunnel_offload_mark { 8190 uint32_t val; 8191 struct { 8192 uint32_t app_reserve:8; 8193 uint32_t table_id:15; 8194 uint32_t transfer:1; 8195 uint32_t _unused_:8; 8196 }; 8197 }; 8198 8199 static bool 8200 mlx5_access_tunnel_offload_db 8201 (struct rte_eth_dev *dev, 8202 bool (*match)(struct rte_eth_dev *, 8203 struct mlx5_flow_tunnel *, const void *), 8204 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8205 void (*miss)(struct rte_eth_dev *, void *), 8206 void *ctx, bool lock_op); 8207 8208 static int 8209 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 8210 struct rte_flow *flow, 8211 const struct rte_flow_attr *attr, 8212 const struct rte_flow_action *app_actions, 8213 uint32_t flow_idx, 8214 const struct mlx5_flow_tunnel *tunnel, 8215 struct tunnel_default_miss_ctx *ctx, 8216 struct rte_flow_error *error) 8217 { 8218 struct mlx5_priv *priv = dev->data->dev_private; 8219 struct mlx5_flow *dev_flow; 8220 struct rte_flow_attr miss_attr = *attr; 8221 const struct rte_flow_item miss_items[2] = { 8222 { 8223 .type = RTE_FLOW_ITEM_TYPE_ETH, 8224 .spec = NULL, 8225 .last = NULL, 8226 .mask = NULL 8227 }, 8228 { 8229 .type = RTE_FLOW_ITEM_TYPE_END, 8230 .spec = NULL, 8231 .last = NULL, 8232 .mask = NULL 8233 } 8234 }; 8235 union tunnel_offload_mark mark_id; 8236 struct rte_flow_action_mark miss_mark; 8237 struct rte_flow_action miss_actions[3] = { 8238 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 8239 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 8240 }; 8241 const struct rte_flow_action_jump *jump_data; 8242 uint32_t i, flow_table = 0; /* prevent compilation warning */ 8243 struct flow_grp_info grp_info = { 8244 .external = 1, 8245 .transfer = attr->transfer, 8246 .fdb_def_rule = !!priv->fdb_def_rule, 8247 .std_tbl_fix = 0, 8248 }; 8249 int ret; 8250 8251 if (!attr->transfer) { 8252 uint32_t q_size; 8253 8254 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 8255 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 8256 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 8257 0, SOCKET_ID_ANY); 8258 if (!ctx->queue) 8259 return rte_flow_error_set 8260 (error, ENOMEM, 8261 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8262 NULL, "invalid default miss RSS"); 8263 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8264 ctx->action_rss.level = 0, 8265 ctx->action_rss.types = priv->rss_conf.rss_hf, 8266 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 8267 ctx->action_rss.queue_num = priv->reta_idx_n, 8268 ctx->action_rss.key = priv->rss_conf.rss_key, 8269 ctx->action_rss.queue = ctx->queue; 8270 if (!priv->reta_idx_n || !priv->rxqs_n) 8271 return rte_flow_error_set 8272 (error, EINVAL, 8273 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8274 NULL, "invalid port configuration"); 8275 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 8276 ctx->action_rss.types = 0; 8277 for (i = 0; i != priv->reta_idx_n; ++i) 8278 ctx->queue[i] = (*priv->reta_idx)[i]; 8279 } else { 8280 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 8281 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 8282 } 8283 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 8284 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 8285 jump_data = app_actions->conf; 8286 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 8287 miss_attr.group = jump_data->group; 8288 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 8289 &flow_table, &grp_info, error); 8290 if (ret) 8291 return rte_flow_error_set(error, EINVAL, 8292 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8293 NULL, "invalid tunnel id"); 8294 mark_id.app_reserve = 0; 8295 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 8296 mark_id.transfer = !!attr->transfer; 8297 mark_id._unused_ = 0; 8298 miss_mark.id = mark_id.val; 8299 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 8300 miss_items, miss_actions, flow_idx, error); 8301 if (!dev_flow) 8302 return -rte_errno; 8303 dev_flow->flow = flow; 8304 dev_flow->external = true; 8305 dev_flow->tunnel = tunnel; 8306 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 8307 /* Subflow object was created, we must include one in the list. */ 8308 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 8309 dev_flow->handle, next); 8310 DRV_LOG(DEBUG, 8311 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 8312 dev->data->port_id, tunnel->app_tunnel.type, 8313 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 8314 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 8315 miss_actions, error); 8316 if (!ret) 8317 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 8318 error); 8319 8320 return ret; 8321 } 8322 8323 static const struct mlx5_flow_tbl_data_entry * 8324 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 8325 { 8326 struct mlx5_priv *priv = dev->data->dev_private; 8327 struct mlx5_dev_ctx_shared *sh = priv->sh; 8328 struct mlx5_hlist_entry *he; 8329 union tunnel_offload_mark mbits = { .val = mark }; 8330 union mlx5_flow_tbl_key table_key = { 8331 { 8332 .level = tunnel_id_to_flow_tbl(mbits.table_id), 8333 .id = 0, 8334 .reserved = 0, 8335 .dummy = 0, 8336 .is_fdb = !!mbits.transfer, 8337 .is_egress = 0, 8338 } 8339 }; 8340 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 8341 return he ? 8342 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 8343 } 8344 8345 static void 8346 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 8347 struct mlx5_hlist_entry *entry) 8348 { 8349 struct mlx5_dev_ctx_shared *sh = list->ctx; 8350 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8351 8352 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8353 tunnel_flow_tbl_to_id(tte->flow_table)); 8354 mlx5_free(tte); 8355 } 8356 8357 static int 8358 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused, 8359 struct mlx5_hlist_entry *entry, 8360 uint64_t key, void *cb_ctx __rte_unused) 8361 { 8362 union tunnel_tbl_key tbl = { 8363 .val = key, 8364 }; 8365 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8366 8367 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 8368 } 8369 8370 static struct mlx5_hlist_entry * 8371 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key, 8372 void *ctx __rte_unused) 8373 { 8374 struct mlx5_dev_ctx_shared *sh = list->ctx; 8375 struct tunnel_tbl_entry *tte; 8376 union tunnel_tbl_key tbl = { 8377 .val = key, 8378 }; 8379 8380 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 8381 sizeof(*tte), 0, 8382 SOCKET_ID_ANY); 8383 if (!tte) 8384 goto err; 8385 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8386 &tte->flow_table); 8387 if (tte->flow_table >= MLX5_MAX_TABLES) { 8388 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 8389 tte->flow_table); 8390 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8391 tte->flow_table); 8392 goto err; 8393 } else if (!tte->flow_table) { 8394 goto err; 8395 } 8396 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 8397 tte->tunnel_id = tbl.tunnel_id; 8398 tte->group = tbl.group; 8399 return &tte->hash; 8400 err: 8401 if (tte) 8402 mlx5_free(tte); 8403 return NULL; 8404 } 8405 8406 static uint32_t 8407 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 8408 const struct mlx5_flow_tunnel *tunnel, 8409 uint32_t group, uint32_t *table, 8410 struct rte_flow_error *error) 8411 { 8412 struct mlx5_hlist_entry *he; 8413 struct tunnel_tbl_entry *tte; 8414 union tunnel_tbl_key key = { 8415 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 8416 .group = group 8417 }; 8418 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8419 struct mlx5_hlist *group_hash; 8420 8421 group_hash = tunnel ? tunnel->groups : thub->groups; 8422 he = mlx5_hlist_register(group_hash, key.val, NULL); 8423 if (!he) 8424 return rte_flow_error_set(error, EINVAL, 8425 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8426 NULL, 8427 "tunnel group index not supported"); 8428 tte = container_of(he, typeof(*tte), hash); 8429 *table = tte->flow_table; 8430 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 8431 dev->data->port_id, key.tunnel_id, group, *table); 8432 return 0; 8433 } 8434 8435 static void 8436 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8437 struct mlx5_flow_tunnel *tunnel) 8438 { 8439 struct mlx5_priv *priv = dev->data->dev_private; 8440 struct mlx5_indexed_pool *ipool; 8441 8442 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8443 dev->data->port_id, tunnel->tunnel_id); 8444 LIST_REMOVE(tunnel, chain); 8445 mlx5_hlist_destroy(tunnel->groups); 8446 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8447 mlx5_ipool_free(ipool, tunnel->tunnel_id); 8448 } 8449 8450 static bool 8451 mlx5_access_tunnel_offload_db 8452 (struct rte_eth_dev *dev, 8453 bool (*match)(struct rte_eth_dev *, 8454 struct mlx5_flow_tunnel *, const void *), 8455 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8456 void (*miss)(struct rte_eth_dev *, void *), 8457 void *ctx, bool lock_op) 8458 { 8459 bool verdict = false; 8460 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8461 struct mlx5_flow_tunnel *tunnel; 8462 8463 rte_spinlock_lock(&thub->sl); 8464 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 8465 verdict = match(dev, tunnel, (const void *)ctx); 8466 if (verdict) 8467 break; 8468 } 8469 if (!lock_op) 8470 rte_spinlock_unlock(&thub->sl); 8471 if (verdict && hit) 8472 hit(dev, tunnel, ctx); 8473 if (!verdict && miss) 8474 miss(dev, ctx); 8475 if (lock_op) 8476 rte_spinlock_unlock(&thub->sl); 8477 8478 return verdict; 8479 } 8480 8481 struct tunnel_db_find_tunnel_id_ctx { 8482 uint32_t tunnel_id; 8483 struct mlx5_flow_tunnel *tunnel; 8484 }; 8485 8486 static bool 8487 find_tunnel_id_match(struct rte_eth_dev *dev, 8488 struct mlx5_flow_tunnel *tunnel, const void *x) 8489 { 8490 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8491 8492 RTE_SET_USED(dev); 8493 return tunnel->tunnel_id == ctx->tunnel_id; 8494 } 8495 8496 static void 8497 find_tunnel_id_hit(struct rte_eth_dev *dev, 8498 struct mlx5_flow_tunnel *tunnel, void *x) 8499 { 8500 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8501 RTE_SET_USED(dev); 8502 ctx->tunnel = tunnel; 8503 } 8504 8505 static struct mlx5_flow_tunnel * 8506 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 8507 { 8508 struct tunnel_db_find_tunnel_id_ctx ctx = { 8509 .tunnel_id = id, 8510 }; 8511 8512 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 8513 find_tunnel_id_hit, NULL, &ctx, true); 8514 8515 return ctx.tunnel; 8516 } 8517 8518 static struct mlx5_flow_tunnel * 8519 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 8520 const struct rte_flow_tunnel *app_tunnel) 8521 { 8522 struct mlx5_priv *priv = dev->data->dev_private; 8523 struct mlx5_indexed_pool *ipool; 8524 struct mlx5_flow_tunnel *tunnel; 8525 uint32_t id; 8526 8527 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8528 tunnel = mlx5_ipool_zmalloc(ipool, &id); 8529 if (!tunnel) 8530 return NULL; 8531 if (id >= MLX5_MAX_TUNNELS) { 8532 mlx5_ipool_free(ipool, id); 8533 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 8534 return NULL; 8535 } 8536 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 8537 mlx5_flow_tunnel_grp2tbl_create_cb, 8538 mlx5_flow_tunnel_grp2tbl_match_cb, 8539 mlx5_flow_tunnel_grp2tbl_remove_cb); 8540 if (!tunnel->groups) { 8541 mlx5_ipool_free(ipool, id); 8542 return NULL; 8543 } 8544 tunnel->groups->ctx = priv->sh; 8545 /* initiate new PMD tunnel */ 8546 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8547 tunnel->tunnel_id = id; 8548 tunnel->action.type = (typeof(tunnel->action.type)) 8549 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8550 tunnel->action.conf = tunnel; 8551 tunnel->item.type = (typeof(tunnel->item.type)) 8552 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8553 tunnel->item.spec = tunnel; 8554 tunnel->item.last = NULL; 8555 tunnel->item.mask = NULL; 8556 8557 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8558 dev->data->port_id, tunnel->tunnel_id); 8559 8560 return tunnel; 8561 } 8562 8563 struct tunnel_db_get_tunnel_ctx { 8564 const struct rte_flow_tunnel *app_tunnel; 8565 struct mlx5_flow_tunnel *tunnel; 8566 }; 8567 8568 static bool get_tunnel_match(struct rte_eth_dev *dev, 8569 struct mlx5_flow_tunnel *tunnel, const void *x) 8570 { 8571 const struct tunnel_db_get_tunnel_ctx *ctx = x; 8572 8573 RTE_SET_USED(dev); 8574 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 8575 sizeof(*ctx->app_tunnel)); 8576 } 8577 8578 static void get_tunnel_hit(struct rte_eth_dev *dev, 8579 struct mlx5_flow_tunnel *tunnel, void *x) 8580 { 8581 /* called under tunnel spinlock protection */ 8582 struct tunnel_db_get_tunnel_ctx *ctx = x; 8583 8584 RTE_SET_USED(dev); 8585 tunnel->refctn++; 8586 ctx->tunnel = tunnel; 8587 } 8588 8589 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 8590 { 8591 /* called under tunnel spinlock protection */ 8592 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8593 struct tunnel_db_get_tunnel_ctx *ctx = x; 8594 8595 rte_spinlock_unlock(&thub->sl); 8596 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 8597 rte_spinlock_lock(&thub->sl); 8598 if (ctx->tunnel) { 8599 ctx->tunnel->refctn = 1; 8600 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 8601 } 8602 } 8603 8604 8605 static int 8606 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 8607 const struct rte_flow_tunnel *app_tunnel, 8608 struct mlx5_flow_tunnel **tunnel) 8609 { 8610 struct tunnel_db_get_tunnel_ctx ctx = { 8611 .app_tunnel = app_tunnel, 8612 }; 8613 8614 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 8615 get_tunnel_miss, &ctx, true); 8616 *tunnel = ctx.tunnel; 8617 return ctx.tunnel ? 0 : -ENOMEM; 8618 } 8619 8620 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 8621 { 8622 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 8623 8624 if (!thub) 8625 return; 8626 if (!LIST_EMPTY(&thub->tunnels)) 8627 DRV_LOG(WARNING, "port %u tunnels present", port_id); 8628 mlx5_hlist_destroy(thub->groups); 8629 mlx5_free(thub); 8630 } 8631 8632 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 8633 { 8634 int err; 8635 struct mlx5_flow_tunnel_hub *thub; 8636 8637 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 8638 0, SOCKET_ID_ANY); 8639 if (!thub) 8640 return -ENOMEM; 8641 LIST_INIT(&thub->tunnels); 8642 rte_spinlock_init(&thub->sl); 8643 thub->groups = mlx5_hlist_create("flow groups", 8644 rte_align32pow2(MLX5_MAX_TABLES), 0, 8645 0, mlx5_flow_tunnel_grp2tbl_create_cb, 8646 mlx5_flow_tunnel_grp2tbl_match_cb, 8647 mlx5_flow_tunnel_grp2tbl_remove_cb); 8648 if (!thub->groups) { 8649 err = -rte_errno; 8650 goto err; 8651 } 8652 thub->groups->ctx = sh; 8653 sh->tunnel_hub = thub; 8654 8655 return 0; 8656 8657 err: 8658 if (thub->groups) 8659 mlx5_hlist_destroy(thub->groups); 8660 if (thub) 8661 mlx5_free(thub); 8662 return err; 8663 } 8664 8665 static inline bool 8666 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 8667 struct rte_flow_tunnel *tunnel, 8668 const char *err_msg) 8669 { 8670 err_msg = NULL; 8671 if (!is_tunnel_offload_active(dev)) { 8672 err_msg = "tunnel offload was not activated"; 8673 goto out; 8674 } else if (!tunnel) { 8675 err_msg = "no application tunnel"; 8676 goto out; 8677 } 8678 8679 switch (tunnel->type) { 8680 default: 8681 err_msg = "unsupported tunnel type"; 8682 goto out; 8683 case RTE_FLOW_ITEM_TYPE_VXLAN: 8684 break; 8685 } 8686 8687 out: 8688 return !err_msg; 8689 } 8690 8691 static int 8692 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 8693 struct rte_flow_tunnel *app_tunnel, 8694 struct rte_flow_action **actions, 8695 uint32_t *num_of_actions, 8696 struct rte_flow_error *error) 8697 { 8698 int ret; 8699 struct mlx5_flow_tunnel *tunnel; 8700 const char *err_msg = NULL; 8701 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8702 8703 if (!verdict) 8704 return rte_flow_error_set(error, EINVAL, 8705 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8706 err_msg); 8707 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8708 if (ret < 0) { 8709 return rte_flow_error_set(error, ret, 8710 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8711 "failed to initialize pmd tunnel"); 8712 } 8713 *actions = &tunnel->action; 8714 *num_of_actions = 1; 8715 return 0; 8716 } 8717 8718 static int 8719 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 8720 struct rte_flow_tunnel *app_tunnel, 8721 struct rte_flow_item **items, 8722 uint32_t *num_of_items, 8723 struct rte_flow_error *error) 8724 { 8725 int ret; 8726 struct mlx5_flow_tunnel *tunnel; 8727 const char *err_msg = NULL; 8728 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8729 8730 if (!verdict) 8731 return rte_flow_error_set(error, EINVAL, 8732 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8733 err_msg); 8734 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8735 if (ret < 0) { 8736 return rte_flow_error_set(error, ret, 8737 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8738 "failed to initialize pmd tunnel"); 8739 } 8740 *items = &tunnel->item; 8741 *num_of_items = 1; 8742 return 0; 8743 } 8744 8745 struct tunnel_db_element_release_ctx { 8746 struct rte_flow_item *items; 8747 struct rte_flow_action *actions; 8748 uint32_t num_elements; 8749 struct rte_flow_error *error; 8750 int ret; 8751 }; 8752 8753 static bool 8754 tunnel_element_release_match(struct rte_eth_dev *dev, 8755 struct mlx5_flow_tunnel *tunnel, const void *x) 8756 { 8757 const struct tunnel_db_element_release_ctx *ctx = x; 8758 8759 RTE_SET_USED(dev); 8760 if (ctx->num_elements != 1) 8761 return false; 8762 else if (ctx->items) 8763 return ctx->items == &tunnel->item; 8764 else if (ctx->actions) 8765 return ctx->actions == &tunnel->action; 8766 8767 return false; 8768 } 8769 8770 static void 8771 tunnel_element_release_hit(struct rte_eth_dev *dev, 8772 struct mlx5_flow_tunnel *tunnel, void *x) 8773 { 8774 struct tunnel_db_element_release_ctx *ctx = x; 8775 ctx->ret = 0; 8776 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 8777 mlx5_flow_tunnel_free(dev, tunnel); 8778 } 8779 8780 static void 8781 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 8782 { 8783 struct tunnel_db_element_release_ctx *ctx = x; 8784 RTE_SET_USED(dev); 8785 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 8786 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8787 "invalid argument"); 8788 } 8789 8790 static int 8791 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 8792 struct rte_flow_item *pmd_items, 8793 uint32_t num_items, struct rte_flow_error *err) 8794 { 8795 struct tunnel_db_element_release_ctx ctx = { 8796 .items = pmd_items, 8797 .actions = NULL, 8798 .num_elements = num_items, 8799 .error = err, 8800 }; 8801 8802 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 8803 tunnel_element_release_hit, 8804 tunnel_element_release_miss, &ctx, false); 8805 8806 return ctx.ret; 8807 } 8808 8809 static int 8810 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 8811 struct rte_flow_action *pmd_actions, 8812 uint32_t num_actions, struct rte_flow_error *err) 8813 { 8814 struct tunnel_db_element_release_ctx ctx = { 8815 .items = NULL, 8816 .actions = pmd_actions, 8817 .num_elements = num_actions, 8818 .error = err, 8819 }; 8820 8821 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 8822 tunnel_element_release_hit, 8823 tunnel_element_release_miss, &ctx, false); 8824 8825 return ctx.ret; 8826 } 8827 8828 static int 8829 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 8830 struct rte_mbuf *m, 8831 struct rte_flow_restore_info *info, 8832 struct rte_flow_error *err) 8833 { 8834 uint64_t ol_flags = m->ol_flags; 8835 const struct mlx5_flow_tbl_data_entry *tble; 8836 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 8837 8838 if (!is_tunnel_offload_active(dev)) { 8839 info->flags = 0; 8840 return 0; 8841 } 8842 8843 if ((ol_flags & mask) != mask) 8844 goto err; 8845 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 8846 if (!tble) { 8847 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 8848 dev->data->port_id, m->hash.fdir.hi); 8849 goto err; 8850 } 8851 MLX5_ASSERT(tble->tunnel); 8852 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 8853 info->group_id = tble->group_id; 8854 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 8855 RTE_FLOW_RESTORE_INFO_GROUP_ID | 8856 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 8857 8858 return 0; 8859 8860 err: 8861 return rte_flow_error_set(err, EINVAL, 8862 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8863 "failed to get restore info"); 8864 } 8865 8866 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 8867 static int 8868 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 8869 __rte_unused struct rte_flow_tunnel *app_tunnel, 8870 __rte_unused struct rte_flow_action **actions, 8871 __rte_unused uint32_t *num_of_actions, 8872 __rte_unused struct rte_flow_error *error) 8873 { 8874 return -ENOTSUP; 8875 } 8876 8877 static int 8878 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 8879 __rte_unused struct rte_flow_tunnel *app_tunnel, 8880 __rte_unused struct rte_flow_item **items, 8881 __rte_unused uint32_t *num_of_items, 8882 __rte_unused struct rte_flow_error *error) 8883 { 8884 return -ENOTSUP; 8885 } 8886 8887 static int 8888 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 8889 __rte_unused struct rte_flow_item *pmd_items, 8890 __rte_unused uint32_t num_items, 8891 __rte_unused struct rte_flow_error *err) 8892 { 8893 return -ENOTSUP; 8894 } 8895 8896 static int 8897 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 8898 __rte_unused struct rte_flow_action *pmd_action, 8899 __rte_unused uint32_t num_actions, 8900 __rte_unused struct rte_flow_error *err) 8901 { 8902 return -ENOTSUP; 8903 } 8904 8905 static int 8906 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 8907 __rte_unused struct rte_mbuf *m, 8908 __rte_unused struct rte_flow_restore_info *i, 8909 __rte_unused struct rte_flow_error *err) 8910 { 8911 return -ENOTSUP; 8912 } 8913 8914 static int 8915 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 8916 __rte_unused struct rte_flow *flow, 8917 __rte_unused const struct rte_flow_attr *attr, 8918 __rte_unused const struct rte_flow_action *actions, 8919 __rte_unused uint32_t flow_idx, 8920 __rte_unused const struct mlx5_flow_tunnel *tunnel, 8921 __rte_unused struct tunnel_default_miss_ctx *ctx, 8922 __rte_unused struct rte_flow_error *error) 8923 { 8924 return -ENOTSUP; 8925 } 8926 8927 static struct mlx5_flow_tunnel * 8928 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 8929 __rte_unused uint32_t id) 8930 { 8931 return NULL; 8932 } 8933 8934 static void 8935 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 8936 __rte_unused struct mlx5_flow_tunnel *tunnel) 8937 { 8938 } 8939 8940 static uint32_t 8941 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 8942 __rte_unused const struct mlx5_flow_tunnel *t, 8943 __rte_unused uint32_t group, 8944 __rte_unused uint32_t *table, 8945 struct rte_flow_error *error) 8946 { 8947 return rte_flow_error_set(error, ENOTSUP, 8948 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8949 "tunnel offload requires DV support"); 8950 } 8951 8952 void 8953 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 8954 __rte_unused uint16_t port_id) 8955 { 8956 } 8957 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 8958