1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see ETH_RSS_* definitions). 102 */ 103 uint8_t optional; 104 /**< optional expand field. Default 0 to expand, 1 not go deeper. */ 105 }; 106 107 /** Object returned by mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_rss { 109 uint32_t entries; 110 /**< Number of entries @p patterns and @p priorities. */ 111 struct { 112 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 113 uint32_t priority; /**< Priority offset for each expansion. */ 114 } entry[]; 115 }; 116 117 static enum rte_flow_item_type 118 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 119 { 120 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 121 uint16_t ether_type = 0; 122 uint16_t ether_type_m; 123 uint8_t ip_next_proto = 0; 124 uint8_t ip_next_proto_m; 125 126 if (item == NULL || item->spec == NULL) 127 return ret; 128 switch (item->type) { 129 case RTE_FLOW_ITEM_TYPE_ETH: 130 if (item->mask) 131 ether_type_m = ((const struct rte_flow_item_eth *) 132 (item->mask))->type; 133 else 134 ether_type_m = rte_flow_item_eth_mask.type; 135 if (ether_type_m != RTE_BE16(0xFFFF)) 136 break; 137 ether_type = ((const struct rte_flow_item_eth *) 138 (item->spec))->type; 139 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 140 ret = RTE_FLOW_ITEM_TYPE_IPV4; 141 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 142 ret = RTE_FLOW_ITEM_TYPE_IPV6; 143 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 144 ret = RTE_FLOW_ITEM_TYPE_VLAN; 145 else 146 ret = RTE_FLOW_ITEM_TYPE_END; 147 break; 148 case RTE_FLOW_ITEM_TYPE_VLAN: 149 if (item->mask) 150 ether_type_m = ((const struct rte_flow_item_vlan *) 151 (item->mask))->inner_type; 152 else 153 ether_type_m = rte_flow_item_vlan_mask.inner_type; 154 if (ether_type_m != RTE_BE16(0xFFFF)) 155 break; 156 ether_type = ((const struct rte_flow_item_vlan *) 157 (item->spec))->inner_type; 158 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 159 ret = RTE_FLOW_ITEM_TYPE_IPV4; 160 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 161 ret = RTE_FLOW_ITEM_TYPE_IPV6; 162 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 163 ret = RTE_FLOW_ITEM_TYPE_VLAN; 164 else 165 ret = RTE_FLOW_ITEM_TYPE_END; 166 break; 167 case RTE_FLOW_ITEM_TYPE_IPV4: 168 if (item->mask) 169 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 170 (item->mask))->hdr.next_proto_id; 171 else 172 ip_next_proto_m = 173 rte_flow_item_ipv4_mask.hdr.next_proto_id; 174 if (ip_next_proto_m != 0xFF) 175 break; 176 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 177 (item->spec))->hdr.next_proto_id; 178 if (ip_next_proto == IPPROTO_UDP) 179 ret = RTE_FLOW_ITEM_TYPE_UDP; 180 else if (ip_next_proto == IPPROTO_TCP) 181 ret = RTE_FLOW_ITEM_TYPE_TCP; 182 else if (ip_next_proto == IPPROTO_IP) 183 ret = RTE_FLOW_ITEM_TYPE_IPV4; 184 else if (ip_next_proto == IPPROTO_IPV6) 185 ret = RTE_FLOW_ITEM_TYPE_IPV6; 186 else 187 ret = RTE_FLOW_ITEM_TYPE_END; 188 break; 189 case RTE_FLOW_ITEM_TYPE_IPV6: 190 if (item->mask) 191 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 192 (item->mask))->hdr.proto; 193 else 194 ip_next_proto_m = 195 rte_flow_item_ipv6_mask.hdr.proto; 196 if (ip_next_proto_m != 0xFF) 197 break; 198 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 199 (item->spec))->hdr.proto; 200 if (ip_next_proto == IPPROTO_UDP) 201 ret = RTE_FLOW_ITEM_TYPE_UDP; 202 else if (ip_next_proto == IPPROTO_TCP) 203 ret = RTE_FLOW_ITEM_TYPE_TCP; 204 else if (ip_next_proto == IPPROTO_IP) 205 ret = RTE_FLOW_ITEM_TYPE_IPV4; 206 else if (ip_next_proto == IPPROTO_IPV6) 207 ret = RTE_FLOW_ITEM_TYPE_IPV6; 208 else 209 ret = RTE_FLOW_ITEM_TYPE_END; 210 break; 211 default: 212 ret = RTE_FLOW_ITEM_TYPE_VOID; 213 break; 214 } 215 return ret; 216 } 217 218 #define MLX5_RSS_EXP_ELT_N 16 219 220 /** 221 * Expand RSS flows into several possible flows according to the RSS hash 222 * fields requested and the driver capabilities. 223 * 224 * @param[out] buf 225 * Buffer to store the result expansion. 226 * @param[in] size 227 * Buffer size in bytes. If 0, @p buf can be NULL. 228 * @param[in] pattern 229 * User flow pattern. 230 * @param[in] types 231 * RSS types to expand (see ETH_RSS_* definitions). 232 * @param[in] graph 233 * Input graph to expand @p pattern according to @p types. 234 * @param[in] graph_root_index 235 * Index of root node in @p graph, typically 0. 236 * 237 * @return 238 * A positive value representing the size of @p buf in bytes regardless of 239 * @p size on success, a negative errno value otherwise and rte_errno is 240 * set, the following errors are defined: 241 * 242 * -E2BIG: graph-depth @p graph is too deep. 243 */ 244 static int 245 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 246 const struct rte_flow_item *pattern, uint64_t types, 247 const struct mlx5_flow_expand_node graph[], 248 int graph_root_index) 249 { 250 const struct rte_flow_item *item; 251 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 252 const int *next_node; 253 const int *stack[MLX5_RSS_EXP_ELT_N]; 254 int stack_pos = 0; 255 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 256 unsigned int i; 257 size_t lsize; 258 size_t user_pattern_size = 0; 259 void *addr = NULL; 260 const struct mlx5_flow_expand_node *next = NULL; 261 struct rte_flow_item missed_item; 262 int missed = 0; 263 int elt = 0; 264 const struct rte_flow_item *last_item = NULL; 265 266 memset(&missed_item, 0, sizeof(missed_item)); 267 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 268 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 269 if (lsize <= size) { 270 buf->entry[0].priority = 0; 271 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 272 buf->entries = 0; 273 addr = buf->entry[0].pattern; 274 } 275 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 276 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 277 last_item = item; 278 for (i = 0; node->next && node->next[i]; ++i) { 279 next = &graph[node->next[i]]; 280 if (next->type == item->type) 281 break; 282 } 283 if (next) 284 node = next; 285 user_pattern_size += sizeof(*item); 286 } 287 user_pattern_size += sizeof(*item); /* Handle END item. */ 288 lsize += user_pattern_size; 289 /* Copy the user pattern in the first entry of the buffer. */ 290 if (lsize <= size) { 291 rte_memcpy(addr, pattern, user_pattern_size); 292 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 293 buf->entries = 1; 294 } 295 /* Start expanding. */ 296 memset(flow_items, 0, sizeof(flow_items)); 297 user_pattern_size -= sizeof(*item); 298 /* 299 * Check if the last valid item has spec set, need complete pattern, 300 * and the pattern can be used for expansion. 301 */ 302 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 303 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 304 /* Item type END indicates expansion is not required. */ 305 return lsize; 306 } 307 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 308 next = NULL; 309 missed = 1; 310 for (i = 0; node->next && node->next[i]; ++i) { 311 next = &graph[node->next[i]]; 312 if (next->type == missed_item.type) { 313 flow_items[0].type = missed_item.type; 314 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 315 break; 316 } 317 next = NULL; 318 } 319 } 320 if (next && missed) { 321 elt = 2; /* missed item + item end. */ 322 node = next; 323 lsize += elt * sizeof(*item) + user_pattern_size; 324 if ((node->rss_types & types) && lsize <= size) { 325 buf->entry[buf->entries].priority = 1; 326 buf->entry[buf->entries].pattern = addr; 327 buf->entries++; 328 rte_memcpy(addr, buf->entry[0].pattern, 329 user_pattern_size); 330 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 331 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 332 addr = (void *)(((uintptr_t)addr) + 333 elt * sizeof(*item)); 334 } 335 } 336 memset(flow_items, 0, sizeof(flow_items)); 337 next_node = node->next; 338 stack[stack_pos] = next_node; 339 node = next_node ? &graph[*next_node] : NULL; 340 while (node) { 341 flow_items[stack_pos].type = node->type; 342 if (node->rss_types & types) { 343 /* 344 * compute the number of items to copy from the 345 * expansion and copy it. 346 * When the stack_pos is 0, there are 1 element in it, 347 * plus the addition END item. 348 */ 349 elt = stack_pos + 2; 350 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 351 lsize += elt * sizeof(*item) + user_pattern_size; 352 if (lsize <= size) { 353 size_t n = elt * sizeof(*item); 354 355 buf->entry[buf->entries].priority = 356 stack_pos + 1 + missed; 357 buf->entry[buf->entries].pattern = addr; 358 buf->entries++; 359 rte_memcpy(addr, buf->entry[0].pattern, 360 user_pattern_size); 361 addr = (void *)(((uintptr_t)addr) + 362 user_pattern_size); 363 rte_memcpy(addr, &missed_item, 364 missed * sizeof(*item)); 365 addr = (void *)(((uintptr_t)addr) + 366 missed * sizeof(*item)); 367 rte_memcpy(addr, flow_items, n); 368 addr = (void *)(((uintptr_t)addr) + n); 369 } 370 } 371 /* Go deeper. */ 372 if (!node->optional && node->next) { 373 next_node = node->next; 374 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 375 rte_errno = E2BIG; 376 return -rte_errno; 377 } 378 stack[stack_pos] = next_node; 379 } else if (*(next_node + 1)) { 380 /* Follow up with the next possibility. */ 381 ++next_node; 382 } else { 383 /* Move to the next path. */ 384 if (stack_pos) 385 next_node = stack[--stack_pos]; 386 next_node++; 387 stack[stack_pos] = next_node; 388 } 389 node = *next_node ? &graph[*next_node] : NULL; 390 }; 391 return lsize; 392 } 393 394 enum mlx5_expansion { 395 MLX5_EXPANSION_ROOT, 396 MLX5_EXPANSION_ROOT_OUTER, 397 MLX5_EXPANSION_ROOT_ETH_VLAN, 398 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 399 MLX5_EXPANSION_OUTER_ETH, 400 MLX5_EXPANSION_OUTER_ETH_VLAN, 401 MLX5_EXPANSION_OUTER_VLAN, 402 MLX5_EXPANSION_OUTER_IPV4, 403 MLX5_EXPANSION_OUTER_IPV4_UDP, 404 MLX5_EXPANSION_OUTER_IPV4_TCP, 405 MLX5_EXPANSION_OUTER_IPV6, 406 MLX5_EXPANSION_OUTER_IPV6_UDP, 407 MLX5_EXPANSION_OUTER_IPV6_TCP, 408 MLX5_EXPANSION_VXLAN, 409 MLX5_EXPANSION_VXLAN_GPE, 410 MLX5_EXPANSION_GRE, 411 MLX5_EXPANSION_NVGRE, 412 MLX5_EXPANSION_GRE_KEY, 413 MLX5_EXPANSION_MPLS, 414 MLX5_EXPANSION_ETH, 415 MLX5_EXPANSION_ETH_VLAN, 416 MLX5_EXPANSION_VLAN, 417 MLX5_EXPANSION_IPV4, 418 MLX5_EXPANSION_IPV4_UDP, 419 MLX5_EXPANSION_IPV4_TCP, 420 MLX5_EXPANSION_IPV6, 421 MLX5_EXPANSION_IPV6_UDP, 422 MLX5_EXPANSION_IPV6_TCP, 423 }; 424 425 /** Supported expansion of items. */ 426 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 427 [MLX5_EXPANSION_ROOT] = { 428 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 429 MLX5_EXPANSION_IPV4, 430 MLX5_EXPANSION_IPV6), 431 .type = RTE_FLOW_ITEM_TYPE_END, 432 }, 433 [MLX5_EXPANSION_ROOT_OUTER] = { 434 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 435 MLX5_EXPANSION_OUTER_IPV4, 436 MLX5_EXPANSION_OUTER_IPV6), 437 .type = RTE_FLOW_ITEM_TYPE_END, 438 }, 439 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 440 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 441 .type = RTE_FLOW_ITEM_TYPE_END, 442 }, 443 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 444 .next = MLX5_FLOW_EXPAND_RSS_NEXT 445 (MLX5_EXPANSION_OUTER_ETH_VLAN), 446 .type = RTE_FLOW_ITEM_TYPE_END, 447 }, 448 [MLX5_EXPANSION_OUTER_ETH] = { 449 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 450 MLX5_EXPANSION_OUTER_IPV6, 451 MLX5_EXPANSION_MPLS), 452 .type = RTE_FLOW_ITEM_TYPE_ETH, 453 .rss_types = 0, 454 }, 455 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 456 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 457 .type = RTE_FLOW_ITEM_TYPE_ETH, 458 .rss_types = 0, 459 }, 460 [MLX5_EXPANSION_OUTER_VLAN] = { 461 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 462 MLX5_EXPANSION_OUTER_IPV6), 463 .type = RTE_FLOW_ITEM_TYPE_VLAN, 464 }, 465 [MLX5_EXPANSION_OUTER_IPV4] = { 466 .next = MLX5_FLOW_EXPAND_RSS_NEXT 467 (MLX5_EXPANSION_OUTER_IPV4_UDP, 468 MLX5_EXPANSION_OUTER_IPV4_TCP, 469 MLX5_EXPANSION_GRE, 470 MLX5_EXPANSION_NVGRE, 471 MLX5_EXPANSION_IPV4, 472 MLX5_EXPANSION_IPV6), 473 .type = RTE_FLOW_ITEM_TYPE_IPV4, 474 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 475 ETH_RSS_NONFRAG_IPV4_OTHER, 476 }, 477 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 478 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 479 MLX5_EXPANSION_VXLAN_GPE), 480 .type = RTE_FLOW_ITEM_TYPE_UDP, 481 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 482 }, 483 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 484 .type = RTE_FLOW_ITEM_TYPE_TCP, 485 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 486 }, 487 [MLX5_EXPANSION_OUTER_IPV6] = { 488 .next = MLX5_FLOW_EXPAND_RSS_NEXT 489 (MLX5_EXPANSION_OUTER_IPV6_UDP, 490 MLX5_EXPANSION_OUTER_IPV6_TCP, 491 MLX5_EXPANSION_IPV4, 492 MLX5_EXPANSION_IPV6, 493 MLX5_EXPANSION_GRE, 494 MLX5_EXPANSION_NVGRE), 495 .type = RTE_FLOW_ITEM_TYPE_IPV6, 496 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 497 ETH_RSS_NONFRAG_IPV6_OTHER, 498 }, 499 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 500 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 501 MLX5_EXPANSION_VXLAN_GPE), 502 .type = RTE_FLOW_ITEM_TYPE_UDP, 503 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 504 }, 505 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 506 .type = RTE_FLOW_ITEM_TYPE_TCP, 507 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 508 }, 509 [MLX5_EXPANSION_VXLAN] = { 510 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 511 MLX5_EXPANSION_IPV4, 512 MLX5_EXPANSION_IPV6), 513 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 514 }, 515 [MLX5_EXPANSION_VXLAN_GPE] = { 516 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 517 MLX5_EXPANSION_IPV4, 518 MLX5_EXPANSION_IPV6), 519 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 520 }, 521 [MLX5_EXPANSION_GRE] = { 522 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 523 MLX5_EXPANSION_IPV6, 524 MLX5_EXPANSION_GRE_KEY), 525 .type = RTE_FLOW_ITEM_TYPE_GRE, 526 }, 527 [MLX5_EXPANSION_GRE_KEY] = { 528 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 529 MLX5_EXPANSION_IPV6), 530 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 531 .optional = 1, 532 }, 533 [MLX5_EXPANSION_NVGRE] = { 534 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 535 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 536 }, 537 [MLX5_EXPANSION_MPLS] = { 538 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 539 MLX5_EXPANSION_IPV6), 540 .type = RTE_FLOW_ITEM_TYPE_MPLS, 541 }, 542 [MLX5_EXPANSION_ETH] = { 543 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 544 MLX5_EXPANSION_IPV6), 545 .type = RTE_FLOW_ITEM_TYPE_ETH, 546 }, 547 [MLX5_EXPANSION_ETH_VLAN] = { 548 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 549 .type = RTE_FLOW_ITEM_TYPE_ETH, 550 }, 551 [MLX5_EXPANSION_VLAN] = { 552 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 553 MLX5_EXPANSION_IPV6), 554 .type = RTE_FLOW_ITEM_TYPE_VLAN, 555 }, 556 [MLX5_EXPANSION_IPV4] = { 557 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 558 MLX5_EXPANSION_IPV4_TCP), 559 .type = RTE_FLOW_ITEM_TYPE_IPV4, 560 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 561 ETH_RSS_NONFRAG_IPV4_OTHER, 562 }, 563 [MLX5_EXPANSION_IPV4_UDP] = { 564 .type = RTE_FLOW_ITEM_TYPE_UDP, 565 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 566 }, 567 [MLX5_EXPANSION_IPV4_TCP] = { 568 .type = RTE_FLOW_ITEM_TYPE_TCP, 569 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 570 }, 571 [MLX5_EXPANSION_IPV6] = { 572 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 573 MLX5_EXPANSION_IPV6_TCP), 574 .type = RTE_FLOW_ITEM_TYPE_IPV6, 575 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 576 ETH_RSS_NONFRAG_IPV6_OTHER, 577 }, 578 [MLX5_EXPANSION_IPV6_UDP] = { 579 .type = RTE_FLOW_ITEM_TYPE_UDP, 580 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 581 }, 582 [MLX5_EXPANSION_IPV6_TCP] = { 583 .type = RTE_FLOW_ITEM_TYPE_TCP, 584 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 585 }, 586 }; 587 588 static struct rte_flow_action_handle * 589 mlx5_action_handle_create(struct rte_eth_dev *dev, 590 const struct rte_flow_indir_action_conf *conf, 591 const struct rte_flow_action *action, 592 struct rte_flow_error *error); 593 static int mlx5_action_handle_destroy 594 (struct rte_eth_dev *dev, 595 struct rte_flow_action_handle *handle, 596 struct rte_flow_error *error); 597 static int mlx5_action_handle_update 598 (struct rte_eth_dev *dev, 599 struct rte_flow_action_handle *handle, 600 const void *update, 601 struct rte_flow_error *error); 602 static int mlx5_action_handle_query 603 (struct rte_eth_dev *dev, 604 const struct rte_flow_action_handle *handle, 605 void *data, 606 struct rte_flow_error *error); 607 static int 608 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 609 struct rte_flow_tunnel *app_tunnel, 610 struct rte_flow_action **actions, 611 uint32_t *num_of_actions, 612 struct rte_flow_error *error); 613 static int 614 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 615 struct rte_flow_tunnel *app_tunnel, 616 struct rte_flow_item **items, 617 uint32_t *num_of_items, 618 struct rte_flow_error *error); 619 static int 620 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 621 struct rte_flow_item *pmd_items, 622 uint32_t num_items, struct rte_flow_error *err); 623 static int 624 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 625 struct rte_flow_action *pmd_actions, 626 uint32_t num_actions, 627 struct rte_flow_error *err); 628 static int 629 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 630 struct rte_mbuf *m, 631 struct rte_flow_restore_info *info, 632 struct rte_flow_error *err); 633 634 static const struct rte_flow_ops mlx5_flow_ops = { 635 .validate = mlx5_flow_validate, 636 .create = mlx5_flow_create, 637 .destroy = mlx5_flow_destroy, 638 .flush = mlx5_flow_flush, 639 .isolate = mlx5_flow_isolate, 640 .query = mlx5_flow_query, 641 .dev_dump = mlx5_flow_dev_dump, 642 .get_aged_flows = mlx5_flow_get_aged_flows, 643 .action_handle_create = mlx5_action_handle_create, 644 .action_handle_destroy = mlx5_action_handle_destroy, 645 .action_handle_update = mlx5_action_handle_update, 646 .action_handle_query = mlx5_action_handle_query, 647 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 648 .tunnel_match = mlx5_flow_tunnel_match, 649 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 650 .tunnel_item_release = mlx5_flow_tunnel_item_release, 651 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 652 }; 653 654 /* Tunnel information. */ 655 struct mlx5_flow_tunnel_info { 656 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 657 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 658 }; 659 660 static struct mlx5_flow_tunnel_info tunnels_info[] = { 661 { 662 .tunnel = MLX5_FLOW_LAYER_VXLAN, 663 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 664 }, 665 { 666 .tunnel = MLX5_FLOW_LAYER_GENEVE, 667 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 668 }, 669 { 670 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 671 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 672 }, 673 { 674 .tunnel = MLX5_FLOW_LAYER_GRE, 675 .ptype = RTE_PTYPE_TUNNEL_GRE, 676 }, 677 { 678 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 679 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 680 }, 681 { 682 .tunnel = MLX5_FLOW_LAYER_MPLS, 683 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 684 }, 685 { 686 .tunnel = MLX5_FLOW_LAYER_NVGRE, 687 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 688 }, 689 { 690 .tunnel = MLX5_FLOW_LAYER_IPIP, 691 .ptype = RTE_PTYPE_TUNNEL_IP, 692 }, 693 { 694 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 695 .ptype = RTE_PTYPE_TUNNEL_IP, 696 }, 697 { 698 .tunnel = MLX5_FLOW_LAYER_GTP, 699 .ptype = RTE_PTYPE_TUNNEL_GTPU, 700 }, 701 }; 702 703 704 705 /** 706 * Translate tag ID to register. 707 * 708 * @param[in] dev 709 * Pointer to the Ethernet device structure. 710 * @param[in] feature 711 * The feature that request the register. 712 * @param[in] id 713 * The request register ID. 714 * @param[out] error 715 * Error description in case of any. 716 * 717 * @return 718 * The request register on success, a negative errno 719 * value otherwise and rte_errno is set. 720 */ 721 int 722 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 723 enum mlx5_feature_name feature, 724 uint32_t id, 725 struct rte_flow_error *error) 726 { 727 struct mlx5_priv *priv = dev->data->dev_private; 728 struct mlx5_dev_config *config = &priv->config; 729 enum modify_reg start_reg; 730 bool skip_mtr_reg = false; 731 732 switch (feature) { 733 case MLX5_HAIRPIN_RX: 734 return REG_B; 735 case MLX5_HAIRPIN_TX: 736 return REG_A; 737 case MLX5_METADATA_RX: 738 switch (config->dv_xmeta_en) { 739 case MLX5_XMETA_MODE_LEGACY: 740 return REG_B; 741 case MLX5_XMETA_MODE_META16: 742 return REG_C_0; 743 case MLX5_XMETA_MODE_META32: 744 return REG_C_1; 745 } 746 break; 747 case MLX5_METADATA_TX: 748 return REG_A; 749 case MLX5_METADATA_FDB: 750 switch (config->dv_xmeta_en) { 751 case MLX5_XMETA_MODE_LEGACY: 752 return REG_NON; 753 case MLX5_XMETA_MODE_META16: 754 return REG_C_0; 755 case MLX5_XMETA_MODE_META32: 756 return REG_C_1; 757 } 758 break; 759 case MLX5_FLOW_MARK: 760 switch (config->dv_xmeta_en) { 761 case MLX5_XMETA_MODE_LEGACY: 762 return REG_NON; 763 case MLX5_XMETA_MODE_META16: 764 return REG_C_1; 765 case MLX5_XMETA_MODE_META32: 766 return REG_C_0; 767 } 768 break; 769 case MLX5_MTR_ID: 770 /* 771 * If meter color and meter id share one register, flow match 772 * should use the meter color register for match. 773 */ 774 if (priv->mtr_reg_share) 775 return priv->mtr_color_reg; 776 else 777 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 778 REG_C_3; 779 case MLX5_MTR_COLOR: 780 case MLX5_ASO_FLOW_HIT: 781 case MLX5_ASO_CONNTRACK: 782 /* All features use the same REG_C. */ 783 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 784 return priv->mtr_color_reg; 785 case MLX5_COPY_MARK: 786 /* 787 * Metadata COPY_MARK register using is in meter suffix sub 788 * flow while with meter. It's safe to share the same register. 789 */ 790 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 791 case MLX5_APP_TAG: 792 /* 793 * If meter is enable, it will engage the register for color 794 * match and flow match. If meter color match is not using the 795 * REG_C_2, need to skip the REG_C_x be used by meter color 796 * match. 797 * If meter is disable, free to use all available registers. 798 */ 799 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 800 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 801 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 802 if (id > (uint32_t)(REG_C_7 - start_reg)) 803 return rte_flow_error_set(error, EINVAL, 804 RTE_FLOW_ERROR_TYPE_ITEM, 805 NULL, "invalid tag id"); 806 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 807 return rte_flow_error_set(error, ENOTSUP, 808 RTE_FLOW_ERROR_TYPE_ITEM, 809 NULL, "unsupported tag id"); 810 /* 811 * This case means meter is using the REG_C_x great than 2. 812 * Take care not to conflict with meter color REG_C_x. 813 * If the available index REG_C_y >= REG_C_x, skip the 814 * color register. 815 */ 816 if (skip_mtr_reg && config->flow_mreg_c 817 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 818 if (id >= (uint32_t)(REG_C_7 - start_reg)) 819 return rte_flow_error_set(error, EINVAL, 820 RTE_FLOW_ERROR_TYPE_ITEM, 821 NULL, "invalid tag id"); 822 if (config->flow_mreg_c 823 [id + 1 + start_reg - REG_C_0] != REG_NON) 824 return config->flow_mreg_c 825 [id + 1 + start_reg - REG_C_0]; 826 return rte_flow_error_set(error, ENOTSUP, 827 RTE_FLOW_ERROR_TYPE_ITEM, 828 NULL, "unsupported tag id"); 829 } 830 return config->flow_mreg_c[id + start_reg - REG_C_0]; 831 } 832 MLX5_ASSERT(false); 833 return rte_flow_error_set(error, EINVAL, 834 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 835 NULL, "invalid feature name"); 836 } 837 838 /** 839 * Check extensive flow metadata register support. 840 * 841 * @param dev 842 * Pointer to rte_eth_dev structure. 843 * 844 * @return 845 * True if device supports extensive flow metadata register, otherwise false. 846 */ 847 bool 848 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 849 { 850 struct mlx5_priv *priv = dev->data->dev_private; 851 struct mlx5_dev_config *config = &priv->config; 852 853 /* 854 * Having available reg_c can be regarded inclusively as supporting 855 * extensive flow metadata register, which could mean, 856 * - metadata register copy action by modify header. 857 * - 16 modify header actions is supported. 858 * - reg_c's are preserved across different domain (FDB and NIC) on 859 * packet loopback by flow lookup miss. 860 */ 861 return config->flow_mreg_c[2] != REG_NON; 862 } 863 864 /** 865 * Get the lowest priority. 866 * 867 * @param[in] dev 868 * Pointer to the Ethernet device structure. 869 * @param[in] attributes 870 * Pointer to device flow rule attributes. 871 * 872 * @return 873 * The value of lowest priority of flow. 874 */ 875 uint32_t 876 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 877 const struct rte_flow_attr *attr) 878 { 879 struct mlx5_priv *priv = dev->data->dev_private; 880 881 if (!attr->group && !attr->transfer) 882 return priv->config.flow_prio - 2; 883 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 884 } 885 886 /** 887 * Calculate matcher priority of the flow. 888 * 889 * @param[in] dev 890 * Pointer to the Ethernet device structure. 891 * @param[in] attr 892 * Pointer to device flow rule attributes. 893 * @param[in] subpriority 894 * The priority based on the items. 895 * @return 896 * The matcher priority of the flow. 897 */ 898 uint16_t 899 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 900 const struct rte_flow_attr *attr, 901 uint32_t subpriority) 902 { 903 uint16_t priority = (uint16_t)attr->priority; 904 struct mlx5_priv *priv = dev->data->dev_private; 905 906 if (!attr->group && !attr->transfer) { 907 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 908 priority = priv->config.flow_prio - 1; 909 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 910 } 911 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 912 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 913 return priority * 3 + subpriority; 914 } 915 916 /** 917 * Verify the @p item specifications (spec, last, mask) are compatible with the 918 * NIC capabilities. 919 * 920 * @param[in] item 921 * Item specification. 922 * @param[in] mask 923 * @p item->mask or flow default bit-masks. 924 * @param[in] nic_mask 925 * Bit-masks covering supported fields by the NIC to compare with user mask. 926 * @param[in] size 927 * Bit-masks size in bytes. 928 * @param[in] range_accepted 929 * True if range of values is accepted for specific fields, false otherwise. 930 * @param[out] error 931 * Pointer to error structure. 932 * 933 * @return 934 * 0 on success, a negative errno value otherwise and rte_errno is set. 935 */ 936 int 937 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 938 const uint8_t *mask, 939 const uint8_t *nic_mask, 940 unsigned int size, 941 bool range_accepted, 942 struct rte_flow_error *error) 943 { 944 unsigned int i; 945 946 MLX5_ASSERT(nic_mask); 947 for (i = 0; i < size; ++i) 948 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 949 return rte_flow_error_set(error, ENOTSUP, 950 RTE_FLOW_ERROR_TYPE_ITEM, 951 item, 952 "mask enables non supported" 953 " bits"); 954 if (!item->spec && (item->mask || item->last)) 955 return rte_flow_error_set(error, EINVAL, 956 RTE_FLOW_ERROR_TYPE_ITEM, item, 957 "mask/last without a spec is not" 958 " supported"); 959 if (item->spec && item->last && !range_accepted) { 960 uint8_t spec[size]; 961 uint8_t last[size]; 962 unsigned int i; 963 int ret; 964 965 for (i = 0; i < size; ++i) { 966 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 967 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 968 } 969 ret = memcmp(spec, last, size); 970 if (ret != 0) 971 return rte_flow_error_set(error, EINVAL, 972 RTE_FLOW_ERROR_TYPE_ITEM, 973 item, 974 "range is not valid"); 975 } 976 return 0; 977 } 978 979 /** 980 * Adjust the hash fields according to the @p flow information. 981 * 982 * @param[in] dev_flow. 983 * Pointer to the mlx5_flow. 984 * @param[in] tunnel 985 * 1 when the hash field is for a tunnel item. 986 * @param[in] layer_types 987 * ETH_RSS_* types. 988 * @param[in] hash_fields 989 * Item hash fields. 990 * 991 * @return 992 * The hash fields that should be used. 993 */ 994 uint64_t 995 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 996 int tunnel __rte_unused, uint64_t layer_types, 997 uint64_t hash_fields) 998 { 999 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1000 int rss_request_inner = rss_desc->level >= 2; 1001 1002 /* Check RSS hash level for tunnel. */ 1003 if (tunnel && rss_request_inner) 1004 hash_fields |= IBV_RX_HASH_INNER; 1005 else if (tunnel || rss_request_inner) 1006 return 0; 1007 #endif 1008 /* Check if requested layer matches RSS hash fields. */ 1009 if (!(rss_desc->types & layer_types)) 1010 return 0; 1011 return hash_fields; 1012 } 1013 1014 /** 1015 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1016 * if several tunnel rules are used on this queue, the tunnel ptype will be 1017 * cleared. 1018 * 1019 * @param rxq_ctrl 1020 * Rx queue to update. 1021 */ 1022 static void 1023 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1024 { 1025 unsigned int i; 1026 uint32_t tunnel_ptype = 0; 1027 1028 /* Look up for the ptype to use. */ 1029 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1030 if (!rxq_ctrl->flow_tunnels_n[i]) 1031 continue; 1032 if (!tunnel_ptype) { 1033 tunnel_ptype = tunnels_info[i].ptype; 1034 } else { 1035 tunnel_ptype = 0; 1036 break; 1037 } 1038 } 1039 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1040 } 1041 1042 /** 1043 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1044 * flow. 1045 * 1046 * @param[in] dev 1047 * Pointer to the Ethernet device structure. 1048 * @param[in] dev_handle 1049 * Pointer to device flow handle structure. 1050 */ 1051 void 1052 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1053 struct mlx5_flow_handle *dev_handle) 1054 { 1055 struct mlx5_priv *priv = dev->data->dev_private; 1056 const int mark = dev_handle->mark; 1057 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1058 struct mlx5_ind_table_obj *ind_tbl = NULL; 1059 unsigned int i; 1060 1061 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1062 struct mlx5_hrxq *hrxq; 1063 1064 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1065 dev_handle->rix_hrxq); 1066 if (hrxq) 1067 ind_tbl = hrxq->ind_table; 1068 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1069 struct mlx5_shared_action_rss *shared_rss; 1070 1071 shared_rss = mlx5_ipool_get 1072 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1073 dev_handle->rix_srss); 1074 if (shared_rss) 1075 ind_tbl = shared_rss->ind_tbl; 1076 } 1077 if (!ind_tbl) 1078 return; 1079 for (i = 0; i != ind_tbl->queues_n; ++i) { 1080 int idx = ind_tbl->queues[i]; 1081 struct mlx5_rxq_ctrl *rxq_ctrl = 1082 container_of((*priv->rxqs)[idx], 1083 struct mlx5_rxq_ctrl, rxq); 1084 1085 /* 1086 * To support metadata register copy on Tx loopback, 1087 * this must be always enabled (metadata may arive 1088 * from other port - not from local flows only. 1089 */ 1090 if (priv->config.dv_flow_en && 1091 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1092 mlx5_flow_ext_mreg_supported(dev)) { 1093 rxq_ctrl->rxq.mark = 1; 1094 rxq_ctrl->flow_mark_n = 1; 1095 } else if (mark) { 1096 rxq_ctrl->rxq.mark = 1; 1097 rxq_ctrl->flow_mark_n++; 1098 } 1099 if (tunnel) { 1100 unsigned int j; 1101 1102 /* Increase the counter matching the flow. */ 1103 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1104 if ((tunnels_info[j].tunnel & 1105 dev_handle->layers) == 1106 tunnels_info[j].tunnel) { 1107 rxq_ctrl->flow_tunnels_n[j]++; 1108 break; 1109 } 1110 } 1111 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1112 } 1113 } 1114 } 1115 1116 /** 1117 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1118 * 1119 * @param[in] dev 1120 * Pointer to the Ethernet device structure. 1121 * @param[in] flow 1122 * Pointer to flow structure. 1123 */ 1124 static void 1125 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1126 { 1127 struct mlx5_priv *priv = dev->data->dev_private; 1128 uint32_t handle_idx; 1129 struct mlx5_flow_handle *dev_handle; 1130 1131 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1132 handle_idx, dev_handle, next) 1133 flow_drv_rxq_flags_set(dev, dev_handle); 1134 } 1135 1136 /** 1137 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1138 * device flow if no other flow uses it with the same kind of request. 1139 * 1140 * @param dev 1141 * Pointer to Ethernet device. 1142 * @param[in] dev_handle 1143 * Pointer to the device flow handle structure. 1144 */ 1145 static void 1146 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1147 struct mlx5_flow_handle *dev_handle) 1148 { 1149 struct mlx5_priv *priv = dev->data->dev_private; 1150 const int mark = dev_handle->mark; 1151 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1152 struct mlx5_ind_table_obj *ind_tbl = NULL; 1153 unsigned int i; 1154 1155 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1156 struct mlx5_hrxq *hrxq; 1157 1158 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1159 dev_handle->rix_hrxq); 1160 if (hrxq) 1161 ind_tbl = hrxq->ind_table; 1162 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1163 struct mlx5_shared_action_rss *shared_rss; 1164 1165 shared_rss = mlx5_ipool_get 1166 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1167 dev_handle->rix_srss); 1168 if (shared_rss) 1169 ind_tbl = shared_rss->ind_tbl; 1170 } 1171 if (!ind_tbl) 1172 return; 1173 MLX5_ASSERT(dev->data->dev_started); 1174 for (i = 0; i != ind_tbl->queues_n; ++i) { 1175 int idx = ind_tbl->queues[i]; 1176 struct mlx5_rxq_ctrl *rxq_ctrl = 1177 container_of((*priv->rxqs)[idx], 1178 struct mlx5_rxq_ctrl, rxq); 1179 1180 if (priv->config.dv_flow_en && 1181 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1182 mlx5_flow_ext_mreg_supported(dev)) { 1183 rxq_ctrl->rxq.mark = 1; 1184 rxq_ctrl->flow_mark_n = 1; 1185 } else if (mark) { 1186 rxq_ctrl->flow_mark_n--; 1187 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1188 } 1189 if (tunnel) { 1190 unsigned int j; 1191 1192 /* Decrease the counter matching the flow. */ 1193 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1194 if ((tunnels_info[j].tunnel & 1195 dev_handle->layers) == 1196 tunnels_info[j].tunnel) { 1197 rxq_ctrl->flow_tunnels_n[j]--; 1198 break; 1199 } 1200 } 1201 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1202 } 1203 } 1204 } 1205 1206 /** 1207 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1208 * @p flow if no other flow uses it with the same kind of request. 1209 * 1210 * @param dev 1211 * Pointer to Ethernet device. 1212 * @param[in] flow 1213 * Pointer to the flow. 1214 */ 1215 static void 1216 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1217 { 1218 struct mlx5_priv *priv = dev->data->dev_private; 1219 uint32_t handle_idx; 1220 struct mlx5_flow_handle *dev_handle; 1221 1222 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1223 handle_idx, dev_handle, next) 1224 flow_drv_rxq_flags_trim(dev, dev_handle); 1225 } 1226 1227 /** 1228 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1229 * 1230 * @param dev 1231 * Pointer to Ethernet device. 1232 */ 1233 static void 1234 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1235 { 1236 struct mlx5_priv *priv = dev->data->dev_private; 1237 unsigned int i; 1238 1239 for (i = 0; i != priv->rxqs_n; ++i) { 1240 struct mlx5_rxq_ctrl *rxq_ctrl; 1241 unsigned int j; 1242 1243 if (!(*priv->rxqs)[i]) 1244 continue; 1245 rxq_ctrl = container_of((*priv->rxqs)[i], 1246 struct mlx5_rxq_ctrl, rxq); 1247 rxq_ctrl->flow_mark_n = 0; 1248 rxq_ctrl->rxq.mark = 0; 1249 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1250 rxq_ctrl->flow_tunnels_n[j] = 0; 1251 rxq_ctrl->rxq.tunnel = 0; 1252 } 1253 } 1254 1255 /** 1256 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1257 * 1258 * @param[in] dev 1259 * Pointer to the Ethernet device structure. 1260 */ 1261 void 1262 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1263 { 1264 struct mlx5_priv *priv = dev->data->dev_private; 1265 struct mlx5_rxq_data *data; 1266 unsigned int i; 1267 1268 for (i = 0; i != priv->rxqs_n; ++i) { 1269 if (!(*priv->rxqs)[i]) 1270 continue; 1271 data = (*priv->rxqs)[i]; 1272 if (!rte_flow_dynf_metadata_avail()) { 1273 data->dynf_meta = 0; 1274 data->flow_meta_mask = 0; 1275 data->flow_meta_offset = -1; 1276 data->flow_meta_port_mask = 0; 1277 } else { 1278 data->dynf_meta = 1; 1279 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1280 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1281 data->flow_meta_port_mask = (uint32_t)~0; 1282 if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16) 1283 data->flow_meta_port_mask >>= 16; 1284 } 1285 } 1286 } 1287 1288 /* 1289 * return a pointer to the desired action in the list of actions. 1290 * 1291 * @param[in] actions 1292 * The list of actions to search the action in. 1293 * @param[in] action 1294 * The action to find. 1295 * 1296 * @return 1297 * Pointer to the action in the list, if found. NULL otherwise. 1298 */ 1299 const struct rte_flow_action * 1300 mlx5_flow_find_action(const struct rte_flow_action *actions, 1301 enum rte_flow_action_type action) 1302 { 1303 if (actions == NULL) 1304 return NULL; 1305 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1306 if (actions->type == action) 1307 return actions; 1308 return NULL; 1309 } 1310 1311 /* 1312 * Validate the flag action. 1313 * 1314 * @param[in] action_flags 1315 * Bit-fields that holds the actions detected until now. 1316 * @param[in] attr 1317 * Attributes of flow that includes this action. 1318 * @param[out] error 1319 * Pointer to error structure. 1320 * 1321 * @return 1322 * 0 on success, a negative errno value otherwise and rte_errno is set. 1323 */ 1324 int 1325 mlx5_flow_validate_action_flag(uint64_t action_flags, 1326 const struct rte_flow_attr *attr, 1327 struct rte_flow_error *error) 1328 { 1329 if (action_flags & MLX5_FLOW_ACTION_MARK) 1330 return rte_flow_error_set(error, EINVAL, 1331 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1332 "can't mark and flag in same flow"); 1333 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1334 return rte_flow_error_set(error, EINVAL, 1335 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1336 "can't have 2 flag" 1337 " actions in same flow"); 1338 if (attr->egress) 1339 return rte_flow_error_set(error, ENOTSUP, 1340 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1341 "flag action not supported for " 1342 "egress"); 1343 return 0; 1344 } 1345 1346 /* 1347 * Validate the mark action. 1348 * 1349 * @param[in] action 1350 * Pointer to the queue action. 1351 * @param[in] action_flags 1352 * Bit-fields that holds the actions detected until now. 1353 * @param[in] attr 1354 * Attributes of flow that includes this action. 1355 * @param[out] error 1356 * Pointer to error structure. 1357 * 1358 * @return 1359 * 0 on success, a negative errno value otherwise and rte_errno is set. 1360 */ 1361 int 1362 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1363 uint64_t action_flags, 1364 const struct rte_flow_attr *attr, 1365 struct rte_flow_error *error) 1366 { 1367 const struct rte_flow_action_mark *mark = action->conf; 1368 1369 if (!mark) 1370 return rte_flow_error_set(error, EINVAL, 1371 RTE_FLOW_ERROR_TYPE_ACTION, 1372 action, 1373 "configuration cannot be null"); 1374 if (mark->id >= MLX5_FLOW_MARK_MAX) 1375 return rte_flow_error_set(error, EINVAL, 1376 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1377 &mark->id, 1378 "mark id must in 0 <= id < " 1379 RTE_STR(MLX5_FLOW_MARK_MAX)); 1380 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1381 return rte_flow_error_set(error, EINVAL, 1382 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1383 "can't flag and mark in same flow"); 1384 if (action_flags & MLX5_FLOW_ACTION_MARK) 1385 return rte_flow_error_set(error, EINVAL, 1386 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1387 "can't have 2 mark actions in same" 1388 " flow"); 1389 if (attr->egress) 1390 return rte_flow_error_set(error, ENOTSUP, 1391 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1392 "mark action not supported for " 1393 "egress"); 1394 return 0; 1395 } 1396 1397 /* 1398 * Validate the drop action. 1399 * 1400 * @param[in] action_flags 1401 * Bit-fields that holds the actions detected until now. 1402 * @param[in] attr 1403 * Attributes of flow that includes this action. 1404 * @param[out] error 1405 * Pointer to error structure. 1406 * 1407 * @return 1408 * 0 on success, a negative errno value otherwise and rte_errno is set. 1409 */ 1410 int 1411 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1412 const struct rte_flow_attr *attr, 1413 struct rte_flow_error *error) 1414 { 1415 if (attr->egress) 1416 return rte_flow_error_set(error, ENOTSUP, 1417 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1418 "drop action not supported for " 1419 "egress"); 1420 return 0; 1421 } 1422 1423 /* 1424 * Validate the queue action. 1425 * 1426 * @param[in] action 1427 * Pointer to the queue action. 1428 * @param[in] action_flags 1429 * Bit-fields that holds the actions detected until now. 1430 * @param[in] dev 1431 * Pointer to the Ethernet device structure. 1432 * @param[in] attr 1433 * Attributes of flow that includes this action. 1434 * @param[out] error 1435 * Pointer to error structure. 1436 * 1437 * @return 1438 * 0 on success, a negative errno value otherwise and rte_errno is set. 1439 */ 1440 int 1441 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1442 uint64_t action_flags, 1443 struct rte_eth_dev *dev, 1444 const struct rte_flow_attr *attr, 1445 struct rte_flow_error *error) 1446 { 1447 struct mlx5_priv *priv = dev->data->dev_private; 1448 const struct rte_flow_action_queue *queue = action->conf; 1449 1450 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1451 return rte_flow_error_set(error, EINVAL, 1452 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1453 "can't have 2 fate actions in" 1454 " same flow"); 1455 if (!priv->rxqs_n) 1456 return rte_flow_error_set(error, EINVAL, 1457 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1458 NULL, "No Rx queues configured"); 1459 if (queue->index >= priv->rxqs_n) 1460 return rte_flow_error_set(error, EINVAL, 1461 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1462 &queue->index, 1463 "queue index out of range"); 1464 if (!(*priv->rxqs)[queue->index]) 1465 return rte_flow_error_set(error, EINVAL, 1466 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1467 &queue->index, 1468 "queue is not configured"); 1469 if (attr->egress) 1470 return rte_flow_error_set(error, ENOTSUP, 1471 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1472 "queue action not supported for " 1473 "egress"); 1474 return 0; 1475 } 1476 1477 /* 1478 * Validate the rss action. 1479 * 1480 * @param[in] dev 1481 * Pointer to the Ethernet device structure. 1482 * @param[in] action 1483 * Pointer to the queue action. 1484 * @param[out] error 1485 * Pointer to error structure. 1486 * 1487 * @return 1488 * 0 on success, a negative errno value otherwise and rte_errno is set. 1489 */ 1490 int 1491 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1492 const struct rte_flow_action *action, 1493 struct rte_flow_error *error) 1494 { 1495 struct mlx5_priv *priv = dev->data->dev_private; 1496 const struct rte_flow_action_rss *rss = action->conf; 1497 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1498 unsigned int i; 1499 1500 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1501 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1502 return rte_flow_error_set(error, ENOTSUP, 1503 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1504 &rss->func, 1505 "RSS hash function not supported"); 1506 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1507 if (rss->level > 2) 1508 #else 1509 if (rss->level > 1) 1510 #endif 1511 return rte_flow_error_set(error, ENOTSUP, 1512 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1513 &rss->level, 1514 "tunnel RSS is not supported"); 1515 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1516 if (rss->key_len == 0 && rss->key != NULL) 1517 return rte_flow_error_set(error, ENOTSUP, 1518 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1519 &rss->key_len, 1520 "RSS hash key length 0"); 1521 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1522 return rte_flow_error_set(error, ENOTSUP, 1523 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1524 &rss->key_len, 1525 "RSS hash key too small"); 1526 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1527 return rte_flow_error_set(error, ENOTSUP, 1528 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1529 &rss->key_len, 1530 "RSS hash key too large"); 1531 if (rss->queue_num > priv->config.ind_table_max_size) 1532 return rte_flow_error_set(error, ENOTSUP, 1533 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1534 &rss->queue_num, 1535 "number of queues too large"); 1536 if (rss->types & MLX5_RSS_HF_MASK) 1537 return rte_flow_error_set(error, ENOTSUP, 1538 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1539 &rss->types, 1540 "some RSS protocols are not" 1541 " supported"); 1542 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1543 !(rss->types & ETH_RSS_IP)) 1544 return rte_flow_error_set(error, EINVAL, 1545 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1546 "L3 partial RSS requested but L3 RSS" 1547 " type not specified"); 1548 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1549 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1550 return rte_flow_error_set(error, EINVAL, 1551 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1552 "L4 partial RSS requested but L4 RSS" 1553 " type not specified"); 1554 if (!priv->rxqs_n) 1555 return rte_flow_error_set(error, EINVAL, 1556 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1557 NULL, "No Rx queues configured"); 1558 if (!rss->queue_num) 1559 return rte_flow_error_set(error, EINVAL, 1560 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1561 NULL, "No queues configured"); 1562 for (i = 0; i != rss->queue_num; ++i) { 1563 struct mlx5_rxq_ctrl *rxq_ctrl; 1564 1565 if (rss->queue[i] >= priv->rxqs_n) 1566 return rte_flow_error_set 1567 (error, EINVAL, 1568 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1569 &rss->queue[i], "queue index out of range"); 1570 if (!(*priv->rxqs)[rss->queue[i]]) 1571 return rte_flow_error_set 1572 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1573 &rss->queue[i], "queue is not configured"); 1574 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1575 struct mlx5_rxq_ctrl, rxq); 1576 if (i == 0) 1577 rxq_type = rxq_ctrl->type; 1578 if (rxq_type != rxq_ctrl->type) 1579 return rte_flow_error_set 1580 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1581 &rss->queue[i], 1582 "combining hairpin and regular RSS queues is not supported"); 1583 } 1584 return 0; 1585 } 1586 1587 /* 1588 * Validate the rss action. 1589 * 1590 * @param[in] action 1591 * Pointer to the queue action. 1592 * @param[in] action_flags 1593 * Bit-fields that holds the actions detected until now. 1594 * @param[in] dev 1595 * Pointer to the Ethernet device structure. 1596 * @param[in] attr 1597 * Attributes of flow that includes this action. 1598 * @param[in] item_flags 1599 * Items that were detected. 1600 * @param[out] error 1601 * Pointer to error structure. 1602 * 1603 * @return 1604 * 0 on success, a negative errno value otherwise and rte_errno is set. 1605 */ 1606 int 1607 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1608 uint64_t action_flags, 1609 struct rte_eth_dev *dev, 1610 const struct rte_flow_attr *attr, 1611 uint64_t item_flags, 1612 struct rte_flow_error *error) 1613 { 1614 const struct rte_flow_action_rss *rss = action->conf; 1615 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1616 int ret; 1617 1618 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1619 return rte_flow_error_set(error, EINVAL, 1620 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1621 "can't have 2 fate actions" 1622 " in same flow"); 1623 ret = mlx5_validate_action_rss(dev, action, error); 1624 if (ret) 1625 return ret; 1626 if (attr->egress) 1627 return rte_flow_error_set(error, ENOTSUP, 1628 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1629 "rss action not supported for " 1630 "egress"); 1631 if (rss->level > 1 && !tunnel) 1632 return rte_flow_error_set(error, EINVAL, 1633 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1634 "inner RSS is not supported for " 1635 "non-tunnel flows"); 1636 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1637 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1638 return rte_flow_error_set(error, EINVAL, 1639 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1640 "RSS on eCPRI is not supported now"); 1641 } 1642 return 0; 1643 } 1644 1645 /* 1646 * Validate the default miss action. 1647 * 1648 * @param[in] action_flags 1649 * Bit-fields that holds the actions detected until now. 1650 * @param[out] error 1651 * Pointer to error structure. 1652 * 1653 * @return 1654 * 0 on success, a negative errno value otherwise and rte_errno is set. 1655 */ 1656 int 1657 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1658 const struct rte_flow_attr *attr, 1659 struct rte_flow_error *error) 1660 { 1661 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1662 return rte_flow_error_set(error, EINVAL, 1663 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1664 "can't have 2 fate actions in" 1665 " same flow"); 1666 if (attr->egress) 1667 return rte_flow_error_set(error, ENOTSUP, 1668 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1669 "default miss action not supported " 1670 "for egress"); 1671 if (attr->group) 1672 return rte_flow_error_set(error, ENOTSUP, 1673 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1674 "only group 0 is supported"); 1675 if (attr->transfer) 1676 return rte_flow_error_set(error, ENOTSUP, 1677 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1678 NULL, "transfer is not supported"); 1679 return 0; 1680 } 1681 1682 /* 1683 * Validate the count action. 1684 * 1685 * @param[in] dev 1686 * Pointer to the Ethernet device structure. 1687 * @param[in] attr 1688 * Attributes of flow that includes this action. 1689 * @param[out] error 1690 * Pointer to error structure. 1691 * 1692 * @return 1693 * 0 on success, a negative errno value otherwise and rte_errno is set. 1694 */ 1695 int 1696 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1697 const struct rte_flow_attr *attr, 1698 struct rte_flow_error *error) 1699 { 1700 if (attr->egress) 1701 return rte_flow_error_set(error, ENOTSUP, 1702 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1703 "count action not supported for " 1704 "egress"); 1705 return 0; 1706 } 1707 1708 /* 1709 * Validate the ASO CT action. 1710 * 1711 * @param[in] dev 1712 * Pointer to the Ethernet device structure. 1713 * @param[in] conntrack 1714 * Pointer to the CT action profile. 1715 * @param[out] error 1716 * Pointer to error structure. 1717 * 1718 * @return 1719 * 0 on success, a negative errno value otherwise and rte_errno is set. 1720 */ 1721 int 1722 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1723 const struct rte_flow_action_conntrack *conntrack, 1724 struct rte_flow_error *error) 1725 { 1726 RTE_SET_USED(dev); 1727 1728 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1729 return rte_flow_error_set(error, EINVAL, 1730 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1731 "Invalid CT state"); 1732 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1733 return rte_flow_error_set(error, EINVAL, 1734 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1735 "Invalid last TCP packet flag"); 1736 return 0; 1737 } 1738 1739 /** 1740 * Verify the @p attributes will be correctly understood by the NIC and store 1741 * them in the @p flow if everything is correct. 1742 * 1743 * @param[in] dev 1744 * Pointer to the Ethernet device structure. 1745 * @param[in] attributes 1746 * Pointer to flow attributes 1747 * @param[out] error 1748 * Pointer to error structure. 1749 * 1750 * @return 1751 * 0 on success, a negative errno value otherwise and rte_errno is set. 1752 */ 1753 int 1754 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1755 const struct rte_flow_attr *attributes, 1756 struct rte_flow_error *error) 1757 { 1758 struct mlx5_priv *priv = dev->data->dev_private; 1759 uint32_t priority_max = priv->config.flow_prio - 1; 1760 1761 if (attributes->group) 1762 return rte_flow_error_set(error, ENOTSUP, 1763 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1764 NULL, "groups is not supported"); 1765 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1766 attributes->priority >= priority_max) 1767 return rte_flow_error_set(error, ENOTSUP, 1768 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1769 NULL, "priority out of range"); 1770 if (attributes->egress) 1771 return rte_flow_error_set(error, ENOTSUP, 1772 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1773 "egress is not supported"); 1774 if (attributes->transfer && !priv->config.dv_esw_en) 1775 return rte_flow_error_set(error, ENOTSUP, 1776 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1777 NULL, "transfer is not supported"); 1778 if (!attributes->ingress) 1779 return rte_flow_error_set(error, EINVAL, 1780 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1781 NULL, 1782 "ingress attribute is mandatory"); 1783 return 0; 1784 } 1785 1786 /** 1787 * Validate ICMP6 item. 1788 * 1789 * @param[in] item 1790 * Item specification. 1791 * @param[in] item_flags 1792 * Bit-fields that holds the items detected until now. 1793 * @param[in] ext_vlan_sup 1794 * Whether extended VLAN features are supported or not. 1795 * @param[out] error 1796 * Pointer to error structure. 1797 * 1798 * @return 1799 * 0 on success, a negative errno value otherwise and rte_errno is set. 1800 */ 1801 int 1802 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1803 uint64_t item_flags, 1804 uint8_t target_protocol, 1805 struct rte_flow_error *error) 1806 { 1807 const struct rte_flow_item_icmp6 *mask = item->mask; 1808 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1809 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1810 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1811 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1812 MLX5_FLOW_LAYER_OUTER_L4; 1813 int ret; 1814 1815 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1816 return rte_flow_error_set(error, EINVAL, 1817 RTE_FLOW_ERROR_TYPE_ITEM, item, 1818 "protocol filtering not compatible" 1819 " with ICMP6 layer"); 1820 if (!(item_flags & l3m)) 1821 return rte_flow_error_set(error, EINVAL, 1822 RTE_FLOW_ERROR_TYPE_ITEM, item, 1823 "IPv6 is mandatory to filter on" 1824 " ICMP6"); 1825 if (item_flags & l4m) 1826 return rte_flow_error_set(error, EINVAL, 1827 RTE_FLOW_ERROR_TYPE_ITEM, item, 1828 "multiple L4 layers not supported"); 1829 if (!mask) 1830 mask = &rte_flow_item_icmp6_mask; 1831 ret = mlx5_flow_item_acceptable 1832 (item, (const uint8_t *)mask, 1833 (const uint8_t *)&rte_flow_item_icmp6_mask, 1834 sizeof(struct rte_flow_item_icmp6), 1835 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1836 if (ret < 0) 1837 return ret; 1838 return 0; 1839 } 1840 1841 /** 1842 * Validate ICMP item. 1843 * 1844 * @param[in] item 1845 * Item specification. 1846 * @param[in] item_flags 1847 * Bit-fields that holds the items detected until now. 1848 * @param[out] error 1849 * Pointer to error structure. 1850 * 1851 * @return 1852 * 0 on success, a negative errno value otherwise and rte_errno is set. 1853 */ 1854 int 1855 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1856 uint64_t item_flags, 1857 uint8_t target_protocol, 1858 struct rte_flow_error *error) 1859 { 1860 const struct rte_flow_item_icmp *mask = item->mask; 1861 const struct rte_flow_item_icmp nic_mask = { 1862 .hdr.icmp_type = 0xff, 1863 .hdr.icmp_code = 0xff, 1864 .hdr.icmp_ident = RTE_BE16(0xffff), 1865 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1866 }; 1867 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1868 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1869 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1870 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1871 MLX5_FLOW_LAYER_OUTER_L4; 1872 int ret; 1873 1874 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1875 return rte_flow_error_set(error, EINVAL, 1876 RTE_FLOW_ERROR_TYPE_ITEM, item, 1877 "protocol filtering not compatible" 1878 " with ICMP layer"); 1879 if (!(item_flags & l3m)) 1880 return rte_flow_error_set(error, EINVAL, 1881 RTE_FLOW_ERROR_TYPE_ITEM, item, 1882 "IPv4 is mandatory to filter" 1883 " on ICMP"); 1884 if (item_flags & l4m) 1885 return rte_flow_error_set(error, EINVAL, 1886 RTE_FLOW_ERROR_TYPE_ITEM, item, 1887 "multiple L4 layers not supported"); 1888 if (!mask) 1889 mask = &nic_mask; 1890 ret = mlx5_flow_item_acceptable 1891 (item, (const uint8_t *)mask, 1892 (const uint8_t *)&nic_mask, 1893 sizeof(struct rte_flow_item_icmp), 1894 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1895 if (ret < 0) 1896 return ret; 1897 return 0; 1898 } 1899 1900 /** 1901 * Validate Ethernet item. 1902 * 1903 * @param[in] item 1904 * Item specification. 1905 * @param[in] item_flags 1906 * Bit-fields that holds the items detected until now. 1907 * @param[out] error 1908 * Pointer to error structure. 1909 * 1910 * @return 1911 * 0 on success, a negative errno value otherwise and rte_errno is set. 1912 */ 1913 int 1914 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1915 uint64_t item_flags, bool ext_vlan_sup, 1916 struct rte_flow_error *error) 1917 { 1918 const struct rte_flow_item_eth *mask = item->mask; 1919 const struct rte_flow_item_eth nic_mask = { 1920 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1921 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1922 .type = RTE_BE16(0xffff), 1923 .has_vlan = ext_vlan_sup ? 1 : 0, 1924 }; 1925 int ret; 1926 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1927 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1928 MLX5_FLOW_LAYER_OUTER_L2; 1929 1930 if (item_flags & ethm) 1931 return rte_flow_error_set(error, ENOTSUP, 1932 RTE_FLOW_ERROR_TYPE_ITEM, item, 1933 "multiple L2 layers not supported"); 1934 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1935 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1936 return rte_flow_error_set(error, EINVAL, 1937 RTE_FLOW_ERROR_TYPE_ITEM, item, 1938 "L2 layer should not follow " 1939 "L3 layers"); 1940 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1941 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1942 return rte_flow_error_set(error, EINVAL, 1943 RTE_FLOW_ERROR_TYPE_ITEM, item, 1944 "L2 layer should not follow VLAN"); 1945 if (!mask) 1946 mask = &rte_flow_item_eth_mask; 1947 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1948 (const uint8_t *)&nic_mask, 1949 sizeof(struct rte_flow_item_eth), 1950 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1951 return ret; 1952 } 1953 1954 /** 1955 * Validate VLAN item. 1956 * 1957 * @param[in] item 1958 * Item specification. 1959 * @param[in] item_flags 1960 * Bit-fields that holds the items detected until now. 1961 * @param[in] dev 1962 * Ethernet device flow is being created on. 1963 * @param[out] error 1964 * Pointer to error structure. 1965 * 1966 * @return 1967 * 0 on success, a negative errno value otherwise and rte_errno is set. 1968 */ 1969 int 1970 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1971 uint64_t item_flags, 1972 struct rte_eth_dev *dev, 1973 struct rte_flow_error *error) 1974 { 1975 const struct rte_flow_item_vlan *spec = item->spec; 1976 const struct rte_flow_item_vlan *mask = item->mask; 1977 const struct rte_flow_item_vlan nic_mask = { 1978 .tci = RTE_BE16(UINT16_MAX), 1979 .inner_type = RTE_BE16(UINT16_MAX), 1980 }; 1981 uint16_t vlan_tag = 0; 1982 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1983 int ret; 1984 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1985 MLX5_FLOW_LAYER_INNER_L4) : 1986 (MLX5_FLOW_LAYER_OUTER_L3 | 1987 MLX5_FLOW_LAYER_OUTER_L4); 1988 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1989 MLX5_FLOW_LAYER_OUTER_VLAN; 1990 1991 if (item_flags & vlanm) 1992 return rte_flow_error_set(error, EINVAL, 1993 RTE_FLOW_ERROR_TYPE_ITEM, item, 1994 "multiple VLAN layers not supported"); 1995 else if ((item_flags & l34m) != 0) 1996 return rte_flow_error_set(error, EINVAL, 1997 RTE_FLOW_ERROR_TYPE_ITEM, item, 1998 "VLAN cannot follow L3/L4 layer"); 1999 if (!mask) 2000 mask = &rte_flow_item_vlan_mask; 2001 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2002 (const uint8_t *)&nic_mask, 2003 sizeof(struct rte_flow_item_vlan), 2004 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2005 if (ret) 2006 return ret; 2007 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2008 struct mlx5_priv *priv = dev->data->dev_private; 2009 2010 if (priv->vmwa_context) { 2011 /* 2012 * Non-NULL context means we have a virtual machine 2013 * and SR-IOV enabled, we have to create VLAN interface 2014 * to make hypervisor to setup E-Switch vport 2015 * context correctly. We avoid creating the multiple 2016 * VLAN interfaces, so we cannot support VLAN tag mask. 2017 */ 2018 return rte_flow_error_set(error, EINVAL, 2019 RTE_FLOW_ERROR_TYPE_ITEM, 2020 item, 2021 "VLAN tag mask is not" 2022 " supported in virtual" 2023 " environment"); 2024 } 2025 } 2026 if (spec) { 2027 vlan_tag = spec->tci; 2028 vlan_tag &= mask->tci; 2029 } 2030 /* 2031 * From verbs perspective an empty VLAN is equivalent 2032 * to a packet without VLAN layer. 2033 */ 2034 if (!vlan_tag) 2035 return rte_flow_error_set(error, EINVAL, 2036 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2037 item->spec, 2038 "VLAN cannot be empty"); 2039 return 0; 2040 } 2041 2042 /** 2043 * Validate IPV4 item. 2044 * 2045 * @param[in] item 2046 * Item specification. 2047 * @param[in] item_flags 2048 * Bit-fields that holds the items detected until now. 2049 * @param[in] last_item 2050 * Previous validated item in the pattern items. 2051 * @param[in] ether_type 2052 * Type in the ethernet layer header (including dot1q). 2053 * @param[in] acc_mask 2054 * Acceptable mask, if NULL default internal default mask 2055 * will be used to check whether item fields are supported. 2056 * @param[in] range_accepted 2057 * True if range of values is accepted for specific fields, false otherwise. 2058 * @param[out] error 2059 * Pointer to error structure. 2060 * 2061 * @return 2062 * 0 on success, a negative errno value otherwise and rte_errno is set. 2063 */ 2064 int 2065 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2066 uint64_t item_flags, 2067 uint64_t last_item, 2068 uint16_t ether_type, 2069 const struct rte_flow_item_ipv4 *acc_mask, 2070 bool range_accepted, 2071 struct rte_flow_error *error) 2072 { 2073 const struct rte_flow_item_ipv4 *mask = item->mask; 2074 const struct rte_flow_item_ipv4 *spec = item->spec; 2075 const struct rte_flow_item_ipv4 nic_mask = { 2076 .hdr = { 2077 .src_addr = RTE_BE32(0xffffffff), 2078 .dst_addr = RTE_BE32(0xffffffff), 2079 .type_of_service = 0xff, 2080 .next_proto_id = 0xff, 2081 }, 2082 }; 2083 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2084 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2085 MLX5_FLOW_LAYER_OUTER_L3; 2086 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2087 MLX5_FLOW_LAYER_OUTER_L4; 2088 int ret; 2089 uint8_t next_proto = 0xFF; 2090 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2091 MLX5_FLOW_LAYER_OUTER_VLAN | 2092 MLX5_FLOW_LAYER_INNER_VLAN); 2093 2094 if ((last_item & l2_vlan) && ether_type && 2095 ether_type != RTE_ETHER_TYPE_IPV4) 2096 return rte_flow_error_set(error, EINVAL, 2097 RTE_FLOW_ERROR_TYPE_ITEM, item, 2098 "IPv4 cannot follow L2/VLAN layer " 2099 "which ether type is not IPv4"); 2100 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2101 if (mask && spec) 2102 next_proto = mask->hdr.next_proto_id & 2103 spec->hdr.next_proto_id; 2104 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2105 return rte_flow_error_set(error, EINVAL, 2106 RTE_FLOW_ERROR_TYPE_ITEM, 2107 item, 2108 "multiple tunnel " 2109 "not supported"); 2110 } 2111 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2112 return rte_flow_error_set(error, EINVAL, 2113 RTE_FLOW_ERROR_TYPE_ITEM, item, 2114 "wrong tunnel type - IPv6 specified " 2115 "but IPv4 item provided"); 2116 if (item_flags & l3m) 2117 return rte_flow_error_set(error, ENOTSUP, 2118 RTE_FLOW_ERROR_TYPE_ITEM, item, 2119 "multiple L3 layers not supported"); 2120 else if (item_flags & l4m) 2121 return rte_flow_error_set(error, EINVAL, 2122 RTE_FLOW_ERROR_TYPE_ITEM, item, 2123 "L3 cannot follow an L4 layer."); 2124 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2125 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2126 return rte_flow_error_set(error, EINVAL, 2127 RTE_FLOW_ERROR_TYPE_ITEM, item, 2128 "L3 cannot follow an NVGRE layer."); 2129 if (!mask) 2130 mask = &rte_flow_item_ipv4_mask; 2131 else if (mask->hdr.next_proto_id != 0 && 2132 mask->hdr.next_proto_id != 0xff) 2133 return rte_flow_error_set(error, EINVAL, 2134 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2135 "partial mask is not supported" 2136 " for protocol"); 2137 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2138 acc_mask ? (const uint8_t *)acc_mask 2139 : (const uint8_t *)&nic_mask, 2140 sizeof(struct rte_flow_item_ipv4), 2141 range_accepted, error); 2142 if (ret < 0) 2143 return ret; 2144 return 0; 2145 } 2146 2147 /** 2148 * Validate IPV6 item. 2149 * 2150 * @param[in] item 2151 * Item specification. 2152 * @param[in] item_flags 2153 * Bit-fields that holds the items detected until now. 2154 * @param[in] last_item 2155 * Previous validated item in the pattern items. 2156 * @param[in] ether_type 2157 * Type in the ethernet layer header (including dot1q). 2158 * @param[in] acc_mask 2159 * Acceptable mask, if NULL default internal default mask 2160 * will be used to check whether item fields are supported. 2161 * @param[out] error 2162 * Pointer to error structure. 2163 * 2164 * @return 2165 * 0 on success, a negative errno value otherwise and rte_errno is set. 2166 */ 2167 int 2168 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2169 uint64_t item_flags, 2170 uint64_t last_item, 2171 uint16_t ether_type, 2172 const struct rte_flow_item_ipv6 *acc_mask, 2173 struct rte_flow_error *error) 2174 { 2175 const struct rte_flow_item_ipv6 *mask = item->mask; 2176 const struct rte_flow_item_ipv6 *spec = item->spec; 2177 const struct rte_flow_item_ipv6 nic_mask = { 2178 .hdr = { 2179 .src_addr = 2180 "\xff\xff\xff\xff\xff\xff\xff\xff" 2181 "\xff\xff\xff\xff\xff\xff\xff\xff", 2182 .dst_addr = 2183 "\xff\xff\xff\xff\xff\xff\xff\xff" 2184 "\xff\xff\xff\xff\xff\xff\xff\xff", 2185 .vtc_flow = RTE_BE32(0xffffffff), 2186 .proto = 0xff, 2187 }, 2188 }; 2189 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2190 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2191 MLX5_FLOW_LAYER_OUTER_L3; 2192 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2193 MLX5_FLOW_LAYER_OUTER_L4; 2194 int ret; 2195 uint8_t next_proto = 0xFF; 2196 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2197 MLX5_FLOW_LAYER_OUTER_VLAN | 2198 MLX5_FLOW_LAYER_INNER_VLAN); 2199 2200 if ((last_item & l2_vlan) && ether_type && 2201 ether_type != RTE_ETHER_TYPE_IPV6) 2202 return rte_flow_error_set(error, EINVAL, 2203 RTE_FLOW_ERROR_TYPE_ITEM, item, 2204 "IPv6 cannot follow L2/VLAN layer " 2205 "which ether type is not IPv6"); 2206 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2207 next_proto = spec->hdr.proto; 2208 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 2209 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2210 return rte_flow_error_set(error, EINVAL, 2211 RTE_FLOW_ERROR_TYPE_ITEM, 2212 item, 2213 "multiple tunnel " 2214 "not supported"); 2215 } 2216 if (next_proto == IPPROTO_HOPOPTS || 2217 next_proto == IPPROTO_ROUTING || 2218 next_proto == IPPROTO_FRAGMENT || 2219 next_proto == IPPROTO_ESP || 2220 next_proto == IPPROTO_AH || 2221 next_proto == IPPROTO_DSTOPTS) 2222 return rte_flow_error_set(error, EINVAL, 2223 RTE_FLOW_ERROR_TYPE_ITEM, item, 2224 "IPv6 proto (next header) should " 2225 "not be set as extension header"); 2226 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2227 return rte_flow_error_set(error, EINVAL, 2228 RTE_FLOW_ERROR_TYPE_ITEM, item, 2229 "wrong tunnel type - IPv4 specified " 2230 "but IPv6 item provided"); 2231 if (item_flags & l3m) 2232 return rte_flow_error_set(error, ENOTSUP, 2233 RTE_FLOW_ERROR_TYPE_ITEM, item, 2234 "multiple L3 layers not supported"); 2235 else if (item_flags & l4m) 2236 return rte_flow_error_set(error, EINVAL, 2237 RTE_FLOW_ERROR_TYPE_ITEM, item, 2238 "L3 cannot follow an L4 layer."); 2239 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2240 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2241 return rte_flow_error_set(error, EINVAL, 2242 RTE_FLOW_ERROR_TYPE_ITEM, item, 2243 "L3 cannot follow an NVGRE layer."); 2244 if (!mask) 2245 mask = &rte_flow_item_ipv6_mask; 2246 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2247 acc_mask ? (const uint8_t *)acc_mask 2248 : (const uint8_t *)&nic_mask, 2249 sizeof(struct rte_flow_item_ipv6), 2250 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2251 if (ret < 0) 2252 return ret; 2253 return 0; 2254 } 2255 2256 /** 2257 * Validate UDP item. 2258 * 2259 * @param[in] item 2260 * Item specification. 2261 * @param[in] item_flags 2262 * Bit-fields that holds the items detected until now. 2263 * @param[in] target_protocol 2264 * The next protocol in the previous item. 2265 * @param[in] flow_mask 2266 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2267 * @param[out] error 2268 * Pointer to error structure. 2269 * 2270 * @return 2271 * 0 on success, a negative errno value otherwise and rte_errno is set. 2272 */ 2273 int 2274 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2275 uint64_t item_flags, 2276 uint8_t target_protocol, 2277 struct rte_flow_error *error) 2278 { 2279 const struct rte_flow_item_udp *mask = item->mask; 2280 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2281 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2282 MLX5_FLOW_LAYER_OUTER_L3; 2283 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2284 MLX5_FLOW_LAYER_OUTER_L4; 2285 int ret; 2286 2287 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2288 return rte_flow_error_set(error, EINVAL, 2289 RTE_FLOW_ERROR_TYPE_ITEM, item, 2290 "protocol filtering not compatible" 2291 " with UDP layer"); 2292 if (!(item_flags & l3m)) 2293 return rte_flow_error_set(error, EINVAL, 2294 RTE_FLOW_ERROR_TYPE_ITEM, item, 2295 "L3 is mandatory to filter on L4"); 2296 if (item_flags & l4m) 2297 return rte_flow_error_set(error, EINVAL, 2298 RTE_FLOW_ERROR_TYPE_ITEM, item, 2299 "multiple L4 layers not supported"); 2300 if (!mask) 2301 mask = &rte_flow_item_udp_mask; 2302 ret = mlx5_flow_item_acceptable 2303 (item, (const uint8_t *)mask, 2304 (const uint8_t *)&rte_flow_item_udp_mask, 2305 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2306 error); 2307 if (ret < 0) 2308 return ret; 2309 return 0; 2310 } 2311 2312 /** 2313 * Validate TCP item. 2314 * 2315 * @param[in] item 2316 * Item specification. 2317 * @param[in] item_flags 2318 * Bit-fields that holds the items detected until now. 2319 * @param[in] target_protocol 2320 * The next protocol in the previous item. 2321 * @param[out] error 2322 * Pointer to error structure. 2323 * 2324 * @return 2325 * 0 on success, a negative errno value otherwise and rte_errno is set. 2326 */ 2327 int 2328 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2329 uint64_t item_flags, 2330 uint8_t target_protocol, 2331 const struct rte_flow_item_tcp *flow_mask, 2332 struct rte_flow_error *error) 2333 { 2334 const struct rte_flow_item_tcp *mask = item->mask; 2335 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2336 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2337 MLX5_FLOW_LAYER_OUTER_L3; 2338 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2339 MLX5_FLOW_LAYER_OUTER_L4; 2340 int ret; 2341 2342 MLX5_ASSERT(flow_mask); 2343 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2344 return rte_flow_error_set(error, EINVAL, 2345 RTE_FLOW_ERROR_TYPE_ITEM, item, 2346 "protocol filtering not compatible" 2347 " with TCP layer"); 2348 if (!(item_flags & l3m)) 2349 return rte_flow_error_set(error, EINVAL, 2350 RTE_FLOW_ERROR_TYPE_ITEM, item, 2351 "L3 is mandatory to filter on L4"); 2352 if (item_flags & l4m) 2353 return rte_flow_error_set(error, EINVAL, 2354 RTE_FLOW_ERROR_TYPE_ITEM, item, 2355 "multiple L4 layers not supported"); 2356 if (!mask) 2357 mask = &rte_flow_item_tcp_mask; 2358 ret = mlx5_flow_item_acceptable 2359 (item, (const uint8_t *)mask, 2360 (const uint8_t *)flow_mask, 2361 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2362 error); 2363 if (ret < 0) 2364 return ret; 2365 return 0; 2366 } 2367 2368 /** 2369 * Validate VXLAN item. 2370 * 2371 * @param[in] item 2372 * Item specification. 2373 * @param[in] item_flags 2374 * Bit-fields that holds the items detected until now. 2375 * @param[in] target_protocol 2376 * The next protocol in the previous item. 2377 * @param[out] error 2378 * Pointer to error structure. 2379 * 2380 * @return 2381 * 0 on success, a negative errno value otherwise and rte_errno is set. 2382 */ 2383 int 2384 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2385 uint64_t item_flags, 2386 struct rte_flow_error *error) 2387 { 2388 const struct rte_flow_item_vxlan *spec = item->spec; 2389 const struct rte_flow_item_vxlan *mask = item->mask; 2390 int ret; 2391 union vni { 2392 uint32_t vlan_id; 2393 uint8_t vni[4]; 2394 } id = { .vlan_id = 0, }; 2395 2396 2397 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2398 return rte_flow_error_set(error, ENOTSUP, 2399 RTE_FLOW_ERROR_TYPE_ITEM, item, 2400 "multiple tunnel layers not" 2401 " supported"); 2402 /* 2403 * Verify only UDPv4 is present as defined in 2404 * https://tools.ietf.org/html/rfc7348 2405 */ 2406 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2407 return rte_flow_error_set(error, EINVAL, 2408 RTE_FLOW_ERROR_TYPE_ITEM, item, 2409 "no outer UDP layer found"); 2410 if (!mask) 2411 mask = &rte_flow_item_vxlan_mask; 2412 ret = mlx5_flow_item_acceptable 2413 (item, (const uint8_t *)mask, 2414 (const uint8_t *)&rte_flow_item_vxlan_mask, 2415 sizeof(struct rte_flow_item_vxlan), 2416 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2417 if (ret < 0) 2418 return ret; 2419 if (spec) { 2420 memcpy(&id.vni[1], spec->vni, 3); 2421 memcpy(&id.vni[1], mask->vni, 3); 2422 } 2423 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2424 return rte_flow_error_set(error, ENOTSUP, 2425 RTE_FLOW_ERROR_TYPE_ITEM, item, 2426 "VXLAN tunnel must be fully defined"); 2427 return 0; 2428 } 2429 2430 /** 2431 * Validate VXLAN_GPE item. 2432 * 2433 * @param[in] item 2434 * Item specification. 2435 * @param[in] item_flags 2436 * Bit-fields that holds the items detected until now. 2437 * @param[in] priv 2438 * Pointer to the private data structure. 2439 * @param[in] target_protocol 2440 * The next protocol in the previous item. 2441 * @param[out] error 2442 * Pointer to error structure. 2443 * 2444 * @return 2445 * 0 on success, a negative errno value otherwise and rte_errno is set. 2446 */ 2447 int 2448 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2449 uint64_t item_flags, 2450 struct rte_eth_dev *dev, 2451 struct rte_flow_error *error) 2452 { 2453 struct mlx5_priv *priv = dev->data->dev_private; 2454 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2455 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2456 int ret; 2457 union vni { 2458 uint32_t vlan_id; 2459 uint8_t vni[4]; 2460 } id = { .vlan_id = 0, }; 2461 2462 if (!priv->config.l3_vxlan_en) 2463 return rte_flow_error_set(error, ENOTSUP, 2464 RTE_FLOW_ERROR_TYPE_ITEM, item, 2465 "L3 VXLAN is not enabled by device" 2466 " parameter and/or not configured in" 2467 " firmware"); 2468 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2469 return rte_flow_error_set(error, ENOTSUP, 2470 RTE_FLOW_ERROR_TYPE_ITEM, item, 2471 "multiple tunnel layers not" 2472 " supported"); 2473 /* 2474 * Verify only UDPv4 is present as defined in 2475 * https://tools.ietf.org/html/rfc7348 2476 */ 2477 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2478 return rte_flow_error_set(error, EINVAL, 2479 RTE_FLOW_ERROR_TYPE_ITEM, item, 2480 "no outer UDP layer found"); 2481 if (!mask) 2482 mask = &rte_flow_item_vxlan_gpe_mask; 2483 ret = mlx5_flow_item_acceptable 2484 (item, (const uint8_t *)mask, 2485 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2486 sizeof(struct rte_flow_item_vxlan_gpe), 2487 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2488 if (ret < 0) 2489 return ret; 2490 if (spec) { 2491 if (spec->protocol) 2492 return rte_flow_error_set(error, ENOTSUP, 2493 RTE_FLOW_ERROR_TYPE_ITEM, 2494 item, 2495 "VxLAN-GPE protocol" 2496 " not supported"); 2497 memcpy(&id.vni[1], spec->vni, 3); 2498 memcpy(&id.vni[1], mask->vni, 3); 2499 } 2500 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2501 return rte_flow_error_set(error, ENOTSUP, 2502 RTE_FLOW_ERROR_TYPE_ITEM, item, 2503 "VXLAN-GPE tunnel must be fully" 2504 " defined"); 2505 return 0; 2506 } 2507 /** 2508 * Validate GRE Key item. 2509 * 2510 * @param[in] item 2511 * Item specification. 2512 * @param[in] item_flags 2513 * Bit flags to mark detected items. 2514 * @param[in] gre_item 2515 * Pointer to gre_item 2516 * @param[out] error 2517 * Pointer to error structure. 2518 * 2519 * @return 2520 * 0 on success, a negative errno value otherwise and rte_errno is set. 2521 */ 2522 int 2523 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2524 uint64_t item_flags, 2525 const struct rte_flow_item *gre_item, 2526 struct rte_flow_error *error) 2527 { 2528 const rte_be32_t *mask = item->mask; 2529 int ret = 0; 2530 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2531 const struct rte_flow_item_gre *gre_spec; 2532 const struct rte_flow_item_gre *gre_mask; 2533 2534 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2535 return rte_flow_error_set(error, ENOTSUP, 2536 RTE_FLOW_ERROR_TYPE_ITEM, item, 2537 "Multiple GRE key not support"); 2538 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2539 return rte_flow_error_set(error, ENOTSUP, 2540 RTE_FLOW_ERROR_TYPE_ITEM, item, 2541 "No preceding GRE header"); 2542 if (item_flags & MLX5_FLOW_LAYER_INNER) 2543 return rte_flow_error_set(error, ENOTSUP, 2544 RTE_FLOW_ERROR_TYPE_ITEM, item, 2545 "GRE key following a wrong item"); 2546 gre_mask = gre_item->mask; 2547 if (!gre_mask) 2548 gre_mask = &rte_flow_item_gre_mask; 2549 gre_spec = gre_item->spec; 2550 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2551 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2552 return rte_flow_error_set(error, EINVAL, 2553 RTE_FLOW_ERROR_TYPE_ITEM, item, 2554 "Key bit must be on"); 2555 2556 if (!mask) 2557 mask = &gre_key_default_mask; 2558 ret = mlx5_flow_item_acceptable 2559 (item, (const uint8_t *)mask, 2560 (const uint8_t *)&gre_key_default_mask, 2561 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2562 return ret; 2563 } 2564 2565 /** 2566 * Validate GRE item. 2567 * 2568 * @param[in] item 2569 * Item specification. 2570 * @param[in] item_flags 2571 * Bit flags to mark detected items. 2572 * @param[in] target_protocol 2573 * The next protocol in the previous item. 2574 * @param[out] error 2575 * Pointer to error structure. 2576 * 2577 * @return 2578 * 0 on success, a negative errno value otherwise and rte_errno is set. 2579 */ 2580 int 2581 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2582 uint64_t item_flags, 2583 uint8_t target_protocol, 2584 struct rte_flow_error *error) 2585 { 2586 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2587 const struct rte_flow_item_gre *mask = item->mask; 2588 int ret; 2589 const struct rte_flow_item_gre nic_mask = { 2590 .c_rsvd0_ver = RTE_BE16(0xB000), 2591 .protocol = RTE_BE16(UINT16_MAX), 2592 }; 2593 2594 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2595 return rte_flow_error_set(error, EINVAL, 2596 RTE_FLOW_ERROR_TYPE_ITEM, item, 2597 "protocol filtering not compatible" 2598 " with this GRE layer"); 2599 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2600 return rte_flow_error_set(error, ENOTSUP, 2601 RTE_FLOW_ERROR_TYPE_ITEM, item, 2602 "multiple tunnel layers not" 2603 " supported"); 2604 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2605 return rte_flow_error_set(error, ENOTSUP, 2606 RTE_FLOW_ERROR_TYPE_ITEM, item, 2607 "L3 Layer is missing"); 2608 if (!mask) 2609 mask = &rte_flow_item_gre_mask; 2610 ret = mlx5_flow_item_acceptable 2611 (item, (const uint8_t *)mask, 2612 (const uint8_t *)&nic_mask, 2613 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2614 error); 2615 if (ret < 0) 2616 return ret; 2617 #ifndef HAVE_MLX5DV_DR 2618 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2619 if (spec && (spec->protocol & mask->protocol)) 2620 return rte_flow_error_set(error, ENOTSUP, 2621 RTE_FLOW_ERROR_TYPE_ITEM, item, 2622 "without MPLS support the" 2623 " specification cannot be used for" 2624 " filtering"); 2625 #endif 2626 #endif 2627 return 0; 2628 } 2629 2630 /** 2631 * Validate Geneve item. 2632 * 2633 * @param[in] item 2634 * Item specification. 2635 * @param[in] itemFlags 2636 * Bit-fields that holds the items detected until now. 2637 * @param[in] enPriv 2638 * Pointer to the private data structure. 2639 * @param[out] error 2640 * Pointer to error structure. 2641 * 2642 * @return 2643 * 0 on success, a negative errno value otherwise and rte_errno is set. 2644 */ 2645 2646 int 2647 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2648 uint64_t item_flags, 2649 struct rte_eth_dev *dev, 2650 struct rte_flow_error *error) 2651 { 2652 struct mlx5_priv *priv = dev->data->dev_private; 2653 const struct rte_flow_item_geneve *spec = item->spec; 2654 const struct rte_flow_item_geneve *mask = item->mask; 2655 int ret; 2656 uint16_t gbhdr; 2657 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2658 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2659 const struct rte_flow_item_geneve nic_mask = { 2660 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2661 .vni = "\xff\xff\xff", 2662 .protocol = RTE_BE16(UINT16_MAX), 2663 }; 2664 2665 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2666 return rte_flow_error_set(error, ENOTSUP, 2667 RTE_FLOW_ERROR_TYPE_ITEM, item, 2668 "L3 Geneve is not enabled by device" 2669 " parameter and/or not configured in" 2670 " firmware"); 2671 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2672 return rte_flow_error_set(error, ENOTSUP, 2673 RTE_FLOW_ERROR_TYPE_ITEM, item, 2674 "multiple tunnel layers not" 2675 " supported"); 2676 /* 2677 * Verify only UDPv4 is present as defined in 2678 * https://tools.ietf.org/html/rfc7348 2679 */ 2680 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2681 return rte_flow_error_set(error, EINVAL, 2682 RTE_FLOW_ERROR_TYPE_ITEM, item, 2683 "no outer UDP layer found"); 2684 if (!mask) 2685 mask = &rte_flow_item_geneve_mask; 2686 ret = mlx5_flow_item_acceptable 2687 (item, (const uint8_t *)mask, 2688 (const uint8_t *)&nic_mask, 2689 sizeof(struct rte_flow_item_geneve), 2690 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2691 if (ret) 2692 return ret; 2693 if (spec) { 2694 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2695 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2696 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2697 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2698 return rte_flow_error_set(error, ENOTSUP, 2699 RTE_FLOW_ERROR_TYPE_ITEM, 2700 item, 2701 "Geneve protocol unsupported" 2702 " fields are being used"); 2703 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2704 return rte_flow_error_set 2705 (error, ENOTSUP, 2706 RTE_FLOW_ERROR_TYPE_ITEM, 2707 item, 2708 "Unsupported Geneve options length"); 2709 } 2710 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2711 return rte_flow_error_set 2712 (error, ENOTSUP, 2713 RTE_FLOW_ERROR_TYPE_ITEM, item, 2714 "Geneve tunnel must be fully defined"); 2715 return 0; 2716 } 2717 2718 /** 2719 * Validate Geneve TLV option item. 2720 * 2721 * @param[in] item 2722 * Item specification. 2723 * @param[in] last_item 2724 * Previous validated item in the pattern items. 2725 * @param[in] geneve_item 2726 * Previous GENEVE item specification. 2727 * @param[in] dev 2728 * Pointer to the rte_eth_dev structure. 2729 * @param[out] error 2730 * Pointer to error structure. 2731 * 2732 * @return 2733 * 0 on success, a negative errno value otherwise and rte_errno is set. 2734 */ 2735 int 2736 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2737 uint64_t last_item, 2738 const struct rte_flow_item *geneve_item, 2739 struct rte_eth_dev *dev, 2740 struct rte_flow_error *error) 2741 { 2742 struct mlx5_priv *priv = dev->data->dev_private; 2743 struct mlx5_dev_ctx_shared *sh = priv->sh; 2744 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2745 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2746 uint8_t data_max_supported = 2747 hca_attr->max_geneve_tlv_option_data_len * 4; 2748 struct mlx5_dev_config *config = &priv->config; 2749 const struct rte_flow_item_geneve *geneve_spec; 2750 const struct rte_flow_item_geneve *geneve_mask; 2751 const struct rte_flow_item_geneve_opt *spec = item->spec; 2752 const struct rte_flow_item_geneve_opt *mask = item->mask; 2753 unsigned int i; 2754 unsigned int data_len; 2755 uint8_t tlv_option_len; 2756 uint16_t optlen_m, optlen_v; 2757 const struct rte_flow_item_geneve_opt full_mask = { 2758 .option_class = RTE_BE16(0xffff), 2759 .option_type = 0xff, 2760 .option_len = 0x1f, 2761 }; 2762 2763 if (!mask) 2764 mask = &rte_flow_item_geneve_opt_mask; 2765 if (!spec) 2766 return rte_flow_error_set 2767 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2768 "Geneve TLV opt class/type/length must be specified"); 2769 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2770 return rte_flow_error_set 2771 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2772 "Geneve TLV opt length exceeeds the limit (31)"); 2773 /* Check if class type and length masks are full. */ 2774 if (full_mask.option_class != mask->option_class || 2775 full_mask.option_type != mask->option_type || 2776 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2777 return rte_flow_error_set 2778 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2779 "Geneve TLV opt class/type/length masks must be full"); 2780 /* Check if length is supported */ 2781 if ((uint32_t)spec->option_len > 2782 config->hca_attr.max_geneve_tlv_option_data_len) 2783 return rte_flow_error_set 2784 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2785 "Geneve TLV opt length not supported"); 2786 if (config->hca_attr.max_geneve_tlv_options > 1) 2787 DRV_LOG(DEBUG, 2788 "max_geneve_tlv_options supports more than 1 option"); 2789 /* Check GENEVE item preceding. */ 2790 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2791 return rte_flow_error_set 2792 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2793 "Geneve opt item must be preceded with Geneve item"); 2794 geneve_spec = geneve_item->spec; 2795 geneve_mask = geneve_item->mask ? geneve_item->mask : 2796 &rte_flow_item_geneve_mask; 2797 /* Check if GENEVE TLV option size doesn't exceed option length */ 2798 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2799 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2800 tlv_option_len = spec->option_len & mask->option_len; 2801 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2802 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2803 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2804 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2805 if ((optlen_v & optlen_m) <= tlv_option_len) 2806 return rte_flow_error_set 2807 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2808 "GENEVE TLV option length exceeds optlen"); 2809 } 2810 /* Check if length is 0 or data is 0. */ 2811 if (spec->data == NULL || spec->option_len == 0) 2812 return rte_flow_error_set 2813 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2814 "Geneve TLV opt with zero data/length not supported"); 2815 /* Check not all data & mask are 0. */ 2816 data_len = spec->option_len * 4; 2817 if (mask->data == NULL) { 2818 for (i = 0; i < data_len; i++) 2819 if (spec->data[i]) 2820 break; 2821 if (i == data_len) 2822 return rte_flow_error_set(error, ENOTSUP, 2823 RTE_FLOW_ERROR_TYPE_ITEM, item, 2824 "Can't match on Geneve option data 0"); 2825 } else { 2826 for (i = 0; i < data_len; i++) 2827 if (spec->data[i] & mask->data[i]) 2828 break; 2829 if (i == data_len) 2830 return rte_flow_error_set(error, ENOTSUP, 2831 RTE_FLOW_ERROR_TYPE_ITEM, item, 2832 "Can't match on Geneve option data and mask 0"); 2833 /* Check data mask supported. */ 2834 for (i = data_max_supported; i < data_len ; i++) 2835 if (mask->data[i]) 2836 return rte_flow_error_set(error, ENOTSUP, 2837 RTE_FLOW_ERROR_TYPE_ITEM, item, 2838 "Data mask is of unsupported size"); 2839 } 2840 /* Check GENEVE option is supported in NIC. */ 2841 if (!config->hca_attr.geneve_tlv_opt) 2842 return rte_flow_error_set 2843 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2844 "Geneve TLV opt not supported"); 2845 /* Check if we already have geneve option with different type/class. */ 2846 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2847 geneve_opt_resource = sh->geneve_tlv_option_resource; 2848 if (geneve_opt_resource != NULL) 2849 if (geneve_opt_resource->option_class != spec->option_class || 2850 geneve_opt_resource->option_type != spec->option_type || 2851 geneve_opt_resource->length != spec->option_len) { 2852 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2853 return rte_flow_error_set(error, ENOTSUP, 2854 RTE_FLOW_ERROR_TYPE_ITEM, item, 2855 "Only one Geneve TLV option supported"); 2856 } 2857 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2858 return 0; 2859 } 2860 2861 /** 2862 * Validate MPLS item. 2863 * 2864 * @param[in] dev 2865 * Pointer to the rte_eth_dev structure. 2866 * @param[in] item 2867 * Item specification. 2868 * @param[in] item_flags 2869 * Bit-fields that holds the items detected until now. 2870 * @param[in] prev_layer 2871 * The protocol layer indicated in previous item. 2872 * @param[out] error 2873 * Pointer to error structure. 2874 * 2875 * @return 2876 * 0 on success, a negative errno value otherwise and rte_errno is set. 2877 */ 2878 int 2879 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2880 const struct rte_flow_item *item __rte_unused, 2881 uint64_t item_flags __rte_unused, 2882 uint64_t prev_layer __rte_unused, 2883 struct rte_flow_error *error) 2884 { 2885 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2886 const struct rte_flow_item_mpls *mask = item->mask; 2887 struct mlx5_priv *priv = dev->data->dev_private; 2888 int ret; 2889 2890 if (!priv->config.mpls_en) 2891 return rte_flow_error_set(error, ENOTSUP, 2892 RTE_FLOW_ERROR_TYPE_ITEM, item, 2893 "MPLS not supported or" 2894 " disabled in firmware" 2895 " configuration."); 2896 /* MPLS over IP, UDP, GRE is allowed */ 2897 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2898 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2899 MLX5_FLOW_LAYER_GRE | 2900 MLX5_FLOW_LAYER_GRE_KEY))) 2901 return rte_flow_error_set(error, EINVAL, 2902 RTE_FLOW_ERROR_TYPE_ITEM, item, 2903 "protocol filtering not compatible" 2904 " with MPLS layer"); 2905 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2906 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2907 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2908 return rte_flow_error_set(error, ENOTSUP, 2909 RTE_FLOW_ERROR_TYPE_ITEM, item, 2910 "multiple tunnel layers not" 2911 " supported"); 2912 if (!mask) 2913 mask = &rte_flow_item_mpls_mask; 2914 ret = mlx5_flow_item_acceptable 2915 (item, (const uint8_t *)mask, 2916 (const uint8_t *)&rte_flow_item_mpls_mask, 2917 sizeof(struct rte_flow_item_mpls), 2918 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2919 if (ret < 0) 2920 return ret; 2921 return 0; 2922 #else 2923 return rte_flow_error_set(error, ENOTSUP, 2924 RTE_FLOW_ERROR_TYPE_ITEM, item, 2925 "MPLS is not supported by Verbs, please" 2926 " update."); 2927 #endif 2928 } 2929 2930 /** 2931 * Validate NVGRE item. 2932 * 2933 * @param[in] item 2934 * Item specification. 2935 * @param[in] item_flags 2936 * Bit flags to mark detected items. 2937 * @param[in] target_protocol 2938 * The next protocol in the previous item. 2939 * @param[out] error 2940 * Pointer to error structure. 2941 * 2942 * @return 2943 * 0 on success, a negative errno value otherwise and rte_errno is set. 2944 */ 2945 int 2946 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2947 uint64_t item_flags, 2948 uint8_t target_protocol, 2949 struct rte_flow_error *error) 2950 { 2951 const struct rte_flow_item_nvgre *mask = item->mask; 2952 int ret; 2953 2954 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2955 return rte_flow_error_set(error, EINVAL, 2956 RTE_FLOW_ERROR_TYPE_ITEM, item, 2957 "protocol filtering not compatible" 2958 " with this GRE layer"); 2959 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2960 return rte_flow_error_set(error, ENOTSUP, 2961 RTE_FLOW_ERROR_TYPE_ITEM, item, 2962 "multiple tunnel layers not" 2963 " supported"); 2964 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2965 return rte_flow_error_set(error, ENOTSUP, 2966 RTE_FLOW_ERROR_TYPE_ITEM, item, 2967 "L3 Layer is missing"); 2968 if (!mask) 2969 mask = &rte_flow_item_nvgre_mask; 2970 ret = mlx5_flow_item_acceptable 2971 (item, (const uint8_t *)mask, 2972 (const uint8_t *)&rte_flow_item_nvgre_mask, 2973 sizeof(struct rte_flow_item_nvgre), 2974 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2975 if (ret < 0) 2976 return ret; 2977 return 0; 2978 } 2979 2980 /** 2981 * Validate eCPRI item. 2982 * 2983 * @param[in] item 2984 * Item specification. 2985 * @param[in] item_flags 2986 * Bit-fields that holds the items detected until now. 2987 * @param[in] last_item 2988 * Previous validated item in the pattern items. 2989 * @param[in] ether_type 2990 * Type in the ethernet layer header (including dot1q). 2991 * @param[in] acc_mask 2992 * Acceptable mask, if NULL default internal default mask 2993 * will be used to check whether item fields are supported. 2994 * @param[out] error 2995 * Pointer to error structure. 2996 * 2997 * @return 2998 * 0 on success, a negative errno value otherwise and rte_errno is set. 2999 */ 3000 int 3001 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3002 uint64_t item_flags, 3003 uint64_t last_item, 3004 uint16_t ether_type, 3005 const struct rte_flow_item_ecpri *acc_mask, 3006 struct rte_flow_error *error) 3007 { 3008 const struct rte_flow_item_ecpri *mask = item->mask; 3009 const struct rte_flow_item_ecpri nic_mask = { 3010 .hdr = { 3011 .common = { 3012 .u32 = 3013 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3014 .type = 0xFF, 3015 }).u32), 3016 }, 3017 .dummy[0] = 0xFFFFFFFF, 3018 }, 3019 }; 3020 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3021 MLX5_FLOW_LAYER_OUTER_VLAN); 3022 struct rte_flow_item_ecpri mask_lo; 3023 3024 if (!(last_item & outer_l2_vlan) && 3025 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3026 return rte_flow_error_set(error, EINVAL, 3027 RTE_FLOW_ERROR_TYPE_ITEM, item, 3028 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3029 if ((last_item & outer_l2_vlan) && ether_type && 3030 ether_type != RTE_ETHER_TYPE_ECPRI) 3031 return rte_flow_error_set(error, EINVAL, 3032 RTE_FLOW_ERROR_TYPE_ITEM, item, 3033 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3034 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3035 return rte_flow_error_set(error, EINVAL, 3036 RTE_FLOW_ERROR_TYPE_ITEM, item, 3037 "eCPRI with tunnel is not supported right now"); 3038 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3039 return rte_flow_error_set(error, ENOTSUP, 3040 RTE_FLOW_ERROR_TYPE_ITEM, item, 3041 "multiple L3 layers not supported"); 3042 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3043 return rte_flow_error_set(error, EINVAL, 3044 RTE_FLOW_ERROR_TYPE_ITEM, item, 3045 "eCPRI cannot coexist with a TCP layer"); 3046 /* In specification, eCPRI could be over UDP layer. */ 3047 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3048 return rte_flow_error_set(error, EINVAL, 3049 RTE_FLOW_ERROR_TYPE_ITEM, item, 3050 "eCPRI over UDP layer is not yet supported right now"); 3051 /* Mask for type field in common header could be zero. */ 3052 if (!mask) 3053 mask = &rte_flow_item_ecpri_mask; 3054 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3055 /* Input mask is in big-endian format. */ 3056 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3057 return rte_flow_error_set(error, EINVAL, 3058 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3059 "partial mask is not supported for protocol"); 3060 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3061 return rte_flow_error_set(error, EINVAL, 3062 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3063 "message header mask must be after a type mask"); 3064 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3065 acc_mask ? (const uint8_t *)acc_mask 3066 : (const uint8_t *)&nic_mask, 3067 sizeof(struct rte_flow_item_ecpri), 3068 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3069 } 3070 3071 /** 3072 * Release resource related QUEUE/RSS action split. 3073 * 3074 * @param dev 3075 * Pointer to Ethernet device. 3076 * @param flow 3077 * Flow to release id's from. 3078 */ 3079 static void 3080 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 3081 struct rte_flow *flow) 3082 { 3083 struct mlx5_priv *priv = dev->data->dev_private; 3084 uint32_t handle_idx; 3085 struct mlx5_flow_handle *dev_handle; 3086 3087 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 3088 handle_idx, dev_handle, next) 3089 if (dev_handle->split_flow_id && 3090 !dev_handle->is_meter_flow_id) 3091 mlx5_ipool_free(priv->sh->ipool 3092 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 3093 dev_handle->split_flow_id); 3094 } 3095 3096 static int 3097 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3098 const struct rte_flow_attr *attr __rte_unused, 3099 const struct rte_flow_item items[] __rte_unused, 3100 const struct rte_flow_action actions[] __rte_unused, 3101 bool external __rte_unused, 3102 int hairpin __rte_unused, 3103 struct rte_flow_error *error) 3104 { 3105 return rte_flow_error_set(error, ENOTSUP, 3106 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3107 } 3108 3109 static struct mlx5_flow * 3110 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3111 const struct rte_flow_attr *attr __rte_unused, 3112 const struct rte_flow_item items[] __rte_unused, 3113 const struct rte_flow_action actions[] __rte_unused, 3114 struct rte_flow_error *error) 3115 { 3116 rte_flow_error_set(error, ENOTSUP, 3117 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3118 return NULL; 3119 } 3120 3121 static int 3122 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3123 struct mlx5_flow *dev_flow __rte_unused, 3124 const struct rte_flow_attr *attr __rte_unused, 3125 const struct rte_flow_item items[] __rte_unused, 3126 const struct rte_flow_action actions[] __rte_unused, 3127 struct rte_flow_error *error) 3128 { 3129 return rte_flow_error_set(error, ENOTSUP, 3130 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3131 } 3132 3133 static int 3134 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3135 struct rte_flow *flow __rte_unused, 3136 struct rte_flow_error *error) 3137 { 3138 return rte_flow_error_set(error, ENOTSUP, 3139 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3140 } 3141 3142 static void 3143 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3144 struct rte_flow *flow __rte_unused) 3145 { 3146 } 3147 3148 static void 3149 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3150 struct rte_flow *flow __rte_unused) 3151 { 3152 } 3153 3154 static int 3155 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3156 struct rte_flow *flow __rte_unused, 3157 const struct rte_flow_action *actions __rte_unused, 3158 void *data __rte_unused, 3159 struct rte_flow_error *error) 3160 { 3161 return rte_flow_error_set(error, ENOTSUP, 3162 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3163 } 3164 3165 static int 3166 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3167 uint32_t domains __rte_unused, 3168 uint32_t flags __rte_unused) 3169 { 3170 return 0; 3171 } 3172 3173 /* Void driver to protect from null pointer reference. */ 3174 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3175 .validate = flow_null_validate, 3176 .prepare = flow_null_prepare, 3177 .translate = flow_null_translate, 3178 .apply = flow_null_apply, 3179 .remove = flow_null_remove, 3180 .destroy = flow_null_destroy, 3181 .query = flow_null_query, 3182 .sync_domain = flow_null_sync_domain, 3183 }; 3184 3185 /** 3186 * Select flow driver type according to flow attributes and device 3187 * configuration. 3188 * 3189 * @param[in] dev 3190 * Pointer to the dev structure. 3191 * @param[in] attr 3192 * Pointer to the flow attributes. 3193 * 3194 * @return 3195 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3196 */ 3197 static enum mlx5_flow_drv_type 3198 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3199 { 3200 struct mlx5_priv *priv = dev->data->dev_private; 3201 /* The OS can determine first a specific flow type (DV, VERBS) */ 3202 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3203 3204 if (type != MLX5_FLOW_TYPE_MAX) 3205 return type; 3206 /* If no OS specific type - continue with DV/VERBS selection */ 3207 if (attr->transfer && priv->config.dv_esw_en) 3208 type = MLX5_FLOW_TYPE_DV; 3209 if (!attr->transfer) 3210 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3211 MLX5_FLOW_TYPE_VERBS; 3212 return type; 3213 } 3214 3215 #define flow_get_drv_ops(type) flow_drv_ops[type] 3216 3217 /** 3218 * Flow driver validation API. This abstracts calling driver specific functions. 3219 * The type of flow driver is determined according to flow attributes. 3220 * 3221 * @param[in] dev 3222 * Pointer to the dev structure. 3223 * @param[in] attr 3224 * Pointer to the flow attributes. 3225 * @param[in] items 3226 * Pointer to the list of items. 3227 * @param[in] actions 3228 * Pointer to the list of actions. 3229 * @param[in] external 3230 * This flow rule is created by request external to PMD. 3231 * @param[in] hairpin 3232 * Number of hairpin TX actions, 0 means classic flow. 3233 * @param[out] error 3234 * Pointer to the error structure. 3235 * 3236 * @return 3237 * 0 on success, a negative errno value otherwise and rte_errno is set. 3238 */ 3239 static inline int 3240 flow_drv_validate(struct rte_eth_dev *dev, 3241 const struct rte_flow_attr *attr, 3242 const struct rte_flow_item items[], 3243 const struct rte_flow_action actions[], 3244 bool external, int hairpin, struct rte_flow_error *error) 3245 { 3246 const struct mlx5_flow_driver_ops *fops; 3247 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3248 3249 fops = flow_get_drv_ops(type); 3250 return fops->validate(dev, attr, items, actions, external, 3251 hairpin, error); 3252 } 3253 3254 /** 3255 * Flow driver preparation API. This abstracts calling driver specific 3256 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3257 * calculates the size of memory required for device flow, allocates the memory, 3258 * initializes the device flow and returns the pointer. 3259 * 3260 * @note 3261 * This function initializes device flow structure such as dv or verbs in 3262 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3263 * rest. For example, adding returning device flow to flow->dev_flow list and 3264 * setting backward reference to the flow should be done out of this function. 3265 * layers field is not filled either. 3266 * 3267 * @param[in] dev 3268 * Pointer to the dev structure. 3269 * @param[in] attr 3270 * Pointer to the flow attributes. 3271 * @param[in] items 3272 * Pointer to the list of items. 3273 * @param[in] actions 3274 * Pointer to the list of actions. 3275 * @param[in] flow_idx 3276 * This memory pool index to the flow. 3277 * @param[out] error 3278 * Pointer to the error structure. 3279 * 3280 * @return 3281 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3282 */ 3283 static inline struct mlx5_flow * 3284 flow_drv_prepare(struct rte_eth_dev *dev, 3285 const struct rte_flow *flow, 3286 const struct rte_flow_attr *attr, 3287 const struct rte_flow_item items[], 3288 const struct rte_flow_action actions[], 3289 uint32_t flow_idx, 3290 struct rte_flow_error *error) 3291 { 3292 const struct mlx5_flow_driver_ops *fops; 3293 enum mlx5_flow_drv_type type = flow->drv_type; 3294 struct mlx5_flow *mlx5_flow = NULL; 3295 3296 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3297 fops = flow_get_drv_ops(type); 3298 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3299 if (mlx5_flow) 3300 mlx5_flow->flow_idx = flow_idx; 3301 return mlx5_flow; 3302 } 3303 3304 /** 3305 * Flow driver translation API. This abstracts calling driver specific 3306 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3307 * translates a generic flow into a driver flow. flow_drv_prepare() must 3308 * precede. 3309 * 3310 * @note 3311 * dev_flow->layers could be filled as a result of parsing during translation 3312 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3313 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3314 * flow->actions could be overwritten even though all the expanded dev_flows 3315 * have the same actions. 3316 * 3317 * @param[in] dev 3318 * Pointer to the rte dev structure. 3319 * @param[in, out] dev_flow 3320 * Pointer to the mlx5 flow. 3321 * @param[in] attr 3322 * Pointer to the flow attributes. 3323 * @param[in] items 3324 * Pointer to the list of items. 3325 * @param[in] actions 3326 * Pointer to the list of actions. 3327 * @param[out] error 3328 * Pointer to the error structure. 3329 * 3330 * @return 3331 * 0 on success, a negative errno value otherwise and rte_errno is set. 3332 */ 3333 static inline int 3334 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3335 const struct rte_flow_attr *attr, 3336 const struct rte_flow_item items[], 3337 const struct rte_flow_action actions[], 3338 struct rte_flow_error *error) 3339 { 3340 const struct mlx5_flow_driver_ops *fops; 3341 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3342 3343 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3344 fops = flow_get_drv_ops(type); 3345 return fops->translate(dev, dev_flow, attr, items, actions, error); 3346 } 3347 3348 /** 3349 * Flow driver apply API. This abstracts calling driver specific functions. 3350 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3351 * translated driver flows on to device. flow_drv_translate() must precede. 3352 * 3353 * @param[in] dev 3354 * Pointer to Ethernet device structure. 3355 * @param[in, out] flow 3356 * Pointer to flow structure. 3357 * @param[out] error 3358 * Pointer to error structure. 3359 * 3360 * @return 3361 * 0 on success, a negative errno value otherwise and rte_errno is set. 3362 */ 3363 static inline int 3364 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3365 struct rte_flow_error *error) 3366 { 3367 const struct mlx5_flow_driver_ops *fops; 3368 enum mlx5_flow_drv_type type = flow->drv_type; 3369 3370 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3371 fops = flow_get_drv_ops(type); 3372 return fops->apply(dev, flow, error); 3373 } 3374 3375 /** 3376 * Flow driver destroy API. This abstracts calling driver specific functions. 3377 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3378 * on device and releases resources of the flow. 3379 * 3380 * @param[in] dev 3381 * Pointer to Ethernet device. 3382 * @param[in, out] flow 3383 * Pointer to flow structure. 3384 */ 3385 static inline void 3386 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3387 { 3388 const struct mlx5_flow_driver_ops *fops; 3389 enum mlx5_flow_drv_type type = flow->drv_type; 3390 3391 flow_mreg_split_qrss_release(dev, flow); 3392 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3393 fops = flow_get_drv_ops(type); 3394 fops->destroy(dev, flow); 3395 } 3396 3397 /** 3398 * Flow driver find RSS policy tbl API. This abstracts calling driver 3399 * specific functions. Parent flow (rte_flow) should have driver 3400 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3401 * 3402 * @param[in] dev 3403 * Pointer to Ethernet device. 3404 * @param[in, out] flow 3405 * Pointer to flow structure. 3406 * @param[in] policy 3407 * Pointer to meter policy table. 3408 * @param[in] rss_desc 3409 * Pointer to rss_desc 3410 */ 3411 static struct mlx5_flow_meter_sub_policy * 3412 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3413 struct rte_flow *flow, 3414 struct mlx5_flow_meter_policy *policy, 3415 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3416 { 3417 const struct mlx5_flow_driver_ops *fops; 3418 enum mlx5_flow_drv_type type = flow->drv_type; 3419 3420 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3421 fops = flow_get_drv_ops(type); 3422 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3423 } 3424 3425 /** 3426 * Get RSS action from the action list. 3427 * 3428 * @param[in] dev 3429 * Pointer to Ethernet device. 3430 * @param[in] actions 3431 * Pointer to the list of actions. 3432 * @param[in] flow 3433 * Parent flow structure pointer. 3434 * 3435 * @return 3436 * Pointer to the RSS action if exist, else return NULL. 3437 */ 3438 static const struct rte_flow_action_rss* 3439 flow_get_rss_action(struct rte_eth_dev *dev, 3440 const struct rte_flow_action actions[]) 3441 { 3442 struct mlx5_priv *priv = dev->data->dev_private; 3443 const struct rte_flow_action_rss *rss = NULL; 3444 3445 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3446 switch (actions->type) { 3447 case RTE_FLOW_ACTION_TYPE_RSS: 3448 rss = actions->conf; 3449 break; 3450 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3451 { 3452 const struct rte_flow_action_sample *sample = 3453 actions->conf; 3454 const struct rte_flow_action *act = sample->actions; 3455 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3456 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3457 rss = act->conf; 3458 break; 3459 } 3460 case RTE_FLOW_ACTION_TYPE_METER: 3461 { 3462 uint32_t mtr_idx; 3463 struct mlx5_flow_meter_info *fm; 3464 struct mlx5_flow_meter_policy *policy; 3465 const struct rte_flow_action_meter *mtr = actions->conf; 3466 3467 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3468 if (fm) { 3469 policy = mlx5_flow_meter_policy_find(dev, 3470 fm->policy_id, NULL); 3471 if (policy && policy->is_rss) 3472 rss = 3473 policy->act_cnt[RTE_COLOR_GREEN].rss->conf; 3474 } 3475 break; 3476 } 3477 default: 3478 break; 3479 } 3480 } 3481 return rss; 3482 } 3483 3484 /** 3485 * Get ASO age action by index. 3486 * 3487 * @param[in] dev 3488 * Pointer to the Ethernet device structure. 3489 * @param[in] age_idx 3490 * Index to the ASO age action. 3491 * 3492 * @return 3493 * The specified ASO age action. 3494 */ 3495 struct mlx5_aso_age_action* 3496 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3497 { 3498 uint16_t pool_idx = age_idx & UINT16_MAX; 3499 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3500 struct mlx5_priv *priv = dev->data->dev_private; 3501 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3502 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3503 3504 return &pool->actions[offset - 1]; 3505 } 3506 3507 /* maps indirect action to translated direct in some actions array */ 3508 struct mlx5_translated_action_handle { 3509 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3510 int index; /**< Index in related array of rte_flow_action. */ 3511 }; 3512 3513 /** 3514 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3515 * direct action if translation possible. 3516 * This functionality used to run same execution path for both direct and 3517 * indirect actions on flow create. All necessary preparations for indirect 3518 * action handling should be performed on *handle* actions list returned 3519 * from this call. 3520 * 3521 * @param[in] dev 3522 * Pointer to Ethernet device. 3523 * @param[in] actions 3524 * List of actions to translate. 3525 * @param[out] handle 3526 * List to store translated indirect action object handles. 3527 * @param[in, out] indir_n 3528 * Size of *handle* array. On return should be updated with number of 3529 * indirect actions retrieved from the *actions* list. 3530 * @param[out] translated_actions 3531 * List of actions where all indirect actions were translated to direct 3532 * if possible. NULL if no translation took place. 3533 * @param[out] error 3534 * Pointer to the error structure. 3535 * 3536 * @return 3537 * 0 on success, a negative errno value otherwise and rte_errno is set. 3538 */ 3539 static int 3540 flow_action_handles_translate(struct rte_eth_dev *dev, 3541 const struct rte_flow_action actions[], 3542 struct mlx5_translated_action_handle *handle, 3543 int *indir_n, 3544 struct rte_flow_action **translated_actions, 3545 struct rte_flow_error *error) 3546 { 3547 struct mlx5_priv *priv = dev->data->dev_private; 3548 struct rte_flow_action *translated = NULL; 3549 size_t actions_size; 3550 int n; 3551 int copied_n = 0; 3552 struct mlx5_translated_action_handle *handle_end = NULL; 3553 3554 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3555 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3556 continue; 3557 if (copied_n == *indir_n) { 3558 return rte_flow_error_set 3559 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3560 NULL, "too many shared actions"); 3561 } 3562 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3563 sizeof(actions[n].conf)); 3564 handle[copied_n].index = n; 3565 copied_n++; 3566 } 3567 n++; 3568 *indir_n = copied_n; 3569 if (!copied_n) 3570 return 0; 3571 actions_size = sizeof(struct rte_flow_action) * n; 3572 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3573 if (!translated) { 3574 rte_errno = ENOMEM; 3575 return -ENOMEM; 3576 } 3577 memcpy(translated, actions, actions_size); 3578 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3579 struct mlx5_shared_action_rss *shared_rss; 3580 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3581 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3582 uint32_t idx = act_idx & 3583 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3584 3585 switch (type) { 3586 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3587 shared_rss = mlx5_ipool_get 3588 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3589 translated[handle->index].type = 3590 RTE_FLOW_ACTION_TYPE_RSS; 3591 translated[handle->index].conf = 3592 &shared_rss->origin; 3593 break; 3594 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3595 translated[handle->index].type = 3596 (enum rte_flow_action_type) 3597 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3598 translated[handle->index].conf = (void *)(uintptr_t)idx; 3599 break; 3600 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3601 if (priv->sh->flow_hit_aso_en) { 3602 translated[handle->index].type = 3603 (enum rte_flow_action_type) 3604 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3605 translated[handle->index].conf = 3606 (void *)(uintptr_t)idx; 3607 break; 3608 } 3609 /* Fall-through */ 3610 case MLX5_INDIRECT_ACTION_TYPE_CT: 3611 if (priv->sh->ct_aso_en) { 3612 translated[handle->index].type = 3613 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3614 translated[handle->index].conf = 3615 (void *)(uintptr_t)idx; 3616 break; 3617 } 3618 /* Fall-through */ 3619 default: 3620 mlx5_free(translated); 3621 return rte_flow_error_set 3622 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3623 NULL, "invalid indirect action type"); 3624 } 3625 } 3626 *translated_actions = translated; 3627 return 0; 3628 } 3629 3630 /** 3631 * Get Shared RSS action from the action list. 3632 * 3633 * @param[in] dev 3634 * Pointer to Ethernet device. 3635 * @param[in] shared 3636 * Pointer to the list of actions. 3637 * @param[in] shared_n 3638 * Actions list length. 3639 * 3640 * @return 3641 * The MLX5 RSS action ID if exists, otherwise return 0. 3642 */ 3643 static uint32_t 3644 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3645 struct mlx5_translated_action_handle *handle, 3646 int shared_n) 3647 { 3648 struct mlx5_translated_action_handle *handle_end; 3649 struct mlx5_priv *priv = dev->data->dev_private; 3650 struct mlx5_shared_action_rss *shared_rss; 3651 3652 3653 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3654 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3655 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3656 uint32_t idx = act_idx & 3657 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3658 switch (type) { 3659 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3660 shared_rss = mlx5_ipool_get 3661 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3662 idx); 3663 __atomic_add_fetch(&shared_rss->refcnt, 1, 3664 __ATOMIC_RELAXED); 3665 return idx; 3666 default: 3667 break; 3668 } 3669 } 3670 return 0; 3671 } 3672 3673 static unsigned int 3674 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3675 { 3676 const struct rte_flow_item *item; 3677 unsigned int has_vlan = 0; 3678 3679 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3680 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3681 has_vlan = 1; 3682 break; 3683 } 3684 } 3685 if (has_vlan) 3686 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3687 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3688 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3689 MLX5_EXPANSION_ROOT_OUTER; 3690 } 3691 3692 /** 3693 * Get layer flags from the prefix flow. 3694 * 3695 * Some flows may be split to several subflows, the prefix subflow gets the 3696 * match items and the suffix sub flow gets the actions. 3697 * Some actions need the user defined match item flags to get the detail for 3698 * the action. 3699 * This function helps the suffix flow to get the item layer flags from prefix 3700 * subflow. 3701 * 3702 * @param[in] dev_flow 3703 * Pointer the created preifx subflow. 3704 * 3705 * @return 3706 * The layers get from prefix subflow. 3707 */ 3708 static inline uint64_t 3709 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3710 { 3711 uint64_t layers = 0; 3712 3713 /* 3714 * Layers bits could be localization, but usually the compiler will 3715 * help to do the optimization work for source code. 3716 * If no decap actions, use the layers directly. 3717 */ 3718 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3719 return dev_flow->handle->layers; 3720 /* Convert L3 layers with decap action. */ 3721 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3722 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3723 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3724 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3725 /* Convert L4 layers with decap action. */ 3726 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3727 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3728 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3729 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3730 return layers; 3731 } 3732 3733 /** 3734 * Get metadata split action information. 3735 * 3736 * @param[in] actions 3737 * Pointer to the list of actions. 3738 * @param[out] qrss 3739 * Pointer to the return pointer. 3740 * @param[out] qrss_type 3741 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3742 * if no QUEUE/RSS is found. 3743 * @param[out] encap_idx 3744 * Pointer to the index of the encap action if exists, otherwise the last 3745 * action index. 3746 * 3747 * @return 3748 * Total number of actions. 3749 */ 3750 static int 3751 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3752 const struct rte_flow_action **qrss, 3753 int *encap_idx) 3754 { 3755 const struct rte_flow_action_raw_encap *raw_encap; 3756 int actions_n = 0; 3757 int raw_decap_idx = -1; 3758 3759 *encap_idx = -1; 3760 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3761 switch (actions->type) { 3762 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3763 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3764 *encap_idx = actions_n; 3765 break; 3766 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3767 raw_decap_idx = actions_n; 3768 break; 3769 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3770 raw_encap = actions->conf; 3771 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3772 *encap_idx = raw_decap_idx != -1 ? 3773 raw_decap_idx : actions_n; 3774 break; 3775 case RTE_FLOW_ACTION_TYPE_QUEUE: 3776 case RTE_FLOW_ACTION_TYPE_RSS: 3777 *qrss = actions; 3778 break; 3779 default: 3780 break; 3781 } 3782 actions_n++; 3783 } 3784 if (*encap_idx == -1) 3785 *encap_idx = actions_n; 3786 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3787 return actions_n + 1; 3788 } 3789 3790 /** 3791 * Check if the action will change packet. 3792 * 3793 * @param dev 3794 * Pointer to Ethernet device. 3795 * @param[in] type 3796 * action type. 3797 * 3798 * @return 3799 * true if action will change packet, false otherwise. 3800 */ 3801 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 3802 enum rte_flow_action_type type) 3803 { 3804 struct mlx5_priv *priv = dev->data->dev_private; 3805 3806 switch (type) { 3807 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3808 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3809 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3810 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3811 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3812 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3813 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3814 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3815 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3816 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3817 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3818 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3819 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3820 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3821 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 3822 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 3823 case RTE_FLOW_ACTION_TYPE_SET_META: 3824 case RTE_FLOW_ACTION_TYPE_SET_TAG: 3825 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 3826 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3827 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3828 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3829 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3830 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3831 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3832 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3833 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3834 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3835 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 3836 return true; 3837 case RTE_FLOW_ACTION_TYPE_FLAG: 3838 case RTE_FLOW_ACTION_TYPE_MARK: 3839 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 3840 return true; 3841 else 3842 return false; 3843 default: 3844 return false; 3845 } 3846 } 3847 3848 /** 3849 * Check meter action from the action list. 3850 * 3851 * @param dev 3852 * Pointer to Ethernet device. 3853 * @param[in] actions 3854 * Pointer to the list of actions. 3855 * @param[out] has_mtr 3856 * Pointer to the meter exist flag. 3857 * @param[out] has_modify 3858 * Pointer to the flag showing there's packet change action. 3859 * @param[out] meter_id 3860 * Pointer to the meter id. 3861 * 3862 * @return 3863 * Total number of actions. 3864 */ 3865 static int 3866 flow_check_meter_action(struct rte_eth_dev *dev, 3867 const struct rte_flow_action actions[], 3868 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 3869 { 3870 const struct rte_flow_action_meter *mtr = NULL; 3871 int actions_n = 0; 3872 3873 MLX5_ASSERT(has_mtr); 3874 *has_mtr = false; 3875 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3876 switch (actions->type) { 3877 case RTE_FLOW_ACTION_TYPE_METER: 3878 mtr = actions->conf; 3879 *meter_id = mtr->mtr_id; 3880 *has_mtr = true; 3881 break; 3882 default: 3883 break; 3884 } 3885 if (!*has_mtr) 3886 *has_modify |= flow_check_modify_action_type(dev, 3887 actions->type); 3888 actions_n++; 3889 } 3890 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3891 return actions_n + 1; 3892 } 3893 3894 /** 3895 * Check if the flow should be split due to hairpin. 3896 * The reason for the split is that in current HW we can't 3897 * support encap and push-vlan on Rx, so if a flow contains 3898 * these actions we move it to Tx. 3899 * 3900 * @param dev 3901 * Pointer to Ethernet device. 3902 * @param[in] attr 3903 * Flow rule attributes. 3904 * @param[in] actions 3905 * Associated actions (list terminated by the END action). 3906 * 3907 * @return 3908 * > 0 the number of actions and the flow should be split, 3909 * 0 when no split required. 3910 */ 3911 static int 3912 flow_check_hairpin_split(struct rte_eth_dev *dev, 3913 const struct rte_flow_attr *attr, 3914 const struct rte_flow_action actions[]) 3915 { 3916 int queue_action = 0; 3917 int action_n = 0; 3918 int split = 0; 3919 const struct rte_flow_action_queue *queue; 3920 const struct rte_flow_action_rss *rss; 3921 const struct rte_flow_action_raw_encap *raw_encap; 3922 const struct rte_eth_hairpin_conf *conf; 3923 3924 if (!attr->ingress) 3925 return 0; 3926 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3927 switch (actions->type) { 3928 case RTE_FLOW_ACTION_TYPE_QUEUE: 3929 queue = actions->conf; 3930 if (queue == NULL) 3931 return 0; 3932 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3933 if (conf == NULL || conf->tx_explicit != 0) 3934 return 0; 3935 queue_action = 1; 3936 action_n++; 3937 break; 3938 case RTE_FLOW_ACTION_TYPE_RSS: 3939 rss = actions->conf; 3940 if (rss == NULL || rss->queue_num == 0) 3941 return 0; 3942 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3943 if (conf == NULL || conf->tx_explicit != 0) 3944 return 0; 3945 queue_action = 1; 3946 action_n++; 3947 break; 3948 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3949 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3950 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3951 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3952 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3953 split++; 3954 action_n++; 3955 break; 3956 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3957 raw_encap = actions->conf; 3958 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3959 split++; 3960 action_n++; 3961 break; 3962 default: 3963 action_n++; 3964 break; 3965 } 3966 } 3967 if (split && queue_action) 3968 return action_n; 3969 return 0; 3970 } 3971 3972 /* Declare flow create/destroy prototype in advance. */ 3973 static uint32_t 3974 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3975 const struct rte_flow_attr *attr, 3976 const struct rte_flow_item items[], 3977 const struct rte_flow_action actions[], 3978 bool external, struct rte_flow_error *error); 3979 3980 static void 3981 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3982 uint32_t flow_idx); 3983 3984 int 3985 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused, 3986 struct mlx5_hlist_entry *entry, 3987 uint64_t key, void *cb_ctx __rte_unused) 3988 { 3989 struct mlx5_flow_mreg_copy_resource *mcp_res = 3990 container_of(entry, typeof(*mcp_res), hlist_ent); 3991 3992 return mcp_res->mark_id != key; 3993 } 3994 3995 struct mlx5_hlist_entry * 3996 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 3997 void *cb_ctx) 3998 { 3999 struct rte_eth_dev *dev = list->ctx; 4000 struct mlx5_priv *priv = dev->data->dev_private; 4001 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4002 struct mlx5_flow_mreg_copy_resource *mcp_res; 4003 struct rte_flow_error *error = ctx->error; 4004 uint32_t idx = 0; 4005 int ret; 4006 uint32_t mark_id = key; 4007 struct rte_flow_attr attr = { 4008 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4009 .ingress = 1, 4010 }; 4011 struct mlx5_rte_flow_item_tag tag_spec = { 4012 .data = mark_id, 4013 }; 4014 struct rte_flow_item items[] = { 4015 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4016 }; 4017 struct rte_flow_action_mark ftag = { 4018 .id = mark_id, 4019 }; 4020 struct mlx5_flow_action_copy_mreg cp_mreg = { 4021 .dst = REG_B, 4022 .src = REG_NON, 4023 }; 4024 struct rte_flow_action_jump jump = { 4025 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4026 }; 4027 struct rte_flow_action actions[] = { 4028 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4029 }; 4030 4031 /* Fill the register fileds in the flow. */ 4032 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4033 if (ret < 0) 4034 return NULL; 4035 tag_spec.id = ret; 4036 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4037 if (ret < 0) 4038 return NULL; 4039 cp_mreg.src = ret; 4040 /* Provide the full width of FLAG specific value. */ 4041 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4042 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4043 /* Build a new flow. */ 4044 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4045 items[0] = (struct rte_flow_item){ 4046 .type = (enum rte_flow_item_type) 4047 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4048 .spec = &tag_spec, 4049 }; 4050 items[1] = (struct rte_flow_item){ 4051 .type = RTE_FLOW_ITEM_TYPE_END, 4052 }; 4053 actions[0] = (struct rte_flow_action){ 4054 .type = (enum rte_flow_action_type) 4055 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4056 .conf = &ftag, 4057 }; 4058 actions[1] = (struct rte_flow_action){ 4059 .type = (enum rte_flow_action_type) 4060 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4061 .conf = &cp_mreg, 4062 }; 4063 actions[2] = (struct rte_flow_action){ 4064 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4065 .conf = &jump, 4066 }; 4067 actions[3] = (struct rte_flow_action){ 4068 .type = RTE_FLOW_ACTION_TYPE_END, 4069 }; 4070 } else { 4071 /* Default rule, wildcard match. */ 4072 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4073 items[0] = (struct rte_flow_item){ 4074 .type = RTE_FLOW_ITEM_TYPE_END, 4075 }; 4076 actions[0] = (struct rte_flow_action){ 4077 .type = (enum rte_flow_action_type) 4078 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4079 .conf = &cp_mreg, 4080 }; 4081 actions[1] = (struct rte_flow_action){ 4082 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4083 .conf = &jump, 4084 }; 4085 actions[2] = (struct rte_flow_action){ 4086 .type = RTE_FLOW_ACTION_TYPE_END, 4087 }; 4088 } 4089 /* Build a new entry. */ 4090 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4091 if (!mcp_res) { 4092 rte_errno = ENOMEM; 4093 return NULL; 4094 } 4095 mcp_res->idx = idx; 4096 mcp_res->mark_id = mark_id; 4097 /* 4098 * The copy Flows are not included in any list. There 4099 * ones are referenced from other Flows and can not 4100 * be applied, removed, deleted in ardbitrary order 4101 * by list traversing. 4102 */ 4103 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 4104 actions, false, error); 4105 if (!mcp_res->rix_flow) { 4106 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4107 return NULL; 4108 } 4109 return &mcp_res->hlist_ent; 4110 } 4111 4112 /** 4113 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4114 * 4115 * As mark_id is unique, if there's already a registered flow for the mark_id, 4116 * return by increasing the reference counter of the resource. Otherwise, create 4117 * the resource (mcp_res) and flow. 4118 * 4119 * Flow looks like, 4120 * - If ingress port is ANY and reg_c[1] is mark_id, 4121 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4122 * 4123 * For default flow (zero mark_id), flow is like, 4124 * - If ingress port is ANY, 4125 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4126 * 4127 * @param dev 4128 * Pointer to Ethernet device. 4129 * @param mark_id 4130 * ID of MARK action, zero means default flow for META. 4131 * @param[out] error 4132 * Perform verbose error reporting if not NULL. 4133 * 4134 * @return 4135 * Associated resource on success, NULL otherwise and rte_errno is set. 4136 */ 4137 static struct mlx5_flow_mreg_copy_resource * 4138 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4139 struct rte_flow_error *error) 4140 { 4141 struct mlx5_priv *priv = dev->data->dev_private; 4142 struct mlx5_hlist_entry *entry; 4143 struct mlx5_flow_cb_ctx ctx = { 4144 .dev = dev, 4145 .error = error, 4146 }; 4147 4148 /* Check if already registered. */ 4149 MLX5_ASSERT(priv->mreg_cp_tbl); 4150 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4151 if (!entry) 4152 return NULL; 4153 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4154 hlist_ent); 4155 } 4156 4157 void 4158 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 4159 { 4160 struct mlx5_flow_mreg_copy_resource *mcp_res = 4161 container_of(entry, typeof(*mcp_res), hlist_ent); 4162 struct rte_eth_dev *dev = list->ctx; 4163 struct mlx5_priv *priv = dev->data->dev_private; 4164 4165 MLX5_ASSERT(mcp_res->rix_flow); 4166 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 4167 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4168 } 4169 4170 /** 4171 * Release flow in RX_CP_TBL. 4172 * 4173 * @param dev 4174 * Pointer to Ethernet device. 4175 * @flow 4176 * Parent flow for wich copying is provided. 4177 */ 4178 static void 4179 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4180 struct rte_flow *flow) 4181 { 4182 struct mlx5_flow_mreg_copy_resource *mcp_res; 4183 struct mlx5_priv *priv = dev->data->dev_private; 4184 4185 if (!flow->rix_mreg_copy) 4186 return; 4187 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4188 flow->rix_mreg_copy); 4189 if (!mcp_res || !priv->mreg_cp_tbl) 4190 return; 4191 MLX5_ASSERT(mcp_res->rix_flow); 4192 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4193 flow->rix_mreg_copy = 0; 4194 } 4195 4196 /** 4197 * Remove the default copy action from RX_CP_TBL. 4198 * 4199 * This functions is called in the mlx5_dev_start(). No thread safe 4200 * is guaranteed. 4201 * 4202 * @param dev 4203 * Pointer to Ethernet device. 4204 */ 4205 static void 4206 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4207 { 4208 struct mlx5_hlist_entry *entry; 4209 struct mlx5_priv *priv = dev->data->dev_private; 4210 4211 /* Check if default flow is registered. */ 4212 if (!priv->mreg_cp_tbl) 4213 return; 4214 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 4215 MLX5_DEFAULT_COPY_ID, NULL); 4216 if (!entry) 4217 return; 4218 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4219 } 4220 4221 /** 4222 * Add the default copy action in in RX_CP_TBL. 4223 * 4224 * This functions is called in the mlx5_dev_start(). No thread safe 4225 * is guaranteed. 4226 * 4227 * @param dev 4228 * Pointer to Ethernet device. 4229 * @param[out] error 4230 * Perform verbose error reporting if not NULL. 4231 * 4232 * @return 4233 * 0 for success, negative value otherwise and rte_errno is set. 4234 */ 4235 static int 4236 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4237 struct rte_flow_error *error) 4238 { 4239 struct mlx5_priv *priv = dev->data->dev_private; 4240 struct mlx5_flow_mreg_copy_resource *mcp_res; 4241 4242 /* Check whether extensive metadata feature is engaged. */ 4243 if (!priv->config.dv_flow_en || 4244 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4245 !mlx5_flow_ext_mreg_supported(dev) || 4246 !priv->sh->dv_regc0_mask) 4247 return 0; 4248 /* 4249 * Add default mreg copy flow may be called multiple time, but 4250 * only be called once in stop. Avoid register it twice. 4251 */ 4252 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 4253 return 0; 4254 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 4255 if (!mcp_res) 4256 return -rte_errno; 4257 return 0; 4258 } 4259 4260 /** 4261 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4262 * 4263 * All the flow having Q/RSS action should be split by 4264 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4265 * performs the following, 4266 * - CQE->flow_tag := reg_c[1] (MARK) 4267 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4268 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4269 * but there should be a flow per each MARK ID set by MARK action. 4270 * 4271 * For the aforementioned reason, if there's a MARK action in flow's action 4272 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4273 * the MARK ID to CQE's flow_tag like, 4274 * - If reg_c[1] is mark_id, 4275 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4276 * 4277 * For SET_META action which stores value in reg_c[0], as the destination is 4278 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4279 * MARK ID means the default flow. The default flow looks like, 4280 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4281 * 4282 * @param dev 4283 * Pointer to Ethernet device. 4284 * @param flow 4285 * Pointer to flow structure. 4286 * @param[in] actions 4287 * Pointer to the list of actions. 4288 * @param[out] error 4289 * Perform verbose error reporting if not NULL. 4290 * 4291 * @return 4292 * 0 on success, negative value otherwise and rte_errno is set. 4293 */ 4294 static int 4295 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4296 struct rte_flow *flow, 4297 const struct rte_flow_action *actions, 4298 struct rte_flow_error *error) 4299 { 4300 struct mlx5_priv *priv = dev->data->dev_private; 4301 struct mlx5_dev_config *config = &priv->config; 4302 struct mlx5_flow_mreg_copy_resource *mcp_res; 4303 const struct rte_flow_action_mark *mark; 4304 4305 /* Check whether extensive metadata feature is engaged. */ 4306 if (!config->dv_flow_en || 4307 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4308 !mlx5_flow_ext_mreg_supported(dev) || 4309 !priv->sh->dv_regc0_mask) 4310 return 0; 4311 /* Find MARK action. */ 4312 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4313 switch (actions->type) { 4314 case RTE_FLOW_ACTION_TYPE_FLAG: 4315 mcp_res = flow_mreg_add_copy_action 4316 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4317 if (!mcp_res) 4318 return -rte_errno; 4319 flow->rix_mreg_copy = mcp_res->idx; 4320 return 0; 4321 case RTE_FLOW_ACTION_TYPE_MARK: 4322 mark = (const struct rte_flow_action_mark *) 4323 actions->conf; 4324 mcp_res = 4325 flow_mreg_add_copy_action(dev, mark->id, error); 4326 if (!mcp_res) 4327 return -rte_errno; 4328 flow->rix_mreg_copy = mcp_res->idx; 4329 return 0; 4330 default: 4331 break; 4332 } 4333 } 4334 return 0; 4335 } 4336 4337 #define MLX5_MAX_SPLIT_ACTIONS 24 4338 #define MLX5_MAX_SPLIT_ITEMS 24 4339 4340 /** 4341 * Split the hairpin flow. 4342 * Since HW can't support encap and push-vlan on Rx, we move these 4343 * actions to Tx. 4344 * If the count action is after the encap then we also 4345 * move the count action. in this case the count will also measure 4346 * the outer bytes. 4347 * 4348 * @param dev 4349 * Pointer to Ethernet device. 4350 * @param[in] actions 4351 * Associated actions (list terminated by the END action). 4352 * @param[out] actions_rx 4353 * Rx flow actions. 4354 * @param[out] actions_tx 4355 * Tx flow actions.. 4356 * @param[out] pattern_tx 4357 * The pattern items for the Tx flow. 4358 * @param[out] flow_id 4359 * The flow ID connected to this flow. 4360 * 4361 * @return 4362 * 0 on success. 4363 */ 4364 static int 4365 flow_hairpin_split(struct rte_eth_dev *dev, 4366 const struct rte_flow_action actions[], 4367 struct rte_flow_action actions_rx[], 4368 struct rte_flow_action actions_tx[], 4369 struct rte_flow_item pattern_tx[], 4370 uint32_t flow_id) 4371 { 4372 const struct rte_flow_action_raw_encap *raw_encap; 4373 const struct rte_flow_action_raw_decap *raw_decap; 4374 struct mlx5_rte_flow_action_set_tag *set_tag; 4375 struct rte_flow_action *tag_action; 4376 struct mlx5_rte_flow_item_tag *tag_item; 4377 struct rte_flow_item *item; 4378 char *addr; 4379 int encap = 0; 4380 4381 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4382 switch (actions->type) { 4383 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4384 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4385 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4386 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4387 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4388 rte_memcpy(actions_tx, actions, 4389 sizeof(struct rte_flow_action)); 4390 actions_tx++; 4391 break; 4392 case RTE_FLOW_ACTION_TYPE_COUNT: 4393 if (encap) { 4394 rte_memcpy(actions_tx, actions, 4395 sizeof(struct rte_flow_action)); 4396 actions_tx++; 4397 } else { 4398 rte_memcpy(actions_rx, actions, 4399 sizeof(struct rte_flow_action)); 4400 actions_rx++; 4401 } 4402 break; 4403 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4404 raw_encap = actions->conf; 4405 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4406 memcpy(actions_tx, actions, 4407 sizeof(struct rte_flow_action)); 4408 actions_tx++; 4409 encap = 1; 4410 } else { 4411 rte_memcpy(actions_rx, actions, 4412 sizeof(struct rte_flow_action)); 4413 actions_rx++; 4414 } 4415 break; 4416 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4417 raw_decap = actions->conf; 4418 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4419 memcpy(actions_tx, actions, 4420 sizeof(struct rte_flow_action)); 4421 actions_tx++; 4422 } else { 4423 rte_memcpy(actions_rx, actions, 4424 sizeof(struct rte_flow_action)); 4425 actions_rx++; 4426 } 4427 break; 4428 default: 4429 rte_memcpy(actions_rx, actions, 4430 sizeof(struct rte_flow_action)); 4431 actions_rx++; 4432 break; 4433 } 4434 } 4435 /* Add set meta action and end action for the Rx flow. */ 4436 tag_action = actions_rx; 4437 tag_action->type = (enum rte_flow_action_type) 4438 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4439 actions_rx++; 4440 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4441 actions_rx++; 4442 set_tag = (void *)actions_rx; 4443 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4444 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4445 .data = flow_id, 4446 }; 4447 MLX5_ASSERT(set_tag->id > REG_NON); 4448 tag_action->conf = set_tag; 4449 /* Create Tx item list. */ 4450 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4451 addr = (void *)&pattern_tx[2]; 4452 item = pattern_tx; 4453 item->type = (enum rte_flow_item_type) 4454 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4455 tag_item = (void *)addr; 4456 tag_item->data = flow_id; 4457 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4458 MLX5_ASSERT(set_tag->id > REG_NON); 4459 item->spec = tag_item; 4460 addr += sizeof(struct mlx5_rte_flow_item_tag); 4461 tag_item = (void *)addr; 4462 tag_item->data = UINT32_MAX; 4463 tag_item->id = UINT16_MAX; 4464 item->mask = tag_item; 4465 item->last = NULL; 4466 item++; 4467 item->type = RTE_FLOW_ITEM_TYPE_END; 4468 return 0; 4469 } 4470 4471 /** 4472 * The last stage of splitting chain, just creates the subflow 4473 * without any modification. 4474 * 4475 * @param[in] dev 4476 * Pointer to Ethernet device. 4477 * @param[in] flow 4478 * Parent flow structure pointer. 4479 * @param[in, out] sub_flow 4480 * Pointer to return the created subflow, may be NULL. 4481 * @param[in] attr 4482 * Flow rule attributes. 4483 * @param[in] items 4484 * Pattern specification (list terminated by the END pattern item). 4485 * @param[in] actions 4486 * Associated actions (list terminated by the END action). 4487 * @param[in] flow_split_info 4488 * Pointer to flow split info structure. 4489 * @param[out] error 4490 * Perform verbose error reporting if not NULL. 4491 * @return 4492 * 0 on success, negative value otherwise 4493 */ 4494 static int 4495 flow_create_split_inner(struct rte_eth_dev *dev, 4496 struct rte_flow *flow, 4497 struct mlx5_flow **sub_flow, 4498 const struct rte_flow_attr *attr, 4499 const struct rte_flow_item items[], 4500 const struct rte_flow_action actions[], 4501 struct mlx5_flow_split_info *flow_split_info, 4502 struct rte_flow_error *error) 4503 { 4504 struct mlx5_flow *dev_flow; 4505 4506 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4507 flow_split_info->flow_idx, error); 4508 if (!dev_flow) 4509 return -rte_errno; 4510 dev_flow->flow = flow; 4511 dev_flow->external = flow_split_info->external; 4512 dev_flow->skip_scale = flow_split_info->skip_scale; 4513 /* Subflow object was created, we must include one in the list. */ 4514 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4515 dev_flow->handle, next); 4516 /* 4517 * If dev_flow is as one of the suffix flow, some actions in suffix 4518 * flow may need some user defined item layer flags, and pass the 4519 * Metadate rxq mark flag to suffix flow as well. 4520 */ 4521 if (flow_split_info->prefix_layers) 4522 dev_flow->handle->layers = flow_split_info->prefix_layers; 4523 if (flow_split_info->prefix_mark) 4524 dev_flow->handle->mark = 1; 4525 if (sub_flow) 4526 *sub_flow = dev_flow; 4527 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4528 dev_flow->dv.table_id = flow_split_info->table_id; 4529 #endif 4530 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4531 } 4532 4533 /** 4534 * Get the sub policy of a meter. 4535 * 4536 * @param[in] dev 4537 * Pointer to Ethernet device. 4538 * @param[in] flow 4539 * Parent flow structure pointer. 4540 * @param[in] policy_id; 4541 * Meter Policy id. 4542 * @param[in] attr 4543 * Flow rule attributes. 4544 * @param[in] items 4545 * Pattern specification (list terminated by the END pattern item). 4546 * @param[out] error 4547 * Perform verbose error reporting if not NULL. 4548 * 4549 * @return 4550 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4551 */ 4552 static struct mlx5_flow_meter_sub_policy * 4553 get_meter_sub_policy(struct rte_eth_dev *dev, 4554 struct rte_flow *flow, 4555 uint32_t policy_id, 4556 const struct rte_flow_attr *attr, 4557 const struct rte_flow_item items[], 4558 struct rte_flow_error *error) 4559 { 4560 struct mlx5_flow_meter_policy *policy; 4561 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4562 4563 policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 4564 if (!policy) { 4565 rte_flow_error_set(error, EINVAL, 4566 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4567 "Failed to find Meter Policy."); 4568 goto exit; 4569 } 4570 if (policy->is_rss || 4571 (policy->is_queue && 4572 !policy->sub_policys[MLX5_MTR_DOMAIN_INGRESS][0]->rix_hrxq[0])) { 4573 struct mlx5_flow_workspace *wks = 4574 mlx5_flow_get_thread_workspace(); 4575 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4576 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4577 uint32_t i; 4578 4579 MLX5_ASSERT(wks); 4580 /** 4581 * This is a tmp dev_flow, 4582 * no need to register any matcher for it in translate. 4583 */ 4584 wks->skip_matcher_reg = 1; 4585 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4586 struct mlx5_flow dev_flow = {0}; 4587 struct mlx5_flow_handle dev_handle = { {0} }; 4588 4589 rss_desc_v[i] = wks->rss_desc; 4590 if (policy->is_rss) { 4591 const void *rss_act = 4592 policy->act_cnt[i].rss->conf; 4593 struct rte_flow_action rss_actions[2] = { 4594 [0] = { 4595 .type = RTE_FLOW_ACTION_TYPE_RSS, 4596 .conf = rss_act 4597 }, 4598 [1] = { 4599 .type = RTE_FLOW_ACTION_TYPE_END, 4600 .conf = NULL 4601 } 4602 }; 4603 4604 dev_flow.handle = &dev_handle; 4605 dev_flow.ingress = attr->ingress; 4606 dev_flow.flow = flow; 4607 dev_flow.external = 0; 4608 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4609 dev_flow.dv.transfer = attr->transfer; 4610 #endif 4611 /** 4612 * Translate RSS action to get rss hash fields. 4613 */ 4614 if (flow_drv_translate(dev, &dev_flow, attr, 4615 items, rss_actions, error)) 4616 goto exit; 4617 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4618 rss_desc_v[i].hash_fields = 4619 dev_flow.hash_fields; 4620 rss_desc_v[i].queue_num = 4621 rss_desc_v[i].hash_fields ? 4622 rss_desc_v[i].queue_num : 1; 4623 } else { 4624 /* This is queue action. */ 4625 rss_desc_v[i].key_len = 0; 4626 rss_desc_v[i].hash_fields = 0; 4627 rss_desc_v[i].queue = 4628 &policy->act_cnt[i].queue; 4629 rss_desc_v[i].queue_num = 1; 4630 } 4631 rss_desc[i] = &rss_desc_v[i]; 4632 } 4633 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4634 flow, policy, rss_desc); 4635 } else { 4636 enum mlx5_meter_domain mtr_domain = 4637 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4638 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4639 MLX5_MTR_DOMAIN_INGRESS; 4640 sub_policy = policy->sub_policys[mtr_domain][0]; 4641 } 4642 if (!sub_policy) { 4643 rte_flow_error_set(error, EINVAL, 4644 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4645 "Failed to get meter sub-policy."); 4646 goto exit; 4647 } 4648 exit: 4649 return sub_policy; 4650 } 4651 4652 /** 4653 * Split the meter flow. 4654 * 4655 * As meter flow will split to three sub flow, other than meter 4656 * action, the other actions make sense to only meter accepts 4657 * the packet. If it need to be dropped, no other additional 4658 * actions should be take. 4659 * 4660 * One kind of special action which decapsulates the L3 tunnel 4661 * header will be in the prefix sub flow, as not to take the 4662 * L3 tunnel header into account. 4663 * 4664 * @param[in] dev 4665 * Pointer to Ethernet device. 4666 * @param[in] flow 4667 * Parent flow structure pointer. 4668 * @param[in] fm 4669 * Pointer to flow meter structure. 4670 * @param[in] attr 4671 * Flow rule attributes. 4672 * @param[in] items 4673 * Pattern specification (list terminated by the END pattern item). 4674 * @param[out] sfx_items 4675 * Suffix flow match items (list terminated by the END pattern item). 4676 * @param[in] actions 4677 * Associated actions (list terminated by the END action). 4678 * @param[out] actions_sfx 4679 * Suffix flow actions. 4680 * @param[out] actions_pre 4681 * Prefix flow actions. 4682 * @param[out] mtr_flow_id 4683 * Pointer to meter flow id. 4684 * @param[out] error 4685 * Perform verbose error reporting if not NULL. 4686 * 4687 * @return 4688 * 0 on success, a negative errno value otherwise and rte_errno is set. 4689 */ 4690 static int 4691 flow_meter_split_prep(struct rte_eth_dev *dev, 4692 struct rte_flow *flow, 4693 struct mlx5_flow_meter_info *fm, 4694 const struct rte_flow_attr *attr, 4695 const struct rte_flow_item items[], 4696 struct rte_flow_item sfx_items[], 4697 const struct rte_flow_action actions[], 4698 struct rte_flow_action actions_sfx[], 4699 struct rte_flow_action actions_pre[], 4700 uint32_t *mtr_flow_id, 4701 struct rte_flow_error *error) 4702 { 4703 struct mlx5_priv *priv = dev->data->dev_private; 4704 struct rte_flow_action *tag_action = NULL; 4705 struct rte_flow_item *tag_item; 4706 struct mlx5_rte_flow_action_set_tag *set_tag; 4707 const struct rte_flow_action_raw_encap *raw_encap; 4708 const struct rte_flow_action_raw_decap *raw_decap; 4709 struct mlx5_rte_flow_item_tag *tag_item_spec; 4710 struct mlx5_rte_flow_item_tag *tag_item_mask; 4711 uint32_t tag_id = 0; 4712 bool copy_vlan = false; 4713 struct rte_flow_action *hw_mtr_action; 4714 struct rte_flow_action *action_pre_head = NULL; 4715 bool mtr_first = priv->sh->meter_aso_en && 4716 (attr->egress || 4717 (attr->transfer && priv->representor_id != UINT16_MAX)); 4718 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 4719 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 4720 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 4721 uint32_t flow_id = 0; 4722 uint32_t flow_id_reversed = 0; 4723 uint8_t flow_id_bits = 0; 4724 int shift; 4725 4726 /* For ASO meter, meter must be before tag in TX direction. */ 4727 if (mtr_first) { 4728 action_pre_head = actions_pre++; 4729 /* Leave space for tag action. */ 4730 tag_action = actions_pre++; 4731 } 4732 /* Prepare the actions for prefix and suffix flow. */ 4733 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4734 struct rte_flow_action *action_cur = NULL; 4735 4736 switch (actions->type) { 4737 case RTE_FLOW_ACTION_TYPE_METER: 4738 if (mtr_first) { 4739 action_cur = action_pre_head; 4740 } else { 4741 /* Leave space for tag action. */ 4742 tag_action = actions_pre++; 4743 action_cur = actions_pre++; 4744 } 4745 break; 4746 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4747 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4748 action_cur = actions_pre++; 4749 break; 4750 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4751 raw_encap = actions->conf; 4752 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4753 action_cur = actions_pre++; 4754 break; 4755 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4756 raw_decap = actions->conf; 4757 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4758 action_cur = actions_pre++; 4759 break; 4760 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4761 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4762 copy_vlan = true; 4763 break; 4764 default: 4765 break; 4766 } 4767 if (!action_cur) 4768 action_cur = (fm->def_policy) ? 4769 actions_sfx++ : actions_pre++; 4770 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 4771 } 4772 /* Add end action to the actions. */ 4773 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4774 if (priv->sh->meter_aso_en) { 4775 /** 4776 * For ASO meter, need to add an extra jump action explicitly, 4777 * to jump from meter to policer table. 4778 */ 4779 struct mlx5_flow_meter_sub_policy *sub_policy; 4780 struct mlx5_flow_tbl_data_entry *tbl_data; 4781 4782 if (!fm->def_policy) { 4783 sub_policy = get_meter_sub_policy(dev, flow, 4784 fm->policy_id, attr, 4785 items, error); 4786 if (!sub_policy) 4787 return -rte_errno; 4788 } else { 4789 enum mlx5_meter_domain mtr_domain = 4790 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4791 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4792 MLX5_MTR_DOMAIN_INGRESS; 4793 4794 sub_policy = 4795 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 4796 } 4797 tbl_data = container_of(sub_policy->tbl_rsc, 4798 struct mlx5_flow_tbl_data_entry, tbl); 4799 hw_mtr_action = actions_pre++; 4800 hw_mtr_action->type = (enum rte_flow_action_type) 4801 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 4802 hw_mtr_action->conf = tbl_data->jump.action; 4803 } 4804 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4805 actions_pre++; 4806 if (!tag_action) 4807 return rte_flow_error_set(error, ENOMEM, 4808 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4809 "No tag action space."); 4810 if (!mtr_flow_id) { 4811 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 4812 goto exit; 4813 } 4814 /* Only default-policy Meter creates mtr flow id. */ 4815 if (fm->def_policy) { 4816 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 4817 if (!tag_id) 4818 return rte_flow_error_set(error, ENOMEM, 4819 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4820 "Failed to allocate meter flow id."); 4821 flow_id = tag_id - 1; 4822 flow_id_bits = (!flow_id) ? 1 : 4823 (MLX5_REG_BITS - __builtin_clz(flow_id)); 4824 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 4825 mtr_reg_bits) { 4826 mlx5_ipool_free(fm->flow_ipool, tag_id); 4827 return rte_flow_error_set(error, EINVAL, 4828 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4829 "Meter flow id exceeds max limit."); 4830 } 4831 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 4832 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 4833 } 4834 /* Prepare the suffix subflow items. */ 4835 tag_item = sfx_items++; 4836 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4837 int item_type = items->type; 4838 4839 switch (item_type) { 4840 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4841 memcpy(sfx_items, items, sizeof(*sfx_items)); 4842 sfx_items++; 4843 break; 4844 case RTE_FLOW_ITEM_TYPE_VLAN: 4845 if (copy_vlan) { 4846 memcpy(sfx_items, items, sizeof(*sfx_items)); 4847 /* 4848 * Convert to internal match item, it is used 4849 * for vlan push and set vid. 4850 */ 4851 sfx_items->type = (enum rte_flow_item_type) 4852 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4853 sfx_items++; 4854 } 4855 break; 4856 default: 4857 break; 4858 } 4859 } 4860 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4861 sfx_items++; 4862 /* Build tag actions and items for meter_id/meter flow_id. */ 4863 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 4864 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4865 tag_item_mask = tag_item_spec + 1; 4866 /* Both flow_id and meter_id share the same register. */ 4867 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4868 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 4869 0, error), 4870 .offset = mtr_id_offset, 4871 .length = mtr_reg_bits, 4872 .data = flow->meter, 4873 }; 4874 /* 4875 * The color Reg bits used by flow_id are growing from 4876 * msb to lsb, so must do bit reverse for flow_id val in RegC. 4877 */ 4878 for (shift = 0; shift < flow_id_bits; shift++) 4879 flow_id_reversed = (flow_id_reversed << 1) | 4880 ((flow_id >> shift) & 0x1); 4881 set_tag->data |= 4882 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 4883 tag_item_spec->id = set_tag->id; 4884 tag_item_spec->data = set_tag->data << mtr_id_offset; 4885 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 4886 tag_action->type = (enum rte_flow_action_type) 4887 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4888 tag_action->conf = set_tag; 4889 tag_item->type = (enum rte_flow_item_type) 4890 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4891 tag_item->spec = tag_item_spec; 4892 tag_item->last = NULL; 4893 tag_item->mask = tag_item_mask; 4894 exit: 4895 if (mtr_flow_id) 4896 *mtr_flow_id = tag_id; 4897 return 0; 4898 } 4899 4900 /** 4901 * Split action list having QUEUE/RSS for metadata register copy. 4902 * 4903 * Once Q/RSS action is detected in user's action list, the flow action 4904 * should be split in order to copy metadata registers, which will happen in 4905 * RX_CP_TBL like, 4906 * - CQE->flow_tag := reg_c[1] (MARK) 4907 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4908 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4909 * This is because the last action of each flow must be a terminal action 4910 * (QUEUE, RSS or DROP). 4911 * 4912 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4913 * stored and kept in the mlx5_flow structure per each sub_flow. 4914 * 4915 * The Q/RSS action is replaced with, 4916 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4917 * And the following JUMP action is added at the end, 4918 * - JUMP, to RX_CP_TBL. 4919 * 4920 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4921 * flow_create_split_metadata() routine. The flow will look like, 4922 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4923 * 4924 * @param dev 4925 * Pointer to Ethernet device. 4926 * @param[out] split_actions 4927 * Pointer to store split actions to jump to CP_TBL. 4928 * @param[in] actions 4929 * Pointer to the list of original flow actions. 4930 * @param[in] qrss 4931 * Pointer to the Q/RSS action. 4932 * @param[in] actions_n 4933 * Number of original actions. 4934 * @param[out] error 4935 * Perform verbose error reporting if not NULL. 4936 * 4937 * @return 4938 * non-zero unique flow_id on success, otherwise 0 and 4939 * error/rte_error are set. 4940 */ 4941 static uint32_t 4942 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4943 struct rte_flow_action *split_actions, 4944 const struct rte_flow_action *actions, 4945 const struct rte_flow_action *qrss, 4946 int actions_n, struct rte_flow_error *error) 4947 { 4948 struct mlx5_priv *priv = dev->data->dev_private; 4949 struct mlx5_rte_flow_action_set_tag *set_tag; 4950 struct rte_flow_action_jump *jump; 4951 const int qrss_idx = qrss - actions; 4952 uint32_t flow_id = 0; 4953 int ret = 0; 4954 4955 /* 4956 * Given actions will be split 4957 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4958 * - Add jump to mreg CP_TBL. 4959 * As a result, there will be one more action. 4960 */ 4961 ++actions_n; 4962 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4963 set_tag = (void *)(split_actions + actions_n); 4964 /* 4965 * If tag action is not set to void(it means we are not the meter 4966 * suffix flow), add the tag action. Since meter suffix flow already 4967 * has the tag added. 4968 */ 4969 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4970 /* 4971 * Allocate the new subflow ID. This one is unique within 4972 * device and not shared with representors. Otherwise, 4973 * we would have to resolve multi-thread access synch 4974 * issue. Each flow on the shared device is appended 4975 * with source vport identifier, so the resulting 4976 * flows will be unique in the shared (by master and 4977 * representors) domain even if they have coinciding 4978 * IDs. 4979 */ 4980 mlx5_ipool_malloc(priv->sh->ipool 4981 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 4982 if (!flow_id) 4983 return rte_flow_error_set(error, ENOMEM, 4984 RTE_FLOW_ERROR_TYPE_ACTION, 4985 NULL, "can't allocate id " 4986 "for split Q/RSS subflow"); 4987 /* Internal SET_TAG action to set flow ID. */ 4988 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4989 .data = flow_id, 4990 }; 4991 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4992 if (ret < 0) 4993 return ret; 4994 set_tag->id = ret; 4995 /* Construct new actions array. */ 4996 /* Replace QUEUE/RSS action. */ 4997 split_actions[qrss_idx] = (struct rte_flow_action){ 4998 .type = (enum rte_flow_action_type) 4999 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5000 .conf = set_tag, 5001 }; 5002 } 5003 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5004 jump = (void *)(set_tag + 1); 5005 *jump = (struct rte_flow_action_jump){ 5006 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5007 }; 5008 split_actions[actions_n - 2] = (struct rte_flow_action){ 5009 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5010 .conf = jump, 5011 }; 5012 split_actions[actions_n - 1] = (struct rte_flow_action){ 5013 .type = RTE_FLOW_ACTION_TYPE_END, 5014 }; 5015 return flow_id; 5016 } 5017 5018 /** 5019 * Extend the given action list for Tx metadata copy. 5020 * 5021 * Copy the given action list to the ext_actions and add flow metadata register 5022 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5023 * 5024 * @param[out] ext_actions 5025 * Pointer to the extended action list. 5026 * @param[in] actions 5027 * Pointer to the list of actions. 5028 * @param[in] actions_n 5029 * Number of actions in the list. 5030 * @param[out] error 5031 * Perform verbose error reporting if not NULL. 5032 * @param[in] encap_idx 5033 * The encap action inndex. 5034 * 5035 * @return 5036 * 0 on success, negative value otherwise 5037 */ 5038 static int 5039 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5040 struct rte_flow_action *ext_actions, 5041 const struct rte_flow_action *actions, 5042 int actions_n, struct rte_flow_error *error, 5043 int encap_idx) 5044 { 5045 struct mlx5_flow_action_copy_mreg *cp_mreg = 5046 (struct mlx5_flow_action_copy_mreg *) 5047 (ext_actions + actions_n + 1); 5048 int ret; 5049 5050 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5051 if (ret < 0) 5052 return ret; 5053 cp_mreg->dst = ret; 5054 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5055 if (ret < 0) 5056 return ret; 5057 cp_mreg->src = ret; 5058 if (encap_idx != 0) 5059 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5060 if (encap_idx == actions_n - 1) { 5061 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5062 .type = (enum rte_flow_action_type) 5063 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5064 .conf = cp_mreg, 5065 }; 5066 ext_actions[actions_n] = (struct rte_flow_action){ 5067 .type = RTE_FLOW_ACTION_TYPE_END, 5068 }; 5069 } else { 5070 ext_actions[encap_idx] = (struct rte_flow_action){ 5071 .type = (enum rte_flow_action_type) 5072 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5073 .conf = cp_mreg, 5074 }; 5075 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5076 sizeof(*ext_actions) * (actions_n - encap_idx)); 5077 } 5078 return 0; 5079 } 5080 5081 /** 5082 * Check the match action from the action list. 5083 * 5084 * @param[in] actions 5085 * Pointer to the list of actions. 5086 * @param[in] attr 5087 * Flow rule attributes. 5088 * @param[in] action 5089 * The action to be check if exist. 5090 * @param[out] match_action_pos 5091 * Pointer to the position of the matched action if exists, otherwise is -1. 5092 * @param[out] qrss_action_pos 5093 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5094 * @param[out] modify_after_mirror 5095 * Pointer to the flag of modify action after FDB mirroring. 5096 * 5097 * @return 5098 * > 0 the total number of actions. 5099 * 0 if not found match action in action list. 5100 */ 5101 static int 5102 flow_check_match_action(const struct rte_flow_action actions[], 5103 const struct rte_flow_attr *attr, 5104 enum rte_flow_action_type action, 5105 int *match_action_pos, int *qrss_action_pos, 5106 int *modify_after_mirror) 5107 { 5108 const struct rte_flow_action_sample *sample; 5109 int actions_n = 0; 5110 uint32_t ratio = 0; 5111 int sub_type = 0; 5112 int flag = 0; 5113 int fdb_mirror = 0; 5114 5115 *match_action_pos = -1; 5116 *qrss_action_pos = -1; 5117 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5118 if (actions->type == action) { 5119 flag = 1; 5120 *match_action_pos = actions_n; 5121 } 5122 switch (actions->type) { 5123 case RTE_FLOW_ACTION_TYPE_QUEUE: 5124 case RTE_FLOW_ACTION_TYPE_RSS: 5125 *qrss_action_pos = actions_n; 5126 break; 5127 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5128 sample = actions->conf; 5129 ratio = sample->ratio; 5130 sub_type = ((const struct rte_flow_action *) 5131 (sample->actions))->type; 5132 if (ratio == 1 && attr->transfer) 5133 fdb_mirror = 1; 5134 break; 5135 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5136 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5137 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5138 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5139 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5140 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5141 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5142 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5143 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5144 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5145 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5146 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5147 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5148 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5149 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5150 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5151 case RTE_FLOW_ACTION_TYPE_FLAG: 5152 case RTE_FLOW_ACTION_TYPE_MARK: 5153 case RTE_FLOW_ACTION_TYPE_SET_META: 5154 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5155 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5156 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5157 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5158 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5159 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5160 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5161 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5162 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5163 case RTE_FLOW_ACTION_TYPE_METER: 5164 if (fdb_mirror) 5165 *modify_after_mirror = 1; 5166 break; 5167 default: 5168 break; 5169 } 5170 actions_n++; 5171 } 5172 if (flag && fdb_mirror && !*modify_after_mirror) { 5173 /* FDB mirroring uses the destination array to implement 5174 * instead of FLOW_SAMPLER object. 5175 */ 5176 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5177 flag = 0; 5178 } 5179 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5180 return flag ? actions_n + 1 : 0; 5181 } 5182 5183 #define SAMPLE_SUFFIX_ITEM 2 5184 5185 /** 5186 * Split the sample flow. 5187 * 5188 * As sample flow will split to two sub flow, sample flow with 5189 * sample action, the other actions will move to new suffix flow. 5190 * 5191 * Also add unique tag id with tag action in the sample flow, 5192 * the same tag id will be as match in the suffix flow. 5193 * 5194 * @param dev 5195 * Pointer to Ethernet device. 5196 * @param[in] add_tag 5197 * Add extra tag action flag. 5198 * @param[out] sfx_items 5199 * Suffix flow match items (list terminated by the END pattern item). 5200 * @param[in] actions 5201 * Associated actions (list terminated by the END action). 5202 * @param[out] actions_sfx 5203 * Suffix flow actions. 5204 * @param[out] actions_pre 5205 * Prefix flow actions. 5206 * @param[in] actions_n 5207 * The total number of actions. 5208 * @param[in] sample_action_pos 5209 * The sample action position. 5210 * @param[in] qrss_action_pos 5211 * The Queue/RSS action position. 5212 * @param[in] jump_table 5213 * Add extra jump action flag. 5214 * @param[out] error 5215 * Perform verbose error reporting if not NULL. 5216 * 5217 * @return 5218 * 0 on success, or unique flow_id, a negative errno value 5219 * otherwise and rte_errno is set. 5220 */ 5221 static int 5222 flow_sample_split_prep(struct rte_eth_dev *dev, 5223 int add_tag, 5224 struct rte_flow_item sfx_items[], 5225 const struct rte_flow_action actions[], 5226 struct rte_flow_action actions_sfx[], 5227 struct rte_flow_action actions_pre[], 5228 int actions_n, 5229 int sample_action_pos, 5230 int qrss_action_pos, 5231 int jump_table, 5232 struct rte_flow_error *error) 5233 { 5234 struct mlx5_priv *priv = dev->data->dev_private; 5235 struct mlx5_rte_flow_action_set_tag *set_tag; 5236 struct mlx5_rte_flow_item_tag *tag_spec; 5237 struct mlx5_rte_flow_item_tag *tag_mask; 5238 struct rte_flow_action_jump *jump_action; 5239 uint32_t tag_id = 0; 5240 int index; 5241 int append_index = 0; 5242 int ret; 5243 5244 if (sample_action_pos < 0) 5245 return rte_flow_error_set(error, EINVAL, 5246 RTE_FLOW_ERROR_TYPE_ACTION, 5247 NULL, "invalid position of sample " 5248 "action in list"); 5249 /* Prepare the actions for prefix and suffix flow. */ 5250 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5251 index = qrss_action_pos; 5252 /* Put the preceding the Queue/RSS action into prefix flow. */ 5253 if (index != 0) 5254 memcpy(actions_pre, actions, 5255 sizeof(struct rte_flow_action) * index); 5256 /* Put others preceding the sample action into prefix flow. */ 5257 if (sample_action_pos > index + 1) 5258 memcpy(actions_pre + index, actions + index + 1, 5259 sizeof(struct rte_flow_action) * 5260 (sample_action_pos - index - 1)); 5261 index = sample_action_pos - 1; 5262 /* Put Queue/RSS action into Suffix flow. */ 5263 memcpy(actions_sfx, actions + qrss_action_pos, 5264 sizeof(struct rte_flow_action)); 5265 actions_sfx++; 5266 } else { 5267 index = sample_action_pos; 5268 if (index != 0) 5269 memcpy(actions_pre, actions, 5270 sizeof(struct rte_flow_action) * index); 5271 } 5272 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5273 * For CX6DX and above, metadata registers Cx preserve their value, 5274 * add an extra tag action for NIC-RX and E-Switch Domain. 5275 */ 5276 if (add_tag) { 5277 /* Prepare the prefix tag action. */ 5278 append_index++; 5279 set_tag = (void *)(actions_pre + actions_n + append_index); 5280 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5281 if (ret < 0) 5282 return ret; 5283 mlx5_ipool_malloc(priv->sh->ipool 5284 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5285 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5286 .id = ret, 5287 .data = tag_id, 5288 }; 5289 /* Prepare the suffix subflow items. */ 5290 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5291 tag_spec->data = tag_id; 5292 tag_spec->id = set_tag->id; 5293 tag_mask = tag_spec + 1; 5294 tag_mask->data = UINT32_MAX; 5295 sfx_items[0] = (struct rte_flow_item){ 5296 .type = (enum rte_flow_item_type) 5297 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5298 .spec = tag_spec, 5299 .last = NULL, 5300 .mask = tag_mask, 5301 }; 5302 sfx_items[1] = (struct rte_flow_item){ 5303 .type = (enum rte_flow_item_type) 5304 RTE_FLOW_ITEM_TYPE_END, 5305 }; 5306 /* Prepare the tag action in prefix subflow. */ 5307 actions_pre[index++] = 5308 (struct rte_flow_action){ 5309 .type = (enum rte_flow_action_type) 5310 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5311 .conf = set_tag, 5312 }; 5313 } 5314 memcpy(actions_pre + index, actions + sample_action_pos, 5315 sizeof(struct rte_flow_action)); 5316 index += 1; 5317 /* For the modify action after the sample action in E-Switch mirroring, 5318 * Add the extra jump action in prefix subflow and jump into the next 5319 * table, then do the modify action in the new table. 5320 */ 5321 if (jump_table) { 5322 /* Prepare the prefix jump action. */ 5323 append_index++; 5324 jump_action = (void *)(actions_pre + actions_n + append_index); 5325 jump_action->group = jump_table; 5326 actions_pre[index++] = 5327 (struct rte_flow_action){ 5328 .type = (enum rte_flow_action_type) 5329 RTE_FLOW_ACTION_TYPE_JUMP, 5330 .conf = jump_action, 5331 }; 5332 } 5333 actions_pre[index] = (struct rte_flow_action){ 5334 .type = (enum rte_flow_action_type) 5335 RTE_FLOW_ACTION_TYPE_END, 5336 }; 5337 /* Put the actions after sample into Suffix flow. */ 5338 memcpy(actions_sfx, actions + sample_action_pos + 1, 5339 sizeof(struct rte_flow_action) * 5340 (actions_n - sample_action_pos - 1)); 5341 return tag_id; 5342 } 5343 5344 /** 5345 * The splitting for metadata feature. 5346 * 5347 * - Q/RSS action on NIC Rx should be split in order to pass by 5348 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5349 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5350 * 5351 * - All the actions on NIC Tx should have a mreg copy action to 5352 * copy reg_a from WQE to reg_c[0]. 5353 * 5354 * @param dev 5355 * Pointer to Ethernet device. 5356 * @param[in] flow 5357 * Parent flow structure pointer. 5358 * @param[in] attr 5359 * Flow rule attributes. 5360 * @param[in] items 5361 * Pattern specification (list terminated by the END pattern item). 5362 * @param[in] actions 5363 * Associated actions (list terminated by the END action). 5364 * @param[in] flow_split_info 5365 * Pointer to flow split info structure. 5366 * @param[out] error 5367 * Perform verbose error reporting if not NULL. 5368 * @return 5369 * 0 on success, negative value otherwise 5370 */ 5371 static int 5372 flow_create_split_metadata(struct rte_eth_dev *dev, 5373 struct rte_flow *flow, 5374 const struct rte_flow_attr *attr, 5375 const struct rte_flow_item items[], 5376 const struct rte_flow_action actions[], 5377 struct mlx5_flow_split_info *flow_split_info, 5378 struct rte_flow_error *error) 5379 { 5380 struct mlx5_priv *priv = dev->data->dev_private; 5381 struct mlx5_dev_config *config = &priv->config; 5382 const struct rte_flow_action *qrss = NULL; 5383 struct rte_flow_action *ext_actions = NULL; 5384 struct mlx5_flow *dev_flow = NULL; 5385 uint32_t qrss_id = 0; 5386 int mtr_sfx = 0; 5387 size_t act_size; 5388 int actions_n; 5389 int encap_idx; 5390 int ret; 5391 5392 /* Check whether extensive metadata feature is engaged. */ 5393 if (!config->dv_flow_en || 5394 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5395 !mlx5_flow_ext_mreg_supported(dev)) 5396 return flow_create_split_inner(dev, flow, NULL, attr, items, 5397 actions, flow_split_info, error); 5398 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5399 &encap_idx); 5400 if (qrss) { 5401 /* Exclude hairpin flows from splitting. */ 5402 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5403 const struct rte_flow_action_queue *queue; 5404 5405 queue = qrss->conf; 5406 if (mlx5_rxq_get_type(dev, queue->index) == 5407 MLX5_RXQ_TYPE_HAIRPIN) 5408 qrss = NULL; 5409 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5410 const struct rte_flow_action_rss *rss; 5411 5412 rss = qrss->conf; 5413 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5414 MLX5_RXQ_TYPE_HAIRPIN) 5415 qrss = NULL; 5416 } 5417 } 5418 if (qrss) { 5419 /* Check if it is in meter suffix table. */ 5420 mtr_sfx = attr->group == (attr->transfer ? 5421 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5422 MLX5_FLOW_TABLE_LEVEL_METER); 5423 /* 5424 * Q/RSS action on NIC Rx should be split in order to pass by 5425 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5426 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5427 */ 5428 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5429 sizeof(struct rte_flow_action_set_tag) + 5430 sizeof(struct rte_flow_action_jump); 5431 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5432 SOCKET_ID_ANY); 5433 if (!ext_actions) 5434 return rte_flow_error_set(error, ENOMEM, 5435 RTE_FLOW_ERROR_TYPE_ACTION, 5436 NULL, "no memory to split " 5437 "metadata flow"); 5438 /* 5439 * If we are the suffix flow of meter, tag already exist. 5440 * Set the tag action to void. 5441 */ 5442 if (mtr_sfx) 5443 ext_actions[qrss - actions].type = 5444 RTE_FLOW_ACTION_TYPE_VOID; 5445 else 5446 ext_actions[qrss - actions].type = 5447 (enum rte_flow_action_type) 5448 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5449 /* 5450 * Create the new actions list with removed Q/RSS action 5451 * and appended set tag and jump to register copy table 5452 * (RX_CP_TBL). We should preallocate unique tag ID here 5453 * in advance, because it is needed for set tag action. 5454 */ 5455 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5456 qrss, actions_n, error); 5457 if (!mtr_sfx && !qrss_id) { 5458 ret = -rte_errno; 5459 goto exit; 5460 } 5461 } else if (attr->egress && !attr->transfer) { 5462 /* 5463 * All the actions on NIC Tx should have a metadata register 5464 * copy action to copy reg_a from WQE to reg_c[meta] 5465 */ 5466 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5467 sizeof(struct mlx5_flow_action_copy_mreg); 5468 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5469 SOCKET_ID_ANY); 5470 if (!ext_actions) 5471 return rte_flow_error_set(error, ENOMEM, 5472 RTE_FLOW_ERROR_TYPE_ACTION, 5473 NULL, "no memory to split " 5474 "metadata flow"); 5475 /* Create the action list appended with copy register. */ 5476 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5477 actions_n, error, encap_idx); 5478 if (ret < 0) 5479 goto exit; 5480 } 5481 /* Add the unmodified original or prefix subflow. */ 5482 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5483 items, ext_actions ? ext_actions : 5484 actions, flow_split_info, error); 5485 if (ret < 0) 5486 goto exit; 5487 MLX5_ASSERT(dev_flow); 5488 if (qrss) { 5489 const struct rte_flow_attr q_attr = { 5490 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5491 .ingress = 1, 5492 }; 5493 /* Internal PMD action to set register. */ 5494 struct mlx5_rte_flow_item_tag q_tag_spec = { 5495 .data = qrss_id, 5496 .id = REG_NON, 5497 }; 5498 struct rte_flow_item q_items[] = { 5499 { 5500 .type = (enum rte_flow_item_type) 5501 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5502 .spec = &q_tag_spec, 5503 .last = NULL, 5504 .mask = NULL, 5505 }, 5506 { 5507 .type = RTE_FLOW_ITEM_TYPE_END, 5508 }, 5509 }; 5510 struct rte_flow_action q_actions[] = { 5511 { 5512 .type = qrss->type, 5513 .conf = qrss->conf, 5514 }, 5515 { 5516 .type = RTE_FLOW_ACTION_TYPE_END, 5517 }, 5518 }; 5519 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5520 5521 /* 5522 * Configure the tag item only if there is no meter subflow. 5523 * Since tag is already marked in the meter suffix subflow 5524 * we can just use the meter suffix items as is. 5525 */ 5526 if (qrss_id) { 5527 /* Not meter subflow. */ 5528 MLX5_ASSERT(!mtr_sfx); 5529 /* 5530 * Put unique id in prefix flow due to it is destroyed 5531 * after suffix flow and id will be freed after there 5532 * is no actual flows with this id and identifier 5533 * reallocation becomes possible (for example, for 5534 * other flows in other threads). 5535 */ 5536 dev_flow->handle->split_flow_id = qrss_id; 5537 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5538 error); 5539 if (ret < 0) 5540 goto exit; 5541 q_tag_spec.id = ret; 5542 } 5543 dev_flow = NULL; 5544 /* Add suffix subflow to execute Q/RSS. */ 5545 flow_split_info->prefix_layers = layers; 5546 flow_split_info->prefix_mark = 0; 5547 ret = flow_create_split_inner(dev, flow, &dev_flow, 5548 &q_attr, mtr_sfx ? items : 5549 q_items, q_actions, 5550 flow_split_info, error); 5551 if (ret < 0) 5552 goto exit; 5553 /* qrss ID should be freed if failed. */ 5554 qrss_id = 0; 5555 MLX5_ASSERT(dev_flow); 5556 } 5557 5558 exit: 5559 /* 5560 * We do not destroy the partially created sub_flows in case of error. 5561 * These ones are included into parent flow list and will be destroyed 5562 * by flow_drv_destroy. 5563 */ 5564 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5565 qrss_id); 5566 mlx5_free(ext_actions); 5567 return ret; 5568 } 5569 5570 /** 5571 * Create meter internal drop flow with the original pattern. 5572 * 5573 * @param dev 5574 * Pointer to Ethernet device. 5575 * @param[in] flow 5576 * Parent flow structure pointer. 5577 * @param[in] attr 5578 * Flow rule attributes. 5579 * @param[in] items 5580 * Pattern specification (list terminated by the END pattern item). 5581 * @param[in] flow_split_info 5582 * Pointer to flow split info structure. 5583 * @param[in] fm 5584 * Pointer to flow meter structure. 5585 * @param[out] error 5586 * Perform verbose error reporting if not NULL. 5587 * @return 5588 * 0 on success, negative value otherwise 5589 */ 5590 static uint32_t 5591 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5592 struct rte_flow *flow, 5593 const struct rte_flow_attr *attr, 5594 const struct rte_flow_item items[], 5595 struct mlx5_flow_split_info *flow_split_info, 5596 struct mlx5_flow_meter_info *fm, 5597 struct rte_flow_error *error) 5598 { 5599 struct mlx5_flow *dev_flow = NULL; 5600 struct rte_flow_attr drop_attr = *attr; 5601 struct rte_flow_action drop_actions[3]; 5602 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5603 5604 MLX5_ASSERT(fm->drop_cnt); 5605 drop_actions[0].type = 5606 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5607 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5608 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5609 drop_actions[1].conf = NULL; 5610 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5611 drop_actions[2].conf = NULL; 5612 drop_split_info.external = false; 5613 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5614 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5615 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5616 return flow_create_split_inner(dev, flow, &dev_flow, 5617 &drop_attr, items, drop_actions, 5618 &drop_split_info, error); 5619 } 5620 5621 /** 5622 * The splitting for meter feature. 5623 * 5624 * - The meter flow will be split to two flows as prefix and 5625 * suffix flow. The packets make sense only it pass the prefix 5626 * meter action. 5627 * 5628 * - Reg_C_5 is used for the packet to match betweend prefix and 5629 * suffix flow. 5630 * 5631 * @param dev 5632 * Pointer to Ethernet device. 5633 * @param[in] flow 5634 * Parent flow structure pointer. 5635 * @param[in] attr 5636 * Flow rule attributes. 5637 * @param[in] items 5638 * Pattern specification (list terminated by the END pattern item). 5639 * @param[in] actions 5640 * Associated actions (list terminated by the END action). 5641 * @param[in] flow_split_info 5642 * Pointer to flow split info structure. 5643 * @param[out] error 5644 * Perform verbose error reporting if not NULL. 5645 * @return 5646 * 0 on success, negative value otherwise 5647 */ 5648 static int 5649 flow_create_split_meter(struct rte_eth_dev *dev, 5650 struct rte_flow *flow, 5651 const struct rte_flow_attr *attr, 5652 const struct rte_flow_item items[], 5653 const struct rte_flow_action actions[], 5654 struct mlx5_flow_split_info *flow_split_info, 5655 struct rte_flow_error *error) 5656 { 5657 struct mlx5_priv *priv = dev->data->dev_private; 5658 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5659 struct rte_flow_action *sfx_actions = NULL; 5660 struct rte_flow_action *pre_actions = NULL; 5661 struct rte_flow_item *sfx_items = NULL; 5662 struct mlx5_flow *dev_flow = NULL; 5663 struct rte_flow_attr sfx_attr = *attr; 5664 struct mlx5_flow_meter_info *fm = NULL; 5665 uint8_t skip_scale_restore; 5666 bool has_mtr = false; 5667 bool has_modify = false; 5668 bool set_mtr_reg = true; 5669 uint32_t meter_id = 0; 5670 uint32_t mtr_idx = 0; 5671 uint32_t mtr_flow_id = 0; 5672 size_t act_size; 5673 size_t item_size; 5674 int actions_n = 0; 5675 int ret = 0; 5676 5677 if (priv->mtr_en) 5678 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 5679 &has_modify, &meter_id); 5680 if (has_mtr) { 5681 if (flow->meter) { 5682 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 5683 if (!fm) 5684 return rte_flow_error_set(error, EINVAL, 5685 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5686 NULL, "Meter not found."); 5687 } else { 5688 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 5689 if (!fm) 5690 return rte_flow_error_set(error, EINVAL, 5691 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5692 NULL, "Meter not found."); 5693 ret = mlx5_flow_meter_attach(priv, fm, 5694 &sfx_attr, error); 5695 if (ret) 5696 return -rte_errno; 5697 flow->meter = mtr_idx; 5698 } 5699 MLX5_ASSERT(wks); 5700 wks->fm = fm; 5701 /* 5702 * If it isn't default-policy Meter, and 5703 * 1. There's no action in flow to change 5704 * packet (modify/encap/decap etc.), OR 5705 * 2. No drop count needed for this meter. 5706 * no need to use regC to save meter id anymore. 5707 */ 5708 if (!fm->def_policy && (!has_modify || !fm->drop_cnt)) 5709 set_mtr_reg = false; 5710 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 5711 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 5712 sizeof(struct mlx5_rte_flow_action_set_tag); 5713 /* Suffix items: tag, vlan, port id, end. */ 5714 #define METER_SUFFIX_ITEM 4 5715 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5716 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5717 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5718 0, SOCKET_ID_ANY); 5719 if (!sfx_actions) 5720 return rte_flow_error_set(error, ENOMEM, 5721 RTE_FLOW_ERROR_TYPE_ACTION, 5722 NULL, "no memory to split " 5723 "meter flow"); 5724 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5725 act_size); 5726 /* There's no suffix flow for meter of non-default policy. */ 5727 if (!fm->def_policy) 5728 pre_actions = sfx_actions + 1; 5729 else 5730 pre_actions = sfx_actions + actions_n; 5731 ret = flow_meter_split_prep(dev, flow, fm, &sfx_attr, 5732 items, sfx_items, actions, 5733 sfx_actions, pre_actions, 5734 (set_mtr_reg ? &mtr_flow_id : NULL), 5735 error); 5736 if (ret) { 5737 ret = -rte_errno; 5738 goto exit; 5739 } 5740 /* Add the prefix subflow. */ 5741 flow_split_info->prefix_mark = 0; 5742 skip_scale_restore = flow_split_info->skip_scale; 5743 flow_split_info->skip_scale |= 5744 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5745 ret = flow_create_split_inner(dev, flow, &dev_flow, 5746 attr, items, pre_actions, 5747 flow_split_info, error); 5748 flow_split_info->skip_scale = skip_scale_restore; 5749 if (ret) { 5750 if (mtr_flow_id) 5751 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 5752 ret = -rte_errno; 5753 goto exit; 5754 } 5755 if (mtr_flow_id) { 5756 dev_flow->handle->split_flow_id = mtr_flow_id; 5757 dev_flow->handle->is_meter_flow_id = 1; 5758 } 5759 if (!fm->def_policy) { 5760 if (!set_mtr_reg && fm->drop_cnt) 5761 ret = 5762 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 5763 &sfx_attr, items, 5764 flow_split_info, 5765 fm, error); 5766 goto exit; 5767 } 5768 /* Setting the sfx group atrr. */ 5769 sfx_attr.group = sfx_attr.transfer ? 5770 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5771 MLX5_FLOW_TABLE_LEVEL_METER; 5772 flow_split_info->prefix_layers = 5773 flow_get_prefix_layer_flags(dev_flow); 5774 flow_split_info->prefix_mark = dev_flow->handle->mark; 5775 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 5776 } 5777 /* Add the prefix subflow. */ 5778 ret = flow_create_split_metadata(dev, flow, 5779 &sfx_attr, sfx_items ? 5780 sfx_items : items, 5781 sfx_actions ? sfx_actions : actions, 5782 flow_split_info, error); 5783 exit: 5784 if (sfx_actions) 5785 mlx5_free(sfx_actions); 5786 return ret; 5787 } 5788 5789 /** 5790 * The splitting for sample feature. 5791 * 5792 * Once Sample action is detected in the action list, the flow actions should 5793 * be split into prefix sub flow and suffix sub flow. 5794 * 5795 * The original items remain in the prefix sub flow, all actions preceding the 5796 * sample action and the sample action itself will be copied to the prefix 5797 * sub flow, the actions following the sample action will be copied to the 5798 * suffix sub flow, Queue action always be located in the suffix sub flow. 5799 * 5800 * In order to make the packet from prefix sub flow matches with suffix sub 5801 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5802 * flow uses tag item with the unique flow id. 5803 * 5804 * @param dev 5805 * Pointer to Ethernet device. 5806 * @param[in] flow 5807 * Parent flow structure pointer. 5808 * @param[in] attr 5809 * Flow rule attributes. 5810 * @param[in] items 5811 * Pattern specification (list terminated by the END pattern item). 5812 * @param[in] actions 5813 * Associated actions (list terminated by the END action). 5814 * @param[in] flow_split_info 5815 * Pointer to flow split info structure. 5816 * @param[out] error 5817 * Perform verbose error reporting if not NULL. 5818 * @return 5819 * 0 on success, negative value otherwise 5820 */ 5821 static int 5822 flow_create_split_sample(struct rte_eth_dev *dev, 5823 struct rte_flow *flow, 5824 const struct rte_flow_attr *attr, 5825 const struct rte_flow_item items[], 5826 const struct rte_flow_action actions[], 5827 struct mlx5_flow_split_info *flow_split_info, 5828 struct rte_flow_error *error) 5829 { 5830 struct mlx5_priv *priv = dev->data->dev_private; 5831 struct rte_flow_action *sfx_actions = NULL; 5832 struct rte_flow_action *pre_actions = NULL; 5833 struct rte_flow_item *sfx_items = NULL; 5834 struct mlx5_flow *dev_flow = NULL; 5835 struct rte_flow_attr sfx_attr = *attr; 5836 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5837 struct mlx5_flow_dv_sample_resource *sample_res; 5838 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5839 struct mlx5_flow_tbl_resource *sfx_tbl; 5840 #endif 5841 size_t act_size; 5842 size_t item_size; 5843 uint32_t fdb_tx = 0; 5844 int32_t tag_id = 0; 5845 int actions_n = 0; 5846 int sample_action_pos; 5847 int qrss_action_pos; 5848 int add_tag = 0; 5849 int modify_after_mirror = 0; 5850 uint16_t jump_table = 0; 5851 const uint32_t next_ft_step = 1; 5852 int ret = 0; 5853 5854 if (priv->sampler_en) 5855 actions_n = flow_check_match_action(actions, attr, 5856 RTE_FLOW_ACTION_TYPE_SAMPLE, 5857 &sample_action_pos, &qrss_action_pos, 5858 &modify_after_mirror); 5859 if (actions_n) { 5860 /* The prefix actions must includes sample, tag, end. */ 5861 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5862 + sizeof(struct mlx5_rte_flow_action_set_tag); 5863 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5864 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5865 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5866 item_size), 0, SOCKET_ID_ANY); 5867 if (!sfx_actions) 5868 return rte_flow_error_set(error, ENOMEM, 5869 RTE_FLOW_ERROR_TYPE_ACTION, 5870 NULL, "no memory to split " 5871 "sample flow"); 5872 /* The representor_id is -1 for uplink. */ 5873 fdb_tx = (attr->transfer && priv->representor_id != -1); 5874 /* 5875 * When reg_c_preserve is set, metadata registers Cx preserve 5876 * their value even through packet duplication. 5877 */ 5878 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 5879 if (add_tag) 5880 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5881 + act_size); 5882 if (modify_after_mirror) 5883 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 5884 next_ft_step; 5885 pre_actions = sfx_actions + actions_n; 5886 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 5887 actions, sfx_actions, 5888 pre_actions, actions_n, 5889 sample_action_pos, 5890 qrss_action_pos, jump_table, 5891 error); 5892 if (tag_id < 0 || (add_tag && !tag_id)) { 5893 ret = -rte_errno; 5894 goto exit; 5895 } 5896 if (modify_after_mirror) 5897 flow_split_info->skip_scale = 5898 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5899 /* Add the prefix subflow. */ 5900 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5901 items, pre_actions, 5902 flow_split_info, error); 5903 if (ret) { 5904 ret = -rte_errno; 5905 goto exit; 5906 } 5907 dev_flow->handle->split_flow_id = tag_id; 5908 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5909 if (!modify_after_mirror) { 5910 /* Set the sfx group attr. */ 5911 sample_res = (struct mlx5_flow_dv_sample_resource *) 5912 dev_flow->dv.sample_res; 5913 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5914 sample_res->normal_path_tbl; 5915 sfx_tbl_data = container_of(sfx_tbl, 5916 struct mlx5_flow_tbl_data_entry, 5917 tbl); 5918 sfx_attr.group = sfx_attr.transfer ? 5919 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 5920 } else { 5921 MLX5_ASSERT(attr->transfer); 5922 sfx_attr.group = jump_table; 5923 } 5924 flow_split_info->prefix_layers = 5925 flow_get_prefix_layer_flags(dev_flow); 5926 flow_split_info->prefix_mark = dev_flow->handle->mark; 5927 /* Suffix group level already be scaled with factor, set 5928 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 5929 * again in translation. 5930 */ 5931 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5932 #endif 5933 } 5934 /* Add the suffix subflow. */ 5935 ret = flow_create_split_meter(dev, flow, &sfx_attr, 5936 sfx_items ? sfx_items : items, 5937 sfx_actions ? sfx_actions : actions, 5938 flow_split_info, error); 5939 exit: 5940 if (sfx_actions) 5941 mlx5_free(sfx_actions); 5942 return ret; 5943 } 5944 5945 /** 5946 * Split the flow to subflow set. The splitters might be linked 5947 * in the chain, like this: 5948 * flow_create_split_outer() calls: 5949 * flow_create_split_meter() calls: 5950 * flow_create_split_metadata(meter_subflow_0) calls: 5951 * flow_create_split_inner(metadata_subflow_0) 5952 * flow_create_split_inner(metadata_subflow_1) 5953 * flow_create_split_inner(metadata_subflow_2) 5954 * flow_create_split_metadata(meter_subflow_1) calls: 5955 * flow_create_split_inner(metadata_subflow_0) 5956 * flow_create_split_inner(metadata_subflow_1) 5957 * flow_create_split_inner(metadata_subflow_2) 5958 * 5959 * This provide flexible way to add new levels of flow splitting. 5960 * The all of successfully created subflows are included to the 5961 * parent flow dev_flow list. 5962 * 5963 * @param dev 5964 * Pointer to Ethernet device. 5965 * @param[in] flow 5966 * Parent flow structure pointer. 5967 * @param[in] attr 5968 * Flow rule attributes. 5969 * @param[in] items 5970 * Pattern specification (list terminated by the END pattern item). 5971 * @param[in] actions 5972 * Associated actions (list terminated by the END action). 5973 * @param[in] flow_split_info 5974 * Pointer to flow split info structure. 5975 * @param[out] error 5976 * Perform verbose error reporting if not NULL. 5977 * @return 5978 * 0 on success, negative value otherwise 5979 */ 5980 static int 5981 flow_create_split_outer(struct rte_eth_dev *dev, 5982 struct rte_flow *flow, 5983 const struct rte_flow_attr *attr, 5984 const struct rte_flow_item items[], 5985 const struct rte_flow_action actions[], 5986 struct mlx5_flow_split_info *flow_split_info, 5987 struct rte_flow_error *error) 5988 { 5989 int ret; 5990 5991 ret = flow_create_split_sample(dev, flow, attr, items, 5992 actions, flow_split_info, error); 5993 MLX5_ASSERT(ret <= 0); 5994 return ret; 5995 } 5996 5997 static inline struct mlx5_flow_tunnel * 5998 flow_tunnel_from_rule(const struct mlx5_flow *flow) 5999 { 6000 struct mlx5_flow_tunnel *tunnel; 6001 6002 #pragma GCC diagnostic push 6003 #pragma GCC diagnostic ignored "-Wcast-qual" 6004 tunnel = (typeof(tunnel))flow->tunnel; 6005 #pragma GCC diagnostic pop 6006 6007 return tunnel; 6008 } 6009 6010 /** 6011 * Adjust flow RSS workspace if needed. 6012 * 6013 * @param wks 6014 * Pointer to thread flow work space. 6015 * @param rss_desc 6016 * Pointer to RSS descriptor. 6017 * @param[in] nrssq_num 6018 * New RSS queue number. 6019 * 6020 * @return 6021 * 0 on success, -1 otherwise and rte_errno is set. 6022 */ 6023 static int 6024 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6025 struct mlx5_flow_rss_desc *rss_desc, 6026 uint32_t nrssq_num) 6027 { 6028 if (likely(nrssq_num <= wks->rssq_num)) 6029 return 0; 6030 rss_desc->queue = realloc(rss_desc->queue, 6031 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6032 if (!rss_desc->queue) { 6033 rte_errno = ENOMEM; 6034 return -1; 6035 } 6036 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6037 return 0; 6038 } 6039 6040 /** 6041 * Create a flow and add it to @p list. 6042 * 6043 * @param dev 6044 * Pointer to Ethernet device. 6045 * @param list 6046 * Pointer to a TAILQ flow list. If this parameter NULL, 6047 * no list insertion occurred, flow is just created, 6048 * this is caller's responsibility to track the 6049 * created flow. 6050 * @param[in] attr 6051 * Flow rule attributes. 6052 * @param[in] items 6053 * Pattern specification (list terminated by the END pattern item). 6054 * @param[in] actions 6055 * Associated actions (list terminated by the END action). 6056 * @param[in] external 6057 * This flow rule is created by request external to PMD. 6058 * @param[out] error 6059 * Perform verbose error reporting if not NULL. 6060 * 6061 * @return 6062 * A flow index on success, 0 otherwise and rte_errno is set. 6063 */ 6064 static uint32_t 6065 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 6066 const struct rte_flow_attr *attr, 6067 const struct rte_flow_item items[], 6068 const struct rte_flow_action original_actions[], 6069 bool external, struct rte_flow_error *error) 6070 { 6071 struct mlx5_priv *priv = dev->data->dev_private; 6072 struct rte_flow *flow = NULL; 6073 struct mlx5_flow *dev_flow; 6074 const struct rte_flow_action_rss *rss = NULL; 6075 struct mlx5_translated_action_handle 6076 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6077 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6078 union { 6079 struct mlx5_flow_expand_rss buf; 6080 uint8_t buffer[2048]; 6081 } expand_buffer; 6082 union { 6083 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6084 uint8_t buffer[2048]; 6085 } actions_rx; 6086 union { 6087 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6088 uint8_t buffer[2048]; 6089 } actions_hairpin_tx; 6090 union { 6091 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6092 uint8_t buffer[2048]; 6093 } items_tx; 6094 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6095 struct mlx5_flow_rss_desc *rss_desc; 6096 const struct rte_flow_action *p_actions_rx; 6097 uint32_t i; 6098 uint32_t idx = 0; 6099 int hairpin_flow; 6100 struct rte_flow_attr attr_tx = { .priority = 0 }; 6101 const struct rte_flow_action *actions; 6102 struct rte_flow_action *translated_actions = NULL; 6103 struct mlx5_flow_tunnel *tunnel; 6104 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6105 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6106 struct mlx5_flow_split_info flow_split_info = { 6107 .external = !!external, 6108 .skip_scale = 0, 6109 .flow_idx = 0, 6110 .prefix_mark = 0, 6111 .prefix_layers = 0, 6112 .table_id = 0 6113 }; 6114 int ret; 6115 6116 MLX5_ASSERT(wks); 6117 rss_desc = &wks->rss_desc; 6118 ret = flow_action_handles_translate(dev, original_actions, 6119 indir_actions, 6120 &indir_actions_n, 6121 &translated_actions, error); 6122 if (ret < 0) { 6123 MLX5_ASSERT(translated_actions == NULL); 6124 return 0; 6125 } 6126 actions = translated_actions ? translated_actions : original_actions; 6127 p_actions_rx = actions; 6128 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6129 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6130 external, hairpin_flow, error); 6131 if (ret < 0) 6132 goto error_before_hairpin_split; 6133 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 6134 if (!flow) { 6135 rte_errno = ENOMEM; 6136 goto error_before_hairpin_split; 6137 } 6138 if (hairpin_flow > 0) { 6139 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6140 rte_errno = EINVAL; 6141 goto error_before_hairpin_split; 6142 } 6143 flow_hairpin_split(dev, actions, actions_rx.actions, 6144 actions_hairpin_tx.actions, items_tx.items, 6145 idx); 6146 p_actions_rx = actions_rx.actions; 6147 } 6148 flow_split_info.flow_idx = idx; 6149 flow->drv_type = flow_get_drv_type(dev, attr); 6150 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6151 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6152 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6153 /* RSS Action only works on NIC RX domain */ 6154 if (attr->ingress && !attr->transfer) 6155 rss = flow_get_rss_action(dev, p_actions_rx); 6156 if (rss) { 6157 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6158 return 0; 6159 /* 6160 * The following information is required by 6161 * mlx5_flow_hashfields_adjust() in advance. 6162 */ 6163 rss_desc->level = rss->level; 6164 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 6165 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 6166 } 6167 flow->dev_handles = 0; 6168 if (rss && rss->types) { 6169 unsigned int graph_root; 6170 6171 graph_root = find_graph_root(items, rss->level); 6172 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6173 items, rss->types, 6174 mlx5_support_expansion, graph_root); 6175 MLX5_ASSERT(ret > 0 && 6176 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6177 } else { 6178 buf->entries = 1; 6179 buf->entry[0].pattern = (void *)(uintptr_t)items; 6180 } 6181 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6182 indir_actions_n); 6183 for (i = 0; i < buf->entries; ++i) { 6184 /* Initialize flow split data. */ 6185 flow_split_info.prefix_layers = 0; 6186 flow_split_info.prefix_mark = 0; 6187 flow_split_info.skip_scale = 0; 6188 /* 6189 * The splitter may create multiple dev_flows, 6190 * depending on configuration. In the simplest 6191 * case it just creates unmodified original flow. 6192 */ 6193 ret = flow_create_split_outer(dev, flow, attr, 6194 buf->entry[i].pattern, 6195 p_actions_rx, &flow_split_info, 6196 error); 6197 if (ret < 0) 6198 goto error; 6199 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6200 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6201 p_actions_rx, 6202 idx, 6203 wks->flows[0].tunnel, 6204 &default_miss_ctx, 6205 error); 6206 if (ret < 0) { 6207 mlx5_free(default_miss_ctx.queue); 6208 goto error; 6209 } 6210 } 6211 } 6212 /* Create the tx flow. */ 6213 if (hairpin_flow) { 6214 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6215 attr_tx.ingress = 0; 6216 attr_tx.egress = 1; 6217 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6218 actions_hairpin_tx.actions, 6219 idx, error); 6220 if (!dev_flow) 6221 goto error; 6222 dev_flow->flow = flow; 6223 dev_flow->external = 0; 6224 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6225 dev_flow->handle, next); 6226 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6227 items_tx.items, 6228 actions_hairpin_tx.actions, error); 6229 if (ret < 0) 6230 goto error; 6231 } 6232 /* 6233 * Update the metadata register copy table. If extensive 6234 * metadata feature is enabled and registers are supported 6235 * we might create the extra rte_flow for each unique 6236 * MARK/FLAG action ID. 6237 * 6238 * The table is updated for ingress Flows only, because 6239 * the egress Flows belong to the different device and 6240 * copy table should be updated in peer NIC Rx domain. 6241 */ 6242 if (attr->ingress && 6243 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6244 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6245 if (ret) 6246 goto error; 6247 } 6248 /* 6249 * If the flow is external (from application) OR device is started, 6250 * OR mreg discover, then apply immediately. 6251 */ 6252 if (external || dev->data->dev_started || 6253 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6254 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6255 ret = flow_drv_apply(dev, flow, error); 6256 if (ret < 0) 6257 goto error; 6258 } 6259 if (list) { 6260 rte_spinlock_lock(&priv->flow_list_lock); 6261 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 6262 flow, next); 6263 rte_spinlock_unlock(&priv->flow_list_lock); 6264 } 6265 flow_rxq_flags_set(dev, flow); 6266 rte_free(translated_actions); 6267 tunnel = flow_tunnel_from_rule(wks->flows); 6268 if (tunnel) { 6269 flow->tunnel = 1; 6270 flow->tunnel_id = tunnel->tunnel_id; 6271 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6272 mlx5_free(default_miss_ctx.queue); 6273 } 6274 mlx5_flow_pop_thread_workspace(); 6275 return idx; 6276 error: 6277 MLX5_ASSERT(flow); 6278 ret = rte_errno; /* Save rte_errno before cleanup. */ 6279 flow_mreg_del_copy_action(dev, flow); 6280 flow_drv_destroy(dev, flow); 6281 if (rss_desc->shared_rss) 6282 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6283 mlx5_ipool_get 6284 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6285 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6286 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 6287 rte_errno = ret; /* Restore rte_errno. */ 6288 ret = rte_errno; 6289 rte_errno = ret; 6290 mlx5_flow_pop_thread_workspace(); 6291 error_before_hairpin_split: 6292 rte_free(translated_actions); 6293 return 0; 6294 } 6295 6296 /** 6297 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6298 * incoming packets to table 1. 6299 * 6300 * Other flow rules, requested for group n, will be created in 6301 * e-switch table n+1. 6302 * Jump action to e-switch group n will be created to group n+1. 6303 * 6304 * Used when working in switchdev mode, to utilise advantages of table 1 6305 * and above. 6306 * 6307 * @param dev 6308 * Pointer to Ethernet device. 6309 * 6310 * @return 6311 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6312 */ 6313 struct rte_flow * 6314 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6315 { 6316 const struct rte_flow_attr attr = { 6317 .group = 0, 6318 .priority = 0, 6319 .ingress = 1, 6320 .egress = 0, 6321 .transfer = 1, 6322 }; 6323 const struct rte_flow_item pattern = { 6324 .type = RTE_FLOW_ITEM_TYPE_END, 6325 }; 6326 struct rte_flow_action_jump jump = { 6327 .group = 1, 6328 }; 6329 const struct rte_flow_action actions[] = { 6330 { 6331 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6332 .conf = &jump, 6333 }, 6334 { 6335 .type = RTE_FLOW_ACTION_TYPE_END, 6336 }, 6337 }; 6338 struct mlx5_priv *priv = dev->data->dev_private; 6339 struct rte_flow_error error; 6340 6341 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 6342 &attr, &pattern, 6343 actions, false, &error); 6344 } 6345 6346 /** 6347 * Validate a flow supported by the NIC. 6348 * 6349 * @see rte_flow_validate() 6350 * @see rte_flow_ops 6351 */ 6352 int 6353 mlx5_flow_validate(struct rte_eth_dev *dev, 6354 const struct rte_flow_attr *attr, 6355 const struct rte_flow_item items[], 6356 const struct rte_flow_action original_actions[], 6357 struct rte_flow_error *error) 6358 { 6359 int hairpin_flow; 6360 struct mlx5_translated_action_handle 6361 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6362 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6363 const struct rte_flow_action *actions; 6364 struct rte_flow_action *translated_actions = NULL; 6365 int ret = flow_action_handles_translate(dev, original_actions, 6366 indir_actions, 6367 &indir_actions_n, 6368 &translated_actions, error); 6369 6370 if (ret) 6371 return ret; 6372 actions = translated_actions ? translated_actions : original_actions; 6373 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6374 ret = flow_drv_validate(dev, attr, items, actions, 6375 true, hairpin_flow, error); 6376 rte_free(translated_actions); 6377 return ret; 6378 } 6379 6380 /** 6381 * Create a flow. 6382 * 6383 * @see rte_flow_create() 6384 * @see rte_flow_ops 6385 */ 6386 struct rte_flow * 6387 mlx5_flow_create(struct rte_eth_dev *dev, 6388 const struct rte_flow_attr *attr, 6389 const struct rte_flow_item items[], 6390 const struct rte_flow_action actions[], 6391 struct rte_flow_error *error) 6392 { 6393 struct mlx5_priv *priv = dev->data->dev_private; 6394 6395 /* 6396 * If the device is not started yet, it is not allowed to created a 6397 * flow from application. PMD default flows and traffic control flows 6398 * are not affected. 6399 */ 6400 if (unlikely(!dev->data->dev_started)) { 6401 DRV_LOG(DEBUG, "port %u is not started when " 6402 "inserting a flow", dev->data->port_id); 6403 rte_flow_error_set(error, ENODEV, 6404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6405 NULL, 6406 "port not started"); 6407 return NULL; 6408 } 6409 6410 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 6411 attr, items, actions, true, error); 6412 } 6413 6414 /** 6415 * Destroy a flow in a list. 6416 * 6417 * @param dev 6418 * Pointer to Ethernet device. 6419 * @param list 6420 * Pointer to the Indexed flow list. If this parameter NULL, 6421 * there is no flow removal from the list. Be noted that as 6422 * flow is add to the indexed list, memory of the indexed 6423 * list points to maybe changed as flow destroyed. 6424 * @param[in] flow_idx 6425 * Index of flow to destroy. 6426 */ 6427 static void 6428 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 6429 uint32_t flow_idx) 6430 { 6431 struct mlx5_priv *priv = dev->data->dev_private; 6432 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6433 [MLX5_IPOOL_RTE_FLOW], flow_idx); 6434 6435 if (!flow) 6436 return; 6437 /* 6438 * Update RX queue flags only if port is started, otherwise it is 6439 * already clean. 6440 */ 6441 if (dev->data->dev_started) 6442 flow_rxq_flags_trim(dev, flow); 6443 flow_drv_destroy(dev, flow); 6444 if (list) { 6445 rte_spinlock_lock(&priv->flow_list_lock); 6446 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 6447 flow_idx, flow, next); 6448 rte_spinlock_unlock(&priv->flow_list_lock); 6449 } 6450 if (flow->tunnel) { 6451 struct mlx5_flow_tunnel *tunnel; 6452 6453 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6454 RTE_VERIFY(tunnel); 6455 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6456 mlx5_flow_tunnel_free(dev, tunnel); 6457 } 6458 flow_mreg_del_copy_action(dev, flow); 6459 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 6460 } 6461 6462 /** 6463 * Destroy all flows. 6464 * 6465 * @param dev 6466 * Pointer to Ethernet device. 6467 * @param list 6468 * Pointer to the Indexed flow list. 6469 * @param active 6470 * If flushing is called avtively. 6471 */ 6472 void 6473 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 6474 { 6475 uint32_t num_flushed = 0; 6476 6477 while (*list) { 6478 flow_list_destroy(dev, list, *list); 6479 num_flushed++; 6480 } 6481 if (active) { 6482 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6483 dev->data->port_id, num_flushed); 6484 } 6485 } 6486 6487 /** 6488 * Stop all default actions for flows. 6489 * 6490 * @param dev 6491 * Pointer to Ethernet device. 6492 */ 6493 void 6494 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6495 { 6496 flow_mreg_del_default_copy_action(dev); 6497 flow_rxq_flags_clear(dev); 6498 } 6499 6500 /** 6501 * Start all default actions for flows. 6502 * 6503 * @param dev 6504 * Pointer to Ethernet device. 6505 * @return 6506 * 0 on success, a negative errno value otherwise and rte_errno is set. 6507 */ 6508 int 6509 mlx5_flow_start_default(struct rte_eth_dev *dev) 6510 { 6511 struct rte_flow_error error; 6512 6513 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6514 return flow_mreg_add_default_copy_action(dev, &error); 6515 } 6516 6517 /** 6518 * Release key of thread specific flow workspace data. 6519 */ 6520 void 6521 flow_release_workspace(void *data) 6522 { 6523 struct mlx5_flow_workspace *wks = data; 6524 struct mlx5_flow_workspace *next; 6525 6526 while (wks) { 6527 next = wks->next; 6528 free(wks->rss_desc.queue); 6529 free(wks); 6530 wks = next; 6531 } 6532 } 6533 6534 /** 6535 * Get thread specific current flow workspace. 6536 * 6537 * @return pointer to thread specific flow workspace data, NULL on error. 6538 */ 6539 struct mlx5_flow_workspace* 6540 mlx5_flow_get_thread_workspace(void) 6541 { 6542 struct mlx5_flow_workspace *data; 6543 6544 data = mlx5_flow_os_get_specific_workspace(); 6545 MLX5_ASSERT(data && data->inuse); 6546 if (!data || !data->inuse) 6547 DRV_LOG(ERR, "flow workspace not initialized."); 6548 return data; 6549 } 6550 6551 /** 6552 * Allocate and init new flow workspace. 6553 * 6554 * @return pointer to flow workspace data, NULL on error. 6555 */ 6556 static struct mlx5_flow_workspace* 6557 flow_alloc_thread_workspace(void) 6558 { 6559 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6560 6561 if (!data) { 6562 DRV_LOG(ERR, "Failed to allocate flow workspace " 6563 "memory."); 6564 return NULL; 6565 } 6566 data->rss_desc.queue = calloc(1, 6567 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6568 if (!data->rss_desc.queue) 6569 goto err; 6570 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6571 return data; 6572 err: 6573 if (data->rss_desc.queue) 6574 free(data->rss_desc.queue); 6575 free(data); 6576 return NULL; 6577 } 6578 6579 /** 6580 * Get new thread specific flow workspace. 6581 * 6582 * If current workspace inuse, create new one and set as current. 6583 * 6584 * @return pointer to thread specific flow workspace data, NULL on error. 6585 */ 6586 static struct mlx5_flow_workspace* 6587 mlx5_flow_push_thread_workspace(void) 6588 { 6589 struct mlx5_flow_workspace *curr; 6590 struct mlx5_flow_workspace *data; 6591 6592 curr = mlx5_flow_os_get_specific_workspace(); 6593 if (!curr) { 6594 data = flow_alloc_thread_workspace(); 6595 if (!data) 6596 return NULL; 6597 } else if (!curr->inuse) { 6598 data = curr; 6599 } else if (curr->next) { 6600 data = curr->next; 6601 } else { 6602 data = flow_alloc_thread_workspace(); 6603 if (!data) 6604 return NULL; 6605 curr->next = data; 6606 data->prev = curr; 6607 } 6608 data->inuse = 1; 6609 data->flow_idx = 0; 6610 /* Set as current workspace */ 6611 if (mlx5_flow_os_set_specific_workspace(data)) 6612 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6613 return data; 6614 } 6615 6616 /** 6617 * Close current thread specific flow workspace. 6618 * 6619 * If previous workspace available, set it as current. 6620 * 6621 * @return pointer to thread specific flow workspace data, NULL on error. 6622 */ 6623 static void 6624 mlx5_flow_pop_thread_workspace(void) 6625 { 6626 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6627 6628 if (!data) 6629 return; 6630 if (!data->inuse) { 6631 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6632 return; 6633 } 6634 data->inuse = 0; 6635 if (!data->prev) 6636 return; 6637 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6638 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6639 } 6640 6641 /** 6642 * Verify the flow list is empty 6643 * 6644 * @param dev 6645 * Pointer to Ethernet device. 6646 * 6647 * @return the number of flows not released. 6648 */ 6649 int 6650 mlx5_flow_verify(struct rte_eth_dev *dev) 6651 { 6652 struct mlx5_priv *priv = dev->data->dev_private; 6653 struct rte_flow *flow; 6654 uint32_t idx; 6655 int ret = 0; 6656 6657 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 6658 flow, next) { 6659 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6660 dev->data->port_id, (void *)flow); 6661 ++ret; 6662 } 6663 return ret; 6664 } 6665 6666 /** 6667 * Enable default hairpin egress flow. 6668 * 6669 * @param dev 6670 * Pointer to Ethernet device. 6671 * @param queue 6672 * The queue index. 6673 * 6674 * @return 6675 * 0 on success, a negative errno value otherwise and rte_errno is set. 6676 */ 6677 int 6678 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6679 uint32_t queue) 6680 { 6681 struct mlx5_priv *priv = dev->data->dev_private; 6682 const struct rte_flow_attr attr = { 6683 .egress = 1, 6684 .priority = 0, 6685 }; 6686 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6687 .queue = queue, 6688 }; 6689 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6690 .queue = UINT32_MAX, 6691 }; 6692 struct rte_flow_item items[] = { 6693 { 6694 .type = (enum rte_flow_item_type) 6695 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6696 .spec = &queue_spec, 6697 .last = NULL, 6698 .mask = &queue_mask, 6699 }, 6700 { 6701 .type = RTE_FLOW_ITEM_TYPE_END, 6702 }, 6703 }; 6704 struct rte_flow_action_jump jump = { 6705 .group = MLX5_HAIRPIN_TX_TABLE, 6706 }; 6707 struct rte_flow_action actions[2]; 6708 uint32_t flow_idx; 6709 struct rte_flow_error error; 6710 6711 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6712 actions[0].conf = &jump; 6713 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6714 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6715 &attr, items, actions, false, &error); 6716 if (!flow_idx) { 6717 DRV_LOG(DEBUG, 6718 "Failed to create ctrl flow: rte_errno(%d)," 6719 " type(%d), message(%s)", 6720 rte_errno, error.type, 6721 error.message ? error.message : " (no stated reason)"); 6722 return -rte_errno; 6723 } 6724 return 0; 6725 } 6726 6727 /** 6728 * Enable a control flow configured from the control plane. 6729 * 6730 * @param dev 6731 * Pointer to Ethernet device. 6732 * @param eth_spec 6733 * An Ethernet flow spec to apply. 6734 * @param eth_mask 6735 * An Ethernet flow mask to apply. 6736 * @param vlan_spec 6737 * A VLAN flow spec to apply. 6738 * @param vlan_mask 6739 * A VLAN flow mask to apply. 6740 * 6741 * @return 6742 * 0 on success, a negative errno value otherwise and rte_errno is set. 6743 */ 6744 int 6745 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6746 struct rte_flow_item_eth *eth_spec, 6747 struct rte_flow_item_eth *eth_mask, 6748 struct rte_flow_item_vlan *vlan_spec, 6749 struct rte_flow_item_vlan *vlan_mask) 6750 { 6751 struct mlx5_priv *priv = dev->data->dev_private; 6752 const struct rte_flow_attr attr = { 6753 .ingress = 1, 6754 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6755 }; 6756 struct rte_flow_item items[] = { 6757 { 6758 .type = RTE_FLOW_ITEM_TYPE_ETH, 6759 .spec = eth_spec, 6760 .last = NULL, 6761 .mask = eth_mask, 6762 }, 6763 { 6764 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6765 RTE_FLOW_ITEM_TYPE_END, 6766 .spec = vlan_spec, 6767 .last = NULL, 6768 .mask = vlan_mask, 6769 }, 6770 { 6771 .type = RTE_FLOW_ITEM_TYPE_END, 6772 }, 6773 }; 6774 uint16_t queue[priv->reta_idx_n]; 6775 struct rte_flow_action_rss action_rss = { 6776 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6777 .level = 0, 6778 .types = priv->rss_conf.rss_hf, 6779 .key_len = priv->rss_conf.rss_key_len, 6780 .queue_num = priv->reta_idx_n, 6781 .key = priv->rss_conf.rss_key, 6782 .queue = queue, 6783 }; 6784 struct rte_flow_action actions[] = { 6785 { 6786 .type = RTE_FLOW_ACTION_TYPE_RSS, 6787 .conf = &action_rss, 6788 }, 6789 { 6790 .type = RTE_FLOW_ACTION_TYPE_END, 6791 }, 6792 }; 6793 uint32_t flow_idx; 6794 struct rte_flow_error error; 6795 unsigned int i; 6796 6797 if (!priv->reta_idx_n || !priv->rxqs_n) { 6798 return 0; 6799 } 6800 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6801 action_rss.types = 0; 6802 for (i = 0; i != priv->reta_idx_n; ++i) 6803 queue[i] = (*priv->reta_idx)[i]; 6804 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6805 &attr, items, actions, false, &error); 6806 if (!flow_idx) 6807 return -rte_errno; 6808 return 0; 6809 } 6810 6811 /** 6812 * Enable a flow control configured from the control plane. 6813 * 6814 * @param dev 6815 * Pointer to Ethernet device. 6816 * @param eth_spec 6817 * An Ethernet flow spec to apply. 6818 * @param eth_mask 6819 * An Ethernet flow mask to apply. 6820 * 6821 * @return 6822 * 0 on success, a negative errno value otherwise and rte_errno is set. 6823 */ 6824 int 6825 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6826 struct rte_flow_item_eth *eth_spec, 6827 struct rte_flow_item_eth *eth_mask) 6828 { 6829 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6830 } 6831 6832 /** 6833 * Create default miss flow rule matching lacp traffic 6834 * 6835 * @param dev 6836 * Pointer to Ethernet device. 6837 * @param eth_spec 6838 * An Ethernet flow spec to apply. 6839 * 6840 * @return 6841 * 0 on success, a negative errno value otherwise and rte_errno is set. 6842 */ 6843 int 6844 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6845 { 6846 struct mlx5_priv *priv = dev->data->dev_private; 6847 /* 6848 * The LACP matching is done by only using ether type since using 6849 * a multicast dst mac causes kernel to give low priority to this flow. 6850 */ 6851 static const struct rte_flow_item_eth lacp_spec = { 6852 .type = RTE_BE16(0x8809), 6853 }; 6854 static const struct rte_flow_item_eth lacp_mask = { 6855 .type = 0xffff, 6856 }; 6857 const struct rte_flow_attr attr = { 6858 .ingress = 1, 6859 }; 6860 struct rte_flow_item items[] = { 6861 { 6862 .type = RTE_FLOW_ITEM_TYPE_ETH, 6863 .spec = &lacp_spec, 6864 .mask = &lacp_mask, 6865 }, 6866 { 6867 .type = RTE_FLOW_ITEM_TYPE_END, 6868 }, 6869 }; 6870 struct rte_flow_action actions[] = { 6871 { 6872 .type = (enum rte_flow_action_type) 6873 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6874 }, 6875 { 6876 .type = RTE_FLOW_ACTION_TYPE_END, 6877 }, 6878 }; 6879 struct rte_flow_error error; 6880 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6881 &attr, items, actions, false, &error); 6882 6883 if (!flow_idx) 6884 return -rte_errno; 6885 return 0; 6886 } 6887 6888 /** 6889 * Destroy a flow. 6890 * 6891 * @see rte_flow_destroy() 6892 * @see rte_flow_ops 6893 */ 6894 int 6895 mlx5_flow_destroy(struct rte_eth_dev *dev, 6896 struct rte_flow *flow, 6897 struct rte_flow_error *error __rte_unused) 6898 { 6899 struct mlx5_priv *priv = dev->data->dev_private; 6900 6901 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6902 return 0; 6903 } 6904 6905 /** 6906 * Destroy all flows. 6907 * 6908 * @see rte_flow_flush() 6909 * @see rte_flow_ops 6910 */ 6911 int 6912 mlx5_flow_flush(struct rte_eth_dev *dev, 6913 struct rte_flow_error *error __rte_unused) 6914 { 6915 struct mlx5_priv *priv = dev->data->dev_private; 6916 6917 mlx5_flow_list_flush(dev, &priv->flows, false); 6918 return 0; 6919 } 6920 6921 /** 6922 * Isolated mode. 6923 * 6924 * @see rte_flow_isolate() 6925 * @see rte_flow_ops 6926 */ 6927 int 6928 mlx5_flow_isolate(struct rte_eth_dev *dev, 6929 int enable, 6930 struct rte_flow_error *error) 6931 { 6932 struct mlx5_priv *priv = dev->data->dev_private; 6933 6934 if (dev->data->dev_started) { 6935 rte_flow_error_set(error, EBUSY, 6936 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6937 NULL, 6938 "port must be stopped first"); 6939 return -rte_errno; 6940 } 6941 priv->isolated = !!enable; 6942 if (enable) 6943 dev->dev_ops = &mlx5_dev_ops_isolate; 6944 else 6945 dev->dev_ops = &mlx5_dev_ops; 6946 6947 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 6948 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 6949 6950 return 0; 6951 } 6952 6953 /** 6954 * Query a flow. 6955 * 6956 * @see rte_flow_query() 6957 * @see rte_flow_ops 6958 */ 6959 static int 6960 flow_drv_query(struct rte_eth_dev *dev, 6961 uint32_t flow_idx, 6962 const struct rte_flow_action *actions, 6963 void *data, 6964 struct rte_flow_error *error) 6965 { 6966 struct mlx5_priv *priv = dev->data->dev_private; 6967 const struct mlx5_flow_driver_ops *fops; 6968 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6969 [MLX5_IPOOL_RTE_FLOW], 6970 flow_idx); 6971 enum mlx5_flow_drv_type ftype; 6972 6973 if (!flow) { 6974 return rte_flow_error_set(error, ENOENT, 6975 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6976 NULL, 6977 "invalid flow handle"); 6978 } 6979 ftype = flow->drv_type; 6980 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 6981 fops = flow_get_drv_ops(ftype); 6982 6983 return fops->query(dev, flow, actions, data, error); 6984 } 6985 6986 /** 6987 * Query a flow. 6988 * 6989 * @see rte_flow_query() 6990 * @see rte_flow_ops 6991 */ 6992 int 6993 mlx5_flow_query(struct rte_eth_dev *dev, 6994 struct rte_flow *flow, 6995 const struct rte_flow_action *actions, 6996 void *data, 6997 struct rte_flow_error *error) 6998 { 6999 int ret; 7000 7001 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7002 error); 7003 if (ret < 0) 7004 return ret; 7005 return 0; 7006 } 7007 7008 /** 7009 * Get rte_flow callbacks. 7010 * 7011 * @param dev 7012 * Pointer to Ethernet device structure. 7013 * @param ops 7014 * Pointer to operation-specific structure. 7015 * 7016 * @return 0 7017 */ 7018 int 7019 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7020 const struct rte_flow_ops **ops) 7021 { 7022 *ops = &mlx5_flow_ops; 7023 return 0; 7024 } 7025 7026 /** 7027 * Validate meter policy actions. 7028 * Dispatcher for action type specific validation. 7029 * 7030 * @param[in] dev 7031 * Pointer to the Ethernet device structure. 7032 * @param[in] action 7033 * The meter policy action object to validate. 7034 * @param[in] attr 7035 * Attributes of flow to determine steering domain. 7036 * @param[out] is_rss 7037 * Is RSS or not. 7038 * @param[out] domain_bitmap 7039 * Domain bitmap. 7040 * @param[out] is_def_policy 7041 * Is default policy or not. 7042 * @param[out] error 7043 * Perform verbose error reporting if not NULL. Initialized in case of 7044 * error only. 7045 * 7046 * @return 7047 * 0 on success, otherwise negative errno value. 7048 */ 7049 int 7050 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7051 const struct rte_flow_action *actions[RTE_COLORS], 7052 struct rte_flow_attr *attr, 7053 bool *is_rss, 7054 uint8_t *domain_bitmap, 7055 bool *is_def_policy, 7056 struct rte_mtr_error *error) 7057 { 7058 const struct mlx5_flow_driver_ops *fops; 7059 7060 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7061 return fops->validate_mtr_acts(dev, actions, attr, 7062 is_rss, domain_bitmap, is_def_policy, error); 7063 } 7064 7065 /** 7066 * Destroy the meter table set. 7067 * 7068 * @param[in] dev 7069 * Pointer to Ethernet device. 7070 * @param[in] mtr_policy 7071 * Meter policy struct. 7072 */ 7073 void 7074 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7075 struct mlx5_flow_meter_policy *mtr_policy) 7076 { 7077 const struct mlx5_flow_driver_ops *fops; 7078 7079 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7080 fops->destroy_mtr_acts(dev, mtr_policy); 7081 } 7082 7083 /** 7084 * Create policy action, lock free, 7085 * (mutex should be acquired by caller). 7086 * Dispatcher for action type specific call. 7087 * 7088 * @param[in] dev 7089 * Pointer to the Ethernet device structure. 7090 * @param[in] mtr_policy 7091 * Meter policy struct. 7092 * @param[in] action 7093 * Action specification used to create meter actions. 7094 * @param[out] error 7095 * Perform verbose error reporting if not NULL. Initialized in case of 7096 * error only. 7097 * 7098 * @return 7099 * 0 on success, otherwise negative errno value. 7100 */ 7101 int 7102 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7103 struct mlx5_flow_meter_policy *mtr_policy, 7104 const struct rte_flow_action *actions[RTE_COLORS], 7105 struct rte_mtr_error *error) 7106 { 7107 const struct mlx5_flow_driver_ops *fops; 7108 7109 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7110 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7111 } 7112 7113 /** 7114 * Create policy rules, lock free, 7115 * (mutex should be acquired by caller). 7116 * Dispatcher for action type specific call. 7117 * 7118 * @param[in] dev 7119 * Pointer to the Ethernet device structure. 7120 * @param[in] mtr_policy 7121 * Meter policy struct. 7122 * 7123 * @return 7124 * 0 on success, -1 otherwise. 7125 */ 7126 int 7127 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7128 struct mlx5_flow_meter_policy *mtr_policy) 7129 { 7130 const struct mlx5_flow_driver_ops *fops; 7131 7132 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7133 return fops->create_policy_rules(dev, mtr_policy); 7134 } 7135 7136 /** 7137 * Destroy policy rules, lock free, 7138 * (mutex should be acquired by caller). 7139 * Dispatcher for action type specific call. 7140 * 7141 * @param[in] dev 7142 * Pointer to the Ethernet device structure. 7143 * @param[in] mtr_policy 7144 * Meter policy struct. 7145 */ 7146 void 7147 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7148 struct mlx5_flow_meter_policy *mtr_policy) 7149 { 7150 const struct mlx5_flow_driver_ops *fops; 7151 7152 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7153 fops->destroy_policy_rules(dev, mtr_policy); 7154 } 7155 7156 /** 7157 * Destroy the default policy table set. 7158 * 7159 * @param[in] dev 7160 * Pointer to Ethernet device. 7161 */ 7162 void 7163 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7164 { 7165 const struct mlx5_flow_driver_ops *fops; 7166 7167 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7168 fops->destroy_def_policy(dev); 7169 } 7170 7171 /** 7172 * Destroy the default policy table set. 7173 * 7174 * @param[in] dev 7175 * Pointer to Ethernet device. 7176 * 7177 * @return 7178 * 0 on success, -1 otherwise. 7179 */ 7180 int 7181 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7182 { 7183 const struct mlx5_flow_driver_ops *fops; 7184 7185 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7186 return fops->create_def_policy(dev); 7187 } 7188 7189 /** 7190 * Create the needed meter and suffix tables. 7191 * 7192 * @param[in] dev 7193 * Pointer to Ethernet device. 7194 * 7195 * @return 7196 * 0 on success, -1 otherwise. 7197 */ 7198 int 7199 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7200 struct mlx5_flow_meter_info *fm, 7201 uint32_t mtr_idx, 7202 uint8_t domain_bitmap) 7203 { 7204 const struct mlx5_flow_driver_ops *fops; 7205 7206 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7207 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7208 } 7209 7210 /** 7211 * Destroy the meter table set. 7212 * 7213 * @param[in] dev 7214 * Pointer to Ethernet device. 7215 * @param[in] tbl 7216 * Pointer to the meter table set. 7217 */ 7218 void 7219 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7220 struct mlx5_flow_meter_info *fm) 7221 { 7222 const struct mlx5_flow_driver_ops *fops; 7223 7224 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7225 fops->destroy_mtr_tbls(dev, fm); 7226 } 7227 7228 /** 7229 * Destroy the global meter drop table. 7230 * 7231 * @param[in] dev 7232 * Pointer to Ethernet device. 7233 */ 7234 void 7235 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7236 { 7237 const struct mlx5_flow_driver_ops *fops; 7238 7239 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7240 fops->destroy_mtr_drop_tbls(dev); 7241 } 7242 7243 /** 7244 * Destroy the sub policy table with RX queue. 7245 * 7246 * @param[in] dev 7247 * Pointer to Ethernet device. 7248 * @param[in] mtr_policy 7249 * Pointer to meter policy table. 7250 */ 7251 void 7252 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7253 struct mlx5_flow_meter_policy *mtr_policy) 7254 { 7255 const struct mlx5_flow_driver_ops *fops; 7256 7257 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7258 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7259 } 7260 7261 /** 7262 * Allocate the needed aso flow meter id. 7263 * 7264 * @param[in] dev 7265 * Pointer to Ethernet device. 7266 * 7267 * @return 7268 * Index to aso flow meter on success, NULL otherwise. 7269 */ 7270 uint32_t 7271 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7272 { 7273 const struct mlx5_flow_driver_ops *fops; 7274 7275 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7276 return fops->create_meter(dev); 7277 } 7278 7279 /** 7280 * Free the aso flow meter id. 7281 * 7282 * @param[in] dev 7283 * Pointer to Ethernet device. 7284 * @param[in] mtr_idx 7285 * Index to aso flow meter to be free. 7286 * 7287 * @return 7288 * 0 on success. 7289 */ 7290 void 7291 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7292 { 7293 const struct mlx5_flow_driver_ops *fops; 7294 7295 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7296 fops->free_meter(dev, mtr_idx); 7297 } 7298 7299 /** 7300 * Allocate a counter. 7301 * 7302 * @param[in] dev 7303 * Pointer to Ethernet device structure. 7304 * 7305 * @return 7306 * Index to allocated counter on success, 0 otherwise. 7307 */ 7308 uint32_t 7309 mlx5_counter_alloc(struct rte_eth_dev *dev) 7310 { 7311 const struct mlx5_flow_driver_ops *fops; 7312 struct rte_flow_attr attr = { .transfer = 0 }; 7313 7314 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7315 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7316 return fops->counter_alloc(dev); 7317 } 7318 DRV_LOG(ERR, 7319 "port %u counter allocate is not supported.", 7320 dev->data->port_id); 7321 return 0; 7322 } 7323 7324 /** 7325 * Free a counter. 7326 * 7327 * @param[in] dev 7328 * Pointer to Ethernet device structure. 7329 * @param[in] cnt 7330 * Index to counter to be free. 7331 */ 7332 void 7333 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7334 { 7335 const struct mlx5_flow_driver_ops *fops; 7336 struct rte_flow_attr attr = { .transfer = 0 }; 7337 7338 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7339 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7340 fops->counter_free(dev, cnt); 7341 return; 7342 } 7343 DRV_LOG(ERR, 7344 "port %u counter free is not supported.", 7345 dev->data->port_id); 7346 } 7347 7348 /** 7349 * Query counter statistics. 7350 * 7351 * @param[in] dev 7352 * Pointer to Ethernet device structure. 7353 * @param[in] cnt 7354 * Index to counter to query. 7355 * @param[in] clear 7356 * Set to clear counter statistics. 7357 * @param[out] pkts 7358 * The counter hits packets number to save. 7359 * @param[out] bytes 7360 * The counter hits bytes number to save. 7361 * 7362 * @return 7363 * 0 on success, a negative errno value otherwise. 7364 */ 7365 int 7366 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7367 bool clear, uint64_t *pkts, uint64_t *bytes) 7368 { 7369 const struct mlx5_flow_driver_ops *fops; 7370 struct rte_flow_attr attr = { .transfer = 0 }; 7371 7372 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7373 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7374 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7375 } 7376 DRV_LOG(ERR, 7377 "port %u counter query is not supported.", 7378 dev->data->port_id); 7379 return -ENOTSUP; 7380 } 7381 7382 /** 7383 * Allocate a new memory for the counter values wrapped by all the needed 7384 * management. 7385 * 7386 * @param[in] sh 7387 * Pointer to mlx5_dev_ctx_shared object. 7388 * 7389 * @return 7390 * 0 on success, a negative errno value otherwise. 7391 */ 7392 static int 7393 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7394 { 7395 struct mlx5_devx_mkey_attr mkey_attr; 7396 struct mlx5_counter_stats_mem_mng *mem_mng; 7397 volatile struct flow_counter_stats *raw_data; 7398 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7399 int size = (sizeof(struct flow_counter_stats) * 7400 MLX5_COUNTERS_PER_POOL + 7401 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7402 sizeof(struct mlx5_counter_stats_mem_mng); 7403 size_t pgsize = rte_mem_page_size(); 7404 uint8_t *mem; 7405 int i; 7406 7407 if (pgsize == (size_t)-1) { 7408 DRV_LOG(ERR, "Failed to get mem page size"); 7409 rte_errno = ENOMEM; 7410 return -ENOMEM; 7411 } 7412 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7413 if (!mem) { 7414 rte_errno = ENOMEM; 7415 return -ENOMEM; 7416 } 7417 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7418 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7419 mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size, 7420 IBV_ACCESS_LOCAL_WRITE); 7421 if (!mem_mng->umem) { 7422 rte_errno = errno; 7423 mlx5_free(mem); 7424 return -rte_errno; 7425 } 7426 memset(&mkey_attr, 0, sizeof(mkey_attr)); 7427 mkey_attr.addr = (uintptr_t)mem; 7428 mkey_attr.size = size; 7429 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7430 mkey_attr.pd = sh->pdn; 7431 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 7432 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 7433 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7434 if (!mem_mng->dm) { 7435 mlx5_os_umem_dereg(mem_mng->umem); 7436 rte_errno = errno; 7437 mlx5_free(mem); 7438 return -rte_errno; 7439 } 7440 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7441 raw_data = (volatile struct flow_counter_stats *)mem; 7442 for (i = 0; i < raws_n; ++i) { 7443 mem_mng->raws[i].mem_mng = mem_mng; 7444 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7445 } 7446 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7447 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7448 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7449 next); 7450 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7451 sh->cmng.mem_mng = mem_mng; 7452 return 0; 7453 } 7454 7455 /** 7456 * Set the statistic memory to the new counter pool. 7457 * 7458 * @param[in] sh 7459 * Pointer to mlx5_dev_ctx_shared object. 7460 * @param[in] pool 7461 * Pointer to the pool to set the statistic memory. 7462 * 7463 * @return 7464 * 0 on success, a negative errno value otherwise. 7465 */ 7466 static int 7467 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7468 struct mlx5_flow_counter_pool *pool) 7469 { 7470 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7471 /* Resize statistic memory once used out. */ 7472 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7473 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7474 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7475 return -1; 7476 } 7477 rte_spinlock_lock(&pool->sl); 7478 pool->raw = cmng->mem_mng->raws + pool->index % 7479 MLX5_CNT_CONTAINER_RESIZE; 7480 rte_spinlock_unlock(&pool->sl); 7481 pool->raw_hw = NULL; 7482 return 0; 7483 } 7484 7485 #define MLX5_POOL_QUERY_FREQ_US 1000000 7486 7487 /** 7488 * Set the periodic procedure for triggering asynchronous batch queries for all 7489 * the counter pools. 7490 * 7491 * @param[in] sh 7492 * Pointer to mlx5_dev_ctx_shared object. 7493 */ 7494 void 7495 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7496 { 7497 uint32_t pools_n, us; 7498 7499 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7500 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7501 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7502 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7503 sh->cmng.query_thread_on = 0; 7504 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7505 } else { 7506 sh->cmng.query_thread_on = 1; 7507 } 7508 } 7509 7510 /** 7511 * The periodic procedure for triggering asynchronous batch queries for all the 7512 * counter pools. This function is probably called by the host thread. 7513 * 7514 * @param[in] arg 7515 * The parameter for the alarm process. 7516 */ 7517 void 7518 mlx5_flow_query_alarm(void *arg) 7519 { 7520 struct mlx5_dev_ctx_shared *sh = arg; 7521 int ret; 7522 uint16_t pool_index = sh->cmng.pool_index; 7523 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7524 struct mlx5_flow_counter_pool *pool; 7525 uint16_t n_valid; 7526 7527 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7528 goto set_alarm; 7529 rte_spinlock_lock(&cmng->pool_update_sl); 7530 pool = cmng->pools[pool_index]; 7531 n_valid = cmng->n_valid; 7532 rte_spinlock_unlock(&cmng->pool_update_sl); 7533 /* Set the statistic memory to the new created pool. */ 7534 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7535 goto set_alarm; 7536 if (pool->raw_hw) 7537 /* There is a pool query in progress. */ 7538 goto set_alarm; 7539 pool->raw_hw = 7540 LIST_FIRST(&sh->cmng.free_stat_raws); 7541 if (!pool->raw_hw) 7542 /* No free counter statistics raw memory. */ 7543 goto set_alarm; 7544 /* 7545 * Identify the counters released between query trigger and query 7546 * handle more efficiently. The counter released in this gap period 7547 * should wait for a new round of query as the new arrived packets 7548 * will not be taken into account. 7549 */ 7550 pool->query_gen++; 7551 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7552 MLX5_COUNTERS_PER_POOL, 7553 NULL, NULL, 7554 pool->raw_hw->mem_mng->dm->id, 7555 (void *)(uintptr_t) 7556 pool->raw_hw->data, 7557 sh->devx_comp, 7558 (uint64_t)(uintptr_t)pool); 7559 if (ret) { 7560 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7561 " %d", pool->min_dcs->id); 7562 pool->raw_hw = NULL; 7563 goto set_alarm; 7564 } 7565 LIST_REMOVE(pool->raw_hw, next); 7566 sh->cmng.pending_queries++; 7567 pool_index++; 7568 if (pool_index >= n_valid) 7569 pool_index = 0; 7570 set_alarm: 7571 sh->cmng.pool_index = pool_index; 7572 mlx5_set_query_alarm(sh); 7573 } 7574 7575 /** 7576 * Check and callback event for new aged flow in the counter pool 7577 * 7578 * @param[in] sh 7579 * Pointer to mlx5_dev_ctx_shared object. 7580 * @param[in] pool 7581 * Pointer to Current counter pool. 7582 */ 7583 static void 7584 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7585 struct mlx5_flow_counter_pool *pool) 7586 { 7587 struct mlx5_priv *priv; 7588 struct mlx5_flow_counter *cnt; 7589 struct mlx5_age_info *age_info; 7590 struct mlx5_age_param *age_param; 7591 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7592 struct mlx5_counter_stats_raw *prev = pool->raw; 7593 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7594 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7595 uint16_t expected = AGE_CANDIDATE; 7596 uint32_t i; 7597 7598 pool->time_of_last_age_check = curr_time; 7599 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7600 cnt = MLX5_POOL_GET_CNT(pool, i); 7601 age_param = MLX5_CNT_TO_AGE(cnt); 7602 if (__atomic_load_n(&age_param->state, 7603 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7604 continue; 7605 if (cur->data[i].hits != prev->data[i].hits) { 7606 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7607 __ATOMIC_RELAXED); 7608 continue; 7609 } 7610 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7611 time_delta, 7612 __ATOMIC_RELAXED) <= age_param->timeout) 7613 continue; 7614 /** 7615 * Hold the lock first, or if between the 7616 * state AGE_TMOUT and tailq operation the 7617 * release happened, the release procedure 7618 * may delete a non-existent tailq node. 7619 */ 7620 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7621 age_info = GET_PORT_AGE_INFO(priv); 7622 rte_spinlock_lock(&age_info->aged_sl); 7623 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7624 AGE_TMOUT, false, 7625 __ATOMIC_RELAXED, 7626 __ATOMIC_RELAXED)) { 7627 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7628 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7629 } 7630 rte_spinlock_unlock(&age_info->aged_sl); 7631 } 7632 mlx5_age_event_prepare(sh); 7633 } 7634 7635 /** 7636 * Handler for the HW respond about ready values from an asynchronous batch 7637 * query. This function is probably called by the host thread. 7638 * 7639 * @param[in] sh 7640 * The pointer to the shared device context. 7641 * @param[in] async_id 7642 * The Devx async ID. 7643 * @param[in] status 7644 * The status of the completion. 7645 */ 7646 void 7647 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7648 uint64_t async_id, int status) 7649 { 7650 struct mlx5_flow_counter_pool *pool = 7651 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7652 struct mlx5_counter_stats_raw *raw_to_free; 7653 uint8_t query_gen = pool->query_gen ^ 1; 7654 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7655 enum mlx5_counter_type cnt_type = 7656 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7657 MLX5_COUNTER_TYPE_ORIGIN; 7658 7659 if (unlikely(status)) { 7660 raw_to_free = pool->raw_hw; 7661 } else { 7662 raw_to_free = pool->raw; 7663 if (pool->is_aged) 7664 mlx5_flow_aging_check(sh, pool); 7665 rte_spinlock_lock(&pool->sl); 7666 pool->raw = pool->raw_hw; 7667 rte_spinlock_unlock(&pool->sl); 7668 /* Be sure the new raw counters data is updated in memory. */ 7669 rte_io_wmb(); 7670 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7671 rte_spinlock_lock(&cmng->csl[cnt_type]); 7672 TAILQ_CONCAT(&cmng->counters[cnt_type], 7673 &pool->counters[query_gen], next); 7674 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7675 } 7676 } 7677 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7678 pool->raw_hw = NULL; 7679 sh->cmng.pending_queries--; 7680 } 7681 7682 static int 7683 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7684 const struct flow_grp_info *grp_info, 7685 struct rte_flow_error *error) 7686 { 7687 if (grp_info->transfer && grp_info->external && 7688 grp_info->fdb_def_rule) { 7689 if (group == UINT32_MAX) 7690 return rte_flow_error_set 7691 (error, EINVAL, 7692 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7693 NULL, 7694 "group index not supported"); 7695 *table = group + 1; 7696 } else { 7697 *table = group; 7698 } 7699 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7700 return 0; 7701 } 7702 7703 /** 7704 * Translate the rte_flow group index to HW table value. 7705 * 7706 * If tunnel offload is disabled, all group ids converted to flow table 7707 * id using the standard method. 7708 * If tunnel offload is enabled, group id can be converted using the 7709 * standard or tunnel conversion method. Group conversion method 7710 * selection depends on flags in `grp_info` parameter: 7711 * - Internal (grp_info.external == 0) groups conversion uses the 7712 * standard method. 7713 * - Group ids in JUMP action converted with the tunnel conversion. 7714 * - Group id in rule attribute conversion depends on a rule type and 7715 * group id value: 7716 * ** non zero group attributes converted with the tunnel method 7717 * ** zero group attribute in non-tunnel rule is converted using the 7718 * standard method - there's only one root table 7719 * ** zero group attribute in steer tunnel rule is converted with the 7720 * standard method - single root table 7721 * ** zero group attribute in match tunnel rule is a special OvS 7722 * case: that value is used for portability reasons. That group 7723 * id is converted with the tunnel conversion method. 7724 * 7725 * @param[in] dev 7726 * Port device 7727 * @param[in] tunnel 7728 * PMD tunnel offload object 7729 * @param[in] group 7730 * rte_flow group index value. 7731 * @param[out] table 7732 * HW table value. 7733 * @param[in] grp_info 7734 * flags used for conversion 7735 * @param[out] error 7736 * Pointer to error structure. 7737 * 7738 * @return 7739 * 0 on success, a negative errno value otherwise and rte_errno is set. 7740 */ 7741 int 7742 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7743 const struct mlx5_flow_tunnel *tunnel, 7744 uint32_t group, uint32_t *table, 7745 const struct flow_grp_info *grp_info, 7746 struct rte_flow_error *error) 7747 { 7748 int ret; 7749 bool standard_translation; 7750 7751 if (!grp_info->skip_scale && grp_info->external && 7752 group < MLX5_MAX_TABLES_EXTERNAL) 7753 group *= MLX5_FLOW_TABLE_FACTOR; 7754 if (is_tunnel_offload_active(dev)) { 7755 standard_translation = !grp_info->external || 7756 grp_info->std_tbl_fix; 7757 } else { 7758 standard_translation = true; 7759 } 7760 DRV_LOG(DEBUG, 7761 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 7762 dev->data->port_id, group, grp_info->transfer, 7763 grp_info->external, grp_info->fdb_def_rule, 7764 standard_translation ? "STANDARD" : "TUNNEL"); 7765 if (standard_translation) 7766 ret = flow_group_to_table(dev->data->port_id, group, table, 7767 grp_info, error); 7768 else 7769 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7770 table, error); 7771 7772 return ret; 7773 } 7774 7775 /** 7776 * Discover availability of metadata reg_c's. 7777 * 7778 * Iteratively use test flows to check availability. 7779 * 7780 * @param[in] dev 7781 * Pointer to the Ethernet device structure. 7782 * 7783 * @return 7784 * 0 on success, a negative errno value otherwise and rte_errno is set. 7785 */ 7786 int 7787 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7788 { 7789 struct mlx5_priv *priv = dev->data->dev_private; 7790 struct mlx5_dev_config *config = &priv->config; 7791 enum modify_reg idx; 7792 int n = 0; 7793 7794 /* reg_c[0] and reg_c[1] are reserved. */ 7795 config->flow_mreg_c[n++] = REG_C_0; 7796 config->flow_mreg_c[n++] = REG_C_1; 7797 /* Discover availability of other reg_c's. */ 7798 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7799 struct rte_flow_attr attr = { 7800 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7801 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7802 .ingress = 1, 7803 }; 7804 struct rte_flow_item items[] = { 7805 [0] = { 7806 .type = RTE_FLOW_ITEM_TYPE_END, 7807 }, 7808 }; 7809 struct rte_flow_action actions[] = { 7810 [0] = { 7811 .type = (enum rte_flow_action_type) 7812 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7813 .conf = &(struct mlx5_flow_action_copy_mreg){ 7814 .src = REG_C_1, 7815 .dst = idx, 7816 }, 7817 }, 7818 [1] = { 7819 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7820 .conf = &(struct rte_flow_action_jump){ 7821 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7822 }, 7823 }, 7824 [2] = { 7825 .type = RTE_FLOW_ACTION_TYPE_END, 7826 }, 7827 }; 7828 uint32_t flow_idx; 7829 struct rte_flow *flow; 7830 struct rte_flow_error error; 7831 7832 if (!config->dv_flow_en) 7833 break; 7834 /* Create internal flow, validation skips copy action. */ 7835 flow_idx = flow_list_create(dev, NULL, &attr, items, 7836 actions, false, &error); 7837 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 7838 flow_idx); 7839 if (!flow) 7840 continue; 7841 config->flow_mreg_c[n++] = idx; 7842 flow_list_destroy(dev, NULL, flow_idx); 7843 } 7844 for (; n < MLX5_MREG_C_NUM; ++n) 7845 config->flow_mreg_c[n] = REG_NON; 7846 return 0; 7847 } 7848 7849 /** 7850 * Dump flow raw hw data to file 7851 * 7852 * @param[in] dev 7853 * The pointer to Ethernet device. 7854 * @param[in] file 7855 * A pointer to a file for output. 7856 * @param[out] error 7857 * Perform verbose error reporting if not NULL. PMDs initialize this 7858 * structure in case of error only. 7859 * @return 7860 * 0 on success, a nagative value otherwise. 7861 */ 7862 int 7863 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 7864 FILE *file, 7865 struct rte_flow_error *error __rte_unused) 7866 { 7867 struct mlx5_priv *priv = dev->data->dev_private; 7868 struct mlx5_dev_ctx_shared *sh = priv->sh; 7869 uint32_t handle_idx; 7870 int ret; 7871 struct mlx5_flow_handle *dh; 7872 struct rte_flow *flow; 7873 7874 if (!priv->config.dv_flow_en) { 7875 if (fputs("device dv flow disabled\n", file) <= 0) 7876 return -errno; 7877 return -ENOTSUP; 7878 } 7879 7880 /* dump all */ 7881 if (!flow_idx) 7882 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 7883 sh->rx_domain, 7884 sh->tx_domain, file); 7885 /* dump one */ 7886 flow = mlx5_ipool_get(priv->sh->ipool 7887 [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx); 7888 if (!flow) 7889 return -ENOENT; 7890 7891 handle_idx = flow->dev_handles; 7892 while (handle_idx) { 7893 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 7894 handle_idx); 7895 if (!dh) 7896 return -ENOENT; 7897 if (dh->drv_flow) { 7898 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 7899 file); 7900 if (ret) 7901 return -ENOENT; 7902 } 7903 handle_idx = dh->next.next; 7904 } 7905 return 0; 7906 } 7907 7908 /** 7909 * Get aged-out flows. 7910 * 7911 * @param[in] dev 7912 * Pointer to the Ethernet device structure. 7913 * @param[in] context 7914 * The address of an array of pointers to the aged-out flows contexts. 7915 * @param[in] nb_countexts 7916 * The length of context array pointers. 7917 * @param[out] error 7918 * Perform verbose error reporting if not NULL. Initialized in case of 7919 * error only. 7920 * 7921 * @return 7922 * how many contexts get in success, otherwise negative errno value. 7923 * if nb_contexts is 0, return the amount of all aged contexts. 7924 * if nb_contexts is not 0 , return the amount of aged flows reported 7925 * in the context array. 7926 */ 7927 int 7928 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 7929 uint32_t nb_contexts, struct rte_flow_error *error) 7930 { 7931 const struct mlx5_flow_driver_ops *fops; 7932 struct rte_flow_attr attr = { .transfer = 0 }; 7933 7934 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7935 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7936 return fops->get_aged_flows(dev, contexts, nb_contexts, 7937 error); 7938 } 7939 DRV_LOG(ERR, 7940 "port %u get aged flows is not supported.", 7941 dev->data->port_id); 7942 return -ENOTSUP; 7943 } 7944 7945 /* Wrapper for driver action_validate op callback */ 7946 static int 7947 flow_drv_action_validate(struct rte_eth_dev *dev, 7948 const struct rte_flow_indir_action_conf *conf, 7949 const struct rte_flow_action *action, 7950 const struct mlx5_flow_driver_ops *fops, 7951 struct rte_flow_error *error) 7952 { 7953 static const char err_msg[] = "indirect action validation unsupported"; 7954 7955 if (!fops->action_validate) { 7956 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7957 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7958 NULL, err_msg); 7959 return -rte_errno; 7960 } 7961 return fops->action_validate(dev, conf, action, error); 7962 } 7963 7964 /** 7965 * Destroys the shared action by handle. 7966 * 7967 * @param dev 7968 * Pointer to Ethernet device structure. 7969 * @param[in] handle 7970 * Handle for the indirect action object to be destroyed. 7971 * @param[out] error 7972 * Perform verbose error reporting if not NULL. PMDs initialize this 7973 * structure in case of error only. 7974 * 7975 * @return 7976 * 0 on success, a negative errno value otherwise and rte_errno is set. 7977 * 7978 * @note: wrapper for driver action_create op callback. 7979 */ 7980 static int 7981 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 7982 struct rte_flow_action_handle *handle, 7983 struct rte_flow_error *error) 7984 { 7985 static const char err_msg[] = "indirect action destruction unsupported"; 7986 struct rte_flow_attr attr = { .transfer = 0 }; 7987 const struct mlx5_flow_driver_ops *fops = 7988 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7989 7990 if (!fops->action_destroy) { 7991 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7992 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7993 NULL, err_msg); 7994 return -rte_errno; 7995 } 7996 return fops->action_destroy(dev, handle, error); 7997 } 7998 7999 /* Wrapper for driver action_destroy op callback */ 8000 static int 8001 flow_drv_action_update(struct rte_eth_dev *dev, 8002 struct rte_flow_action_handle *handle, 8003 const void *update, 8004 const struct mlx5_flow_driver_ops *fops, 8005 struct rte_flow_error *error) 8006 { 8007 static const char err_msg[] = "indirect action update unsupported"; 8008 8009 if (!fops->action_update) { 8010 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8011 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8012 NULL, err_msg); 8013 return -rte_errno; 8014 } 8015 return fops->action_update(dev, handle, update, error); 8016 } 8017 8018 /* Wrapper for driver action_destroy op callback */ 8019 static int 8020 flow_drv_action_query(struct rte_eth_dev *dev, 8021 const struct rte_flow_action_handle *handle, 8022 void *data, 8023 const struct mlx5_flow_driver_ops *fops, 8024 struct rte_flow_error *error) 8025 { 8026 static const char err_msg[] = "indirect action query unsupported"; 8027 8028 if (!fops->action_query) { 8029 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8030 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8031 NULL, err_msg); 8032 return -rte_errno; 8033 } 8034 return fops->action_query(dev, handle, data, error); 8035 } 8036 8037 /** 8038 * Create indirect action for reuse in multiple flow rules. 8039 * 8040 * @param dev 8041 * Pointer to Ethernet device structure. 8042 * @param conf 8043 * Pointer to indirect action object configuration. 8044 * @param[in] action 8045 * Action configuration for indirect action object creation. 8046 * @param[out] error 8047 * Perform verbose error reporting if not NULL. PMDs initialize this 8048 * structure in case of error only. 8049 * @return 8050 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8051 */ 8052 static struct rte_flow_action_handle * 8053 mlx5_action_handle_create(struct rte_eth_dev *dev, 8054 const struct rte_flow_indir_action_conf *conf, 8055 const struct rte_flow_action *action, 8056 struct rte_flow_error *error) 8057 { 8058 static const char err_msg[] = "indirect action creation unsupported"; 8059 struct rte_flow_attr attr = { .transfer = 0 }; 8060 const struct mlx5_flow_driver_ops *fops = 8061 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8062 8063 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8064 return NULL; 8065 if (!fops->action_create) { 8066 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8067 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8068 NULL, err_msg); 8069 return NULL; 8070 } 8071 return fops->action_create(dev, conf, action, error); 8072 } 8073 8074 /** 8075 * Updates inplace the indirect action configuration pointed by *handle* 8076 * with the configuration provided as *update* argument. 8077 * The update of the indirect action configuration effects all flow rules 8078 * reusing the action via handle. 8079 * 8080 * @param dev 8081 * Pointer to Ethernet device structure. 8082 * @param[in] handle 8083 * Handle for the indirect action to be updated. 8084 * @param[in] update 8085 * Action specification used to modify the action pointed by handle. 8086 * *update* could be of same type with the action pointed by the *handle* 8087 * handle argument, or some other structures like a wrapper, depending on 8088 * the indirect action type. 8089 * @param[out] error 8090 * Perform verbose error reporting if not NULL. PMDs initialize this 8091 * structure in case of error only. 8092 * 8093 * @return 8094 * 0 on success, a negative errno value otherwise and rte_errno is set. 8095 */ 8096 static int 8097 mlx5_action_handle_update(struct rte_eth_dev *dev, 8098 struct rte_flow_action_handle *handle, 8099 const void *update, 8100 struct rte_flow_error *error) 8101 { 8102 struct rte_flow_attr attr = { .transfer = 0 }; 8103 const struct mlx5_flow_driver_ops *fops = 8104 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8105 int ret; 8106 8107 ret = flow_drv_action_validate(dev, NULL, 8108 (const struct rte_flow_action *)update, fops, error); 8109 if (ret) 8110 return ret; 8111 return flow_drv_action_update(dev, handle, update, fops, 8112 error); 8113 } 8114 8115 /** 8116 * Query the indirect action by handle. 8117 * 8118 * This function allows retrieving action-specific data such as counters. 8119 * Data is gathered by special action which may be present/referenced in 8120 * more than one flow rule definition. 8121 * 8122 * see @RTE_FLOW_ACTION_TYPE_COUNT 8123 * 8124 * @param dev 8125 * Pointer to Ethernet device structure. 8126 * @param[in] handle 8127 * Handle for the indirect action to query. 8128 * @param[in, out] data 8129 * Pointer to storage for the associated query data type. 8130 * @param[out] error 8131 * Perform verbose error reporting if not NULL. PMDs initialize this 8132 * structure in case of error only. 8133 * 8134 * @return 8135 * 0 on success, a negative errno value otherwise and rte_errno is set. 8136 */ 8137 static int 8138 mlx5_action_handle_query(struct rte_eth_dev *dev, 8139 const struct rte_flow_action_handle *handle, 8140 void *data, 8141 struct rte_flow_error *error) 8142 { 8143 struct rte_flow_attr attr = { .transfer = 0 }; 8144 const struct mlx5_flow_driver_ops *fops = 8145 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8146 8147 return flow_drv_action_query(dev, handle, data, fops, error); 8148 } 8149 8150 /** 8151 * Destroy all indirect actions (shared RSS). 8152 * 8153 * @param dev 8154 * Pointer to Ethernet device. 8155 * 8156 * @return 8157 * 0 on success, a negative errno value otherwise and rte_errno is set. 8158 */ 8159 int 8160 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8161 { 8162 struct rte_flow_error error; 8163 struct mlx5_priv *priv = dev->data->dev_private; 8164 struct mlx5_shared_action_rss *shared_rss; 8165 int ret = 0; 8166 uint32_t idx; 8167 8168 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8169 priv->rss_shared_actions, idx, shared_rss, next) { 8170 ret |= mlx5_action_handle_destroy(dev, 8171 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8172 } 8173 return ret; 8174 } 8175 8176 #ifndef HAVE_MLX5DV_DR 8177 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8178 #else 8179 #define MLX5_DOMAIN_SYNC_FLOW \ 8180 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8181 #endif 8182 8183 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8184 { 8185 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8186 const struct mlx5_flow_driver_ops *fops; 8187 int ret; 8188 struct rte_flow_attr attr = { .transfer = 0 }; 8189 8190 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8191 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8192 if (ret > 0) 8193 ret = -ret; 8194 return ret; 8195 } 8196 8197 const struct mlx5_flow_tunnel * 8198 mlx5_get_tof(const struct rte_flow_item *item, 8199 const struct rte_flow_action *action, 8200 enum mlx5_tof_rule_type *rule_type) 8201 { 8202 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 8203 if (item->type == (typeof(item->type)) 8204 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 8205 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 8206 return flow_items_to_tunnel(item); 8207 } 8208 } 8209 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 8210 if (action->type == (typeof(action->type)) 8211 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 8212 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 8213 return flow_actions_to_tunnel(action); 8214 } 8215 } 8216 return NULL; 8217 } 8218 8219 /** 8220 * tunnel offload functionalilty is defined for DV environment only 8221 */ 8222 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8223 __extension__ 8224 union tunnel_offload_mark { 8225 uint32_t val; 8226 struct { 8227 uint32_t app_reserve:8; 8228 uint32_t table_id:15; 8229 uint32_t transfer:1; 8230 uint32_t _unused_:8; 8231 }; 8232 }; 8233 8234 static bool 8235 mlx5_access_tunnel_offload_db 8236 (struct rte_eth_dev *dev, 8237 bool (*match)(struct rte_eth_dev *, 8238 struct mlx5_flow_tunnel *, const void *), 8239 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8240 void (*miss)(struct rte_eth_dev *, void *), 8241 void *ctx, bool lock_op); 8242 8243 static int 8244 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 8245 struct rte_flow *flow, 8246 const struct rte_flow_attr *attr, 8247 const struct rte_flow_action *app_actions, 8248 uint32_t flow_idx, 8249 const struct mlx5_flow_tunnel *tunnel, 8250 struct tunnel_default_miss_ctx *ctx, 8251 struct rte_flow_error *error) 8252 { 8253 struct mlx5_priv *priv = dev->data->dev_private; 8254 struct mlx5_flow *dev_flow; 8255 struct rte_flow_attr miss_attr = *attr; 8256 const struct rte_flow_item miss_items[2] = { 8257 { 8258 .type = RTE_FLOW_ITEM_TYPE_ETH, 8259 .spec = NULL, 8260 .last = NULL, 8261 .mask = NULL 8262 }, 8263 { 8264 .type = RTE_FLOW_ITEM_TYPE_END, 8265 .spec = NULL, 8266 .last = NULL, 8267 .mask = NULL 8268 } 8269 }; 8270 union tunnel_offload_mark mark_id; 8271 struct rte_flow_action_mark miss_mark; 8272 struct rte_flow_action miss_actions[3] = { 8273 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 8274 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 8275 }; 8276 const struct rte_flow_action_jump *jump_data; 8277 uint32_t i, flow_table = 0; /* prevent compilation warning */ 8278 struct flow_grp_info grp_info = { 8279 .external = 1, 8280 .transfer = attr->transfer, 8281 .fdb_def_rule = !!priv->fdb_def_rule, 8282 .std_tbl_fix = 0, 8283 }; 8284 int ret; 8285 8286 if (!attr->transfer) { 8287 uint32_t q_size; 8288 8289 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 8290 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 8291 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 8292 0, SOCKET_ID_ANY); 8293 if (!ctx->queue) 8294 return rte_flow_error_set 8295 (error, ENOMEM, 8296 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8297 NULL, "invalid default miss RSS"); 8298 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8299 ctx->action_rss.level = 0, 8300 ctx->action_rss.types = priv->rss_conf.rss_hf, 8301 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 8302 ctx->action_rss.queue_num = priv->reta_idx_n, 8303 ctx->action_rss.key = priv->rss_conf.rss_key, 8304 ctx->action_rss.queue = ctx->queue; 8305 if (!priv->reta_idx_n || !priv->rxqs_n) 8306 return rte_flow_error_set 8307 (error, EINVAL, 8308 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8309 NULL, "invalid port configuration"); 8310 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 8311 ctx->action_rss.types = 0; 8312 for (i = 0; i != priv->reta_idx_n; ++i) 8313 ctx->queue[i] = (*priv->reta_idx)[i]; 8314 } else { 8315 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 8316 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 8317 } 8318 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 8319 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 8320 jump_data = app_actions->conf; 8321 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 8322 miss_attr.group = jump_data->group; 8323 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 8324 &flow_table, &grp_info, error); 8325 if (ret) 8326 return rte_flow_error_set(error, EINVAL, 8327 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8328 NULL, "invalid tunnel id"); 8329 mark_id.app_reserve = 0; 8330 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 8331 mark_id.transfer = !!attr->transfer; 8332 mark_id._unused_ = 0; 8333 miss_mark.id = mark_id.val; 8334 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 8335 miss_items, miss_actions, flow_idx, error); 8336 if (!dev_flow) 8337 return -rte_errno; 8338 dev_flow->flow = flow; 8339 dev_flow->external = true; 8340 dev_flow->tunnel = tunnel; 8341 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 8342 /* Subflow object was created, we must include one in the list. */ 8343 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 8344 dev_flow->handle, next); 8345 DRV_LOG(DEBUG, 8346 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 8347 dev->data->port_id, tunnel->app_tunnel.type, 8348 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 8349 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 8350 miss_actions, error); 8351 if (!ret) 8352 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 8353 error); 8354 8355 return ret; 8356 } 8357 8358 static const struct mlx5_flow_tbl_data_entry * 8359 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 8360 { 8361 struct mlx5_priv *priv = dev->data->dev_private; 8362 struct mlx5_dev_ctx_shared *sh = priv->sh; 8363 struct mlx5_hlist_entry *he; 8364 union tunnel_offload_mark mbits = { .val = mark }; 8365 union mlx5_flow_tbl_key table_key = { 8366 { 8367 .level = tunnel_id_to_flow_tbl(mbits.table_id), 8368 .id = 0, 8369 .reserved = 0, 8370 .dummy = 0, 8371 .is_fdb = !!mbits.transfer, 8372 .is_egress = 0, 8373 } 8374 }; 8375 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 8376 return he ? 8377 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 8378 } 8379 8380 static void 8381 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 8382 struct mlx5_hlist_entry *entry) 8383 { 8384 struct mlx5_dev_ctx_shared *sh = list->ctx; 8385 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8386 8387 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8388 tunnel_flow_tbl_to_id(tte->flow_table)); 8389 mlx5_free(tte); 8390 } 8391 8392 static int 8393 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused, 8394 struct mlx5_hlist_entry *entry, 8395 uint64_t key, void *cb_ctx __rte_unused) 8396 { 8397 union tunnel_tbl_key tbl = { 8398 .val = key, 8399 }; 8400 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8401 8402 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 8403 } 8404 8405 static struct mlx5_hlist_entry * 8406 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key, 8407 void *ctx __rte_unused) 8408 { 8409 struct mlx5_dev_ctx_shared *sh = list->ctx; 8410 struct tunnel_tbl_entry *tte; 8411 union tunnel_tbl_key tbl = { 8412 .val = key, 8413 }; 8414 8415 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 8416 sizeof(*tte), 0, 8417 SOCKET_ID_ANY); 8418 if (!tte) 8419 goto err; 8420 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8421 &tte->flow_table); 8422 if (tte->flow_table >= MLX5_MAX_TABLES) { 8423 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 8424 tte->flow_table); 8425 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8426 tte->flow_table); 8427 goto err; 8428 } else if (!tte->flow_table) { 8429 goto err; 8430 } 8431 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 8432 tte->tunnel_id = tbl.tunnel_id; 8433 tte->group = tbl.group; 8434 return &tte->hash; 8435 err: 8436 if (tte) 8437 mlx5_free(tte); 8438 return NULL; 8439 } 8440 8441 static uint32_t 8442 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 8443 const struct mlx5_flow_tunnel *tunnel, 8444 uint32_t group, uint32_t *table, 8445 struct rte_flow_error *error) 8446 { 8447 struct mlx5_hlist_entry *he; 8448 struct tunnel_tbl_entry *tte; 8449 union tunnel_tbl_key key = { 8450 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 8451 .group = group 8452 }; 8453 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8454 struct mlx5_hlist *group_hash; 8455 8456 group_hash = tunnel ? tunnel->groups : thub->groups; 8457 he = mlx5_hlist_register(group_hash, key.val, NULL); 8458 if (!he) 8459 return rte_flow_error_set(error, EINVAL, 8460 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8461 NULL, 8462 "tunnel group index not supported"); 8463 tte = container_of(he, typeof(*tte), hash); 8464 *table = tte->flow_table; 8465 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 8466 dev->data->port_id, key.tunnel_id, group, *table); 8467 return 0; 8468 } 8469 8470 static void 8471 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8472 struct mlx5_flow_tunnel *tunnel) 8473 { 8474 struct mlx5_priv *priv = dev->data->dev_private; 8475 struct mlx5_indexed_pool *ipool; 8476 8477 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8478 dev->data->port_id, tunnel->tunnel_id); 8479 LIST_REMOVE(tunnel, chain); 8480 mlx5_hlist_destroy(tunnel->groups); 8481 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8482 mlx5_ipool_free(ipool, tunnel->tunnel_id); 8483 } 8484 8485 static bool 8486 mlx5_access_tunnel_offload_db 8487 (struct rte_eth_dev *dev, 8488 bool (*match)(struct rte_eth_dev *, 8489 struct mlx5_flow_tunnel *, const void *), 8490 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8491 void (*miss)(struct rte_eth_dev *, void *), 8492 void *ctx, bool lock_op) 8493 { 8494 bool verdict = false; 8495 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8496 struct mlx5_flow_tunnel *tunnel; 8497 8498 rte_spinlock_lock(&thub->sl); 8499 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 8500 verdict = match(dev, tunnel, (const void *)ctx); 8501 if (verdict) 8502 break; 8503 } 8504 if (!lock_op) 8505 rte_spinlock_unlock(&thub->sl); 8506 if (verdict && hit) 8507 hit(dev, tunnel, ctx); 8508 if (!verdict && miss) 8509 miss(dev, ctx); 8510 if (lock_op) 8511 rte_spinlock_unlock(&thub->sl); 8512 8513 return verdict; 8514 } 8515 8516 struct tunnel_db_find_tunnel_id_ctx { 8517 uint32_t tunnel_id; 8518 struct mlx5_flow_tunnel *tunnel; 8519 }; 8520 8521 static bool 8522 find_tunnel_id_match(struct rte_eth_dev *dev, 8523 struct mlx5_flow_tunnel *tunnel, const void *x) 8524 { 8525 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8526 8527 RTE_SET_USED(dev); 8528 return tunnel->tunnel_id == ctx->tunnel_id; 8529 } 8530 8531 static void 8532 find_tunnel_id_hit(struct rte_eth_dev *dev, 8533 struct mlx5_flow_tunnel *tunnel, void *x) 8534 { 8535 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8536 RTE_SET_USED(dev); 8537 ctx->tunnel = tunnel; 8538 } 8539 8540 static struct mlx5_flow_tunnel * 8541 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 8542 { 8543 struct tunnel_db_find_tunnel_id_ctx ctx = { 8544 .tunnel_id = id, 8545 }; 8546 8547 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 8548 find_tunnel_id_hit, NULL, &ctx, true); 8549 8550 return ctx.tunnel; 8551 } 8552 8553 static struct mlx5_flow_tunnel * 8554 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 8555 const struct rte_flow_tunnel *app_tunnel) 8556 { 8557 struct mlx5_priv *priv = dev->data->dev_private; 8558 struct mlx5_indexed_pool *ipool; 8559 struct mlx5_flow_tunnel *tunnel; 8560 uint32_t id; 8561 8562 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8563 tunnel = mlx5_ipool_zmalloc(ipool, &id); 8564 if (!tunnel) 8565 return NULL; 8566 if (id >= MLX5_MAX_TUNNELS) { 8567 mlx5_ipool_free(ipool, id); 8568 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 8569 return NULL; 8570 } 8571 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 8572 mlx5_flow_tunnel_grp2tbl_create_cb, 8573 mlx5_flow_tunnel_grp2tbl_match_cb, 8574 mlx5_flow_tunnel_grp2tbl_remove_cb); 8575 if (!tunnel->groups) { 8576 mlx5_ipool_free(ipool, id); 8577 return NULL; 8578 } 8579 tunnel->groups->ctx = priv->sh; 8580 /* initiate new PMD tunnel */ 8581 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8582 tunnel->tunnel_id = id; 8583 tunnel->action.type = (typeof(tunnel->action.type)) 8584 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8585 tunnel->action.conf = tunnel; 8586 tunnel->item.type = (typeof(tunnel->item.type)) 8587 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8588 tunnel->item.spec = tunnel; 8589 tunnel->item.last = NULL; 8590 tunnel->item.mask = NULL; 8591 8592 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8593 dev->data->port_id, tunnel->tunnel_id); 8594 8595 return tunnel; 8596 } 8597 8598 struct tunnel_db_get_tunnel_ctx { 8599 const struct rte_flow_tunnel *app_tunnel; 8600 struct mlx5_flow_tunnel *tunnel; 8601 }; 8602 8603 static bool get_tunnel_match(struct rte_eth_dev *dev, 8604 struct mlx5_flow_tunnel *tunnel, const void *x) 8605 { 8606 const struct tunnel_db_get_tunnel_ctx *ctx = x; 8607 8608 RTE_SET_USED(dev); 8609 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 8610 sizeof(*ctx->app_tunnel)); 8611 } 8612 8613 static void get_tunnel_hit(struct rte_eth_dev *dev, 8614 struct mlx5_flow_tunnel *tunnel, void *x) 8615 { 8616 /* called under tunnel spinlock protection */ 8617 struct tunnel_db_get_tunnel_ctx *ctx = x; 8618 8619 RTE_SET_USED(dev); 8620 tunnel->refctn++; 8621 ctx->tunnel = tunnel; 8622 } 8623 8624 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 8625 { 8626 /* called under tunnel spinlock protection */ 8627 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8628 struct tunnel_db_get_tunnel_ctx *ctx = x; 8629 8630 rte_spinlock_unlock(&thub->sl); 8631 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 8632 rte_spinlock_lock(&thub->sl); 8633 if (ctx->tunnel) { 8634 ctx->tunnel->refctn = 1; 8635 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 8636 } 8637 } 8638 8639 8640 static int 8641 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 8642 const struct rte_flow_tunnel *app_tunnel, 8643 struct mlx5_flow_tunnel **tunnel) 8644 { 8645 struct tunnel_db_get_tunnel_ctx ctx = { 8646 .app_tunnel = app_tunnel, 8647 }; 8648 8649 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 8650 get_tunnel_miss, &ctx, true); 8651 *tunnel = ctx.tunnel; 8652 return ctx.tunnel ? 0 : -ENOMEM; 8653 } 8654 8655 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 8656 { 8657 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 8658 8659 if (!thub) 8660 return; 8661 if (!LIST_EMPTY(&thub->tunnels)) 8662 DRV_LOG(WARNING, "port %u tunnels present", port_id); 8663 mlx5_hlist_destroy(thub->groups); 8664 mlx5_free(thub); 8665 } 8666 8667 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 8668 { 8669 int err; 8670 struct mlx5_flow_tunnel_hub *thub; 8671 8672 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 8673 0, SOCKET_ID_ANY); 8674 if (!thub) 8675 return -ENOMEM; 8676 LIST_INIT(&thub->tunnels); 8677 rte_spinlock_init(&thub->sl); 8678 thub->groups = mlx5_hlist_create("flow groups", 8679 rte_align32pow2(MLX5_MAX_TABLES), 0, 8680 0, mlx5_flow_tunnel_grp2tbl_create_cb, 8681 mlx5_flow_tunnel_grp2tbl_match_cb, 8682 mlx5_flow_tunnel_grp2tbl_remove_cb); 8683 if (!thub->groups) { 8684 err = -rte_errno; 8685 goto err; 8686 } 8687 thub->groups->ctx = sh; 8688 sh->tunnel_hub = thub; 8689 8690 return 0; 8691 8692 err: 8693 if (thub->groups) 8694 mlx5_hlist_destroy(thub->groups); 8695 if (thub) 8696 mlx5_free(thub); 8697 return err; 8698 } 8699 8700 static inline bool 8701 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 8702 struct rte_flow_tunnel *tunnel, 8703 const char *err_msg) 8704 { 8705 err_msg = NULL; 8706 if (!is_tunnel_offload_active(dev)) { 8707 err_msg = "tunnel offload was not activated"; 8708 goto out; 8709 } else if (!tunnel) { 8710 err_msg = "no application tunnel"; 8711 goto out; 8712 } 8713 8714 switch (tunnel->type) { 8715 default: 8716 err_msg = "unsupported tunnel type"; 8717 goto out; 8718 case RTE_FLOW_ITEM_TYPE_VXLAN: 8719 break; 8720 } 8721 8722 out: 8723 return !err_msg; 8724 } 8725 8726 static int 8727 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 8728 struct rte_flow_tunnel *app_tunnel, 8729 struct rte_flow_action **actions, 8730 uint32_t *num_of_actions, 8731 struct rte_flow_error *error) 8732 { 8733 int ret; 8734 struct mlx5_flow_tunnel *tunnel; 8735 const char *err_msg = NULL; 8736 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8737 8738 if (!verdict) 8739 return rte_flow_error_set(error, EINVAL, 8740 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8741 err_msg); 8742 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8743 if (ret < 0) { 8744 return rte_flow_error_set(error, ret, 8745 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8746 "failed to initialize pmd tunnel"); 8747 } 8748 *actions = &tunnel->action; 8749 *num_of_actions = 1; 8750 return 0; 8751 } 8752 8753 static int 8754 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 8755 struct rte_flow_tunnel *app_tunnel, 8756 struct rte_flow_item **items, 8757 uint32_t *num_of_items, 8758 struct rte_flow_error *error) 8759 { 8760 int ret; 8761 struct mlx5_flow_tunnel *tunnel; 8762 const char *err_msg = NULL; 8763 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8764 8765 if (!verdict) 8766 return rte_flow_error_set(error, EINVAL, 8767 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8768 err_msg); 8769 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8770 if (ret < 0) { 8771 return rte_flow_error_set(error, ret, 8772 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8773 "failed to initialize pmd tunnel"); 8774 } 8775 *items = &tunnel->item; 8776 *num_of_items = 1; 8777 return 0; 8778 } 8779 8780 struct tunnel_db_element_release_ctx { 8781 struct rte_flow_item *items; 8782 struct rte_flow_action *actions; 8783 uint32_t num_elements; 8784 struct rte_flow_error *error; 8785 int ret; 8786 }; 8787 8788 static bool 8789 tunnel_element_release_match(struct rte_eth_dev *dev, 8790 struct mlx5_flow_tunnel *tunnel, const void *x) 8791 { 8792 const struct tunnel_db_element_release_ctx *ctx = x; 8793 8794 RTE_SET_USED(dev); 8795 if (ctx->num_elements != 1) 8796 return false; 8797 else if (ctx->items) 8798 return ctx->items == &tunnel->item; 8799 else if (ctx->actions) 8800 return ctx->actions == &tunnel->action; 8801 8802 return false; 8803 } 8804 8805 static void 8806 tunnel_element_release_hit(struct rte_eth_dev *dev, 8807 struct mlx5_flow_tunnel *tunnel, void *x) 8808 { 8809 struct tunnel_db_element_release_ctx *ctx = x; 8810 ctx->ret = 0; 8811 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 8812 mlx5_flow_tunnel_free(dev, tunnel); 8813 } 8814 8815 static void 8816 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 8817 { 8818 struct tunnel_db_element_release_ctx *ctx = x; 8819 RTE_SET_USED(dev); 8820 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 8821 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8822 "invalid argument"); 8823 } 8824 8825 static int 8826 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 8827 struct rte_flow_item *pmd_items, 8828 uint32_t num_items, struct rte_flow_error *err) 8829 { 8830 struct tunnel_db_element_release_ctx ctx = { 8831 .items = pmd_items, 8832 .actions = NULL, 8833 .num_elements = num_items, 8834 .error = err, 8835 }; 8836 8837 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 8838 tunnel_element_release_hit, 8839 tunnel_element_release_miss, &ctx, false); 8840 8841 return ctx.ret; 8842 } 8843 8844 static int 8845 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 8846 struct rte_flow_action *pmd_actions, 8847 uint32_t num_actions, struct rte_flow_error *err) 8848 { 8849 struct tunnel_db_element_release_ctx ctx = { 8850 .items = NULL, 8851 .actions = pmd_actions, 8852 .num_elements = num_actions, 8853 .error = err, 8854 }; 8855 8856 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 8857 tunnel_element_release_hit, 8858 tunnel_element_release_miss, &ctx, false); 8859 8860 return ctx.ret; 8861 } 8862 8863 static int 8864 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 8865 struct rte_mbuf *m, 8866 struct rte_flow_restore_info *info, 8867 struct rte_flow_error *err) 8868 { 8869 uint64_t ol_flags = m->ol_flags; 8870 const struct mlx5_flow_tbl_data_entry *tble; 8871 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 8872 8873 if (!is_tunnel_offload_active(dev)) { 8874 info->flags = 0; 8875 return 0; 8876 } 8877 8878 if ((ol_flags & mask) != mask) 8879 goto err; 8880 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 8881 if (!tble) { 8882 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 8883 dev->data->port_id, m->hash.fdir.hi); 8884 goto err; 8885 } 8886 MLX5_ASSERT(tble->tunnel); 8887 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 8888 info->group_id = tble->group_id; 8889 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 8890 RTE_FLOW_RESTORE_INFO_GROUP_ID | 8891 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 8892 8893 return 0; 8894 8895 err: 8896 return rte_flow_error_set(err, EINVAL, 8897 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8898 "failed to get restore info"); 8899 } 8900 8901 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 8902 static int 8903 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 8904 __rte_unused struct rte_flow_tunnel *app_tunnel, 8905 __rte_unused struct rte_flow_action **actions, 8906 __rte_unused uint32_t *num_of_actions, 8907 __rte_unused struct rte_flow_error *error) 8908 { 8909 return -ENOTSUP; 8910 } 8911 8912 static int 8913 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 8914 __rte_unused struct rte_flow_tunnel *app_tunnel, 8915 __rte_unused struct rte_flow_item **items, 8916 __rte_unused uint32_t *num_of_items, 8917 __rte_unused struct rte_flow_error *error) 8918 { 8919 return -ENOTSUP; 8920 } 8921 8922 static int 8923 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 8924 __rte_unused struct rte_flow_item *pmd_items, 8925 __rte_unused uint32_t num_items, 8926 __rte_unused struct rte_flow_error *err) 8927 { 8928 return -ENOTSUP; 8929 } 8930 8931 static int 8932 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 8933 __rte_unused struct rte_flow_action *pmd_action, 8934 __rte_unused uint32_t num_actions, 8935 __rte_unused struct rte_flow_error *err) 8936 { 8937 return -ENOTSUP; 8938 } 8939 8940 static int 8941 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 8942 __rte_unused struct rte_mbuf *m, 8943 __rte_unused struct rte_flow_restore_info *i, 8944 __rte_unused struct rte_flow_error *err) 8945 { 8946 return -ENOTSUP; 8947 } 8948 8949 static int 8950 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 8951 __rte_unused struct rte_flow *flow, 8952 __rte_unused const struct rte_flow_attr *attr, 8953 __rte_unused const struct rte_flow_action *actions, 8954 __rte_unused uint32_t flow_idx, 8955 __rte_unused const struct mlx5_flow_tunnel *tunnel, 8956 __rte_unused struct tunnel_default_miss_ctx *ctx, 8957 __rte_unused struct rte_flow_error *error) 8958 { 8959 return -ENOTSUP; 8960 } 8961 8962 static struct mlx5_flow_tunnel * 8963 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 8964 __rte_unused uint32_t id) 8965 { 8966 return NULL; 8967 } 8968 8969 static void 8970 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 8971 __rte_unused struct mlx5_flow_tunnel *tunnel) 8972 { 8973 } 8974 8975 static uint32_t 8976 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 8977 __rte_unused const struct mlx5_flow_tunnel *t, 8978 __rte_unused uint32_t group, 8979 __rte_unused uint32_t *table, 8980 struct rte_flow_error *error) 8981 { 8982 return rte_flow_error_set(error, ENOTSUP, 8983 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8984 "tunnel offload requires DV support"); 8985 } 8986 8987 void 8988 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 8989 __rte_unused uint16_t port_id) 8990 { 8991 } 8992 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 8993