1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 #include <rte_common.h> 14 #include <rte_ether.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_eal_paging.h> 17 #include <rte_flow.h> 18 #include <rte_cycles.h> 19 #include <rte_flow_driver.h> 20 #include <rte_malloc.h> 21 #include <rte_ip.h> 22 23 #include <mlx5_glue.h> 24 #include <mlx5_devx_cmds.h> 25 #include <mlx5_prm.h> 26 #include <mlx5_malloc.h> 27 28 #include "mlx5_defs.h" 29 #include "mlx5.h" 30 #include "mlx5_flow.h" 31 #include "mlx5_flow_os.h" 32 #include "mlx5_rxtx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 struct tunnel_default_miss_ctx *ctx, 54 struct rte_flow_error *error); 55 static struct mlx5_flow_tunnel * 56 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 57 static void 58 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 59 static uint32_t 60 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 61 const struct mlx5_flow_tunnel *tunnel, 62 uint32_t group, uint32_t *table, 63 struct rte_flow_error *error); 64 65 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 66 static void mlx5_flow_pop_thread_workspace(void); 67 68 69 /** Device flow drivers. */ 70 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 71 72 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 73 74 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 75 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 76 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 77 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 78 #endif 79 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 80 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 81 }; 82 83 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 84 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 85 (const int []){ \ 86 __VA_ARGS__, 0, \ 87 } 88 89 /** Node object of input graph for mlx5_flow_expand_rss(). */ 90 struct mlx5_flow_expand_node { 91 const int *const next; 92 /**< 93 * List of next node indexes. Index 0 is interpreted as a terminator. 94 */ 95 const enum rte_flow_item_type type; 96 /**< Pattern item type of current node. */ 97 uint64_t rss_types; 98 /**< 99 * RSS types bit-field associated with this node 100 * (see ETH_RSS_* definitions). 101 */ 102 }; 103 104 /** Object returned by mlx5_flow_expand_rss(). */ 105 struct mlx5_flow_expand_rss { 106 uint32_t entries; 107 /**< Number of entries @p patterns and @p priorities. */ 108 struct { 109 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 110 uint32_t priority; /**< Priority offset for each expansion. */ 111 } entry[]; 112 }; 113 114 static enum rte_flow_item_type 115 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 116 { 117 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 118 uint16_t ether_type = 0; 119 uint16_t ether_type_m; 120 uint8_t ip_next_proto = 0; 121 uint8_t ip_next_proto_m; 122 123 if (item == NULL || item->spec == NULL) 124 return ret; 125 switch (item->type) { 126 case RTE_FLOW_ITEM_TYPE_ETH: 127 if (item->mask) 128 ether_type_m = ((const struct rte_flow_item_eth *) 129 (item->mask))->type; 130 else 131 ether_type_m = rte_flow_item_eth_mask.type; 132 if (ether_type_m != RTE_BE16(0xFFFF)) 133 break; 134 ether_type = ((const struct rte_flow_item_eth *) 135 (item->spec))->type; 136 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 137 ret = RTE_FLOW_ITEM_TYPE_IPV4; 138 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 139 ret = RTE_FLOW_ITEM_TYPE_IPV6; 140 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 141 ret = RTE_FLOW_ITEM_TYPE_VLAN; 142 else 143 ret = RTE_FLOW_ITEM_TYPE_END; 144 break; 145 case RTE_FLOW_ITEM_TYPE_VLAN: 146 if (item->mask) 147 ether_type_m = ((const struct rte_flow_item_vlan *) 148 (item->mask))->inner_type; 149 else 150 ether_type_m = rte_flow_item_vlan_mask.inner_type; 151 if (ether_type_m != RTE_BE16(0xFFFF)) 152 break; 153 ether_type = ((const struct rte_flow_item_vlan *) 154 (item->spec))->inner_type; 155 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 156 ret = RTE_FLOW_ITEM_TYPE_IPV4; 157 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 158 ret = RTE_FLOW_ITEM_TYPE_IPV6; 159 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 160 ret = RTE_FLOW_ITEM_TYPE_VLAN; 161 else 162 ret = RTE_FLOW_ITEM_TYPE_END; 163 break; 164 case RTE_FLOW_ITEM_TYPE_IPV4: 165 if (item->mask) 166 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 167 (item->mask))->hdr.next_proto_id; 168 else 169 ip_next_proto_m = 170 rte_flow_item_ipv4_mask.hdr.next_proto_id; 171 if (ip_next_proto_m != 0xFF) 172 break; 173 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 174 (item->spec))->hdr.next_proto_id; 175 if (ip_next_proto == IPPROTO_UDP) 176 ret = RTE_FLOW_ITEM_TYPE_UDP; 177 else if (ip_next_proto == IPPROTO_TCP) 178 ret = RTE_FLOW_ITEM_TYPE_TCP; 179 else if (ip_next_proto == IPPROTO_IP) 180 ret = RTE_FLOW_ITEM_TYPE_IPV4; 181 else if (ip_next_proto == IPPROTO_IPV6) 182 ret = RTE_FLOW_ITEM_TYPE_IPV6; 183 else 184 ret = RTE_FLOW_ITEM_TYPE_END; 185 break; 186 case RTE_FLOW_ITEM_TYPE_IPV6: 187 if (item->mask) 188 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 189 (item->mask))->hdr.proto; 190 else 191 ip_next_proto_m = 192 rte_flow_item_ipv6_mask.hdr.proto; 193 if (ip_next_proto_m != 0xFF) 194 break; 195 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 196 (item->spec))->hdr.proto; 197 if (ip_next_proto == IPPROTO_UDP) 198 ret = RTE_FLOW_ITEM_TYPE_UDP; 199 else if (ip_next_proto == IPPROTO_TCP) 200 ret = RTE_FLOW_ITEM_TYPE_TCP; 201 else if (ip_next_proto == IPPROTO_IP) 202 ret = RTE_FLOW_ITEM_TYPE_IPV4; 203 else if (ip_next_proto == IPPROTO_IPV6) 204 ret = RTE_FLOW_ITEM_TYPE_IPV6; 205 else 206 ret = RTE_FLOW_ITEM_TYPE_END; 207 break; 208 default: 209 ret = RTE_FLOW_ITEM_TYPE_VOID; 210 break; 211 } 212 return ret; 213 } 214 215 /** 216 * Expand RSS flows into several possible flows according to the RSS hash 217 * fields requested and the driver capabilities. 218 * 219 * @param[out] buf 220 * Buffer to store the result expansion. 221 * @param[in] size 222 * Buffer size in bytes. If 0, @p buf can be NULL. 223 * @param[in] pattern 224 * User flow pattern. 225 * @param[in] types 226 * RSS types to expand (see ETH_RSS_* definitions). 227 * @param[in] graph 228 * Input graph to expand @p pattern according to @p types. 229 * @param[in] graph_root_index 230 * Index of root node in @p graph, typically 0. 231 * 232 * @return 233 * A positive value representing the size of @p buf in bytes regardless of 234 * @p size on success, a negative errno value otherwise and rte_errno is 235 * set, the following errors are defined: 236 * 237 * -E2BIG: graph-depth @p graph is too deep. 238 */ 239 static int 240 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 241 const struct rte_flow_item *pattern, uint64_t types, 242 const struct mlx5_flow_expand_node graph[], 243 int graph_root_index) 244 { 245 const int elt_n = 8; 246 const struct rte_flow_item *item; 247 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 248 const int *next_node; 249 const int *stack[elt_n]; 250 int stack_pos = 0; 251 struct rte_flow_item flow_items[elt_n]; 252 unsigned int i; 253 size_t lsize; 254 size_t user_pattern_size = 0; 255 void *addr = NULL; 256 const struct mlx5_flow_expand_node *next = NULL; 257 struct rte_flow_item missed_item; 258 int missed = 0; 259 int elt = 0; 260 const struct rte_flow_item *last_item = NULL; 261 262 memset(&missed_item, 0, sizeof(missed_item)); 263 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 264 elt_n * sizeof(buf->entry[0]); 265 if (lsize <= size) { 266 buf->entry[0].priority = 0; 267 buf->entry[0].pattern = (void *)&buf->entry[elt_n]; 268 buf->entries = 0; 269 addr = buf->entry[0].pattern; 270 } 271 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 272 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 273 last_item = item; 274 for (i = 0; node->next && node->next[i]; ++i) { 275 next = &graph[node->next[i]]; 276 if (next->type == item->type) 277 break; 278 } 279 if (next) 280 node = next; 281 user_pattern_size += sizeof(*item); 282 } 283 user_pattern_size += sizeof(*item); /* Handle END item. */ 284 lsize += user_pattern_size; 285 /* Copy the user pattern in the first entry of the buffer. */ 286 if (lsize <= size) { 287 rte_memcpy(addr, pattern, user_pattern_size); 288 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 289 buf->entries = 1; 290 } 291 /* Start expanding. */ 292 memset(flow_items, 0, sizeof(flow_items)); 293 user_pattern_size -= sizeof(*item); 294 /* 295 * Check if the last valid item has spec set, need complete pattern, 296 * and the pattern can be used for expansion. 297 */ 298 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 299 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 300 /* Item type END indicates expansion is not required. */ 301 return lsize; 302 } 303 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 304 next = NULL; 305 missed = 1; 306 for (i = 0; node->next && node->next[i]; ++i) { 307 next = &graph[node->next[i]]; 308 if (next->type == missed_item.type) { 309 flow_items[0].type = missed_item.type; 310 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 311 break; 312 } 313 next = NULL; 314 } 315 } 316 if (next && missed) { 317 elt = 2; /* missed item + item end. */ 318 node = next; 319 lsize += elt * sizeof(*item) + user_pattern_size; 320 if ((node->rss_types & types) && lsize <= size) { 321 buf->entry[buf->entries].priority = 1; 322 buf->entry[buf->entries].pattern = addr; 323 buf->entries++; 324 rte_memcpy(addr, buf->entry[0].pattern, 325 user_pattern_size); 326 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 327 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 328 addr = (void *)(((uintptr_t)addr) + 329 elt * sizeof(*item)); 330 } 331 } 332 memset(flow_items, 0, sizeof(flow_items)); 333 next_node = node->next; 334 stack[stack_pos] = next_node; 335 node = next_node ? &graph[*next_node] : NULL; 336 while (node) { 337 flow_items[stack_pos].type = node->type; 338 if (node->rss_types & types) { 339 /* 340 * compute the number of items to copy from the 341 * expansion and copy it. 342 * When the stack_pos is 0, there are 1 element in it, 343 * plus the addition END item. 344 */ 345 elt = stack_pos + 2; 346 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 347 lsize += elt * sizeof(*item) + user_pattern_size; 348 if (lsize <= size) { 349 size_t n = elt * sizeof(*item); 350 351 buf->entry[buf->entries].priority = 352 stack_pos + 1 + missed; 353 buf->entry[buf->entries].pattern = addr; 354 buf->entries++; 355 rte_memcpy(addr, buf->entry[0].pattern, 356 user_pattern_size); 357 addr = (void *)(((uintptr_t)addr) + 358 user_pattern_size); 359 rte_memcpy(addr, &missed_item, 360 missed * sizeof(*item)); 361 addr = (void *)(((uintptr_t)addr) + 362 missed * sizeof(*item)); 363 rte_memcpy(addr, flow_items, n); 364 addr = (void *)(((uintptr_t)addr) + n); 365 } 366 } 367 /* Go deeper. */ 368 if (node->next) { 369 next_node = node->next; 370 if (stack_pos++ == elt_n) { 371 rte_errno = E2BIG; 372 return -rte_errno; 373 } 374 stack[stack_pos] = next_node; 375 } else if (*(next_node + 1)) { 376 /* Follow up with the next possibility. */ 377 ++next_node; 378 } else { 379 /* Move to the next path. */ 380 if (stack_pos) 381 next_node = stack[--stack_pos]; 382 next_node++; 383 stack[stack_pos] = next_node; 384 } 385 node = *next_node ? &graph[*next_node] : NULL; 386 }; 387 /* no expanded flows but we have missed item, create one rule for it */ 388 if (buf->entries == 1 && missed != 0) { 389 elt = 2; 390 lsize += elt * sizeof(*item) + user_pattern_size; 391 if (lsize <= size) { 392 buf->entry[buf->entries].priority = 1; 393 buf->entry[buf->entries].pattern = addr; 394 buf->entries++; 395 flow_items[0].type = missed_item.type; 396 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 397 rte_memcpy(addr, buf->entry[0].pattern, 398 user_pattern_size); 399 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 400 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 401 } 402 } 403 return lsize; 404 } 405 406 enum mlx5_expansion { 407 MLX5_EXPANSION_ROOT, 408 MLX5_EXPANSION_ROOT_OUTER, 409 MLX5_EXPANSION_ROOT_ETH_VLAN, 410 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 411 MLX5_EXPANSION_OUTER_ETH, 412 MLX5_EXPANSION_OUTER_ETH_VLAN, 413 MLX5_EXPANSION_OUTER_VLAN, 414 MLX5_EXPANSION_OUTER_IPV4, 415 MLX5_EXPANSION_OUTER_IPV4_UDP, 416 MLX5_EXPANSION_OUTER_IPV4_TCP, 417 MLX5_EXPANSION_OUTER_IPV6, 418 MLX5_EXPANSION_OUTER_IPV6_UDP, 419 MLX5_EXPANSION_OUTER_IPV6_TCP, 420 MLX5_EXPANSION_VXLAN, 421 MLX5_EXPANSION_VXLAN_GPE, 422 MLX5_EXPANSION_GRE, 423 MLX5_EXPANSION_MPLS, 424 MLX5_EXPANSION_ETH, 425 MLX5_EXPANSION_ETH_VLAN, 426 MLX5_EXPANSION_VLAN, 427 MLX5_EXPANSION_IPV4, 428 MLX5_EXPANSION_IPV4_UDP, 429 MLX5_EXPANSION_IPV4_TCP, 430 MLX5_EXPANSION_IPV6, 431 MLX5_EXPANSION_IPV6_UDP, 432 MLX5_EXPANSION_IPV6_TCP, 433 }; 434 435 /** Supported expansion of items. */ 436 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 437 [MLX5_EXPANSION_ROOT] = { 438 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 439 MLX5_EXPANSION_IPV4, 440 MLX5_EXPANSION_IPV6), 441 .type = RTE_FLOW_ITEM_TYPE_END, 442 }, 443 [MLX5_EXPANSION_ROOT_OUTER] = { 444 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 445 MLX5_EXPANSION_OUTER_IPV4, 446 MLX5_EXPANSION_OUTER_IPV6), 447 .type = RTE_FLOW_ITEM_TYPE_END, 448 }, 449 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 450 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 451 .type = RTE_FLOW_ITEM_TYPE_END, 452 }, 453 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 454 .next = MLX5_FLOW_EXPAND_RSS_NEXT 455 (MLX5_EXPANSION_OUTER_ETH_VLAN), 456 .type = RTE_FLOW_ITEM_TYPE_END, 457 }, 458 [MLX5_EXPANSION_OUTER_ETH] = { 459 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 460 MLX5_EXPANSION_OUTER_IPV6, 461 MLX5_EXPANSION_MPLS), 462 .type = RTE_FLOW_ITEM_TYPE_ETH, 463 .rss_types = 0, 464 }, 465 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 466 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 467 .type = RTE_FLOW_ITEM_TYPE_ETH, 468 .rss_types = 0, 469 }, 470 [MLX5_EXPANSION_OUTER_VLAN] = { 471 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 472 MLX5_EXPANSION_OUTER_IPV6), 473 .type = RTE_FLOW_ITEM_TYPE_VLAN, 474 }, 475 [MLX5_EXPANSION_OUTER_IPV4] = { 476 .next = MLX5_FLOW_EXPAND_RSS_NEXT 477 (MLX5_EXPANSION_OUTER_IPV4_UDP, 478 MLX5_EXPANSION_OUTER_IPV4_TCP, 479 MLX5_EXPANSION_GRE, 480 MLX5_EXPANSION_IPV4, 481 MLX5_EXPANSION_IPV6), 482 .type = RTE_FLOW_ITEM_TYPE_IPV4, 483 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 484 ETH_RSS_NONFRAG_IPV4_OTHER, 485 }, 486 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 487 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 488 MLX5_EXPANSION_VXLAN_GPE), 489 .type = RTE_FLOW_ITEM_TYPE_UDP, 490 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 491 }, 492 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 493 .type = RTE_FLOW_ITEM_TYPE_TCP, 494 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 495 }, 496 [MLX5_EXPANSION_OUTER_IPV6] = { 497 .next = MLX5_FLOW_EXPAND_RSS_NEXT 498 (MLX5_EXPANSION_OUTER_IPV6_UDP, 499 MLX5_EXPANSION_OUTER_IPV6_TCP, 500 MLX5_EXPANSION_IPV4, 501 MLX5_EXPANSION_IPV6), 502 .type = RTE_FLOW_ITEM_TYPE_IPV6, 503 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 504 ETH_RSS_NONFRAG_IPV6_OTHER, 505 }, 506 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 507 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 508 MLX5_EXPANSION_VXLAN_GPE), 509 .type = RTE_FLOW_ITEM_TYPE_UDP, 510 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 511 }, 512 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 513 .type = RTE_FLOW_ITEM_TYPE_TCP, 514 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 515 }, 516 [MLX5_EXPANSION_VXLAN] = { 517 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 518 MLX5_EXPANSION_IPV4, 519 MLX5_EXPANSION_IPV6), 520 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 521 }, 522 [MLX5_EXPANSION_VXLAN_GPE] = { 523 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 524 MLX5_EXPANSION_IPV4, 525 MLX5_EXPANSION_IPV6), 526 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 527 }, 528 [MLX5_EXPANSION_GRE] = { 529 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 530 .type = RTE_FLOW_ITEM_TYPE_GRE, 531 }, 532 [MLX5_EXPANSION_MPLS] = { 533 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 534 MLX5_EXPANSION_IPV6), 535 .type = RTE_FLOW_ITEM_TYPE_MPLS, 536 }, 537 [MLX5_EXPANSION_ETH] = { 538 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 539 MLX5_EXPANSION_IPV6), 540 .type = RTE_FLOW_ITEM_TYPE_ETH, 541 }, 542 [MLX5_EXPANSION_ETH_VLAN] = { 543 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 544 .type = RTE_FLOW_ITEM_TYPE_ETH, 545 }, 546 [MLX5_EXPANSION_VLAN] = { 547 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 548 MLX5_EXPANSION_IPV6), 549 .type = RTE_FLOW_ITEM_TYPE_VLAN, 550 }, 551 [MLX5_EXPANSION_IPV4] = { 552 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 553 MLX5_EXPANSION_IPV4_TCP), 554 .type = RTE_FLOW_ITEM_TYPE_IPV4, 555 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 556 ETH_RSS_NONFRAG_IPV4_OTHER, 557 }, 558 [MLX5_EXPANSION_IPV4_UDP] = { 559 .type = RTE_FLOW_ITEM_TYPE_UDP, 560 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 561 }, 562 [MLX5_EXPANSION_IPV4_TCP] = { 563 .type = RTE_FLOW_ITEM_TYPE_TCP, 564 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 565 }, 566 [MLX5_EXPANSION_IPV6] = { 567 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 568 MLX5_EXPANSION_IPV6_TCP), 569 .type = RTE_FLOW_ITEM_TYPE_IPV6, 570 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 571 ETH_RSS_NONFRAG_IPV6_OTHER, 572 }, 573 [MLX5_EXPANSION_IPV6_UDP] = { 574 .type = RTE_FLOW_ITEM_TYPE_UDP, 575 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 576 }, 577 [MLX5_EXPANSION_IPV6_TCP] = { 578 .type = RTE_FLOW_ITEM_TYPE_TCP, 579 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 580 }, 581 }; 582 583 static struct rte_flow_shared_action * 584 mlx5_shared_action_create(struct rte_eth_dev *dev, 585 const struct rte_flow_shared_action_conf *conf, 586 const struct rte_flow_action *action, 587 struct rte_flow_error *error); 588 static int mlx5_shared_action_destroy 589 (struct rte_eth_dev *dev, 590 struct rte_flow_shared_action *shared_action, 591 struct rte_flow_error *error); 592 static int mlx5_shared_action_update 593 (struct rte_eth_dev *dev, 594 struct rte_flow_shared_action *shared_action, 595 const struct rte_flow_action *action, 596 struct rte_flow_error *error); 597 static int mlx5_shared_action_query 598 (struct rte_eth_dev *dev, 599 const struct rte_flow_shared_action *action, 600 void *data, 601 struct rte_flow_error *error); 602 static int 603 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 604 struct rte_flow_tunnel *app_tunnel, 605 struct rte_flow_action **actions, 606 uint32_t *num_of_actions, 607 struct rte_flow_error *error); 608 static int 609 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 610 struct rte_flow_tunnel *app_tunnel, 611 struct rte_flow_item **items, 612 uint32_t *num_of_items, 613 struct rte_flow_error *error); 614 static int 615 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 616 struct rte_flow_item *pmd_items, 617 uint32_t num_items, struct rte_flow_error *err); 618 static int 619 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 620 struct rte_flow_action *pmd_actions, 621 uint32_t num_actions, 622 struct rte_flow_error *err); 623 static int 624 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 625 struct rte_mbuf *m, 626 struct rte_flow_restore_info *info, 627 struct rte_flow_error *err); 628 629 static const struct rte_flow_ops mlx5_flow_ops = { 630 .validate = mlx5_flow_validate, 631 .create = mlx5_flow_create, 632 .destroy = mlx5_flow_destroy, 633 .flush = mlx5_flow_flush, 634 .isolate = mlx5_flow_isolate, 635 .query = mlx5_flow_query, 636 .dev_dump = mlx5_flow_dev_dump, 637 .get_aged_flows = mlx5_flow_get_aged_flows, 638 .shared_action_create = mlx5_shared_action_create, 639 .shared_action_destroy = mlx5_shared_action_destroy, 640 .shared_action_update = mlx5_shared_action_update, 641 .shared_action_query = mlx5_shared_action_query, 642 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 643 .tunnel_match = mlx5_flow_tunnel_match, 644 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 645 .tunnel_item_release = mlx5_flow_tunnel_item_release, 646 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 647 }; 648 649 /* Tunnel information. */ 650 struct mlx5_flow_tunnel_info { 651 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 652 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 653 }; 654 655 static struct mlx5_flow_tunnel_info tunnels_info[] = { 656 { 657 .tunnel = MLX5_FLOW_LAYER_VXLAN, 658 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 659 }, 660 { 661 .tunnel = MLX5_FLOW_LAYER_GENEVE, 662 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 663 }, 664 { 665 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 666 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 667 }, 668 { 669 .tunnel = MLX5_FLOW_LAYER_GRE, 670 .ptype = RTE_PTYPE_TUNNEL_GRE, 671 }, 672 { 673 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 674 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 675 }, 676 { 677 .tunnel = MLX5_FLOW_LAYER_MPLS, 678 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 679 }, 680 { 681 .tunnel = MLX5_FLOW_LAYER_NVGRE, 682 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 683 }, 684 { 685 .tunnel = MLX5_FLOW_LAYER_IPIP, 686 .ptype = RTE_PTYPE_TUNNEL_IP, 687 }, 688 { 689 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 690 .ptype = RTE_PTYPE_TUNNEL_IP, 691 }, 692 { 693 .tunnel = MLX5_FLOW_LAYER_GTP, 694 .ptype = RTE_PTYPE_TUNNEL_GTPU, 695 }, 696 }; 697 698 /* Key of thread specific flow workspace data. */ 699 static pthread_key_t key_workspace; 700 701 /* Thread specific flow workspace data once initialization data. */ 702 static pthread_once_t key_workspace_init; 703 704 705 /** 706 * Translate tag ID to register. 707 * 708 * @param[in] dev 709 * Pointer to the Ethernet device structure. 710 * @param[in] feature 711 * The feature that request the register. 712 * @param[in] id 713 * The request register ID. 714 * @param[out] error 715 * Error description in case of any. 716 * 717 * @return 718 * The request register on success, a negative errno 719 * value otherwise and rte_errno is set. 720 */ 721 int 722 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 723 enum mlx5_feature_name feature, 724 uint32_t id, 725 struct rte_flow_error *error) 726 { 727 struct mlx5_priv *priv = dev->data->dev_private; 728 struct mlx5_dev_config *config = &priv->config; 729 enum modify_reg start_reg; 730 bool skip_mtr_reg = false; 731 732 switch (feature) { 733 case MLX5_HAIRPIN_RX: 734 return REG_B; 735 case MLX5_HAIRPIN_TX: 736 return REG_A; 737 case MLX5_METADATA_RX: 738 switch (config->dv_xmeta_en) { 739 case MLX5_XMETA_MODE_LEGACY: 740 return REG_B; 741 case MLX5_XMETA_MODE_META16: 742 return REG_C_0; 743 case MLX5_XMETA_MODE_META32: 744 return REG_C_1; 745 } 746 break; 747 case MLX5_METADATA_TX: 748 return REG_A; 749 case MLX5_METADATA_FDB: 750 switch (config->dv_xmeta_en) { 751 case MLX5_XMETA_MODE_LEGACY: 752 return REG_NON; 753 case MLX5_XMETA_MODE_META16: 754 return REG_C_0; 755 case MLX5_XMETA_MODE_META32: 756 return REG_C_1; 757 } 758 break; 759 case MLX5_FLOW_MARK: 760 switch (config->dv_xmeta_en) { 761 case MLX5_XMETA_MODE_LEGACY: 762 return REG_NON; 763 case MLX5_XMETA_MODE_META16: 764 return REG_C_1; 765 case MLX5_XMETA_MODE_META32: 766 return REG_C_0; 767 } 768 break; 769 case MLX5_MTR_SFX: 770 /* 771 * If meter color and flow match share one register, flow match 772 * should use the meter color register for match. 773 */ 774 if (priv->mtr_reg_share) 775 return priv->mtr_color_reg; 776 else 777 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 778 REG_C_3; 779 case MLX5_MTR_COLOR: 780 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 781 return priv->mtr_color_reg; 782 case MLX5_COPY_MARK: 783 /* 784 * Metadata COPY_MARK register using is in meter suffix sub 785 * flow while with meter. It's safe to share the same register. 786 */ 787 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 788 case MLX5_APP_TAG: 789 /* 790 * If meter is enable, it will engage the register for color 791 * match and flow match. If meter color match is not using the 792 * REG_C_2, need to skip the REG_C_x be used by meter color 793 * match. 794 * If meter is disable, free to use all available registers. 795 */ 796 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 797 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 798 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 799 if (id > (REG_C_7 - start_reg)) 800 return rte_flow_error_set(error, EINVAL, 801 RTE_FLOW_ERROR_TYPE_ITEM, 802 NULL, "invalid tag id"); 803 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 804 return rte_flow_error_set(error, ENOTSUP, 805 RTE_FLOW_ERROR_TYPE_ITEM, 806 NULL, "unsupported tag id"); 807 /* 808 * This case means meter is using the REG_C_x great than 2. 809 * Take care not to conflict with meter color REG_C_x. 810 * If the available index REG_C_y >= REG_C_x, skip the 811 * color register. 812 */ 813 if (skip_mtr_reg && config->flow_mreg_c 814 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 815 if (id >= (REG_C_7 - start_reg)) 816 return rte_flow_error_set(error, EINVAL, 817 RTE_FLOW_ERROR_TYPE_ITEM, 818 NULL, "invalid tag id"); 819 if (config->flow_mreg_c 820 [id + 1 + start_reg - REG_C_0] != REG_NON) 821 return config->flow_mreg_c 822 [id + 1 + start_reg - REG_C_0]; 823 return rte_flow_error_set(error, ENOTSUP, 824 RTE_FLOW_ERROR_TYPE_ITEM, 825 NULL, "unsupported tag id"); 826 } 827 return config->flow_mreg_c[id + start_reg - REG_C_0]; 828 } 829 MLX5_ASSERT(false); 830 return rte_flow_error_set(error, EINVAL, 831 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 832 NULL, "invalid feature name"); 833 } 834 835 /** 836 * Check extensive flow metadata register support. 837 * 838 * @param dev 839 * Pointer to rte_eth_dev structure. 840 * 841 * @return 842 * True if device supports extensive flow metadata register, otherwise false. 843 */ 844 bool 845 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 846 { 847 struct mlx5_priv *priv = dev->data->dev_private; 848 struct mlx5_dev_config *config = &priv->config; 849 850 /* 851 * Having available reg_c can be regarded inclusively as supporting 852 * extensive flow metadata register, which could mean, 853 * - metadata register copy action by modify header. 854 * - 16 modify header actions is supported. 855 * - reg_c's are preserved across different domain (FDB and NIC) on 856 * packet loopback by flow lookup miss. 857 */ 858 return config->flow_mreg_c[2] != REG_NON; 859 } 860 861 /** 862 * Verify the @p item specifications (spec, last, mask) are compatible with the 863 * NIC capabilities. 864 * 865 * @param[in] item 866 * Item specification. 867 * @param[in] mask 868 * @p item->mask or flow default bit-masks. 869 * @param[in] nic_mask 870 * Bit-masks covering supported fields by the NIC to compare with user mask. 871 * @param[in] size 872 * Bit-masks size in bytes. 873 * @param[in] range_accepted 874 * True if range of values is accepted for specific fields, false otherwise. 875 * @param[out] error 876 * Pointer to error structure. 877 * 878 * @return 879 * 0 on success, a negative errno value otherwise and rte_errno is set. 880 */ 881 int 882 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 883 const uint8_t *mask, 884 const uint8_t *nic_mask, 885 unsigned int size, 886 bool range_accepted, 887 struct rte_flow_error *error) 888 { 889 unsigned int i; 890 891 MLX5_ASSERT(nic_mask); 892 for (i = 0; i < size; ++i) 893 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 894 return rte_flow_error_set(error, ENOTSUP, 895 RTE_FLOW_ERROR_TYPE_ITEM, 896 item, 897 "mask enables non supported" 898 " bits"); 899 if (!item->spec && (item->mask || item->last)) 900 return rte_flow_error_set(error, EINVAL, 901 RTE_FLOW_ERROR_TYPE_ITEM, item, 902 "mask/last without a spec is not" 903 " supported"); 904 if (item->spec && item->last && !range_accepted) { 905 uint8_t spec[size]; 906 uint8_t last[size]; 907 unsigned int i; 908 int ret; 909 910 for (i = 0; i < size; ++i) { 911 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 912 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 913 } 914 ret = memcmp(spec, last, size); 915 if (ret != 0) 916 return rte_flow_error_set(error, EINVAL, 917 RTE_FLOW_ERROR_TYPE_ITEM, 918 item, 919 "range is not valid"); 920 } 921 return 0; 922 } 923 924 /** 925 * Adjust the hash fields according to the @p flow information. 926 * 927 * @param[in] dev_flow. 928 * Pointer to the mlx5_flow. 929 * @param[in] tunnel 930 * 1 when the hash field is for a tunnel item. 931 * @param[in] layer_types 932 * ETH_RSS_* types. 933 * @param[in] hash_fields 934 * Item hash fields. 935 * 936 * @return 937 * The hash fields that should be used. 938 */ 939 uint64_t 940 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 941 int tunnel __rte_unused, uint64_t layer_types, 942 uint64_t hash_fields) 943 { 944 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 945 int rss_request_inner = rss_desc->level >= 2; 946 947 /* Check RSS hash level for tunnel. */ 948 if (tunnel && rss_request_inner) 949 hash_fields |= IBV_RX_HASH_INNER; 950 else if (tunnel || rss_request_inner) 951 return 0; 952 #endif 953 /* Check if requested layer matches RSS hash fields. */ 954 if (!(rss_desc->types & layer_types)) 955 return 0; 956 return hash_fields; 957 } 958 959 /** 960 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 961 * if several tunnel rules are used on this queue, the tunnel ptype will be 962 * cleared. 963 * 964 * @param rxq_ctrl 965 * Rx queue to update. 966 */ 967 static void 968 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 969 { 970 unsigned int i; 971 uint32_t tunnel_ptype = 0; 972 973 /* Look up for the ptype to use. */ 974 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 975 if (!rxq_ctrl->flow_tunnels_n[i]) 976 continue; 977 if (!tunnel_ptype) { 978 tunnel_ptype = tunnels_info[i].ptype; 979 } else { 980 tunnel_ptype = 0; 981 break; 982 } 983 } 984 rxq_ctrl->rxq.tunnel = tunnel_ptype; 985 } 986 987 /** 988 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 989 * flow. 990 * 991 * @param[in] dev 992 * Pointer to the Ethernet device structure. 993 * @param[in] dev_handle 994 * Pointer to device flow handle structure. 995 */ 996 static void 997 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 998 struct mlx5_flow_handle *dev_handle) 999 { 1000 struct mlx5_priv *priv = dev->data->dev_private; 1001 const int mark = dev_handle->mark; 1002 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1003 struct mlx5_hrxq *hrxq; 1004 unsigned int i; 1005 1006 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1007 return; 1008 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1009 dev_handle->rix_hrxq); 1010 if (!hrxq) 1011 return; 1012 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1013 int idx = hrxq->ind_table->queues[i]; 1014 struct mlx5_rxq_ctrl *rxq_ctrl = 1015 container_of((*priv->rxqs)[idx], 1016 struct mlx5_rxq_ctrl, rxq); 1017 1018 /* 1019 * To support metadata register copy on Tx loopback, 1020 * this must be always enabled (metadata may arive 1021 * from other port - not from local flows only. 1022 */ 1023 if (priv->config.dv_flow_en && 1024 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1025 mlx5_flow_ext_mreg_supported(dev)) { 1026 rxq_ctrl->rxq.mark = 1; 1027 rxq_ctrl->flow_mark_n = 1; 1028 } else if (mark) { 1029 rxq_ctrl->rxq.mark = 1; 1030 rxq_ctrl->flow_mark_n++; 1031 } 1032 if (tunnel) { 1033 unsigned int j; 1034 1035 /* Increase the counter matching the flow. */ 1036 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1037 if ((tunnels_info[j].tunnel & 1038 dev_handle->layers) == 1039 tunnels_info[j].tunnel) { 1040 rxq_ctrl->flow_tunnels_n[j]++; 1041 break; 1042 } 1043 } 1044 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1045 } 1046 } 1047 } 1048 1049 /** 1050 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1051 * 1052 * @param[in] dev 1053 * Pointer to the Ethernet device structure. 1054 * @param[in] flow 1055 * Pointer to flow structure. 1056 */ 1057 static void 1058 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1059 { 1060 struct mlx5_priv *priv = dev->data->dev_private; 1061 uint32_t handle_idx; 1062 struct mlx5_flow_handle *dev_handle; 1063 1064 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1065 handle_idx, dev_handle, next) 1066 flow_drv_rxq_flags_set(dev, dev_handle); 1067 } 1068 1069 /** 1070 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1071 * device flow if no other flow uses it with the same kind of request. 1072 * 1073 * @param dev 1074 * Pointer to Ethernet device. 1075 * @param[in] dev_handle 1076 * Pointer to the device flow handle structure. 1077 */ 1078 static void 1079 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1080 struct mlx5_flow_handle *dev_handle) 1081 { 1082 struct mlx5_priv *priv = dev->data->dev_private; 1083 const int mark = dev_handle->mark; 1084 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1085 struct mlx5_hrxq *hrxq; 1086 unsigned int i; 1087 1088 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1089 return; 1090 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1091 dev_handle->rix_hrxq); 1092 if (!hrxq) 1093 return; 1094 MLX5_ASSERT(dev->data->dev_started); 1095 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1096 int idx = hrxq->ind_table->queues[i]; 1097 struct mlx5_rxq_ctrl *rxq_ctrl = 1098 container_of((*priv->rxqs)[idx], 1099 struct mlx5_rxq_ctrl, rxq); 1100 1101 if (priv->config.dv_flow_en && 1102 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1103 mlx5_flow_ext_mreg_supported(dev)) { 1104 rxq_ctrl->rxq.mark = 1; 1105 rxq_ctrl->flow_mark_n = 1; 1106 } else if (mark) { 1107 rxq_ctrl->flow_mark_n--; 1108 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1109 } 1110 if (tunnel) { 1111 unsigned int j; 1112 1113 /* Decrease the counter matching the flow. */ 1114 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1115 if ((tunnels_info[j].tunnel & 1116 dev_handle->layers) == 1117 tunnels_info[j].tunnel) { 1118 rxq_ctrl->flow_tunnels_n[j]--; 1119 break; 1120 } 1121 } 1122 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1123 } 1124 } 1125 } 1126 1127 /** 1128 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1129 * @p flow if no other flow uses it with the same kind of request. 1130 * 1131 * @param dev 1132 * Pointer to Ethernet device. 1133 * @param[in] flow 1134 * Pointer to the flow. 1135 */ 1136 static void 1137 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1138 { 1139 struct mlx5_priv *priv = dev->data->dev_private; 1140 uint32_t handle_idx; 1141 struct mlx5_flow_handle *dev_handle; 1142 1143 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1144 handle_idx, dev_handle, next) 1145 flow_drv_rxq_flags_trim(dev, dev_handle); 1146 } 1147 1148 /** 1149 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1150 * 1151 * @param dev 1152 * Pointer to Ethernet device. 1153 */ 1154 static void 1155 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1156 { 1157 struct mlx5_priv *priv = dev->data->dev_private; 1158 unsigned int i; 1159 1160 for (i = 0; i != priv->rxqs_n; ++i) { 1161 struct mlx5_rxq_ctrl *rxq_ctrl; 1162 unsigned int j; 1163 1164 if (!(*priv->rxqs)[i]) 1165 continue; 1166 rxq_ctrl = container_of((*priv->rxqs)[i], 1167 struct mlx5_rxq_ctrl, rxq); 1168 rxq_ctrl->flow_mark_n = 0; 1169 rxq_ctrl->rxq.mark = 0; 1170 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1171 rxq_ctrl->flow_tunnels_n[j] = 0; 1172 rxq_ctrl->rxq.tunnel = 0; 1173 } 1174 } 1175 1176 /** 1177 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1178 * 1179 * @param[in] dev 1180 * Pointer to the Ethernet device structure. 1181 */ 1182 void 1183 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1184 { 1185 struct mlx5_priv *priv = dev->data->dev_private; 1186 struct mlx5_rxq_data *data; 1187 unsigned int i; 1188 1189 for (i = 0; i != priv->rxqs_n; ++i) { 1190 if (!(*priv->rxqs)[i]) 1191 continue; 1192 data = (*priv->rxqs)[i]; 1193 if (!rte_flow_dynf_metadata_avail()) { 1194 data->dynf_meta = 0; 1195 data->flow_meta_mask = 0; 1196 data->flow_meta_offset = -1; 1197 } else { 1198 data->dynf_meta = 1; 1199 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1200 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1201 } 1202 } 1203 } 1204 1205 /* 1206 * return a pointer to the desired action in the list of actions. 1207 * 1208 * @param[in] actions 1209 * The list of actions to search the action in. 1210 * @param[in] action 1211 * The action to find. 1212 * 1213 * @return 1214 * Pointer to the action in the list, if found. NULL otherwise. 1215 */ 1216 const struct rte_flow_action * 1217 mlx5_flow_find_action(const struct rte_flow_action *actions, 1218 enum rte_flow_action_type action) 1219 { 1220 if (actions == NULL) 1221 return NULL; 1222 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1223 if (actions->type == action) 1224 return actions; 1225 return NULL; 1226 } 1227 1228 /* 1229 * Validate the flag action. 1230 * 1231 * @param[in] action_flags 1232 * Bit-fields that holds the actions detected until now. 1233 * @param[in] attr 1234 * Attributes of flow that includes this action. 1235 * @param[out] error 1236 * Pointer to error structure. 1237 * 1238 * @return 1239 * 0 on success, a negative errno value otherwise and rte_errno is set. 1240 */ 1241 int 1242 mlx5_flow_validate_action_flag(uint64_t action_flags, 1243 const struct rte_flow_attr *attr, 1244 struct rte_flow_error *error) 1245 { 1246 if (action_flags & MLX5_FLOW_ACTION_MARK) 1247 return rte_flow_error_set(error, EINVAL, 1248 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1249 "can't mark and flag in same flow"); 1250 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1251 return rte_flow_error_set(error, EINVAL, 1252 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1253 "can't have 2 flag" 1254 " actions in same flow"); 1255 if (attr->egress) 1256 return rte_flow_error_set(error, ENOTSUP, 1257 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1258 "flag action not supported for " 1259 "egress"); 1260 return 0; 1261 } 1262 1263 /* 1264 * Validate the mark action. 1265 * 1266 * @param[in] action 1267 * Pointer to the queue action. 1268 * @param[in] action_flags 1269 * Bit-fields that holds the actions detected until now. 1270 * @param[in] attr 1271 * Attributes of flow that includes this action. 1272 * @param[out] error 1273 * Pointer to error structure. 1274 * 1275 * @return 1276 * 0 on success, a negative errno value otherwise and rte_errno is set. 1277 */ 1278 int 1279 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1280 uint64_t action_flags, 1281 const struct rte_flow_attr *attr, 1282 struct rte_flow_error *error) 1283 { 1284 const struct rte_flow_action_mark *mark = action->conf; 1285 1286 if (!mark) 1287 return rte_flow_error_set(error, EINVAL, 1288 RTE_FLOW_ERROR_TYPE_ACTION, 1289 action, 1290 "configuration cannot be null"); 1291 if (mark->id >= MLX5_FLOW_MARK_MAX) 1292 return rte_flow_error_set(error, EINVAL, 1293 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1294 &mark->id, 1295 "mark id must in 0 <= id < " 1296 RTE_STR(MLX5_FLOW_MARK_MAX)); 1297 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1298 return rte_flow_error_set(error, EINVAL, 1299 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1300 "can't flag and mark in same flow"); 1301 if (action_flags & MLX5_FLOW_ACTION_MARK) 1302 return rte_flow_error_set(error, EINVAL, 1303 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1304 "can't have 2 mark actions in same" 1305 " flow"); 1306 if (attr->egress) 1307 return rte_flow_error_set(error, ENOTSUP, 1308 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1309 "mark action not supported for " 1310 "egress"); 1311 return 0; 1312 } 1313 1314 /* 1315 * Validate the drop action. 1316 * 1317 * @param[in] action_flags 1318 * Bit-fields that holds the actions detected until now. 1319 * @param[in] attr 1320 * Attributes of flow that includes this action. 1321 * @param[out] error 1322 * Pointer to error structure. 1323 * 1324 * @return 1325 * 0 on success, a negative errno value otherwise and rte_errno is set. 1326 */ 1327 int 1328 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1329 const struct rte_flow_attr *attr, 1330 struct rte_flow_error *error) 1331 { 1332 if (attr->egress) 1333 return rte_flow_error_set(error, ENOTSUP, 1334 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1335 "drop action not supported for " 1336 "egress"); 1337 return 0; 1338 } 1339 1340 /* 1341 * Validate the queue action. 1342 * 1343 * @param[in] action 1344 * Pointer to the queue action. 1345 * @param[in] action_flags 1346 * Bit-fields that holds the actions detected until now. 1347 * @param[in] dev 1348 * Pointer to the Ethernet device structure. 1349 * @param[in] attr 1350 * Attributes of flow that includes this action. 1351 * @param[out] error 1352 * Pointer to error structure. 1353 * 1354 * @return 1355 * 0 on success, a negative errno value otherwise and rte_errno is set. 1356 */ 1357 int 1358 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1359 uint64_t action_flags, 1360 struct rte_eth_dev *dev, 1361 const struct rte_flow_attr *attr, 1362 struct rte_flow_error *error) 1363 { 1364 struct mlx5_priv *priv = dev->data->dev_private; 1365 const struct rte_flow_action_queue *queue = action->conf; 1366 1367 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1368 return rte_flow_error_set(error, EINVAL, 1369 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1370 "can't have 2 fate actions in" 1371 " same flow"); 1372 if (!priv->rxqs_n) 1373 return rte_flow_error_set(error, EINVAL, 1374 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1375 NULL, "No Rx queues configured"); 1376 if (queue->index >= priv->rxqs_n) 1377 return rte_flow_error_set(error, EINVAL, 1378 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1379 &queue->index, 1380 "queue index out of range"); 1381 if (!(*priv->rxqs)[queue->index]) 1382 return rte_flow_error_set(error, EINVAL, 1383 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1384 &queue->index, 1385 "queue is not configured"); 1386 if (attr->egress) 1387 return rte_flow_error_set(error, ENOTSUP, 1388 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1389 "queue action not supported for " 1390 "egress"); 1391 return 0; 1392 } 1393 1394 /* 1395 * Validate the rss action. 1396 * 1397 * @param[in] dev 1398 * Pointer to the Ethernet device structure. 1399 * @param[in] action 1400 * Pointer to the queue action. 1401 * @param[out] error 1402 * Pointer to error structure. 1403 * 1404 * @return 1405 * 0 on success, a negative errno value otherwise and rte_errno is set. 1406 */ 1407 int 1408 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1409 const struct rte_flow_action *action, 1410 struct rte_flow_error *error) 1411 { 1412 struct mlx5_priv *priv = dev->data->dev_private; 1413 const struct rte_flow_action_rss *rss = action->conf; 1414 unsigned int i; 1415 1416 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1417 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1418 return rte_flow_error_set(error, ENOTSUP, 1419 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1420 &rss->func, 1421 "RSS hash function not supported"); 1422 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1423 if (rss->level > 2) 1424 #else 1425 if (rss->level > 1) 1426 #endif 1427 return rte_flow_error_set(error, ENOTSUP, 1428 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1429 &rss->level, 1430 "tunnel RSS is not supported"); 1431 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1432 if (rss->key_len == 0 && rss->key != NULL) 1433 return rte_flow_error_set(error, ENOTSUP, 1434 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1435 &rss->key_len, 1436 "RSS hash key length 0"); 1437 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1438 return rte_flow_error_set(error, ENOTSUP, 1439 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1440 &rss->key_len, 1441 "RSS hash key too small"); 1442 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1443 return rte_flow_error_set(error, ENOTSUP, 1444 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1445 &rss->key_len, 1446 "RSS hash key too large"); 1447 if (rss->queue_num > priv->config.ind_table_max_size) 1448 return rte_flow_error_set(error, ENOTSUP, 1449 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1450 &rss->queue_num, 1451 "number of queues too large"); 1452 if (rss->types & MLX5_RSS_HF_MASK) 1453 return rte_flow_error_set(error, ENOTSUP, 1454 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1455 &rss->types, 1456 "some RSS protocols are not" 1457 " supported"); 1458 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1459 !(rss->types & ETH_RSS_IP)) 1460 return rte_flow_error_set(error, EINVAL, 1461 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1462 "L3 partial RSS requested but L3 RSS" 1463 " type not specified"); 1464 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1465 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1466 return rte_flow_error_set(error, EINVAL, 1467 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1468 "L4 partial RSS requested but L4 RSS" 1469 " type not specified"); 1470 if (!priv->rxqs_n) 1471 return rte_flow_error_set(error, EINVAL, 1472 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1473 NULL, "No Rx queues configured"); 1474 if (!rss->queue_num) 1475 return rte_flow_error_set(error, EINVAL, 1476 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1477 NULL, "No queues configured"); 1478 for (i = 0; i != rss->queue_num; ++i) { 1479 if (rss->queue[i] >= priv->rxqs_n) 1480 return rte_flow_error_set 1481 (error, EINVAL, 1482 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1483 &rss->queue[i], "queue index out of range"); 1484 if (!(*priv->rxqs)[rss->queue[i]]) 1485 return rte_flow_error_set 1486 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1487 &rss->queue[i], "queue is not configured"); 1488 } 1489 return 0; 1490 } 1491 1492 /* 1493 * Validate the rss action. 1494 * 1495 * @param[in] action 1496 * Pointer to the queue action. 1497 * @param[in] action_flags 1498 * Bit-fields that holds the actions detected until now. 1499 * @param[in] dev 1500 * Pointer to the Ethernet device structure. 1501 * @param[in] attr 1502 * Attributes of flow that includes this action. 1503 * @param[in] item_flags 1504 * Items that were detected. 1505 * @param[out] error 1506 * Pointer to error structure. 1507 * 1508 * @return 1509 * 0 on success, a negative errno value otherwise and rte_errno is set. 1510 */ 1511 int 1512 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1513 uint64_t action_flags, 1514 struct rte_eth_dev *dev, 1515 const struct rte_flow_attr *attr, 1516 uint64_t item_flags, 1517 struct rte_flow_error *error) 1518 { 1519 const struct rte_flow_action_rss *rss = action->conf; 1520 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1521 int ret; 1522 1523 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1524 return rte_flow_error_set(error, EINVAL, 1525 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1526 "can't have 2 fate actions" 1527 " in same flow"); 1528 ret = mlx5_validate_action_rss(dev, action, error); 1529 if (ret) 1530 return ret; 1531 if (attr->egress) 1532 return rte_flow_error_set(error, ENOTSUP, 1533 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1534 "rss action not supported for " 1535 "egress"); 1536 if (rss->level > 1 && !tunnel) 1537 return rte_flow_error_set(error, EINVAL, 1538 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1539 "inner RSS is not supported for " 1540 "non-tunnel flows"); 1541 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1542 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1543 return rte_flow_error_set(error, EINVAL, 1544 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1545 "RSS on eCPRI is not supported now"); 1546 } 1547 return 0; 1548 } 1549 1550 /* 1551 * Validate the default miss action. 1552 * 1553 * @param[in] action_flags 1554 * Bit-fields that holds the actions detected until now. 1555 * @param[out] error 1556 * Pointer to error structure. 1557 * 1558 * @return 1559 * 0 on success, a negative errno value otherwise and rte_errno is set. 1560 */ 1561 int 1562 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1563 const struct rte_flow_attr *attr, 1564 struct rte_flow_error *error) 1565 { 1566 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1567 return rte_flow_error_set(error, EINVAL, 1568 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1569 "can't have 2 fate actions in" 1570 " same flow"); 1571 if (attr->egress) 1572 return rte_flow_error_set(error, ENOTSUP, 1573 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1574 "default miss action not supported " 1575 "for egress"); 1576 if (attr->group) 1577 return rte_flow_error_set(error, ENOTSUP, 1578 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1579 "only group 0 is supported"); 1580 if (attr->transfer) 1581 return rte_flow_error_set(error, ENOTSUP, 1582 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1583 NULL, "transfer is not supported"); 1584 return 0; 1585 } 1586 1587 /* 1588 * Validate the count action. 1589 * 1590 * @param[in] dev 1591 * Pointer to the Ethernet device structure. 1592 * @param[in] attr 1593 * Attributes of flow that includes this action. 1594 * @param[out] error 1595 * Pointer to error structure. 1596 * 1597 * @return 1598 * 0 on success, a negative errno value otherwise and rte_errno is set. 1599 */ 1600 int 1601 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1602 const struct rte_flow_attr *attr, 1603 struct rte_flow_error *error) 1604 { 1605 if (attr->egress) 1606 return rte_flow_error_set(error, ENOTSUP, 1607 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1608 "count action not supported for " 1609 "egress"); 1610 return 0; 1611 } 1612 1613 /** 1614 * Verify the @p attributes will be correctly understood by the NIC and store 1615 * them in the @p flow if everything is correct. 1616 * 1617 * @param[in] dev 1618 * Pointer to the Ethernet device structure. 1619 * @param[in] attributes 1620 * Pointer to flow attributes 1621 * @param[out] error 1622 * Pointer to error structure. 1623 * 1624 * @return 1625 * 0 on success, a negative errno value otherwise and rte_errno is set. 1626 */ 1627 int 1628 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1629 const struct rte_flow_attr *attributes, 1630 struct rte_flow_error *error) 1631 { 1632 struct mlx5_priv *priv = dev->data->dev_private; 1633 uint32_t priority_max = priv->config.flow_prio - 1; 1634 1635 if (attributes->group) 1636 return rte_flow_error_set(error, ENOTSUP, 1637 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1638 NULL, "groups is not supported"); 1639 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1640 attributes->priority >= priority_max) 1641 return rte_flow_error_set(error, ENOTSUP, 1642 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1643 NULL, "priority out of range"); 1644 if (attributes->egress) 1645 return rte_flow_error_set(error, ENOTSUP, 1646 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1647 "egress is not supported"); 1648 if (attributes->transfer && !priv->config.dv_esw_en) 1649 return rte_flow_error_set(error, ENOTSUP, 1650 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1651 NULL, "transfer is not supported"); 1652 if (!attributes->ingress) 1653 return rte_flow_error_set(error, EINVAL, 1654 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1655 NULL, 1656 "ingress attribute is mandatory"); 1657 return 0; 1658 } 1659 1660 /** 1661 * Validate ICMP6 item. 1662 * 1663 * @param[in] item 1664 * Item specification. 1665 * @param[in] item_flags 1666 * Bit-fields that holds the items detected until now. 1667 * @param[in] ext_vlan_sup 1668 * Whether extended VLAN features are supported or not. 1669 * @param[out] error 1670 * Pointer to error structure. 1671 * 1672 * @return 1673 * 0 on success, a negative errno value otherwise and rte_errno is set. 1674 */ 1675 int 1676 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1677 uint64_t item_flags, 1678 uint8_t target_protocol, 1679 struct rte_flow_error *error) 1680 { 1681 const struct rte_flow_item_icmp6 *mask = item->mask; 1682 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1683 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1684 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1685 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1686 MLX5_FLOW_LAYER_OUTER_L4; 1687 int ret; 1688 1689 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1690 return rte_flow_error_set(error, EINVAL, 1691 RTE_FLOW_ERROR_TYPE_ITEM, item, 1692 "protocol filtering not compatible" 1693 " with ICMP6 layer"); 1694 if (!(item_flags & l3m)) 1695 return rte_flow_error_set(error, EINVAL, 1696 RTE_FLOW_ERROR_TYPE_ITEM, item, 1697 "IPv6 is mandatory to filter on" 1698 " ICMP6"); 1699 if (item_flags & l4m) 1700 return rte_flow_error_set(error, EINVAL, 1701 RTE_FLOW_ERROR_TYPE_ITEM, item, 1702 "multiple L4 layers not supported"); 1703 if (!mask) 1704 mask = &rte_flow_item_icmp6_mask; 1705 ret = mlx5_flow_item_acceptable 1706 (item, (const uint8_t *)mask, 1707 (const uint8_t *)&rte_flow_item_icmp6_mask, 1708 sizeof(struct rte_flow_item_icmp6), 1709 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1710 if (ret < 0) 1711 return ret; 1712 return 0; 1713 } 1714 1715 /** 1716 * Validate ICMP item. 1717 * 1718 * @param[in] item 1719 * Item specification. 1720 * @param[in] item_flags 1721 * Bit-fields that holds the items detected until now. 1722 * @param[out] error 1723 * Pointer to error structure. 1724 * 1725 * @return 1726 * 0 on success, a negative errno value otherwise and rte_errno is set. 1727 */ 1728 int 1729 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1730 uint64_t item_flags, 1731 uint8_t target_protocol, 1732 struct rte_flow_error *error) 1733 { 1734 const struct rte_flow_item_icmp *mask = item->mask; 1735 const struct rte_flow_item_icmp nic_mask = { 1736 .hdr.icmp_type = 0xff, 1737 .hdr.icmp_code = 0xff, 1738 .hdr.icmp_ident = RTE_BE16(0xffff), 1739 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1740 }; 1741 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1742 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1743 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1744 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1745 MLX5_FLOW_LAYER_OUTER_L4; 1746 int ret; 1747 1748 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1749 return rte_flow_error_set(error, EINVAL, 1750 RTE_FLOW_ERROR_TYPE_ITEM, item, 1751 "protocol filtering not compatible" 1752 " with ICMP layer"); 1753 if (!(item_flags & l3m)) 1754 return rte_flow_error_set(error, EINVAL, 1755 RTE_FLOW_ERROR_TYPE_ITEM, item, 1756 "IPv4 is mandatory to filter" 1757 " on ICMP"); 1758 if (item_flags & l4m) 1759 return rte_flow_error_set(error, EINVAL, 1760 RTE_FLOW_ERROR_TYPE_ITEM, item, 1761 "multiple L4 layers not supported"); 1762 if (!mask) 1763 mask = &nic_mask; 1764 ret = mlx5_flow_item_acceptable 1765 (item, (const uint8_t *)mask, 1766 (const uint8_t *)&nic_mask, 1767 sizeof(struct rte_flow_item_icmp), 1768 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1769 if (ret < 0) 1770 return ret; 1771 return 0; 1772 } 1773 1774 /** 1775 * Validate Ethernet item. 1776 * 1777 * @param[in] item 1778 * Item specification. 1779 * @param[in] item_flags 1780 * Bit-fields that holds the items detected until now. 1781 * @param[out] error 1782 * Pointer to error structure. 1783 * 1784 * @return 1785 * 0 on success, a negative errno value otherwise and rte_errno is set. 1786 */ 1787 int 1788 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1789 uint64_t item_flags, bool ext_vlan_sup, 1790 struct rte_flow_error *error) 1791 { 1792 const struct rte_flow_item_eth *mask = item->mask; 1793 const struct rte_flow_item_eth nic_mask = { 1794 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1795 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1796 .type = RTE_BE16(0xffff), 1797 .has_vlan = ext_vlan_sup ? 1 : 0, 1798 }; 1799 int ret; 1800 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1801 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1802 MLX5_FLOW_LAYER_OUTER_L2; 1803 1804 if (item_flags & ethm) 1805 return rte_flow_error_set(error, ENOTSUP, 1806 RTE_FLOW_ERROR_TYPE_ITEM, item, 1807 "multiple L2 layers not supported"); 1808 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1809 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1810 return rte_flow_error_set(error, EINVAL, 1811 RTE_FLOW_ERROR_TYPE_ITEM, item, 1812 "L2 layer should not follow " 1813 "L3 layers"); 1814 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1815 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1816 return rte_flow_error_set(error, EINVAL, 1817 RTE_FLOW_ERROR_TYPE_ITEM, item, 1818 "L2 layer should not follow VLAN"); 1819 if (!mask) 1820 mask = &rte_flow_item_eth_mask; 1821 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1822 (const uint8_t *)&nic_mask, 1823 sizeof(struct rte_flow_item_eth), 1824 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1825 return ret; 1826 } 1827 1828 /** 1829 * Validate VLAN item. 1830 * 1831 * @param[in] item 1832 * Item specification. 1833 * @param[in] item_flags 1834 * Bit-fields that holds the items detected until now. 1835 * @param[in] dev 1836 * Ethernet device flow is being created on. 1837 * @param[out] error 1838 * Pointer to error structure. 1839 * 1840 * @return 1841 * 0 on success, a negative errno value otherwise and rte_errno is set. 1842 */ 1843 int 1844 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1845 uint64_t item_flags, 1846 struct rte_eth_dev *dev, 1847 struct rte_flow_error *error) 1848 { 1849 const struct rte_flow_item_vlan *spec = item->spec; 1850 const struct rte_flow_item_vlan *mask = item->mask; 1851 const struct rte_flow_item_vlan nic_mask = { 1852 .tci = RTE_BE16(UINT16_MAX), 1853 .inner_type = RTE_BE16(UINT16_MAX), 1854 }; 1855 uint16_t vlan_tag = 0; 1856 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1857 int ret; 1858 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1859 MLX5_FLOW_LAYER_INNER_L4) : 1860 (MLX5_FLOW_LAYER_OUTER_L3 | 1861 MLX5_FLOW_LAYER_OUTER_L4); 1862 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1863 MLX5_FLOW_LAYER_OUTER_VLAN; 1864 1865 if (item_flags & vlanm) 1866 return rte_flow_error_set(error, EINVAL, 1867 RTE_FLOW_ERROR_TYPE_ITEM, item, 1868 "multiple VLAN layers not supported"); 1869 else if ((item_flags & l34m) != 0) 1870 return rte_flow_error_set(error, EINVAL, 1871 RTE_FLOW_ERROR_TYPE_ITEM, item, 1872 "VLAN cannot follow L3/L4 layer"); 1873 if (!mask) 1874 mask = &rte_flow_item_vlan_mask; 1875 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1876 (const uint8_t *)&nic_mask, 1877 sizeof(struct rte_flow_item_vlan), 1878 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1879 if (ret) 1880 return ret; 1881 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1882 struct mlx5_priv *priv = dev->data->dev_private; 1883 1884 if (priv->vmwa_context) { 1885 /* 1886 * Non-NULL context means we have a virtual machine 1887 * and SR-IOV enabled, we have to create VLAN interface 1888 * to make hypervisor to setup E-Switch vport 1889 * context correctly. We avoid creating the multiple 1890 * VLAN interfaces, so we cannot support VLAN tag mask. 1891 */ 1892 return rte_flow_error_set(error, EINVAL, 1893 RTE_FLOW_ERROR_TYPE_ITEM, 1894 item, 1895 "VLAN tag mask is not" 1896 " supported in virtual" 1897 " environment"); 1898 } 1899 } 1900 if (spec) { 1901 vlan_tag = spec->tci; 1902 vlan_tag &= mask->tci; 1903 } 1904 /* 1905 * From verbs perspective an empty VLAN is equivalent 1906 * to a packet without VLAN layer. 1907 */ 1908 if (!vlan_tag) 1909 return rte_flow_error_set(error, EINVAL, 1910 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1911 item->spec, 1912 "VLAN cannot be empty"); 1913 return 0; 1914 } 1915 1916 /** 1917 * Validate IPV4 item. 1918 * 1919 * @param[in] item 1920 * Item specification. 1921 * @param[in] item_flags 1922 * Bit-fields that holds the items detected until now. 1923 * @param[in] last_item 1924 * Previous validated item in the pattern items. 1925 * @param[in] ether_type 1926 * Type in the ethernet layer header (including dot1q). 1927 * @param[in] acc_mask 1928 * Acceptable mask, if NULL default internal default mask 1929 * will be used to check whether item fields are supported. 1930 * @param[in] range_accepted 1931 * True if range of values is accepted for specific fields, false otherwise. 1932 * @param[out] error 1933 * Pointer to error structure. 1934 * 1935 * @return 1936 * 0 on success, a negative errno value otherwise and rte_errno is set. 1937 */ 1938 int 1939 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1940 uint64_t item_flags, 1941 uint64_t last_item, 1942 uint16_t ether_type, 1943 const struct rte_flow_item_ipv4 *acc_mask, 1944 bool range_accepted, 1945 struct rte_flow_error *error) 1946 { 1947 const struct rte_flow_item_ipv4 *mask = item->mask; 1948 const struct rte_flow_item_ipv4 *spec = item->spec; 1949 const struct rte_flow_item_ipv4 nic_mask = { 1950 .hdr = { 1951 .src_addr = RTE_BE32(0xffffffff), 1952 .dst_addr = RTE_BE32(0xffffffff), 1953 .type_of_service = 0xff, 1954 .next_proto_id = 0xff, 1955 }, 1956 }; 1957 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1958 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1959 MLX5_FLOW_LAYER_OUTER_L3; 1960 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1961 MLX5_FLOW_LAYER_OUTER_L4; 1962 int ret; 1963 uint8_t next_proto = 0xFF; 1964 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1965 MLX5_FLOW_LAYER_OUTER_VLAN | 1966 MLX5_FLOW_LAYER_INNER_VLAN); 1967 1968 if ((last_item & l2_vlan) && ether_type && 1969 ether_type != RTE_ETHER_TYPE_IPV4) 1970 return rte_flow_error_set(error, EINVAL, 1971 RTE_FLOW_ERROR_TYPE_ITEM, item, 1972 "IPv4 cannot follow L2/VLAN layer " 1973 "which ether type is not IPv4"); 1974 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1975 if (mask && spec) 1976 next_proto = mask->hdr.next_proto_id & 1977 spec->hdr.next_proto_id; 1978 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1979 return rte_flow_error_set(error, EINVAL, 1980 RTE_FLOW_ERROR_TYPE_ITEM, 1981 item, 1982 "multiple tunnel " 1983 "not supported"); 1984 } 1985 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1986 return rte_flow_error_set(error, EINVAL, 1987 RTE_FLOW_ERROR_TYPE_ITEM, item, 1988 "wrong tunnel type - IPv6 specified " 1989 "but IPv4 item provided"); 1990 if (item_flags & l3m) 1991 return rte_flow_error_set(error, ENOTSUP, 1992 RTE_FLOW_ERROR_TYPE_ITEM, item, 1993 "multiple L3 layers not supported"); 1994 else if (item_flags & l4m) 1995 return rte_flow_error_set(error, EINVAL, 1996 RTE_FLOW_ERROR_TYPE_ITEM, item, 1997 "L3 cannot follow an L4 layer."); 1998 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1999 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2000 return rte_flow_error_set(error, EINVAL, 2001 RTE_FLOW_ERROR_TYPE_ITEM, item, 2002 "L3 cannot follow an NVGRE layer."); 2003 if (!mask) 2004 mask = &rte_flow_item_ipv4_mask; 2005 else if (mask->hdr.next_proto_id != 0 && 2006 mask->hdr.next_proto_id != 0xff) 2007 return rte_flow_error_set(error, EINVAL, 2008 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2009 "partial mask is not supported" 2010 " for protocol"); 2011 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2012 acc_mask ? (const uint8_t *)acc_mask 2013 : (const uint8_t *)&nic_mask, 2014 sizeof(struct rte_flow_item_ipv4), 2015 range_accepted, error); 2016 if (ret < 0) 2017 return ret; 2018 return 0; 2019 } 2020 2021 /** 2022 * Validate IPV6 item. 2023 * 2024 * @param[in] item 2025 * Item specification. 2026 * @param[in] item_flags 2027 * Bit-fields that holds the items detected until now. 2028 * @param[in] last_item 2029 * Previous validated item in the pattern items. 2030 * @param[in] ether_type 2031 * Type in the ethernet layer header (including dot1q). 2032 * @param[in] acc_mask 2033 * Acceptable mask, if NULL default internal default mask 2034 * will be used to check whether item fields are supported. 2035 * @param[out] error 2036 * Pointer to error structure. 2037 * 2038 * @return 2039 * 0 on success, a negative errno value otherwise and rte_errno is set. 2040 */ 2041 int 2042 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2043 uint64_t item_flags, 2044 uint64_t last_item, 2045 uint16_t ether_type, 2046 const struct rte_flow_item_ipv6 *acc_mask, 2047 struct rte_flow_error *error) 2048 { 2049 const struct rte_flow_item_ipv6 *mask = item->mask; 2050 const struct rte_flow_item_ipv6 *spec = item->spec; 2051 const struct rte_flow_item_ipv6 nic_mask = { 2052 .hdr = { 2053 .src_addr = 2054 "\xff\xff\xff\xff\xff\xff\xff\xff" 2055 "\xff\xff\xff\xff\xff\xff\xff\xff", 2056 .dst_addr = 2057 "\xff\xff\xff\xff\xff\xff\xff\xff" 2058 "\xff\xff\xff\xff\xff\xff\xff\xff", 2059 .vtc_flow = RTE_BE32(0xffffffff), 2060 .proto = 0xff, 2061 }, 2062 }; 2063 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2064 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2065 MLX5_FLOW_LAYER_OUTER_L3; 2066 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2067 MLX5_FLOW_LAYER_OUTER_L4; 2068 int ret; 2069 uint8_t next_proto = 0xFF; 2070 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2071 MLX5_FLOW_LAYER_OUTER_VLAN | 2072 MLX5_FLOW_LAYER_INNER_VLAN); 2073 2074 if ((last_item & l2_vlan) && ether_type && 2075 ether_type != RTE_ETHER_TYPE_IPV6) 2076 return rte_flow_error_set(error, EINVAL, 2077 RTE_FLOW_ERROR_TYPE_ITEM, item, 2078 "IPv6 cannot follow L2/VLAN layer " 2079 "which ether type is not IPv6"); 2080 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2081 next_proto = spec->hdr.proto; 2082 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 2083 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2084 return rte_flow_error_set(error, EINVAL, 2085 RTE_FLOW_ERROR_TYPE_ITEM, 2086 item, 2087 "multiple tunnel " 2088 "not supported"); 2089 } 2090 if (next_proto == IPPROTO_HOPOPTS || 2091 next_proto == IPPROTO_ROUTING || 2092 next_proto == IPPROTO_FRAGMENT || 2093 next_proto == IPPROTO_ESP || 2094 next_proto == IPPROTO_AH || 2095 next_proto == IPPROTO_DSTOPTS) 2096 return rte_flow_error_set(error, EINVAL, 2097 RTE_FLOW_ERROR_TYPE_ITEM, item, 2098 "IPv6 proto (next header) should " 2099 "not be set as extension header"); 2100 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2101 return rte_flow_error_set(error, EINVAL, 2102 RTE_FLOW_ERROR_TYPE_ITEM, item, 2103 "wrong tunnel type - IPv4 specified " 2104 "but IPv6 item provided"); 2105 if (item_flags & l3m) 2106 return rte_flow_error_set(error, ENOTSUP, 2107 RTE_FLOW_ERROR_TYPE_ITEM, item, 2108 "multiple L3 layers not supported"); 2109 else if (item_flags & l4m) 2110 return rte_flow_error_set(error, EINVAL, 2111 RTE_FLOW_ERROR_TYPE_ITEM, item, 2112 "L3 cannot follow an L4 layer."); 2113 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2114 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2115 return rte_flow_error_set(error, EINVAL, 2116 RTE_FLOW_ERROR_TYPE_ITEM, item, 2117 "L3 cannot follow an NVGRE layer."); 2118 if (!mask) 2119 mask = &rte_flow_item_ipv6_mask; 2120 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2121 acc_mask ? (const uint8_t *)acc_mask 2122 : (const uint8_t *)&nic_mask, 2123 sizeof(struct rte_flow_item_ipv6), 2124 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2125 if (ret < 0) 2126 return ret; 2127 return 0; 2128 } 2129 2130 /** 2131 * Validate UDP item. 2132 * 2133 * @param[in] item 2134 * Item specification. 2135 * @param[in] item_flags 2136 * Bit-fields that holds the items detected until now. 2137 * @param[in] target_protocol 2138 * The next protocol in the previous item. 2139 * @param[in] flow_mask 2140 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2141 * @param[out] error 2142 * Pointer to error structure. 2143 * 2144 * @return 2145 * 0 on success, a negative errno value otherwise and rte_errno is set. 2146 */ 2147 int 2148 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2149 uint64_t item_flags, 2150 uint8_t target_protocol, 2151 struct rte_flow_error *error) 2152 { 2153 const struct rte_flow_item_udp *mask = item->mask; 2154 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2155 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2156 MLX5_FLOW_LAYER_OUTER_L3; 2157 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2158 MLX5_FLOW_LAYER_OUTER_L4; 2159 int ret; 2160 2161 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2162 return rte_flow_error_set(error, EINVAL, 2163 RTE_FLOW_ERROR_TYPE_ITEM, item, 2164 "protocol filtering not compatible" 2165 " with UDP layer"); 2166 if (!(item_flags & l3m)) 2167 return rte_flow_error_set(error, EINVAL, 2168 RTE_FLOW_ERROR_TYPE_ITEM, item, 2169 "L3 is mandatory to filter on L4"); 2170 if (item_flags & l4m) 2171 return rte_flow_error_set(error, EINVAL, 2172 RTE_FLOW_ERROR_TYPE_ITEM, item, 2173 "multiple L4 layers not supported"); 2174 if (!mask) 2175 mask = &rte_flow_item_udp_mask; 2176 ret = mlx5_flow_item_acceptable 2177 (item, (const uint8_t *)mask, 2178 (const uint8_t *)&rte_flow_item_udp_mask, 2179 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2180 error); 2181 if (ret < 0) 2182 return ret; 2183 return 0; 2184 } 2185 2186 /** 2187 * Validate TCP item. 2188 * 2189 * @param[in] item 2190 * Item specification. 2191 * @param[in] item_flags 2192 * Bit-fields that holds the items detected until now. 2193 * @param[in] target_protocol 2194 * The next protocol in the previous item. 2195 * @param[out] error 2196 * Pointer to error structure. 2197 * 2198 * @return 2199 * 0 on success, a negative errno value otherwise and rte_errno is set. 2200 */ 2201 int 2202 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2203 uint64_t item_flags, 2204 uint8_t target_protocol, 2205 const struct rte_flow_item_tcp *flow_mask, 2206 struct rte_flow_error *error) 2207 { 2208 const struct rte_flow_item_tcp *mask = item->mask; 2209 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2210 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2211 MLX5_FLOW_LAYER_OUTER_L3; 2212 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2213 MLX5_FLOW_LAYER_OUTER_L4; 2214 int ret; 2215 2216 MLX5_ASSERT(flow_mask); 2217 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2218 return rte_flow_error_set(error, EINVAL, 2219 RTE_FLOW_ERROR_TYPE_ITEM, item, 2220 "protocol filtering not compatible" 2221 " with TCP layer"); 2222 if (!(item_flags & l3m)) 2223 return rte_flow_error_set(error, EINVAL, 2224 RTE_FLOW_ERROR_TYPE_ITEM, item, 2225 "L3 is mandatory to filter on L4"); 2226 if (item_flags & l4m) 2227 return rte_flow_error_set(error, EINVAL, 2228 RTE_FLOW_ERROR_TYPE_ITEM, item, 2229 "multiple L4 layers not supported"); 2230 if (!mask) 2231 mask = &rte_flow_item_tcp_mask; 2232 ret = mlx5_flow_item_acceptable 2233 (item, (const uint8_t *)mask, 2234 (const uint8_t *)flow_mask, 2235 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2236 error); 2237 if (ret < 0) 2238 return ret; 2239 return 0; 2240 } 2241 2242 /** 2243 * Validate VXLAN item. 2244 * 2245 * @param[in] item 2246 * Item specification. 2247 * @param[in] item_flags 2248 * Bit-fields that holds the items detected until now. 2249 * @param[in] target_protocol 2250 * The next protocol in the previous item. 2251 * @param[out] error 2252 * Pointer to error structure. 2253 * 2254 * @return 2255 * 0 on success, a negative errno value otherwise and rte_errno is set. 2256 */ 2257 int 2258 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2259 uint64_t item_flags, 2260 struct rte_flow_error *error) 2261 { 2262 const struct rte_flow_item_vxlan *spec = item->spec; 2263 const struct rte_flow_item_vxlan *mask = item->mask; 2264 int ret; 2265 union vni { 2266 uint32_t vlan_id; 2267 uint8_t vni[4]; 2268 } id = { .vlan_id = 0, }; 2269 2270 2271 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2272 return rte_flow_error_set(error, ENOTSUP, 2273 RTE_FLOW_ERROR_TYPE_ITEM, item, 2274 "multiple tunnel layers not" 2275 " supported"); 2276 /* 2277 * Verify only UDPv4 is present as defined in 2278 * https://tools.ietf.org/html/rfc7348 2279 */ 2280 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2281 return rte_flow_error_set(error, EINVAL, 2282 RTE_FLOW_ERROR_TYPE_ITEM, item, 2283 "no outer UDP layer found"); 2284 if (!mask) 2285 mask = &rte_flow_item_vxlan_mask; 2286 ret = mlx5_flow_item_acceptable 2287 (item, (const uint8_t *)mask, 2288 (const uint8_t *)&rte_flow_item_vxlan_mask, 2289 sizeof(struct rte_flow_item_vxlan), 2290 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2291 if (ret < 0) 2292 return ret; 2293 if (spec) { 2294 memcpy(&id.vni[1], spec->vni, 3); 2295 memcpy(&id.vni[1], mask->vni, 3); 2296 } 2297 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2298 return rte_flow_error_set(error, ENOTSUP, 2299 RTE_FLOW_ERROR_TYPE_ITEM, item, 2300 "VXLAN tunnel must be fully defined"); 2301 return 0; 2302 } 2303 2304 /** 2305 * Validate VXLAN_GPE item. 2306 * 2307 * @param[in] item 2308 * Item specification. 2309 * @param[in] item_flags 2310 * Bit-fields that holds the items detected until now. 2311 * @param[in] priv 2312 * Pointer to the private data structure. 2313 * @param[in] target_protocol 2314 * The next protocol in the previous item. 2315 * @param[out] error 2316 * Pointer to error structure. 2317 * 2318 * @return 2319 * 0 on success, a negative errno value otherwise and rte_errno is set. 2320 */ 2321 int 2322 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2323 uint64_t item_flags, 2324 struct rte_eth_dev *dev, 2325 struct rte_flow_error *error) 2326 { 2327 struct mlx5_priv *priv = dev->data->dev_private; 2328 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2329 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2330 int ret; 2331 union vni { 2332 uint32_t vlan_id; 2333 uint8_t vni[4]; 2334 } id = { .vlan_id = 0, }; 2335 2336 if (!priv->config.l3_vxlan_en) 2337 return rte_flow_error_set(error, ENOTSUP, 2338 RTE_FLOW_ERROR_TYPE_ITEM, item, 2339 "L3 VXLAN is not enabled by device" 2340 " parameter and/or not configured in" 2341 " firmware"); 2342 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2343 return rte_flow_error_set(error, ENOTSUP, 2344 RTE_FLOW_ERROR_TYPE_ITEM, item, 2345 "multiple tunnel layers not" 2346 " supported"); 2347 /* 2348 * Verify only UDPv4 is present as defined in 2349 * https://tools.ietf.org/html/rfc7348 2350 */ 2351 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2352 return rte_flow_error_set(error, EINVAL, 2353 RTE_FLOW_ERROR_TYPE_ITEM, item, 2354 "no outer UDP layer found"); 2355 if (!mask) 2356 mask = &rte_flow_item_vxlan_gpe_mask; 2357 ret = mlx5_flow_item_acceptable 2358 (item, (const uint8_t *)mask, 2359 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2360 sizeof(struct rte_flow_item_vxlan_gpe), 2361 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2362 if (ret < 0) 2363 return ret; 2364 if (spec) { 2365 if (spec->protocol) 2366 return rte_flow_error_set(error, ENOTSUP, 2367 RTE_FLOW_ERROR_TYPE_ITEM, 2368 item, 2369 "VxLAN-GPE protocol" 2370 " not supported"); 2371 memcpy(&id.vni[1], spec->vni, 3); 2372 memcpy(&id.vni[1], mask->vni, 3); 2373 } 2374 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2375 return rte_flow_error_set(error, ENOTSUP, 2376 RTE_FLOW_ERROR_TYPE_ITEM, item, 2377 "VXLAN-GPE tunnel must be fully" 2378 " defined"); 2379 return 0; 2380 } 2381 /** 2382 * Validate GRE Key item. 2383 * 2384 * @param[in] item 2385 * Item specification. 2386 * @param[in] item_flags 2387 * Bit flags to mark detected items. 2388 * @param[in] gre_item 2389 * Pointer to gre_item 2390 * @param[out] error 2391 * Pointer to error structure. 2392 * 2393 * @return 2394 * 0 on success, a negative errno value otherwise and rte_errno is set. 2395 */ 2396 int 2397 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2398 uint64_t item_flags, 2399 const struct rte_flow_item *gre_item, 2400 struct rte_flow_error *error) 2401 { 2402 const rte_be32_t *mask = item->mask; 2403 int ret = 0; 2404 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2405 const struct rte_flow_item_gre *gre_spec; 2406 const struct rte_flow_item_gre *gre_mask; 2407 2408 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2409 return rte_flow_error_set(error, ENOTSUP, 2410 RTE_FLOW_ERROR_TYPE_ITEM, item, 2411 "Multiple GRE key not support"); 2412 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2413 return rte_flow_error_set(error, ENOTSUP, 2414 RTE_FLOW_ERROR_TYPE_ITEM, item, 2415 "No preceding GRE header"); 2416 if (item_flags & MLX5_FLOW_LAYER_INNER) 2417 return rte_flow_error_set(error, ENOTSUP, 2418 RTE_FLOW_ERROR_TYPE_ITEM, item, 2419 "GRE key following a wrong item"); 2420 gre_mask = gre_item->mask; 2421 if (!gre_mask) 2422 gre_mask = &rte_flow_item_gre_mask; 2423 gre_spec = gre_item->spec; 2424 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2425 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2426 return rte_flow_error_set(error, EINVAL, 2427 RTE_FLOW_ERROR_TYPE_ITEM, item, 2428 "Key bit must be on"); 2429 2430 if (!mask) 2431 mask = &gre_key_default_mask; 2432 ret = mlx5_flow_item_acceptable 2433 (item, (const uint8_t *)mask, 2434 (const uint8_t *)&gre_key_default_mask, 2435 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2436 return ret; 2437 } 2438 2439 /** 2440 * Validate GRE item. 2441 * 2442 * @param[in] item 2443 * Item specification. 2444 * @param[in] item_flags 2445 * Bit flags to mark detected items. 2446 * @param[in] target_protocol 2447 * The next protocol in the previous item. 2448 * @param[out] error 2449 * Pointer to error structure. 2450 * 2451 * @return 2452 * 0 on success, a negative errno value otherwise and rte_errno is set. 2453 */ 2454 int 2455 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2456 uint64_t item_flags, 2457 uint8_t target_protocol, 2458 struct rte_flow_error *error) 2459 { 2460 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2461 const struct rte_flow_item_gre *mask = item->mask; 2462 int ret; 2463 const struct rte_flow_item_gre nic_mask = { 2464 .c_rsvd0_ver = RTE_BE16(0xB000), 2465 .protocol = RTE_BE16(UINT16_MAX), 2466 }; 2467 2468 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2469 return rte_flow_error_set(error, EINVAL, 2470 RTE_FLOW_ERROR_TYPE_ITEM, item, 2471 "protocol filtering not compatible" 2472 " with this GRE layer"); 2473 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2474 return rte_flow_error_set(error, ENOTSUP, 2475 RTE_FLOW_ERROR_TYPE_ITEM, item, 2476 "multiple tunnel layers not" 2477 " supported"); 2478 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2479 return rte_flow_error_set(error, ENOTSUP, 2480 RTE_FLOW_ERROR_TYPE_ITEM, item, 2481 "L3 Layer is missing"); 2482 if (!mask) 2483 mask = &rte_flow_item_gre_mask; 2484 ret = mlx5_flow_item_acceptable 2485 (item, (const uint8_t *)mask, 2486 (const uint8_t *)&nic_mask, 2487 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2488 error); 2489 if (ret < 0) 2490 return ret; 2491 #ifndef HAVE_MLX5DV_DR 2492 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2493 if (spec && (spec->protocol & mask->protocol)) 2494 return rte_flow_error_set(error, ENOTSUP, 2495 RTE_FLOW_ERROR_TYPE_ITEM, item, 2496 "without MPLS support the" 2497 " specification cannot be used for" 2498 " filtering"); 2499 #endif 2500 #endif 2501 return 0; 2502 } 2503 2504 /** 2505 * Validate Geneve item. 2506 * 2507 * @param[in] item 2508 * Item specification. 2509 * @param[in] itemFlags 2510 * Bit-fields that holds the items detected until now. 2511 * @param[in] enPriv 2512 * Pointer to the private data structure. 2513 * @param[out] error 2514 * Pointer to error structure. 2515 * 2516 * @return 2517 * 0 on success, a negative errno value otherwise and rte_errno is set. 2518 */ 2519 2520 int 2521 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2522 uint64_t item_flags, 2523 struct rte_eth_dev *dev, 2524 struct rte_flow_error *error) 2525 { 2526 struct mlx5_priv *priv = dev->data->dev_private; 2527 const struct rte_flow_item_geneve *spec = item->spec; 2528 const struct rte_flow_item_geneve *mask = item->mask; 2529 int ret; 2530 uint16_t gbhdr; 2531 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2532 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2533 const struct rte_flow_item_geneve nic_mask = { 2534 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2535 .vni = "\xff\xff\xff", 2536 .protocol = RTE_BE16(UINT16_MAX), 2537 }; 2538 2539 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2540 return rte_flow_error_set(error, ENOTSUP, 2541 RTE_FLOW_ERROR_TYPE_ITEM, item, 2542 "L3 Geneve is not enabled by device" 2543 " parameter and/or not configured in" 2544 " firmware"); 2545 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2546 return rte_flow_error_set(error, ENOTSUP, 2547 RTE_FLOW_ERROR_TYPE_ITEM, item, 2548 "multiple tunnel layers not" 2549 " supported"); 2550 /* 2551 * Verify only UDPv4 is present as defined in 2552 * https://tools.ietf.org/html/rfc7348 2553 */ 2554 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2555 return rte_flow_error_set(error, EINVAL, 2556 RTE_FLOW_ERROR_TYPE_ITEM, item, 2557 "no outer UDP layer found"); 2558 if (!mask) 2559 mask = &rte_flow_item_geneve_mask; 2560 ret = mlx5_flow_item_acceptable 2561 (item, (const uint8_t *)mask, 2562 (const uint8_t *)&nic_mask, 2563 sizeof(struct rte_flow_item_geneve), 2564 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2565 if (ret) 2566 return ret; 2567 if (spec) { 2568 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2569 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2570 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2571 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2572 return rte_flow_error_set(error, ENOTSUP, 2573 RTE_FLOW_ERROR_TYPE_ITEM, 2574 item, 2575 "Geneve protocol unsupported" 2576 " fields are being used"); 2577 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2578 return rte_flow_error_set 2579 (error, ENOTSUP, 2580 RTE_FLOW_ERROR_TYPE_ITEM, 2581 item, 2582 "Unsupported Geneve options length"); 2583 } 2584 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2585 return rte_flow_error_set 2586 (error, ENOTSUP, 2587 RTE_FLOW_ERROR_TYPE_ITEM, item, 2588 "Geneve tunnel must be fully defined"); 2589 return 0; 2590 } 2591 2592 /** 2593 * Validate MPLS item. 2594 * 2595 * @param[in] dev 2596 * Pointer to the rte_eth_dev structure. 2597 * @param[in] item 2598 * Item specification. 2599 * @param[in] item_flags 2600 * Bit-fields that holds the items detected until now. 2601 * @param[in] prev_layer 2602 * The protocol layer indicated in previous item. 2603 * @param[out] error 2604 * Pointer to error structure. 2605 * 2606 * @return 2607 * 0 on success, a negative errno value otherwise and rte_errno is set. 2608 */ 2609 int 2610 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2611 const struct rte_flow_item *item __rte_unused, 2612 uint64_t item_flags __rte_unused, 2613 uint64_t prev_layer __rte_unused, 2614 struct rte_flow_error *error) 2615 { 2616 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2617 const struct rte_flow_item_mpls *mask = item->mask; 2618 struct mlx5_priv *priv = dev->data->dev_private; 2619 int ret; 2620 2621 if (!priv->config.mpls_en) 2622 return rte_flow_error_set(error, ENOTSUP, 2623 RTE_FLOW_ERROR_TYPE_ITEM, item, 2624 "MPLS not supported or" 2625 " disabled in firmware" 2626 " configuration."); 2627 /* MPLS over IP, UDP, GRE is allowed */ 2628 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2629 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2630 MLX5_FLOW_LAYER_GRE | 2631 MLX5_FLOW_LAYER_GRE_KEY))) 2632 return rte_flow_error_set(error, EINVAL, 2633 RTE_FLOW_ERROR_TYPE_ITEM, item, 2634 "protocol filtering not compatible" 2635 " with MPLS layer"); 2636 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2637 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2638 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2639 return rte_flow_error_set(error, ENOTSUP, 2640 RTE_FLOW_ERROR_TYPE_ITEM, item, 2641 "multiple tunnel layers not" 2642 " supported"); 2643 if (!mask) 2644 mask = &rte_flow_item_mpls_mask; 2645 ret = mlx5_flow_item_acceptable 2646 (item, (const uint8_t *)mask, 2647 (const uint8_t *)&rte_flow_item_mpls_mask, 2648 sizeof(struct rte_flow_item_mpls), 2649 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2650 if (ret < 0) 2651 return ret; 2652 return 0; 2653 #else 2654 return rte_flow_error_set(error, ENOTSUP, 2655 RTE_FLOW_ERROR_TYPE_ITEM, item, 2656 "MPLS is not supported by Verbs, please" 2657 " update."); 2658 #endif 2659 } 2660 2661 /** 2662 * Validate NVGRE item. 2663 * 2664 * @param[in] item 2665 * Item specification. 2666 * @param[in] item_flags 2667 * Bit flags to mark detected items. 2668 * @param[in] target_protocol 2669 * The next protocol in the previous item. 2670 * @param[out] error 2671 * Pointer to error structure. 2672 * 2673 * @return 2674 * 0 on success, a negative errno value otherwise and rte_errno is set. 2675 */ 2676 int 2677 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2678 uint64_t item_flags, 2679 uint8_t target_protocol, 2680 struct rte_flow_error *error) 2681 { 2682 const struct rte_flow_item_nvgre *mask = item->mask; 2683 int ret; 2684 2685 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2686 return rte_flow_error_set(error, EINVAL, 2687 RTE_FLOW_ERROR_TYPE_ITEM, item, 2688 "protocol filtering not compatible" 2689 " with this GRE layer"); 2690 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2691 return rte_flow_error_set(error, ENOTSUP, 2692 RTE_FLOW_ERROR_TYPE_ITEM, item, 2693 "multiple tunnel layers not" 2694 " supported"); 2695 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2696 return rte_flow_error_set(error, ENOTSUP, 2697 RTE_FLOW_ERROR_TYPE_ITEM, item, 2698 "L3 Layer is missing"); 2699 if (!mask) 2700 mask = &rte_flow_item_nvgre_mask; 2701 ret = mlx5_flow_item_acceptable 2702 (item, (const uint8_t *)mask, 2703 (const uint8_t *)&rte_flow_item_nvgre_mask, 2704 sizeof(struct rte_flow_item_nvgre), 2705 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2706 if (ret < 0) 2707 return ret; 2708 return 0; 2709 } 2710 2711 /** 2712 * Validate eCPRI item. 2713 * 2714 * @param[in] item 2715 * Item specification. 2716 * @param[in] item_flags 2717 * Bit-fields that holds the items detected until now. 2718 * @param[in] last_item 2719 * Previous validated item in the pattern items. 2720 * @param[in] ether_type 2721 * Type in the ethernet layer header (including dot1q). 2722 * @param[in] acc_mask 2723 * Acceptable mask, if NULL default internal default mask 2724 * will be used to check whether item fields are supported. 2725 * @param[out] error 2726 * Pointer to error structure. 2727 * 2728 * @return 2729 * 0 on success, a negative errno value otherwise and rte_errno is set. 2730 */ 2731 int 2732 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 2733 uint64_t item_flags, 2734 uint64_t last_item, 2735 uint16_t ether_type, 2736 const struct rte_flow_item_ecpri *acc_mask, 2737 struct rte_flow_error *error) 2738 { 2739 const struct rte_flow_item_ecpri *mask = item->mask; 2740 const struct rte_flow_item_ecpri nic_mask = { 2741 .hdr = { 2742 .common = { 2743 .u32 = 2744 RTE_BE32(((const struct rte_ecpri_common_hdr) { 2745 .type = 0xFF, 2746 }).u32), 2747 }, 2748 .dummy[0] = 0xFFFFFFFF, 2749 }, 2750 }; 2751 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 2752 MLX5_FLOW_LAYER_OUTER_VLAN); 2753 struct rte_flow_item_ecpri mask_lo; 2754 2755 if (!(last_item & outer_l2_vlan) && 2756 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 2757 return rte_flow_error_set(error, EINVAL, 2758 RTE_FLOW_ERROR_TYPE_ITEM, item, 2759 "eCPRI can only follow L2/VLAN layer or UDP layer"); 2760 if ((last_item & outer_l2_vlan) && ether_type && 2761 ether_type != RTE_ETHER_TYPE_ECPRI) 2762 return rte_flow_error_set(error, EINVAL, 2763 RTE_FLOW_ERROR_TYPE_ITEM, item, 2764 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 2765 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2766 return rte_flow_error_set(error, EINVAL, 2767 RTE_FLOW_ERROR_TYPE_ITEM, item, 2768 "eCPRI with tunnel is not supported right now"); 2769 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 2770 return rte_flow_error_set(error, ENOTSUP, 2771 RTE_FLOW_ERROR_TYPE_ITEM, item, 2772 "multiple L3 layers not supported"); 2773 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 2774 return rte_flow_error_set(error, EINVAL, 2775 RTE_FLOW_ERROR_TYPE_ITEM, item, 2776 "eCPRI cannot coexist with a TCP layer"); 2777 /* In specification, eCPRI could be over UDP layer. */ 2778 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 2779 return rte_flow_error_set(error, EINVAL, 2780 RTE_FLOW_ERROR_TYPE_ITEM, item, 2781 "eCPRI over UDP layer is not yet supported right now"); 2782 /* Mask for type field in common header could be zero. */ 2783 if (!mask) 2784 mask = &rte_flow_item_ecpri_mask; 2785 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 2786 /* Input mask is in big-endian format. */ 2787 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 2788 return rte_flow_error_set(error, EINVAL, 2789 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2790 "partial mask is not supported for protocol"); 2791 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 2792 return rte_flow_error_set(error, EINVAL, 2793 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2794 "message header mask must be after a type mask"); 2795 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2796 acc_mask ? (const uint8_t *)acc_mask 2797 : (const uint8_t *)&nic_mask, 2798 sizeof(struct rte_flow_item_ecpri), 2799 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2800 } 2801 2802 /** 2803 * Release resource related QUEUE/RSS action split. 2804 * 2805 * @param dev 2806 * Pointer to Ethernet device. 2807 * @param flow 2808 * Flow to release id's from. 2809 */ 2810 static void 2811 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2812 struct rte_flow *flow) 2813 { 2814 struct mlx5_priv *priv = dev->data->dev_private; 2815 uint32_t handle_idx; 2816 struct mlx5_flow_handle *dev_handle; 2817 2818 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2819 handle_idx, dev_handle, next) 2820 if (dev_handle->split_flow_id) 2821 mlx5_ipool_free(priv->sh->ipool 2822 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 2823 dev_handle->split_flow_id); 2824 } 2825 2826 static int 2827 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2828 const struct rte_flow_attr *attr __rte_unused, 2829 const struct rte_flow_item items[] __rte_unused, 2830 const struct rte_flow_action actions[] __rte_unused, 2831 bool external __rte_unused, 2832 int hairpin __rte_unused, 2833 struct rte_flow_error *error) 2834 { 2835 return rte_flow_error_set(error, ENOTSUP, 2836 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2837 } 2838 2839 static struct mlx5_flow * 2840 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2841 const struct rte_flow_attr *attr __rte_unused, 2842 const struct rte_flow_item items[] __rte_unused, 2843 const struct rte_flow_action actions[] __rte_unused, 2844 struct rte_flow_error *error) 2845 { 2846 rte_flow_error_set(error, ENOTSUP, 2847 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2848 return NULL; 2849 } 2850 2851 static int 2852 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2853 struct mlx5_flow *dev_flow __rte_unused, 2854 const struct rte_flow_attr *attr __rte_unused, 2855 const struct rte_flow_item items[] __rte_unused, 2856 const struct rte_flow_action actions[] __rte_unused, 2857 struct rte_flow_error *error) 2858 { 2859 return rte_flow_error_set(error, ENOTSUP, 2860 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2861 } 2862 2863 static int 2864 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2865 struct rte_flow *flow __rte_unused, 2866 struct rte_flow_error *error) 2867 { 2868 return rte_flow_error_set(error, ENOTSUP, 2869 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2870 } 2871 2872 static void 2873 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2874 struct rte_flow *flow __rte_unused) 2875 { 2876 } 2877 2878 static void 2879 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2880 struct rte_flow *flow __rte_unused) 2881 { 2882 } 2883 2884 static int 2885 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2886 struct rte_flow *flow __rte_unused, 2887 const struct rte_flow_action *actions __rte_unused, 2888 void *data __rte_unused, 2889 struct rte_flow_error *error) 2890 { 2891 return rte_flow_error_set(error, ENOTSUP, 2892 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2893 } 2894 2895 static int 2896 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 2897 uint32_t domains __rte_unused, 2898 uint32_t flags __rte_unused) 2899 { 2900 return 0; 2901 } 2902 2903 /* Void driver to protect from null pointer reference. */ 2904 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2905 .validate = flow_null_validate, 2906 .prepare = flow_null_prepare, 2907 .translate = flow_null_translate, 2908 .apply = flow_null_apply, 2909 .remove = flow_null_remove, 2910 .destroy = flow_null_destroy, 2911 .query = flow_null_query, 2912 .sync_domain = flow_null_sync_domain, 2913 }; 2914 2915 /** 2916 * Select flow driver type according to flow attributes and device 2917 * configuration. 2918 * 2919 * @param[in] dev 2920 * Pointer to the dev structure. 2921 * @param[in] attr 2922 * Pointer to the flow attributes. 2923 * 2924 * @return 2925 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2926 */ 2927 static enum mlx5_flow_drv_type 2928 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2929 { 2930 struct mlx5_priv *priv = dev->data->dev_private; 2931 /* The OS can determine first a specific flow type (DV, VERBS) */ 2932 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 2933 2934 if (type != MLX5_FLOW_TYPE_MAX) 2935 return type; 2936 /* If no OS specific type - continue with DV/VERBS selection */ 2937 if (attr->transfer && priv->config.dv_esw_en) 2938 type = MLX5_FLOW_TYPE_DV; 2939 if (!attr->transfer) 2940 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2941 MLX5_FLOW_TYPE_VERBS; 2942 return type; 2943 } 2944 2945 #define flow_get_drv_ops(type) flow_drv_ops[type] 2946 2947 /** 2948 * Flow driver validation API. This abstracts calling driver specific functions. 2949 * The type of flow driver is determined according to flow attributes. 2950 * 2951 * @param[in] dev 2952 * Pointer to the dev structure. 2953 * @param[in] attr 2954 * Pointer to the flow attributes. 2955 * @param[in] items 2956 * Pointer to the list of items. 2957 * @param[in] actions 2958 * Pointer to the list of actions. 2959 * @param[in] external 2960 * This flow rule is created by request external to PMD. 2961 * @param[in] hairpin 2962 * Number of hairpin TX actions, 0 means classic flow. 2963 * @param[out] error 2964 * Pointer to the error structure. 2965 * 2966 * @return 2967 * 0 on success, a negative errno value otherwise and rte_errno is set. 2968 */ 2969 static inline int 2970 flow_drv_validate(struct rte_eth_dev *dev, 2971 const struct rte_flow_attr *attr, 2972 const struct rte_flow_item items[], 2973 const struct rte_flow_action actions[], 2974 bool external, int hairpin, struct rte_flow_error *error) 2975 { 2976 const struct mlx5_flow_driver_ops *fops; 2977 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2978 2979 fops = flow_get_drv_ops(type); 2980 return fops->validate(dev, attr, items, actions, external, 2981 hairpin, error); 2982 } 2983 2984 /** 2985 * Flow driver preparation API. This abstracts calling driver specific 2986 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2987 * calculates the size of memory required for device flow, allocates the memory, 2988 * initializes the device flow and returns the pointer. 2989 * 2990 * @note 2991 * This function initializes device flow structure such as dv or verbs in 2992 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2993 * rest. For example, adding returning device flow to flow->dev_flow list and 2994 * setting backward reference to the flow should be done out of this function. 2995 * layers field is not filled either. 2996 * 2997 * @param[in] dev 2998 * Pointer to the dev structure. 2999 * @param[in] attr 3000 * Pointer to the flow attributes. 3001 * @param[in] items 3002 * Pointer to the list of items. 3003 * @param[in] actions 3004 * Pointer to the list of actions. 3005 * @param[in] flow_idx 3006 * This memory pool index to the flow. 3007 * @param[out] error 3008 * Pointer to the error structure. 3009 * 3010 * @return 3011 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3012 */ 3013 static inline struct mlx5_flow * 3014 flow_drv_prepare(struct rte_eth_dev *dev, 3015 const struct rte_flow *flow, 3016 const struct rte_flow_attr *attr, 3017 const struct rte_flow_item items[], 3018 const struct rte_flow_action actions[], 3019 uint32_t flow_idx, 3020 struct rte_flow_error *error) 3021 { 3022 const struct mlx5_flow_driver_ops *fops; 3023 enum mlx5_flow_drv_type type = flow->drv_type; 3024 struct mlx5_flow *mlx5_flow = NULL; 3025 3026 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3027 fops = flow_get_drv_ops(type); 3028 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3029 if (mlx5_flow) 3030 mlx5_flow->flow_idx = flow_idx; 3031 return mlx5_flow; 3032 } 3033 3034 /** 3035 * Flow driver translation API. This abstracts calling driver specific 3036 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3037 * translates a generic flow into a driver flow. flow_drv_prepare() must 3038 * precede. 3039 * 3040 * @note 3041 * dev_flow->layers could be filled as a result of parsing during translation 3042 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3043 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3044 * flow->actions could be overwritten even though all the expanded dev_flows 3045 * have the same actions. 3046 * 3047 * @param[in] dev 3048 * Pointer to the rte dev structure. 3049 * @param[in, out] dev_flow 3050 * Pointer to the mlx5 flow. 3051 * @param[in] attr 3052 * Pointer to the flow attributes. 3053 * @param[in] items 3054 * Pointer to the list of items. 3055 * @param[in] actions 3056 * Pointer to the list of actions. 3057 * @param[out] error 3058 * Pointer to the error structure. 3059 * 3060 * @return 3061 * 0 on success, a negative errno value otherwise and rte_errno is set. 3062 */ 3063 static inline int 3064 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3065 const struct rte_flow_attr *attr, 3066 const struct rte_flow_item items[], 3067 const struct rte_flow_action actions[], 3068 struct rte_flow_error *error) 3069 { 3070 const struct mlx5_flow_driver_ops *fops; 3071 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3072 3073 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3074 fops = flow_get_drv_ops(type); 3075 return fops->translate(dev, dev_flow, attr, items, actions, error); 3076 } 3077 3078 /** 3079 * Flow driver apply API. This abstracts calling driver specific functions. 3080 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3081 * translated driver flows on to device. flow_drv_translate() must precede. 3082 * 3083 * @param[in] dev 3084 * Pointer to Ethernet device structure. 3085 * @param[in, out] flow 3086 * Pointer to flow structure. 3087 * @param[out] error 3088 * Pointer to error structure. 3089 * 3090 * @return 3091 * 0 on success, a negative errno value otherwise and rte_errno is set. 3092 */ 3093 static inline int 3094 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3095 struct rte_flow_error *error) 3096 { 3097 const struct mlx5_flow_driver_ops *fops; 3098 enum mlx5_flow_drv_type type = flow->drv_type; 3099 3100 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3101 fops = flow_get_drv_ops(type); 3102 return fops->apply(dev, flow, error); 3103 } 3104 3105 /** 3106 * Flow driver destroy API. This abstracts calling driver specific functions. 3107 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3108 * on device and releases resources of the flow. 3109 * 3110 * @param[in] dev 3111 * Pointer to Ethernet device. 3112 * @param[in, out] flow 3113 * Pointer to flow structure. 3114 */ 3115 static inline void 3116 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3117 { 3118 const struct mlx5_flow_driver_ops *fops; 3119 enum mlx5_flow_drv_type type = flow->drv_type; 3120 3121 flow_mreg_split_qrss_release(dev, flow); 3122 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3123 fops = flow_get_drv_ops(type); 3124 fops->destroy(dev, flow); 3125 } 3126 3127 /** 3128 * Get RSS action from the action list. 3129 * 3130 * @param[in] actions 3131 * Pointer to the list of actions. 3132 * 3133 * @return 3134 * Pointer to the RSS action if exist, else return NULL. 3135 */ 3136 static const struct rte_flow_action_rss* 3137 flow_get_rss_action(const struct rte_flow_action actions[]) 3138 { 3139 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3140 switch (actions->type) { 3141 case RTE_FLOW_ACTION_TYPE_RSS: 3142 return (const struct rte_flow_action_rss *) 3143 actions->conf; 3144 default: 3145 break; 3146 } 3147 } 3148 return NULL; 3149 } 3150 3151 /** 3152 * Get ASO age action by index. 3153 * 3154 * @param[in] dev 3155 * Pointer to the Ethernet device structure. 3156 * @param[in] age_idx 3157 * Index to the ASO age action. 3158 * 3159 * @return 3160 * The specified ASO age action. 3161 */ 3162 struct mlx5_aso_age_action* 3163 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3164 { 3165 uint16_t pool_idx = age_idx & UINT16_MAX; 3166 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3167 struct mlx5_priv *priv = dev->data->dev_private; 3168 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3169 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3170 3171 return &pool->actions[offset - 1]; 3172 } 3173 3174 /* maps shared action to translated non shared in some actions array */ 3175 struct mlx5_translated_shared_action { 3176 struct rte_flow_shared_action *action; /**< Shared action */ 3177 int index; /**< Index in related array of rte_flow_action */ 3178 }; 3179 3180 /** 3181 * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related 3182 * non shared action if translation possible. 3183 * This functionality used to run same execution path for both shared & non 3184 * shared actions on flow create. All necessary preparations for shared 3185 * action handling should be preformed on *shared* actions list returned 3186 * from this call. 3187 * 3188 * @param[in] dev 3189 * Pointer to Ethernet device. 3190 * @param[in] actions 3191 * List of actions to translate. 3192 * @param[out] shared 3193 * List to store translated shared actions. 3194 * @param[in, out] shared_n 3195 * Size of *shared* array. On return should be updated with number of shared 3196 * actions retrieved from the *actions* list. 3197 * @param[out] translated_actions 3198 * List of actions where all shared actions were translated to non shared 3199 * if possible. NULL if no translation took place. 3200 * @param[out] error 3201 * Pointer to the error structure. 3202 * 3203 * @return 3204 * 0 on success, a negative errno value otherwise and rte_errno is set. 3205 */ 3206 static int 3207 flow_shared_actions_translate(struct rte_eth_dev *dev, 3208 const struct rte_flow_action actions[], 3209 struct mlx5_translated_shared_action *shared, 3210 int *shared_n, 3211 struct rte_flow_action **translated_actions, 3212 struct rte_flow_error *error) 3213 { 3214 struct mlx5_priv *priv = dev->data->dev_private; 3215 struct rte_flow_action *translated = NULL; 3216 size_t actions_size; 3217 int n; 3218 int copied_n = 0; 3219 struct mlx5_translated_shared_action *shared_end = NULL; 3220 3221 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3222 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED) 3223 continue; 3224 if (copied_n == *shared_n) { 3225 return rte_flow_error_set 3226 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3227 NULL, "too many shared actions"); 3228 } 3229 rte_memcpy(&shared[copied_n].action, &actions[n].conf, 3230 sizeof(actions[n].conf)); 3231 shared[copied_n].index = n; 3232 copied_n++; 3233 } 3234 n++; 3235 *shared_n = copied_n; 3236 if (!copied_n) 3237 return 0; 3238 actions_size = sizeof(struct rte_flow_action) * n; 3239 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3240 if (!translated) { 3241 rte_errno = ENOMEM; 3242 return -ENOMEM; 3243 } 3244 memcpy(translated, actions, actions_size); 3245 for (shared_end = shared + copied_n; shared < shared_end; shared++) { 3246 struct mlx5_shared_action_rss *shared_rss; 3247 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; 3248 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; 3249 uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) 3250 - 1); 3251 3252 switch (type) { 3253 case MLX5_SHARED_ACTION_TYPE_RSS: 3254 shared_rss = mlx5_ipool_get 3255 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3256 translated[shared->index].type = 3257 RTE_FLOW_ACTION_TYPE_RSS; 3258 translated[shared->index].conf = 3259 &shared_rss->origin; 3260 break; 3261 case MLX5_SHARED_ACTION_TYPE_AGE: 3262 if (priv->sh->flow_hit_aso_en) { 3263 translated[shared->index].type = 3264 (enum rte_flow_action_type) 3265 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3266 translated[shared->index].conf = 3267 (void *)(uintptr_t)idx; 3268 break; 3269 } 3270 /* Fall-through */ 3271 default: 3272 mlx5_free(translated); 3273 return rte_flow_error_set 3274 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3275 NULL, "invalid shared action type"); 3276 } 3277 } 3278 *translated_actions = translated; 3279 return 0; 3280 } 3281 3282 /** 3283 * Get Shared RSS action from the action list. 3284 * 3285 * @param[in] dev 3286 * Pointer to Ethernet device. 3287 * @param[in] shared 3288 * Pointer to the list of actions. 3289 * @param[in] shared_n 3290 * Actions list length. 3291 * 3292 * @return 3293 * The MLX5 RSS action ID if exists, otherwise return 0. 3294 */ 3295 static uint32_t 3296 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3297 struct mlx5_translated_shared_action *shared, 3298 int shared_n) 3299 { 3300 struct mlx5_translated_shared_action *shared_end; 3301 struct mlx5_priv *priv = dev->data->dev_private; 3302 struct mlx5_shared_action_rss *shared_rss; 3303 3304 3305 for (shared_end = shared + shared_n; shared < shared_end; shared++) { 3306 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; 3307 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; 3308 uint32_t idx = act_idx & 3309 ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); 3310 switch (type) { 3311 case MLX5_SHARED_ACTION_TYPE_RSS: 3312 shared_rss = mlx5_ipool_get 3313 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3314 idx); 3315 __atomic_add_fetch(&shared_rss->refcnt, 1, 3316 __ATOMIC_RELAXED); 3317 return idx; 3318 default: 3319 break; 3320 } 3321 } 3322 return 0; 3323 } 3324 3325 static unsigned int 3326 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3327 { 3328 const struct rte_flow_item *item; 3329 unsigned int has_vlan = 0; 3330 3331 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3332 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3333 has_vlan = 1; 3334 break; 3335 } 3336 } 3337 if (has_vlan) 3338 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3339 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3340 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3341 MLX5_EXPANSION_ROOT_OUTER; 3342 } 3343 3344 /** 3345 * Get layer flags from the prefix flow. 3346 * 3347 * Some flows may be split to several subflows, the prefix subflow gets the 3348 * match items and the suffix sub flow gets the actions. 3349 * Some actions need the user defined match item flags to get the detail for 3350 * the action. 3351 * This function helps the suffix flow to get the item layer flags from prefix 3352 * subflow. 3353 * 3354 * @param[in] dev_flow 3355 * Pointer the created preifx subflow. 3356 * 3357 * @return 3358 * The layers get from prefix subflow. 3359 */ 3360 static inline uint64_t 3361 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3362 { 3363 uint64_t layers = 0; 3364 3365 /* 3366 * Layers bits could be localization, but usually the compiler will 3367 * help to do the optimization work for source code. 3368 * If no decap actions, use the layers directly. 3369 */ 3370 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3371 return dev_flow->handle->layers; 3372 /* Convert L3 layers with decap action. */ 3373 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3374 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3375 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3376 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3377 /* Convert L4 layers with decap action. */ 3378 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3379 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3380 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3381 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3382 return layers; 3383 } 3384 3385 /** 3386 * Get metadata split action information. 3387 * 3388 * @param[in] actions 3389 * Pointer to the list of actions. 3390 * @param[out] qrss 3391 * Pointer to the return pointer. 3392 * @param[out] qrss_type 3393 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3394 * if no QUEUE/RSS is found. 3395 * @param[out] encap_idx 3396 * Pointer to the index of the encap action if exists, otherwise the last 3397 * action index. 3398 * 3399 * @return 3400 * Total number of actions. 3401 */ 3402 static int 3403 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3404 const struct rte_flow_action **qrss, 3405 int *encap_idx) 3406 { 3407 const struct rte_flow_action_raw_encap *raw_encap; 3408 int actions_n = 0; 3409 int raw_decap_idx = -1; 3410 3411 *encap_idx = -1; 3412 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3413 switch (actions->type) { 3414 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3415 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3416 *encap_idx = actions_n; 3417 break; 3418 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3419 raw_decap_idx = actions_n; 3420 break; 3421 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3422 raw_encap = actions->conf; 3423 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3424 *encap_idx = raw_decap_idx != -1 ? 3425 raw_decap_idx : actions_n; 3426 break; 3427 case RTE_FLOW_ACTION_TYPE_QUEUE: 3428 case RTE_FLOW_ACTION_TYPE_RSS: 3429 *qrss = actions; 3430 break; 3431 default: 3432 break; 3433 } 3434 actions_n++; 3435 } 3436 if (*encap_idx == -1) 3437 *encap_idx = actions_n; 3438 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3439 return actions_n + 1; 3440 } 3441 3442 /** 3443 * Check meter action from the action list. 3444 * 3445 * @param[in] actions 3446 * Pointer to the list of actions. 3447 * @param[out] mtr 3448 * Pointer to the meter exist flag. 3449 * 3450 * @return 3451 * Total number of actions. 3452 */ 3453 static int 3454 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 3455 { 3456 int actions_n = 0; 3457 3458 MLX5_ASSERT(mtr); 3459 *mtr = 0; 3460 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3461 switch (actions->type) { 3462 case RTE_FLOW_ACTION_TYPE_METER: 3463 *mtr = 1; 3464 break; 3465 default: 3466 break; 3467 } 3468 actions_n++; 3469 } 3470 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3471 return actions_n + 1; 3472 } 3473 3474 /** 3475 * Check if the flow should be split due to hairpin. 3476 * The reason for the split is that in current HW we can't 3477 * support encap and push-vlan on Rx, so if a flow contains 3478 * these actions we move it to Tx. 3479 * 3480 * @param dev 3481 * Pointer to Ethernet device. 3482 * @param[in] attr 3483 * Flow rule attributes. 3484 * @param[in] actions 3485 * Associated actions (list terminated by the END action). 3486 * 3487 * @return 3488 * > 0 the number of actions and the flow should be split, 3489 * 0 when no split required. 3490 */ 3491 static int 3492 flow_check_hairpin_split(struct rte_eth_dev *dev, 3493 const struct rte_flow_attr *attr, 3494 const struct rte_flow_action actions[]) 3495 { 3496 int queue_action = 0; 3497 int action_n = 0; 3498 int split = 0; 3499 const struct rte_flow_action_queue *queue; 3500 const struct rte_flow_action_rss *rss; 3501 const struct rte_flow_action_raw_encap *raw_encap; 3502 const struct rte_eth_hairpin_conf *conf; 3503 3504 if (!attr->ingress) 3505 return 0; 3506 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3507 switch (actions->type) { 3508 case RTE_FLOW_ACTION_TYPE_QUEUE: 3509 queue = actions->conf; 3510 if (queue == NULL) 3511 return 0; 3512 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3513 if (conf != NULL && !!conf->tx_explicit) 3514 return 0; 3515 queue_action = 1; 3516 action_n++; 3517 break; 3518 case RTE_FLOW_ACTION_TYPE_RSS: 3519 rss = actions->conf; 3520 if (rss == NULL || rss->queue_num == 0) 3521 return 0; 3522 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3523 if (conf != NULL && !!conf->tx_explicit) 3524 return 0; 3525 queue_action = 1; 3526 action_n++; 3527 break; 3528 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3529 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3530 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3531 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3532 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3533 split++; 3534 action_n++; 3535 break; 3536 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3537 raw_encap = actions->conf; 3538 if (raw_encap->size > 3539 (sizeof(struct rte_flow_item_eth) + 3540 sizeof(struct rte_flow_item_ipv4))) 3541 split++; 3542 action_n++; 3543 break; 3544 default: 3545 action_n++; 3546 break; 3547 } 3548 } 3549 if (split && queue_action) 3550 return action_n; 3551 return 0; 3552 } 3553 3554 /* Declare flow create/destroy prototype in advance. */ 3555 static uint32_t 3556 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3557 const struct rte_flow_attr *attr, 3558 const struct rte_flow_item items[], 3559 const struct rte_flow_action actions[], 3560 bool external, struct rte_flow_error *error); 3561 3562 static void 3563 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3564 uint32_t flow_idx); 3565 3566 struct mlx5_hlist_entry * 3567 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 3568 void *cb_ctx) 3569 { 3570 struct rte_eth_dev *dev = list->ctx; 3571 struct mlx5_priv *priv = dev->data->dev_private; 3572 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3573 struct mlx5_flow_mreg_copy_resource *mcp_res; 3574 struct rte_flow_error *error = ctx->error; 3575 uint32_t idx = 0; 3576 int ret; 3577 uint32_t mark_id = key; 3578 struct rte_flow_attr attr = { 3579 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3580 .ingress = 1, 3581 }; 3582 struct mlx5_rte_flow_item_tag tag_spec = { 3583 .data = mark_id, 3584 }; 3585 struct rte_flow_item items[] = { 3586 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 3587 }; 3588 struct rte_flow_action_mark ftag = { 3589 .id = mark_id, 3590 }; 3591 struct mlx5_flow_action_copy_mreg cp_mreg = { 3592 .dst = REG_B, 3593 .src = REG_NON, 3594 }; 3595 struct rte_flow_action_jump jump = { 3596 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3597 }; 3598 struct rte_flow_action actions[] = { 3599 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 3600 }; 3601 3602 /* Fill the register fileds in the flow. */ 3603 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3604 if (ret < 0) 3605 return NULL; 3606 tag_spec.id = ret; 3607 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3608 if (ret < 0) 3609 return NULL; 3610 cp_mreg.src = ret; 3611 /* Provide the full width of FLAG specific value. */ 3612 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3613 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3614 /* Build a new flow. */ 3615 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3616 items[0] = (struct rte_flow_item){ 3617 .type = (enum rte_flow_item_type) 3618 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3619 .spec = &tag_spec, 3620 }; 3621 items[1] = (struct rte_flow_item){ 3622 .type = RTE_FLOW_ITEM_TYPE_END, 3623 }; 3624 actions[0] = (struct rte_flow_action){ 3625 .type = (enum rte_flow_action_type) 3626 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3627 .conf = &ftag, 3628 }; 3629 actions[1] = (struct rte_flow_action){ 3630 .type = (enum rte_flow_action_type) 3631 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3632 .conf = &cp_mreg, 3633 }; 3634 actions[2] = (struct rte_flow_action){ 3635 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3636 .conf = &jump, 3637 }; 3638 actions[3] = (struct rte_flow_action){ 3639 .type = RTE_FLOW_ACTION_TYPE_END, 3640 }; 3641 } else { 3642 /* Default rule, wildcard match. */ 3643 attr.priority = MLX5_FLOW_PRIO_RSVD; 3644 items[0] = (struct rte_flow_item){ 3645 .type = RTE_FLOW_ITEM_TYPE_END, 3646 }; 3647 actions[0] = (struct rte_flow_action){ 3648 .type = (enum rte_flow_action_type) 3649 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3650 .conf = &cp_mreg, 3651 }; 3652 actions[1] = (struct rte_flow_action){ 3653 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3654 .conf = &jump, 3655 }; 3656 actions[2] = (struct rte_flow_action){ 3657 .type = RTE_FLOW_ACTION_TYPE_END, 3658 }; 3659 } 3660 /* Build a new entry. */ 3661 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3662 if (!mcp_res) { 3663 rte_errno = ENOMEM; 3664 return NULL; 3665 } 3666 mcp_res->idx = idx; 3667 /* 3668 * The copy Flows are not included in any list. There 3669 * ones are referenced from other Flows and can not 3670 * be applied, removed, deleted in ardbitrary order 3671 * by list traversing. 3672 */ 3673 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3674 actions, false, error); 3675 if (!mcp_res->rix_flow) { 3676 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 3677 return NULL; 3678 } 3679 return &mcp_res->hlist_ent; 3680 } 3681 3682 /** 3683 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3684 * 3685 * As mark_id is unique, if there's already a registered flow for the mark_id, 3686 * return by increasing the reference counter of the resource. Otherwise, create 3687 * the resource (mcp_res) and flow. 3688 * 3689 * Flow looks like, 3690 * - If ingress port is ANY and reg_c[1] is mark_id, 3691 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3692 * 3693 * For default flow (zero mark_id), flow is like, 3694 * - If ingress port is ANY, 3695 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 3696 * 3697 * @param dev 3698 * Pointer to Ethernet device. 3699 * @param mark_id 3700 * ID of MARK action, zero means default flow for META. 3701 * @param[out] error 3702 * Perform verbose error reporting if not NULL. 3703 * 3704 * @return 3705 * Associated resource on success, NULL otherwise and rte_errno is set. 3706 */ 3707 static struct mlx5_flow_mreg_copy_resource * 3708 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 3709 struct rte_flow_error *error) 3710 { 3711 struct mlx5_priv *priv = dev->data->dev_private; 3712 struct mlx5_hlist_entry *entry; 3713 struct mlx5_flow_cb_ctx ctx = { 3714 .dev = dev, 3715 .error = error, 3716 }; 3717 3718 /* Check if already registered. */ 3719 MLX5_ASSERT(priv->mreg_cp_tbl); 3720 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 3721 if (!entry) 3722 return NULL; 3723 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 3724 hlist_ent); 3725 } 3726 3727 void 3728 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 3729 { 3730 struct mlx5_flow_mreg_copy_resource *mcp_res = 3731 container_of(entry, typeof(*mcp_res), hlist_ent); 3732 struct rte_eth_dev *dev = list->ctx; 3733 struct mlx5_priv *priv = dev->data->dev_private; 3734 3735 MLX5_ASSERT(mcp_res->rix_flow); 3736 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3737 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3738 } 3739 3740 /** 3741 * Release flow in RX_CP_TBL. 3742 * 3743 * @param dev 3744 * Pointer to Ethernet device. 3745 * @flow 3746 * Parent flow for wich copying is provided. 3747 */ 3748 static void 3749 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3750 struct rte_flow *flow) 3751 { 3752 struct mlx5_flow_mreg_copy_resource *mcp_res; 3753 struct mlx5_priv *priv = dev->data->dev_private; 3754 3755 if (!flow->rix_mreg_copy) 3756 return; 3757 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3758 flow->rix_mreg_copy); 3759 if (!mcp_res || !priv->mreg_cp_tbl) 3760 return; 3761 MLX5_ASSERT(mcp_res->rix_flow); 3762 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3763 flow->rix_mreg_copy = 0; 3764 } 3765 3766 /** 3767 * Remove the default copy action from RX_CP_TBL. 3768 * 3769 * This functions is called in the mlx5_dev_start(). No thread safe 3770 * is guaranteed. 3771 * 3772 * @param dev 3773 * Pointer to Ethernet device. 3774 */ 3775 static void 3776 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3777 { 3778 struct mlx5_hlist_entry *entry; 3779 struct mlx5_priv *priv = dev->data->dev_private; 3780 3781 /* Check if default flow is registered. */ 3782 if (!priv->mreg_cp_tbl) 3783 return; 3784 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 3785 MLX5_DEFAULT_COPY_ID, NULL); 3786 if (!entry) 3787 return; 3788 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 3789 } 3790 3791 /** 3792 * Add the default copy action in in RX_CP_TBL. 3793 * 3794 * This functions is called in the mlx5_dev_start(). No thread safe 3795 * is guaranteed. 3796 * 3797 * @param dev 3798 * Pointer to Ethernet device. 3799 * @param[out] error 3800 * Perform verbose error reporting if not NULL. 3801 * 3802 * @return 3803 * 0 for success, negative value otherwise and rte_errno is set. 3804 */ 3805 static int 3806 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3807 struct rte_flow_error *error) 3808 { 3809 struct mlx5_priv *priv = dev->data->dev_private; 3810 struct mlx5_flow_mreg_copy_resource *mcp_res; 3811 3812 /* Check whether extensive metadata feature is engaged. */ 3813 if (!priv->config.dv_flow_en || 3814 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3815 !mlx5_flow_ext_mreg_supported(dev) || 3816 !priv->sh->dv_regc0_mask) 3817 return 0; 3818 /* 3819 * Add default mreg copy flow may be called multiple time, but 3820 * only be called once in stop. Avoid register it twice. 3821 */ 3822 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 3823 return 0; 3824 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3825 if (!mcp_res) 3826 return -rte_errno; 3827 return 0; 3828 } 3829 3830 /** 3831 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3832 * 3833 * All the flow having Q/RSS action should be split by 3834 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3835 * performs the following, 3836 * - CQE->flow_tag := reg_c[1] (MARK) 3837 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3838 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3839 * but there should be a flow per each MARK ID set by MARK action. 3840 * 3841 * For the aforementioned reason, if there's a MARK action in flow's action 3842 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3843 * the MARK ID to CQE's flow_tag like, 3844 * - If reg_c[1] is mark_id, 3845 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3846 * 3847 * For SET_META action which stores value in reg_c[0], as the destination is 3848 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3849 * MARK ID means the default flow. The default flow looks like, 3850 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3851 * 3852 * @param dev 3853 * Pointer to Ethernet device. 3854 * @param flow 3855 * Pointer to flow structure. 3856 * @param[in] actions 3857 * Pointer to the list of actions. 3858 * @param[out] error 3859 * Perform verbose error reporting if not NULL. 3860 * 3861 * @return 3862 * 0 on success, negative value otherwise and rte_errno is set. 3863 */ 3864 static int 3865 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3866 struct rte_flow *flow, 3867 const struct rte_flow_action *actions, 3868 struct rte_flow_error *error) 3869 { 3870 struct mlx5_priv *priv = dev->data->dev_private; 3871 struct mlx5_dev_config *config = &priv->config; 3872 struct mlx5_flow_mreg_copy_resource *mcp_res; 3873 const struct rte_flow_action_mark *mark; 3874 3875 /* Check whether extensive metadata feature is engaged. */ 3876 if (!config->dv_flow_en || 3877 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3878 !mlx5_flow_ext_mreg_supported(dev) || 3879 !priv->sh->dv_regc0_mask) 3880 return 0; 3881 /* Find MARK action. */ 3882 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3883 switch (actions->type) { 3884 case RTE_FLOW_ACTION_TYPE_FLAG: 3885 mcp_res = flow_mreg_add_copy_action 3886 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3887 if (!mcp_res) 3888 return -rte_errno; 3889 flow->rix_mreg_copy = mcp_res->idx; 3890 return 0; 3891 case RTE_FLOW_ACTION_TYPE_MARK: 3892 mark = (const struct rte_flow_action_mark *) 3893 actions->conf; 3894 mcp_res = 3895 flow_mreg_add_copy_action(dev, mark->id, error); 3896 if (!mcp_res) 3897 return -rte_errno; 3898 flow->rix_mreg_copy = mcp_res->idx; 3899 return 0; 3900 default: 3901 break; 3902 } 3903 } 3904 return 0; 3905 } 3906 3907 #define MLX5_MAX_SPLIT_ACTIONS 24 3908 #define MLX5_MAX_SPLIT_ITEMS 24 3909 3910 /** 3911 * Split the hairpin flow. 3912 * Since HW can't support encap and push-vlan on Rx, we move these 3913 * actions to Tx. 3914 * If the count action is after the encap then we also 3915 * move the count action. in this case the count will also measure 3916 * the outer bytes. 3917 * 3918 * @param dev 3919 * Pointer to Ethernet device. 3920 * @param[in] actions 3921 * Associated actions (list terminated by the END action). 3922 * @param[out] actions_rx 3923 * Rx flow actions. 3924 * @param[out] actions_tx 3925 * Tx flow actions.. 3926 * @param[out] pattern_tx 3927 * The pattern items for the Tx flow. 3928 * @param[out] flow_id 3929 * The flow ID connected to this flow. 3930 * 3931 * @return 3932 * 0 on success. 3933 */ 3934 static int 3935 flow_hairpin_split(struct rte_eth_dev *dev, 3936 const struct rte_flow_action actions[], 3937 struct rte_flow_action actions_rx[], 3938 struct rte_flow_action actions_tx[], 3939 struct rte_flow_item pattern_tx[], 3940 uint32_t flow_id) 3941 { 3942 const struct rte_flow_action_raw_encap *raw_encap; 3943 const struct rte_flow_action_raw_decap *raw_decap; 3944 struct mlx5_rte_flow_action_set_tag *set_tag; 3945 struct rte_flow_action *tag_action; 3946 struct mlx5_rte_flow_item_tag *tag_item; 3947 struct rte_flow_item *item; 3948 char *addr; 3949 int encap = 0; 3950 3951 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3952 switch (actions->type) { 3953 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3954 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3955 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3956 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3957 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3958 rte_memcpy(actions_tx, actions, 3959 sizeof(struct rte_flow_action)); 3960 actions_tx++; 3961 break; 3962 case RTE_FLOW_ACTION_TYPE_COUNT: 3963 if (encap) { 3964 rte_memcpy(actions_tx, actions, 3965 sizeof(struct rte_flow_action)); 3966 actions_tx++; 3967 } else { 3968 rte_memcpy(actions_rx, actions, 3969 sizeof(struct rte_flow_action)); 3970 actions_rx++; 3971 } 3972 break; 3973 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3974 raw_encap = actions->conf; 3975 if (raw_encap->size > 3976 (sizeof(struct rte_flow_item_eth) + 3977 sizeof(struct rte_flow_item_ipv4))) { 3978 memcpy(actions_tx, actions, 3979 sizeof(struct rte_flow_action)); 3980 actions_tx++; 3981 encap = 1; 3982 } else { 3983 rte_memcpy(actions_rx, actions, 3984 sizeof(struct rte_flow_action)); 3985 actions_rx++; 3986 } 3987 break; 3988 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3989 raw_decap = actions->conf; 3990 if (raw_decap->size < 3991 (sizeof(struct rte_flow_item_eth) + 3992 sizeof(struct rte_flow_item_ipv4))) { 3993 memcpy(actions_tx, actions, 3994 sizeof(struct rte_flow_action)); 3995 actions_tx++; 3996 } else { 3997 rte_memcpy(actions_rx, actions, 3998 sizeof(struct rte_flow_action)); 3999 actions_rx++; 4000 } 4001 break; 4002 default: 4003 rte_memcpy(actions_rx, actions, 4004 sizeof(struct rte_flow_action)); 4005 actions_rx++; 4006 break; 4007 } 4008 } 4009 /* Add set meta action and end action for the Rx flow. */ 4010 tag_action = actions_rx; 4011 tag_action->type = (enum rte_flow_action_type) 4012 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4013 actions_rx++; 4014 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4015 actions_rx++; 4016 set_tag = (void *)actions_rx; 4017 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 4018 MLX5_ASSERT(set_tag->id > REG_NON); 4019 set_tag->data = flow_id; 4020 tag_action->conf = set_tag; 4021 /* Create Tx item list. */ 4022 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4023 addr = (void *)&pattern_tx[2]; 4024 item = pattern_tx; 4025 item->type = (enum rte_flow_item_type) 4026 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4027 tag_item = (void *)addr; 4028 tag_item->data = flow_id; 4029 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4030 MLX5_ASSERT(set_tag->id > REG_NON); 4031 item->spec = tag_item; 4032 addr += sizeof(struct mlx5_rte_flow_item_tag); 4033 tag_item = (void *)addr; 4034 tag_item->data = UINT32_MAX; 4035 tag_item->id = UINT16_MAX; 4036 item->mask = tag_item; 4037 item->last = NULL; 4038 item++; 4039 item->type = RTE_FLOW_ITEM_TYPE_END; 4040 return 0; 4041 } 4042 4043 /** 4044 * The last stage of splitting chain, just creates the subflow 4045 * without any modification. 4046 * 4047 * @param[in] dev 4048 * Pointer to Ethernet device. 4049 * @param[in] flow 4050 * Parent flow structure pointer. 4051 * @param[in, out] sub_flow 4052 * Pointer to return the created subflow, may be NULL. 4053 * @param[in] attr 4054 * Flow rule attributes. 4055 * @param[in] items 4056 * Pattern specification (list terminated by the END pattern item). 4057 * @param[in] actions 4058 * Associated actions (list terminated by the END action). 4059 * @param[in] flow_split_info 4060 * Pointer to flow split info structure. 4061 * @param[out] error 4062 * Perform verbose error reporting if not NULL. 4063 * @return 4064 * 0 on success, negative value otherwise 4065 */ 4066 static int 4067 flow_create_split_inner(struct rte_eth_dev *dev, 4068 struct rte_flow *flow, 4069 struct mlx5_flow **sub_flow, 4070 const struct rte_flow_attr *attr, 4071 const struct rte_flow_item items[], 4072 const struct rte_flow_action actions[], 4073 struct mlx5_flow_split_info *flow_split_info, 4074 struct rte_flow_error *error) 4075 { 4076 struct mlx5_flow *dev_flow; 4077 4078 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4079 flow_split_info->flow_idx, error); 4080 if (!dev_flow) 4081 return -rte_errno; 4082 dev_flow->flow = flow; 4083 dev_flow->external = flow_split_info->external; 4084 dev_flow->skip_scale = flow_split_info->skip_scale; 4085 /* Subflow object was created, we must include one in the list. */ 4086 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4087 dev_flow->handle, next); 4088 /* 4089 * If dev_flow is as one of the suffix flow, some actions in suffix 4090 * flow may need some user defined item layer flags, and pass the 4091 * Metadate rxq mark flag to suffix flow as well. 4092 */ 4093 if (flow_split_info->prefix_layers) 4094 dev_flow->handle->layers = flow_split_info->prefix_layers; 4095 if (flow_split_info->prefix_mark) 4096 dev_flow->handle->mark = 1; 4097 if (sub_flow) 4098 *sub_flow = dev_flow; 4099 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4100 } 4101 4102 /** 4103 * Split the meter flow. 4104 * 4105 * As meter flow will split to three sub flow, other than meter 4106 * action, the other actions make sense to only meter accepts 4107 * the packet. If it need to be dropped, no other additional 4108 * actions should be take. 4109 * 4110 * One kind of special action which decapsulates the L3 tunnel 4111 * header will be in the prefix sub flow, as not to take the 4112 * L3 tunnel header into account. 4113 * 4114 * @param dev 4115 * Pointer to Ethernet device. 4116 * @param[in] items 4117 * Pattern specification (list terminated by the END pattern item). 4118 * @param[out] sfx_items 4119 * Suffix flow match items (list terminated by the END pattern item). 4120 * @param[in] actions 4121 * Associated actions (list terminated by the END action). 4122 * @param[out] actions_sfx 4123 * Suffix flow actions. 4124 * @param[out] actions_pre 4125 * Prefix flow actions. 4126 * @param[out] pattern_sfx 4127 * The pattern items for the suffix flow. 4128 * @param[out] tag_sfx 4129 * Pointer to suffix flow tag. 4130 * 4131 * @return 4132 * 0 on success. 4133 */ 4134 static int 4135 flow_meter_split_prep(struct rte_eth_dev *dev, 4136 const struct rte_flow_item items[], 4137 struct rte_flow_item sfx_items[], 4138 const struct rte_flow_action actions[], 4139 struct rte_flow_action actions_sfx[], 4140 struct rte_flow_action actions_pre[]) 4141 { 4142 struct mlx5_priv *priv = dev->data->dev_private; 4143 struct rte_flow_action *tag_action = NULL; 4144 struct rte_flow_item *tag_item; 4145 struct mlx5_rte_flow_action_set_tag *set_tag; 4146 struct rte_flow_error error; 4147 const struct rte_flow_action_raw_encap *raw_encap; 4148 const struct rte_flow_action_raw_decap *raw_decap; 4149 struct mlx5_rte_flow_item_tag *tag_spec; 4150 struct mlx5_rte_flow_item_tag *tag_mask; 4151 uint32_t tag_id = 0; 4152 bool copy_vlan = false; 4153 4154 /* Prepare the actions for prefix and suffix flow. */ 4155 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4156 struct rte_flow_action **action_cur = NULL; 4157 4158 switch (actions->type) { 4159 case RTE_FLOW_ACTION_TYPE_METER: 4160 /* Add the extra tag action first. */ 4161 tag_action = actions_pre; 4162 tag_action->type = (enum rte_flow_action_type) 4163 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4164 actions_pre++; 4165 action_cur = &actions_pre; 4166 break; 4167 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4168 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4169 action_cur = &actions_pre; 4170 break; 4171 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4172 raw_encap = actions->conf; 4173 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4174 action_cur = &actions_pre; 4175 break; 4176 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4177 raw_decap = actions->conf; 4178 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4179 action_cur = &actions_pre; 4180 break; 4181 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4182 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4183 copy_vlan = true; 4184 break; 4185 default: 4186 break; 4187 } 4188 if (!action_cur) 4189 action_cur = &actions_sfx; 4190 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 4191 (*action_cur)++; 4192 } 4193 /* Add end action to the actions. */ 4194 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4195 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4196 actions_pre++; 4197 /* Set the tag. */ 4198 set_tag = (void *)actions_pre; 4199 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4200 mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 4201 &tag_id); 4202 if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { 4203 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", 4204 dev->data->port_id); 4205 mlx5_ipool_free(priv->sh->ipool 4206 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); 4207 return 0; 4208 } else if (!tag_id) { 4209 return 0; 4210 } 4211 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 4212 assert(tag_action); 4213 tag_action->conf = set_tag; 4214 /* Prepare the suffix subflow items. */ 4215 tag_item = sfx_items++; 4216 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4217 int item_type = items->type; 4218 4219 switch (item_type) { 4220 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4221 memcpy(sfx_items, items, sizeof(*sfx_items)); 4222 sfx_items++; 4223 break; 4224 case RTE_FLOW_ITEM_TYPE_VLAN: 4225 if (copy_vlan) { 4226 memcpy(sfx_items, items, sizeof(*sfx_items)); 4227 /* 4228 * Convert to internal match item, it is used 4229 * for vlan push and set vid. 4230 */ 4231 sfx_items->type = (enum rte_flow_item_type) 4232 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4233 sfx_items++; 4234 } 4235 break; 4236 default: 4237 break; 4238 } 4239 } 4240 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4241 sfx_items++; 4242 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4243 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 4244 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4245 tag_mask = tag_spec + 1; 4246 tag_mask->data = 0xffffff00; 4247 tag_item->type = (enum rte_flow_item_type) 4248 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4249 tag_item->spec = tag_spec; 4250 tag_item->last = NULL; 4251 tag_item->mask = tag_mask; 4252 return tag_id; 4253 } 4254 4255 /** 4256 * Split action list having QUEUE/RSS for metadata register copy. 4257 * 4258 * Once Q/RSS action is detected in user's action list, the flow action 4259 * should be split in order to copy metadata registers, which will happen in 4260 * RX_CP_TBL like, 4261 * - CQE->flow_tag := reg_c[1] (MARK) 4262 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4263 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4264 * This is because the last action of each flow must be a terminal action 4265 * (QUEUE, RSS or DROP). 4266 * 4267 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4268 * stored and kept in the mlx5_flow structure per each sub_flow. 4269 * 4270 * The Q/RSS action is replaced with, 4271 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4272 * And the following JUMP action is added at the end, 4273 * - JUMP, to RX_CP_TBL. 4274 * 4275 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4276 * flow_create_split_metadata() routine. The flow will look like, 4277 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4278 * 4279 * @param dev 4280 * Pointer to Ethernet device. 4281 * @param[out] split_actions 4282 * Pointer to store split actions to jump to CP_TBL. 4283 * @param[in] actions 4284 * Pointer to the list of original flow actions. 4285 * @param[in] qrss 4286 * Pointer to the Q/RSS action. 4287 * @param[in] actions_n 4288 * Number of original actions. 4289 * @param[out] error 4290 * Perform verbose error reporting if not NULL. 4291 * 4292 * @return 4293 * non-zero unique flow_id on success, otherwise 0 and 4294 * error/rte_error are set. 4295 */ 4296 static uint32_t 4297 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4298 struct rte_flow_action *split_actions, 4299 const struct rte_flow_action *actions, 4300 const struct rte_flow_action *qrss, 4301 int actions_n, struct rte_flow_error *error) 4302 { 4303 struct mlx5_priv *priv = dev->data->dev_private; 4304 struct mlx5_rte_flow_action_set_tag *set_tag; 4305 struct rte_flow_action_jump *jump; 4306 const int qrss_idx = qrss - actions; 4307 uint32_t flow_id = 0; 4308 int ret = 0; 4309 4310 /* 4311 * Given actions will be split 4312 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4313 * - Add jump to mreg CP_TBL. 4314 * As a result, there will be one more action. 4315 */ 4316 ++actions_n; 4317 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4318 set_tag = (void *)(split_actions + actions_n); 4319 /* 4320 * If tag action is not set to void(it means we are not the meter 4321 * suffix flow), add the tag action. Since meter suffix flow already 4322 * has the tag added. 4323 */ 4324 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4325 /* 4326 * Allocate the new subflow ID. This one is unique within 4327 * device and not shared with representors. Otherwise, 4328 * we would have to resolve multi-thread access synch 4329 * issue. Each flow on the shared device is appended 4330 * with source vport identifier, so the resulting 4331 * flows will be unique in the shared (by master and 4332 * representors) domain even if they have coinciding 4333 * IDs. 4334 */ 4335 mlx5_ipool_malloc(priv->sh->ipool 4336 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 4337 if (!flow_id) 4338 return rte_flow_error_set(error, ENOMEM, 4339 RTE_FLOW_ERROR_TYPE_ACTION, 4340 NULL, "can't allocate id " 4341 "for split Q/RSS subflow"); 4342 /* Internal SET_TAG action to set flow ID. */ 4343 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4344 .data = flow_id, 4345 }; 4346 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4347 if (ret < 0) 4348 return ret; 4349 set_tag->id = ret; 4350 /* Construct new actions array. */ 4351 /* Replace QUEUE/RSS action. */ 4352 split_actions[qrss_idx] = (struct rte_flow_action){ 4353 .type = (enum rte_flow_action_type) 4354 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4355 .conf = set_tag, 4356 }; 4357 } 4358 /* JUMP action to jump to mreg copy table (CP_TBL). */ 4359 jump = (void *)(set_tag + 1); 4360 *jump = (struct rte_flow_action_jump){ 4361 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4362 }; 4363 split_actions[actions_n - 2] = (struct rte_flow_action){ 4364 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4365 .conf = jump, 4366 }; 4367 split_actions[actions_n - 1] = (struct rte_flow_action){ 4368 .type = RTE_FLOW_ACTION_TYPE_END, 4369 }; 4370 return flow_id; 4371 } 4372 4373 /** 4374 * Extend the given action list for Tx metadata copy. 4375 * 4376 * Copy the given action list to the ext_actions and add flow metadata register 4377 * copy action in order to copy reg_a set by WQE to reg_c[0]. 4378 * 4379 * @param[out] ext_actions 4380 * Pointer to the extended action list. 4381 * @param[in] actions 4382 * Pointer to the list of actions. 4383 * @param[in] actions_n 4384 * Number of actions in the list. 4385 * @param[out] error 4386 * Perform verbose error reporting if not NULL. 4387 * @param[in] encap_idx 4388 * The encap action inndex. 4389 * 4390 * @return 4391 * 0 on success, negative value otherwise 4392 */ 4393 static int 4394 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 4395 struct rte_flow_action *ext_actions, 4396 const struct rte_flow_action *actions, 4397 int actions_n, struct rte_flow_error *error, 4398 int encap_idx) 4399 { 4400 struct mlx5_flow_action_copy_mreg *cp_mreg = 4401 (struct mlx5_flow_action_copy_mreg *) 4402 (ext_actions + actions_n + 1); 4403 int ret; 4404 4405 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4406 if (ret < 0) 4407 return ret; 4408 cp_mreg->dst = ret; 4409 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 4410 if (ret < 0) 4411 return ret; 4412 cp_mreg->src = ret; 4413 if (encap_idx != 0) 4414 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 4415 if (encap_idx == actions_n - 1) { 4416 ext_actions[actions_n - 1] = (struct rte_flow_action){ 4417 .type = (enum rte_flow_action_type) 4418 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4419 .conf = cp_mreg, 4420 }; 4421 ext_actions[actions_n] = (struct rte_flow_action){ 4422 .type = RTE_FLOW_ACTION_TYPE_END, 4423 }; 4424 } else { 4425 ext_actions[encap_idx] = (struct rte_flow_action){ 4426 .type = (enum rte_flow_action_type) 4427 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4428 .conf = cp_mreg, 4429 }; 4430 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 4431 sizeof(*ext_actions) * (actions_n - encap_idx)); 4432 } 4433 return 0; 4434 } 4435 4436 /** 4437 * Check the match action from the action list. 4438 * 4439 * @param[in] actions 4440 * Pointer to the list of actions. 4441 * @param[in] attr 4442 * Flow rule attributes. 4443 * @param[in] action 4444 * The action to be check if exist. 4445 * @param[out] match_action_pos 4446 * Pointer to the position of the matched action if exists, otherwise is -1. 4447 * @param[out] qrss_action_pos 4448 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 4449 * 4450 * @return 4451 * > 0 the total number of actions. 4452 * 0 if not found match action in action list. 4453 */ 4454 static int 4455 flow_check_match_action(const struct rte_flow_action actions[], 4456 const struct rte_flow_attr *attr, 4457 enum rte_flow_action_type action, 4458 int *match_action_pos, int *qrss_action_pos) 4459 { 4460 const struct rte_flow_action_sample *sample; 4461 int actions_n = 0; 4462 int jump_flag = 0; 4463 uint32_t ratio = 0; 4464 int sub_type = 0; 4465 int flag = 0; 4466 4467 *match_action_pos = -1; 4468 *qrss_action_pos = -1; 4469 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4470 if (actions->type == action) { 4471 flag = 1; 4472 *match_action_pos = actions_n; 4473 } 4474 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE || 4475 actions->type == RTE_FLOW_ACTION_TYPE_RSS) 4476 *qrss_action_pos = actions_n; 4477 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP) 4478 jump_flag = 1; 4479 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) { 4480 sample = actions->conf; 4481 ratio = sample->ratio; 4482 sub_type = ((const struct rte_flow_action *) 4483 (sample->actions))->type; 4484 } 4485 actions_n++; 4486 } 4487 if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) { 4488 if (ratio == 1) { 4489 /* JUMP Action not support for Mirroring; 4490 * Mirroring support multi-destination; 4491 */ 4492 if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END) 4493 flag = 0; 4494 } 4495 } 4496 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4497 return flag ? actions_n + 1 : 0; 4498 } 4499 4500 #define SAMPLE_SUFFIX_ITEM 2 4501 4502 /** 4503 * Split the sample flow. 4504 * 4505 * As sample flow will split to two sub flow, sample flow with 4506 * sample action, the other actions will move to new suffix flow. 4507 * 4508 * Also add unique tag id with tag action in the sample flow, 4509 * the same tag id will be as match in the suffix flow. 4510 * 4511 * @param dev 4512 * Pointer to Ethernet device. 4513 * @param[in] fdb_tx 4514 * FDB egress flow flag. 4515 * @param[out] sfx_items 4516 * Suffix flow match items (list terminated by the END pattern item). 4517 * @param[in] actions 4518 * Associated actions (list terminated by the END action). 4519 * @param[out] actions_sfx 4520 * Suffix flow actions. 4521 * @param[out] actions_pre 4522 * Prefix flow actions. 4523 * @param[in] actions_n 4524 * The total number of actions. 4525 * @param[in] sample_action_pos 4526 * The sample action position. 4527 * @param[in] qrss_action_pos 4528 * The Queue/RSS action position. 4529 * @param[out] error 4530 * Perform verbose error reporting if not NULL. 4531 * 4532 * @return 4533 * 0 on success, or unique flow_id, a negative errno value 4534 * otherwise and rte_errno is set. 4535 */ 4536 static int 4537 flow_sample_split_prep(struct rte_eth_dev *dev, 4538 uint32_t fdb_tx, 4539 struct rte_flow_item sfx_items[], 4540 const struct rte_flow_action actions[], 4541 struct rte_flow_action actions_sfx[], 4542 struct rte_flow_action actions_pre[], 4543 int actions_n, 4544 int sample_action_pos, 4545 int qrss_action_pos, 4546 struct rte_flow_error *error) 4547 { 4548 struct mlx5_priv *priv = dev->data->dev_private; 4549 struct mlx5_rte_flow_action_set_tag *set_tag; 4550 struct mlx5_rte_flow_item_tag *tag_spec; 4551 struct mlx5_rte_flow_item_tag *tag_mask; 4552 uint32_t tag_id = 0; 4553 int index; 4554 int ret; 4555 4556 if (sample_action_pos < 0) 4557 return rte_flow_error_set(error, EINVAL, 4558 RTE_FLOW_ERROR_TYPE_ACTION, 4559 NULL, "invalid position of sample " 4560 "action in list"); 4561 if (!fdb_tx) { 4562 /* Prepare the prefix tag action. */ 4563 set_tag = (void *)(actions_pre + actions_n + 1); 4564 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 4565 if (ret < 0) 4566 return ret; 4567 set_tag->id = ret; 4568 mlx5_ipool_malloc(priv->sh->ipool 4569 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 4570 set_tag->data = tag_id; 4571 /* Prepare the suffix subflow items. */ 4572 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 4573 tag_spec->data = tag_id; 4574 tag_spec->id = set_tag->id; 4575 tag_mask = tag_spec + 1; 4576 tag_mask->data = UINT32_MAX; 4577 sfx_items[0] = (struct rte_flow_item){ 4578 .type = (enum rte_flow_item_type) 4579 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4580 .spec = tag_spec, 4581 .last = NULL, 4582 .mask = tag_mask, 4583 }; 4584 sfx_items[1] = (struct rte_flow_item){ 4585 .type = (enum rte_flow_item_type) 4586 RTE_FLOW_ITEM_TYPE_END, 4587 }; 4588 } 4589 /* Prepare the actions for prefix and suffix flow. */ 4590 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 4591 index = qrss_action_pos; 4592 /* Put the preceding the Queue/RSS action into prefix flow. */ 4593 if (index != 0) 4594 memcpy(actions_pre, actions, 4595 sizeof(struct rte_flow_action) * index); 4596 /* Put others preceding the sample action into prefix flow. */ 4597 if (sample_action_pos > index + 1) 4598 memcpy(actions_pre + index, actions + index + 1, 4599 sizeof(struct rte_flow_action) * 4600 (sample_action_pos - index - 1)); 4601 index = sample_action_pos - 1; 4602 /* Put Queue/RSS action into Suffix flow. */ 4603 memcpy(actions_sfx, actions + qrss_action_pos, 4604 sizeof(struct rte_flow_action)); 4605 actions_sfx++; 4606 } else { 4607 index = sample_action_pos; 4608 if (index != 0) 4609 memcpy(actions_pre, actions, 4610 sizeof(struct rte_flow_action) * index); 4611 } 4612 /* Add the extra tag action for NIC-RX and E-Switch ingress. */ 4613 if (!fdb_tx) { 4614 actions_pre[index++] = 4615 (struct rte_flow_action){ 4616 .type = (enum rte_flow_action_type) 4617 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4618 .conf = set_tag, 4619 }; 4620 } 4621 memcpy(actions_pre + index, actions + sample_action_pos, 4622 sizeof(struct rte_flow_action)); 4623 index += 1; 4624 actions_pre[index] = (struct rte_flow_action){ 4625 .type = (enum rte_flow_action_type) 4626 RTE_FLOW_ACTION_TYPE_END, 4627 }; 4628 /* Put the actions after sample into Suffix flow. */ 4629 memcpy(actions_sfx, actions + sample_action_pos + 1, 4630 sizeof(struct rte_flow_action) * 4631 (actions_n - sample_action_pos - 1)); 4632 return tag_id; 4633 } 4634 4635 /** 4636 * The splitting for metadata feature. 4637 * 4638 * - Q/RSS action on NIC Rx should be split in order to pass by 4639 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4640 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4641 * 4642 * - All the actions on NIC Tx should have a mreg copy action to 4643 * copy reg_a from WQE to reg_c[0]. 4644 * 4645 * @param dev 4646 * Pointer to Ethernet device. 4647 * @param[in] flow 4648 * Parent flow structure pointer. 4649 * @param[in] attr 4650 * Flow rule attributes. 4651 * @param[in] items 4652 * Pattern specification (list terminated by the END pattern item). 4653 * @param[in] actions 4654 * Associated actions (list terminated by the END action). 4655 * @param[in] flow_split_info 4656 * Pointer to flow split info structure. 4657 * @param[out] error 4658 * Perform verbose error reporting if not NULL. 4659 * @return 4660 * 0 on success, negative value otherwise 4661 */ 4662 static int 4663 flow_create_split_metadata(struct rte_eth_dev *dev, 4664 struct rte_flow *flow, 4665 const struct rte_flow_attr *attr, 4666 const struct rte_flow_item items[], 4667 const struct rte_flow_action actions[], 4668 struct mlx5_flow_split_info *flow_split_info, 4669 struct rte_flow_error *error) 4670 { 4671 struct mlx5_priv *priv = dev->data->dev_private; 4672 struct mlx5_dev_config *config = &priv->config; 4673 const struct rte_flow_action *qrss = NULL; 4674 struct rte_flow_action *ext_actions = NULL; 4675 struct mlx5_flow *dev_flow = NULL; 4676 uint32_t qrss_id = 0; 4677 int mtr_sfx = 0; 4678 size_t act_size; 4679 int actions_n; 4680 int encap_idx; 4681 int ret; 4682 4683 /* Check whether extensive metadata feature is engaged. */ 4684 if (!config->dv_flow_en || 4685 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4686 !mlx5_flow_ext_mreg_supported(dev)) 4687 return flow_create_split_inner(dev, flow, NULL, attr, items, 4688 actions, flow_split_info, error); 4689 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 4690 &encap_idx); 4691 if (qrss) { 4692 /* Exclude hairpin flows from splitting. */ 4693 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 4694 const struct rte_flow_action_queue *queue; 4695 4696 queue = qrss->conf; 4697 if (mlx5_rxq_get_type(dev, queue->index) == 4698 MLX5_RXQ_TYPE_HAIRPIN) 4699 qrss = NULL; 4700 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 4701 const struct rte_flow_action_rss *rss; 4702 4703 rss = qrss->conf; 4704 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 4705 MLX5_RXQ_TYPE_HAIRPIN) 4706 qrss = NULL; 4707 } 4708 } 4709 if (qrss) { 4710 /* Check if it is in meter suffix table. */ 4711 mtr_sfx = attr->group == (attr->transfer ? 4712 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4713 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 4714 /* 4715 * Q/RSS action on NIC Rx should be split in order to pass by 4716 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4717 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4718 */ 4719 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4720 sizeof(struct rte_flow_action_set_tag) + 4721 sizeof(struct rte_flow_action_jump); 4722 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4723 SOCKET_ID_ANY); 4724 if (!ext_actions) 4725 return rte_flow_error_set(error, ENOMEM, 4726 RTE_FLOW_ERROR_TYPE_ACTION, 4727 NULL, "no memory to split " 4728 "metadata flow"); 4729 /* 4730 * If we are the suffix flow of meter, tag already exist. 4731 * Set the tag action to void. 4732 */ 4733 if (mtr_sfx) 4734 ext_actions[qrss - actions].type = 4735 RTE_FLOW_ACTION_TYPE_VOID; 4736 else 4737 ext_actions[qrss - actions].type = 4738 (enum rte_flow_action_type) 4739 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4740 /* 4741 * Create the new actions list with removed Q/RSS action 4742 * and appended set tag and jump to register copy table 4743 * (RX_CP_TBL). We should preallocate unique tag ID here 4744 * in advance, because it is needed for set tag action. 4745 */ 4746 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 4747 qrss, actions_n, error); 4748 if (!mtr_sfx && !qrss_id) { 4749 ret = -rte_errno; 4750 goto exit; 4751 } 4752 } else if (attr->egress && !attr->transfer) { 4753 /* 4754 * All the actions on NIC Tx should have a metadata register 4755 * copy action to copy reg_a from WQE to reg_c[meta] 4756 */ 4757 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4758 sizeof(struct mlx5_flow_action_copy_mreg); 4759 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4760 SOCKET_ID_ANY); 4761 if (!ext_actions) 4762 return rte_flow_error_set(error, ENOMEM, 4763 RTE_FLOW_ERROR_TYPE_ACTION, 4764 NULL, "no memory to split " 4765 "metadata flow"); 4766 /* Create the action list appended with copy register. */ 4767 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 4768 actions_n, error, encap_idx); 4769 if (ret < 0) 4770 goto exit; 4771 } 4772 /* Add the unmodified original or prefix subflow. */ 4773 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 4774 items, ext_actions ? ext_actions : 4775 actions, flow_split_info, error); 4776 if (ret < 0) 4777 goto exit; 4778 MLX5_ASSERT(dev_flow); 4779 if (qrss) { 4780 const struct rte_flow_attr q_attr = { 4781 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4782 .ingress = 1, 4783 }; 4784 /* Internal PMD action to set register. */ 4785 struct mlx5_rte_flow_item_tag q_tag_spec = { 4786 .data = qrss_id, 4787 .id = REG_NON, 4788 }; 4789 struct rte_flow_item q_items[] = { 4790 { 4791 .type = (enum rte_flow_item_type) 4792 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4793 .spec = &q_tag_spec, 4794 .last = NULL, 4795 .mask = NULL, 4796 }, 4797 { 4798 .type = RTE_FLOW_ITEM_TYPE_END, 4799 }, 4800 }; 4801 struct rte_flow_action q_actions[] = { 4802 { 4803 .type = qrss->type, 4804 .conf = qrss->conf, 4805 }, 4806 { 4807 .type = RTE_FLOW_ACTION_TYPE_END, 4808 }, 4809 }; 4810 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4811 4812 /* 4813 * Configure the tag item only if there is no meter subflow. 4814 * Since tag is already marked in the meter suffix subflow 4815 * we can just use the meter suffix items as is. 4816 */ 4817 if (qrss_id) { 4818 /* Not meter subflow. */ 4819 MLX5_ASSERT(!mtr_sfx); 4820 /* 4821 * Put unique id in prefix flow due to it is destroyed 4822 * after suffix flow and id will be freed after there 4823 * is no actual flows with this id and identifier 4824 * reallocation becomes possible (for example, for 4825 * other flows in other threads). 4826 */ 4827 dev_flow->handle->split_flow_id = qrss_id; 4828 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4829 error); 4830 if (ret < 0) 4831 goto exit; 4832 q_tag_spec.id = ret; 4833 } 4834 dev_flow = NULL; 4835 /* Add suffix subflow to execute Q/RSS. */ 4836 flow_split_info->prefix_layers = layers; 4837 flow_split_info->prefix_mark = 0; 4838 ret = flow_create_split_inner(dev, flow, &dev_flow, 4839 &q_attr, mtr_sfx ? items : 4840 q_items, q_actions, 4841 flow_split_info, error); 4842 if (ret < 0) 4843 goto exit; 4844 /* qrss ID should be freed if failed. */ 4845 qrss_id = 0; 4846 MLX5_ASSERT(dev_flow); 4847 } 4848 4849 exit: 4850 /* 4851 * We do not destroy the partially created sub_flows in case of error. 4852 * These ones are included into parent flow list and will be destroyed 4853 * by flow_drv_destroy. 4854 */ 4855 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 4856 qrss_id); 4857 mlx5_free(ext_actions); 4858 return ret; 4859 } 4860 4861 /** 4862 * The splitting for meter feature. 4863 * 4864 * - The meter flow will be split to two flows as prefix and 4865 * suffix flow. The packets make sense only it pass the prefix 4866 * meter action. 4867 * 4868 * - Reg_C_5 is used for the packet to match betweend prefix and 4869 * suffix flow. 4870 * 4871 * @param dev 4872 * Pointer to Ethernet device. 4873 * @param[in] flow 4874 * Parent flow structure pointer. 4875 * @param[in] attr 4876 * Flow rule attributes. 4877 * @param[in] items 4878 * Pattern specification (list terminated by the END pattern item). 4879 * @param[in] actions 4880 * Associated actions (list terminated by the END action). 4881 * @param[in] flow_split_info 4882 * Pointer to flow split info structure. 4883 * @param[out] error 4884 * Perform verbose error reporting if not NULL. 4885 * @return 4886 * 0 on success, negative value otherwise 4887 */ 4888 static int 4889 flow_create_split_meter(struct rte_eth_dev *dev, 4890 struct rte_flow *flow, 4891 const struct rte_flow_attr *attr, 4892 const struct rte_flow_item items[], 4893 const struct rte_flow_action actions[], 4894 struct mlx5_flow_split_info *flow_split_info, 4895 struct rte_flow_error *error) 4896 { 4897 struct mlx5_priv *priv = dev->data->dev_private; 4898 struct rte_flow_action *sfx_actions = NULL; 4899 struct rte_flow_action *pre_actions = NULL; 4900 struct rte_flow_item *sfx_items = NULL; 4901 struct mlx5_flow *dev_flow = NULL; 4902 struct rte_flow_attr sfx_attr = *attr; 4903 uint32_t mtr = 0; 4904 uint32_t mtr_tag_id = 0; 4905 size_t act_size; 4906 size_t item_size; 4907 int actions_n = 0; 4908 int ret; 4909 4910 if (priv->mtr_en) 4911 actions_n = flow_check_meter_action(actions, &mtr); 4912 if (mtr) { 4913 /* The five prefix actions: meter, decap, encap, tag, end. */ 4914 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4915 sizeof(struct mlx5_rte_flow_action_set_tag); 4916 /* tag, vlan, port id, end. */ 4917 #define METER_SUFFIX_ITEM 4 4918 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4919 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4920 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 4921 0, SOCKET_ID_ANY); 4922 if (!sfx_actions) 4923 return rte_flow_error_set(error, ENOMEM, 4924 RTE_FLOW_ERROR_TYPE_ACTION, 4925 NULL, "no memory to split " 4926 "meter flow"); 4927 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4928 act_size); 4929 pre_actions = sfx_actions + actions_n; 4930 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4931 actions, sfx_actions, 4932 pre_actions); 4933 if (!mtr_tag_id) { 4934 ret = -rte_errno; 4935 goto exit; 4936 } 4937 /* Add the prefix subflow. */ 4938 flow_split_info->prefix_mark = 0; 4939 ret = flow_create_split_inner(dev, flow, &dev_flow, 4940 attr, items, pre_actions, 4941 flow_split_info, error); 4942 if (ret) { 4943 ret = -rte_errno; 4944 goto exit; 4945 } 4946 dev_flow->handle->split_flow_id = mtr_tag_id; 4947 /* Setting the sfx group atrr. */ 4948 sfx_attr.group = sfx_attr.transfer ? 4949 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4950 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4951 flow_split_info->prefix_layers = 4952 flow_get_prefix_layer_flags(dev_flow); 4953 flow_split_info->prefix_mark = dev_flow->handle->mark; 4954 } 4955 /* Add the prefix subflow. */ 4956 ret = flow_create_split_metadata(dev, flow, 4957 &sfx_attr, sfx_items ? 4958 sfx_items : items, 4959 sfx_actions ? sfx_actions : actions, 4960 flow_split_info, error); 4961 exit: 4962 if (sfx_actions) 4963 mlx5_free(sfx_actions); 4964 return ret; 4965 } 4966 4967 /** 4968 * The splitting for sample feature. 4969 * 4970 * Once Sample action is detected in the action list, the flow actions should 4971 * be split into prefix sub flow and suffix sub flow. 4972 * 4973 * The original items remain in the prefix sub flow, all actions preceding the 4974 * sample action and the sample action itself will be copied to the prefix 4975 * sub flow, the actions following the sample action will be copied to the 4976 * suffix sub flow, Queue action always be located in the suffix sub flow. 4977 * 4978 * In order to make the packet from prefix sub flow matches with suffix sub 4979 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 4980 * flow uses tag item with the unique flow id. 4981 * 4982 * @param dev 4983 * Pointer to Ethernet device. 4984 * @param[in] flow 4985 * Parent flow structure pointer. 4986 * @param[in] attr 4987 * Flow rule attributes. 4988 * @param[in] items 4989 * Pattern specification (list terminated by the END pattern item). 4990 * @param[in] actions 4991 * Associated actions (list terminated by the END action). 4992 * @param[in] flow_split_info 4993 * Pointer to flow split info structure. 4994 * @param[out] error 4995 * Perform verbose error reporting if not NULL. 4996 * @return 4997 * 0 on success, negative value otherwise 4998 */ 4999 static int 5000 flow_create_split_sample(struct rte_eth_dev *dev, 5001 struct rte_flow *flow, 5002 const struct rte_flow_attr *attr, 5003 const struct rte_flow_item items[], 5004 const struct rte_flow_action actions[], 5005 struct mlx5_flow_split_info *flow_split_info, 5006 struct rte_flow_error *error) 5007 { 5008 struct mlx5_priv *priv = dev->data->dev_private; 5009 struct rte_flow_action *sfx_actions = NULL; 5010 struct rte_flow_action *pre_actions = NULL; 5011 struct rte_flow_item *sfx_items = NULL; 5012 struct mlx5_flow *dev_flow = NULL; 5013 struct rte_flow_attr sfx_attr = *attr; 5014 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5015 struct mlx5_flow_dv_sample_resource *sample_res; 5016 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5017 struct mlx5_flow_tbl_resource *sfx_tbl; 5018 union mlx5_flow_tbl_key sfx_table_key; 5019 #endif 5020 size_t act_size; 5021 size_t item_size; 5022 uint32_t fdb_tx = 0; 5023 int32_t tag_id = 0; 5024 int actions_n = 0; 5025 int sample_action_pos; 5026 int qrss_action_pos; 5027 int ret = 0; 5028 5029 if (priv->sampler_en) 5030 actions_n = flow_check_match_action(actions, attr, 5031 RTE_FLOW_ACTION_TYPE_SAMPLE, 5032 &sample_action_pos, &qrss_action_pos); 5033 if (actions_n) { 5034 /* The prefix actions must includes sample, tag, end. */ 5035 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5036 + sizeof(struct mlx5_rte_flow_action_set_tag); 5037 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5038 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5039 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5040 item_size), 0, SOCKET_ID_ANY); 5041 if (!sfx_actions) 5042 return rte_flow_error_set(error, ENOMEM, 5043 RTE_FLOW_ERROR_TYPE_ACTION, 5044 NULL, "no memory to split " 5045 "sample flow"); 5046 /* The representor_id is -1 for uplink. */ 5047 fdb_tx = (attr->transfer && priv->representor_id != -1); 5048 if (!fdb_tx) 5049 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5050 + act_size); 5051 pre_actions = sfx_actions + actions_n; 5052 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items, 5053 actions, sfx_actions, 5054 pre_actions, actions_n, 5055 sample_action_pos, 5056 qrss_action_pos, error); 5057 if (tag_id < 0 || (!fdb_tx && !tag_id)) { 5058 ret = -rte_errno; 5059 goto exit; 5060 } 5061 /* Add the prefix subflow. */ 5062 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5063 items, pre_actions, 5064 flow_split_info, error); 5065 if (ret) { 5066 ret = -rte_errno; 5067 goto exit; 5068 } 5069 dev_flow->handle->split_flow_id = tag_id; 5070 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5071 /* Set the sfx group attr. */ 5072 sample_res = (struct mlx5_flow_dv_sample_resource *) 5073 dev_flow->dv.sample_res; 5074 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5075 sample_res->normal_path_tbl; 5076 sfx_tbl_data = container_of(sfx_tbl, 5077 struct mlx5_flow_tbl_data_entry, tbl); 5078 sfx_table_key.v64 = sfx_tbl_data->entry.key; 5079 sfx_attr.group = sfx_attr.transfer ? 5080 (sfx_table_key.table_id - 1) : 5081 sfx_table_key.table_id; 5082 flow_split_info->prefix_layers = 5083 flow_get_prefix_layer_flags(dev_flow); 5084 flow_split_info->prefix_mark = dev_flow->handle->mark; 5085 /* Suffix group level already be scaled with factor, set 5086 * skip_scale to 1 to avoid scale again in translation. 5087 */ 5088 flow_split_info->skip_scale = 1; 5089 #endif 5090 } 5091 /* Add the suffix subflow. */ 5092 ret = flow_create_split_meter(dev, flow, &sfx_attr, 5093 sfx_items ? sfx_items : items, 5094 sfx_actions ? sfx_actions : actions, 5095 flow_split_info, error); 5096 exit: 5097 if (sfx_actions) 5098 mlx5_free(sfx_actions); 5099 return ret; 5100 } 5101 5102 /** 5103 * Split the flow to subflow set. The splitters might be linked 5104 * in the chain, like this: 5105 * flow_create_split_outer() calls: 5106 * flow_create_split_meter() calls: 5107 * flow_create_split_metadata(meter_subflow_0) calls: 5108 * flow_create_split_inner(metadata_subflow_0) 5109 * flow_create_split_inner(metadata_subflow_1) 5110 * flow_create_split_inner(metadata_subflow_2) 5111 * flow_create_split_metadata(meter_subflow_1) calls: 5112 * flow_create_split_inner(metadata_subflow_0) 5113 * flow_create_split_inner(metadata_subflow_1) 5114 * flow_create_split_inner(metadata_subflow_2) 5115 * 5116 * This provide flexible way to add new levels of flow splitting. 5117 * The all of successfully created subflows are included to the 5118 * parent flow dev_flow list. 5119 * 5120 * @param dev 5121 * Pointer to Ethernet device. 5122 * @param[in] flow 5123 * Parent flow structure pointer. 5124 * @param[in] attr 5125 * Flow rule attributes. 5126 * @param[in] items 5127 * Pattern specification (list terminated by the END pattern item). 5128 * @param[in] actions 5129 * Associated actions (list terminated by the END action). 5130 * @param[in] flow_split_info 5131 * Pointer to flow split info structure. 5132 * @param[out] error 5133 * Perform verbose error reporting if not NULL. 5134 * @return 5135 * 0 on success, negative value otherwise 5136 */ 5137 static int 5138 flow_create_split_outer(struct rte_eth_dev *dev, 5139 struct rte_flow *flow, 5140 const struct rte_flow_attr *attr, 5141 const struct rte_flow_item items[], 5142 const struct rte_flow_action actions[], 5143 struct mlx5_flow_split_info *flow_split_info, 5144 struct rte_flow_error *error) 5145 { 5146 int ret; 5147 5148 ret = flow_create_split_sample(dev, flow, attr, items, 5149 actions, flow_split_info, error); 5150 MLX5_ASSERT(ret <= 0); 5151 return ret; 5152 } 5153 5154 static struct mlx5_flow_tunnel * 5155 flow_tunnel_from_rule(struct rte_eth_dev *dev, 5156 const struct rte_flow_attr *attr, 5157 const struct rte_flow_item items[], 5158 const struct rte_flow_action actions[]) 5159 { 5160 struct mlx5_flow_tunnel *tunnel; 5161 5162 #pragma GCC diagnostic push 5163 #pragma GCC diagnostic ignored "-Wcast-qual" 5164 if (is_flow_tunnel_match_rule(dev, attr, items, actions)) 5165 tunnel = (struct mlx5_flow_tunnel *)items[0].spec; 5166 else if (is_flow_tunnel_steer_rule(dev, attr, items, actions)) 5167 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf; 5168 else 5169 tunnel = NULL; 5170 #pragma GCC diagnostic pop 5171 5172 return tunnel; 5173 } 5174 5175 /** 5176 * Adjust flow RSS workspace if needed. 5177 * 5178 * @param wks 5179 * Pointer to thread flow work space. 5180 * @param rss_desc 5181 * Pointer to RSS descriptor. 5182 * @param[in] nrssq_num 5183 * New RSS queue number. 5184 * 5185 * @return 5186 * 0 on success, -1 otherwise and rte_errno is set. 5187 */ 5188 static int 5189 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 5190 struct mlx5_flow_rss_desc *rss_desc, 5191 uint32_t nrssq_num) 5192 { 5193 if (likely(nrssq_num <= wks->rssq_num)) 5194 return 0; 5195 rss_desc->queue = realloc(rss_desc->queue, 5196 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 5197 if (!rss_desc->queue) { 5198 rte_errno = ENOMEM; 5199 return -1; 5200 } 5201 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 5202 return 0; 5203 } 5204 5205 /** 5206 * Create a flow and add it to @p list. 5207 * 5208 * @param dev 5209 * Pointer to Ethernet device. 5210 * @param list 5211 * Pointer to a TAILQ flow list. If this parameter NULL, 5212 * no list insertion occurred, flow is just created, 5213 * this is caller's responsibility to track the 5214 * created flow. 5215 * @param[in] attr 5216 * Flow rule attributes. 5217 * @param[in] items 5218 * Pattern specification (list terminated by the END pattern item). 5219 * @param[in] actions 5220 * Associated actions (list terminated by the END action). 5221 * @param[in] external 5222 * This flow rule is created by request external to PMD. 5223 * @param[out] error 5224 * Perform verbose error reporting if not NULL. 5225 * 5226 * @return 5227 * A flow index on success, 0 otherwise and rte_errno is set. 5228 */ 5229 static uint32_t 5230 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 5231 const struct rte_flow_attr *attr, 5232 const struct rte_flow_item items[], 5233 const struct rte_flow_action original_actions[], 5234 bool external, struct rte_flow_error *error) 5235 { 5236 struct mlx5_priv *priv = dev->data->dev_private; 5237 struct rte_flow *flow = NULL; 5238 struct mlx5_flow *dev_flow; 5239 const struct rte_flow_action_rss *rss; 5240 struct mlx5_translated_shared_action 5241 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5242 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5243 union { 5244 struct mlx5_flow_expand_rss buf; 5245 uint8_t buffer[2048]; 5246 } expand_buffer; 5247 union { 5248 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5249 uint8_t buffer[2048]; 5250 } actions_rx; 5251 union { 5252 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5253 uint8_t buffer[2048]; 5254 } actions_hairpin_tx; 5255 union { 5256 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 5257 uint8_t buffer[2048]; 5258 } items_tx; 5259 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 5260 struct mlx5_flow_rss_desc *rss_desc; 5261 const struct rte_flow_action *p_actions_rx; 5262 uint32_t i; 5263 uint32_t idx = 0; 5264 int hairpin_flow; 5265 struct rte_flow_attr attr_tx = { .priority = 0 }; 5266 const struct rte_flow_action *actions; 5267 struct rte_flow_action *translated_actions = NULL; 5268 struct mlx5_flow_tunnel *tunnel; 5269 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 5270 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 5271 struct mlx5_flow_split_info flow_split_info = { 5272 .external = !!external, 5273 .skip_scale = 0, 5274 .flow_idx = 0, 5275 .prefix_mark = 0, 5276 .prefix_layers = 0 5277 }; 5278 int ret; 5279 5280 MLX5_ASSERT(wks); 5281 rss_desc = &wks->rss_desc; 5282 ret = flow_shared_actions_translate(dev, original_actions, 5283 shared_actions, 5284 &shared_actions_n, 5285 &translated_actions, error); 5286 if (ret < 0) { 5287 MLX5_ASSERT(translated_actions == NULL); 5288 return 0; 5289 } 5290 actions = translated_actions ? translated_actions : original_actions; 5291 p_actions_rx = actions; 5292 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5293 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 5294 external, hairpin_flow, error); 5295 if (ret < 0) 5296 goto error_before_hairpin_split; 5297 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 5298 if (!flow) { 5299 rte_errno = ENOMEM; 5300 goto error_before_hairpin_split; 5301 } 5302 if (hairpin_flow > 0) { 5303 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 5304 rte_errno = EINVAL; 5305 goto error_before_hairpin_split; 5306 } 5307 flow_hairpin_split(dev, actions, actions_rx.actions, 5308 actions_hairpin_tx.actions, items_tx.items, 5309 idx); 5310 p_actions_rx = actions_rx.actions; 5311 } 5312 flow_split_info.flow_idx = idx; 5313 flow->drv_type = flow_get_drv_type(dev, attr); 5314 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 5315 flow->drv_type < MLX5_FLOW_TYPE_MAX); 5316 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 5317 rss = flow_get_rss_action(p_actions_rx); 5318 if (rss) { 5319 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 5320 return 0; 5321 /* 5322 * The following information is required by 5323 * mlx5_flow_hashfields_adjust() in advance. 5324 */ 5325 rss_desc->level = rss->level; 5326 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 5327 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 5328 } 5329 flow->dev_handles = 0; 5330 if (rss && rss->types) { 5331 unsigned int graph_root; 5332 5333 graph_root = find_graph_root(items, rss->level); 5334 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 5335 items, rss->types, 5336 mlx5_support_expansion, graph_root); 5337 MLX5_ASSERT(ret > 0 && 5338 (unsigned int)ret < sizeof(expand_buffer.buffer)); 5339 } else { 5340 buf->entries = 1; 5341 buf->entry[0].pattern = (void *)(uintptr_t)items; 5342 } 5343 rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions, 5344 shared_actions_n); 5345 for (i = 0; i < buf->entries; ++i) { 5346 /* Initialize flow split data. */ 5347 flow_split_info.prefix_layers = 0; 5348 flow_split_info.prefix_mark = 0; 5349 flow_split_info.skip_scale = 0; 5350 /* 5351 * The splitter may create multiple dev_flows, 5352 * depending on configuration. In the simplest 5353 * case it just creates unmodified original flow. 5354 */ 5355 ret = flow_create_split_outer(dev, flow, attr, 5356 buf->entry[i].pattern, 5357 p_actions_rx, &flow_split_info, 5358 error); 5359 if (ret < 0) 5360 goto error; 5361 if (is_flow_tunnel_steer_rule(dev, attr, 5362 buf->entry[i].pattern, 5363 p_actions_rx)) { 5364 ret = flow_tunnel_add_default_miss(dev, flow, attr, 5365 p_actions_rx, 5366 idx, 5367 &default_miss_ctx, 5368 error); 5369 if (ret < 0) { 5370 mlx5_free(default_miss_ctx.queue); 5371 goto error; 5372 } 5373 } 5374 } 5375 /* Create the tx flow. */ 5376 if (hairpin_flow) { 5377 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 5378 attr_tx.ingress = 0; 5379 attr_tx.egress = 1; 5380 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 5381 actions_hairpin_tx.actions, 5382 idx, error); 5383 if (!dev_flow) 5384 goto error; 5385 dev_flow->flow = flow; 5386 dev_flow->external = 0; 5387 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5388 dev_flow->handle, next); 5389 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 5390 items_tx.items, 5391 actions_hairpin_tx.actions, error); 5392 if (ret < 0) 5393 goto error; 5394 } 5395 /* 5396 * Update the metadata register copy table. If extensive 5397 * metadata feature is enabled and registers are supported 5398 * we might create the extra rte_flow for each unique 5399 * MARK/FLAG action ID. 5400 * 5401 * The table is updated for ingress Flows only, because 5402 * the egress Flows belong to the different device and 5403 * copy table should be updated in peer NIC Rx domain. 5404 */ 5405 if (attr->ingress && 5406 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 5407 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 5408 if (ret) 5409 goto error; 5410 } 5411 /* 5412 * If the flow is external (from application) OR device is started, 5413 * OR mreg discover, then apply immediately. 5414 */ 5415 if (external || dev->data->dev_started || 5416 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 5417 attr->priority == MLX5_FLOW_PRIO_RSVD)) { 5418 ret = flow_drv_apply(dev, flow, error); 5419 if (ret < 0) 5420 goto error; 5421 } 5422 if (list) { 5423 rte_spinlock_lock(&priv->flow_list_lock); 5424 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 5425 flow, next); 5426 rte_spinlock_unlock(&priv->flow_list_lock); 5427 } 5428 flow_rxq_flags_set(dev, flow); 5429 rte_free(translated_actions); 5430 tunnel = flow_tunnel_from_rule(dev, attr, items, actions); 5431 if (tunnel) { 5432 flow->tunnel = 1; 5433 flow->tunnel_id = tunnel->tunnel_id; 5434 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 5435 mlx5_free(default_miss_ctx.queue); 5436 } 5437 mlx5_flow_pop_thread_workspace(); 5438 return idx; 5439 error: 5440 MLX5_ASSERT(flow); 5441 ret = rte_errno; /* Save rte_errno before cleanup. */ 5442 flow_mreg_del_copy_action(dev, flow); 5443 flow_drv_destroy(dev, flow); 5444 if (rss_desc->shared_rss) 5445 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 5446 mlx5_ipool_get 5447 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 5448 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 5449 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 5450 rte_errno = ret; /* Restore rte_errno. */ 5451 ret = rte_errno; 5452 rte_errno = ret; 5453 mlx5_flow_pop_thread_workspace(); 5454 error_before_hairpin_split: 5455 rte_free(translated_actions); 5456 return 0; 5457 } 5458 5459 /** 5460 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 5461 * incoming packets to table 1. 5462 * 5463 * Other flow rules, requested for group n, will be created in 5464 * e-switch table n+1. 5465 * Jump action to e-switch group n will be created to group n+1. 5466 * 5467 * Used when working in switchdev mode, to utilise advantages of table 1 5468 * and above. 5469 * 5470 * @param dev 5471 * Pointer to Ethernet device. 5472 * 5473 * @return 5474 * Pointer to flow on success, NULL otherwise and rte_errno is set. 5475 */ 5476 struct rte_flow * 5477 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 5478 { 5479 const struct rte_flow_attr attr = { 5480 .group = 0, 5481 .priority = 0, 5482 .ingress = 1, 5483 .egress = 0, 5484 .transfer = 1, 5485 }; 5486 const struct rte_flow_item pattern = { 5487 .type = RTE_FLOW_ITEM_TYPE_END, 5488 }; 5489 struct rte_flow_action_jump jump = { 5490 .group = 1, 5491 }; 5492 const struct rte_flow_action actions[] = { 5493 { 5494 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5495 .conf = &jump, 5496 }, 5497 { 5498 .type = RTE_FLOW_ACTION_TYPE_END, 5499 }, 5500 }; 5501 struct mlx5_priv *priv = dev->data->dev_private; 5502 struct rte_flow_error error; 5503 5504 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 5505 &attr, &pattern, 5506 actions, false, &error); 5507 } 5508 5509 /** 5510 * Validate a flow supported by the NIC. 5511 * 5512 * @see rte_flow_validate() 5513 * @see rte_flow_ops 5514 */ 5515 int 5516 mlx5_flow_validate(struct rte_eth_dev *dev, 5517 const struct rte_flow_attr *attr, 5518 const struct rte_flow_item items[], 5519 const struct rte_flow_action original_actions[], 5520 struct rte_flow_error *error) 5521 { 5522 int hairpin_flow; 5523 struct mlx5_translated_shared_action 5524 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5525 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5526 const struct rte_flow_action *actions; 5527 struct rte_flow_action *translated_actions = NULL; 5528 int ret = flow_shared_actions_translate(dev, original_actions, 5529 shared_actions, 5530 &shared_actions_n, 5531 &translated_actions, error); 5532 5533 if (ret) 5534 return ret; 5535 actions = translated_actions ? translated_actions : original_actions; 5536 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5537 ret = flow_drv_validate(dev, attr, items, actions, 5538 true, hairpin_flow, error); 5539 rte_free(translated_actions); 5540 return ret; 5541 } 5542 5543 /** 5544 * Create a flow. 5545 * 5546 * @see rte_flow_create() 5547 * @see rte_flow_ops 5548 */ 5549 struct rte_flow * 5550 mlx5_flow_create(struct rte_eth_dev *dev, 5551 const struct rte_flow_attr *attr, 5552 const struct rte_flow_item items[], 5553 const struct rte_flow_action actions[], 5554 struct rte_flow_error *error) 5555 { 5556 struct mlx5_priv *priv = dev->data->dev_private; 5557 5558 /* 5559 * If the device is not started yet, it is not allowed to created a 5560 * flow from application. PMD default flows and traffic control flows 5561 * are not affected. 5562 */ 5563 if (unlikely(!dev->data->dev_started)) { 5564 DRV_LOG(DEBUG, "port %u is not started when " 5565 "inserting a flow", dev->data->port_id); 5566 rte_flow_error_set(error, ENODEV, 5567 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5568 NULL, 5569 "port not started"); 5570 return NULL; 5571 } 5572 5573 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 5574 attr, items, actions, true, error); 5575 } 5576 5577 /** 5578 * Destroy a flow in a list. 5579 * 5580 * @param dev 5581 * Pointer to Ethernet device. 5582 * @param list 5583 * Pointer to the Indexed flow list. If this parameter NULL, 5584 * there is no flow removal from the list. Be noted that as 5585 * flow is add to the indexed list, memory of the indexed 5586 * list points to maybe changed as flow destroyed. 5587 * @param[in] flow_idx 5588 * Index of flow to destroy. 5589 */ 5590 static void 5591 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 5592 uint32_t flow_idx) 5593 { 5594 struct mlx5_priv *priv = dev->data->dev_private; 5595 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5596 [MLX5_IPOOL_RTE_FLOW], flow_idx); 5597 5598 if (!flow) 5599 return; 5600 /* 5601 * Update RX queue flags only if port is started, otherwise it is 5602 * already clean. 5603 */ 5604 if (dev->data->dev_started) 5605 flow_rxq_flags_trim(dev, flow); 5606 flow_drv_destroy(dev, flow); 5607 if (list) { 5608 rte_spinlock_lock(&priv->flow_list_lock); 5609 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 5610 flow_idx, flow, next); 5611 rte_spinlock_unlock(&priv->flow_list_lock); 5612 } 5613 if (flow->tunnel) { 5614 struct mlx5_flow_tunnel *tunnel; 5615 5616 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 5617 RTE_VERIFY(tunnel); 5618 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 5619 mlx5_flow_tunnel_free(dev, tunnel); 5620 } 5621 flow_mreg_del_copy_action(dev, flow); 5622 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5623 } 5624 5625 /** 5626 * Destroy all flows. 5627 * 5628 * @param dev 5629 * Pointer to Ethernet device. 5630 * @param list 5631 * Pointer to the Indexed flow list. 5632 * @param active 5633 * If flushing is called avtively. 5634 */ 5635 void 5636 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 5637 { 5638 uint32_t num_flushed = 0; 5639 5640 while (*list) { 5641 flow_list_destroy(dev, list, *list); 5642 num_flushed++; 5643 } 5644 if (active) { 5645 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 5646 dev->data->port_id, num_flushed); 5647 } 5648 } 5649 5650 /** 5651 * Stop all default actions for flows. 5652 * 5653 * @param dev 5654 * Pointer to Ethernet device. 5655 */ 5656 void 5657 mlx5_flow_stop_default(struct rte_eth_dev *dev) 5658 { 5659 flow_mreg_del_default_copy_action(dev); 5660 flow_rxq_flags_clear(dev); 5661 } 5662 5663 /** 5664 * Start all default actions for flows. 5665 * 5666 * @param dev 5667 * Pointer to Ethernet device. 5668 * @return 5669 * 0 on success, a negative errno value otherwise and rte_errno is set. 5670 */ 5671 int 5672 mlx5_flow_start_default(struct rte_eth_dev *dev) 5673 { 5674 struct rte_flow_error error; 5675 5676 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 5677 return flow_mreg_add_default_copy_action(dev, &error); 5678 } 5679 5680 /** 5681 * Release key of thread specific flow workspace data. 5682 */ 5683 static void 5684 flow_release_workspace(void *data) 5685 { 5686 struct mlx5_flow_workspace *wks = data; 5687 struct mlx5_flow_workspace *next; 5688 5689 while (wks) { 5690 next = wks->next; 5691 free(wks->rss_desc.queue); 5692 free(wks); 5693 wks = next; 5694 } 5695 } 5696 5697 /** 5698 * Initialize key of thread specific flow workspace data. 5699 */ 5700 static void 5701 flow_alloc_workspace(void) 5702 { 5703 if (pthread_key_create(&key_workspace, flow_release_workspace)) 5704 DRV_LOG(ERR, "Can't create flow workspace data thread key."); 5705 } 5706 5707 /** 5708 * Get thread specific current flow workspace. 5709 * 5710 * @return pointer to thread specific flow workspace data, NULL on error. 5711 */ 5712 struct mlx5_flow_workspace* 5713 mlx5_flow_get_thread_workspace(void) 5714 { 5715 struct mlx5_flow_workspace *data; 5716 5717 data = pthread_getspecific(key_workspace); 5718 MLX5_ASSERT(data && data->inuse); 5719 if (!data || !data->inuse) 5720 DRV_LOG(ERR, "flow workspace not initialized."); 5721 return data; 5722 } 5723 5724 /** 5725 * Allocate and init new flow workspace. 5726 * 5727 * @return pointer to flow workspace data, NULL on error. 5728 */ 5729 static struct mlx5_flow_workspace* 5730 flow_alloc_thread_workspace(void) 5731 { 5732 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 5733 5734 if (!data) { 5735 DRV_LOG(ERR, "Failed to allocate flow workspace " 5736 "memory."); 5737 return NULL; 5738 } 5739 data->rss_desc.queue = calloc(1, 5740 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 5741 if (!data->rss_desc.queue) 5742 goto err; 5743 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 5744 return data; 5745 err: 5746 if (data->rss_desc.queue) 5747 free(data->rss_desc.queue); 5748 free(data); 5749 return NULL; 5750 } 5751 5752 /** 5753 * Get new thread specific flow workspace. 5754 * 5755 * If current workspace inuse, create new one and set as current. 5756 * 5757 * @return pointer to thread specific flow workspace data, NULL on error. 5758 */ 5759 static struct mlx5_flow_workspace* 5760 mlx5_flow_push_thread_workspace(void) 5761 { 5762 struct mlx5_flow_workspace *curr; 5763 struct mlx5_flow_workspace *data; 5764 5765 if (pthread_once(&key_workspace_init, flow_alloc_workspace)) { 5766 DRV_LOG(ERR, "Failed to init flow workspace data thread key."); 5767 return NULL; 5768 } 5769 curr = pthread_getspecific(key_workspace); 5770 if (!curr) { 5771 data = flow_alloc_thread_workspace(); 5772 if (!data) 5773 return NULL; 5774 } else if (!curr->inuse) { 5775 data = curr; 5776 } else if (curr->next) { 5777 data = curr->next; 5778 } else { 5779 data = flow_alloc_thread_workspace(); 5780 if (!data) 5781 return NULL; 5782 curr->next = data; 5783 data->prev = curr; 5784 } 5785 data->inuse = 1; 5786 data->flow_idx = 0; 5787 /* Set as current workspace */ 5788 if (pthread_setspecific(key_workspace, data)) 5789 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 5790 return data; 5791 } 5792 5793 /** 5794 * Close current thread specific flow workspace. 5795 * 5796 * If previous workspace available, set it as current. 5797 * 5798 * @return pointer to thread specific flow workspace data, NULL on error. 5799 */ 5800 static void 5801 mlx5_flow_pop_thread_workspace(void) 5802 { 5803 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 5804 5805 if (!data) 5806 return; 5807 if (!data->inuse) { 5808 DRV_LOG(ERR, "Failed to close unused flow workspace."); 5809 return; 5810 } 5811 data->inuse = 0; 5812 if (!data->prev) 5813 return; 5814 if (pthread_setspecific(key_workspace, data->prev)) 5815 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 5816 } 5817 5818 /** 5819 * Verify the flow list is empty 5820 * 5821 * @param dev 5822 * Pointer to Ethernet device. 5823 * 5824 * @return the number of flows not released. 5825 */ 5826 int 5827 mlx5_flow_verify(struct rte_eth_dev *dev) 5828 { 5829 struct mlx5_priv *priv = dev->data->dev_private; 5830 struct rte_flow *flow; 5831 uint32_t idx; 5832 int ret = 0; 5833 5834 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 5835 flow, next) { 5836 DRV_LOG(DEBUG, "port %u flow %p still referenced", 5837 dev->data->port_id, (void *)flow); 5838 ++ret; 5839 } 5840 return ret; 5841 } 5842 5843 /** 5844 * Enable default hairpin egress flow. 5845 * 5846 * @param dev 5847 * Pointer to Ethernet device. 5848 * @param queue 5849 * The queue index. 5850 * 5851 * @return 5852 * 0 on success, a negative errno value otherwise and rte_errno is set. 5853 */ 5854 int 5855 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 5856 uint32_t queue) 5857 { 5858 struct mlx5_priv *priv = dev->data->dev_private; 5859 const struct rte_flow_attr attr = { 5860 .egress = 1, 5861 .priority = 0, 5862 }; 5863 struct mlx5_rte_flow_item_tx_queue queue_spec = { 5864 .queue = queue, 5865 }; 5866 struct mlx5_rte_flow_item_tx_queue queue_mask = { 5867 .queue = UINT32_MAX, 5868 }; 5869 struct rte_flow_item items[] = { 5870 { 5871 .type = (enum rte_flow_item_type) 5872 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 5873 .spec = &queue_spec, 5874 .last = NULL, 5875 .mask = &queue_mask, 5876 }, 5877 { 5878 .type = RTE_FLOW_ITEM_TYPE_END, 5879 }, 5880 }; 5881 struct rte_flow_action_jump jump = { 5882 .group = MLX5_HAIRPIN_TX_TABLE, 5883 }; 5884 struct rte_flow_action actions[2]; 5885 uint32_t flow_idx; 5886 struct rte_flow_error error; 5887 5888 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 5889 actions[0].conf = &jump; 5890 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 5891 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5892 &attr, items, actions, false, &error); 5893 if (!flow_idx) { 5894 DRV_LOG(DEBUG, 5895 "Failed to create ctrl flow: rte_errno(%d)," 5896 " type(%d), message(%s)", 5897 rte_errno, error.type, 5898 error.message ? error.message : " (no stated reason)"); 5899 return -rte_errno; 5900 } 5901 return 0; 5902 } 5903 5904 /** 5905 * Enable a control flow configured from the control plane. 5906 * 5907 * @param dev 5908 * Pointer to Ethernet device. 5909 * @param eth_spec 5910 * An Ethernet flow spec to apply. 5911 * @param eth_mask 5912 * An Ethernet flow mask to apply. 5913 * @param vlan_spec 5914 * A VLAN flow spec to apply. 5915 * @param vlan_mask 5916 * A VLAN flow mask to apply. 5917 * 5918 * @return 5919 * 0 on success, a negative errno value otherwise and rte_errno is set. 5920 */ 5921 int 5922 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 5923 struct rte_flow_item_eth *eth_spec, 5924 struct rte_flow_item_eth *eth_mask, 5925 struct rte_flow_item_vlan *vlan_spec, 5926 struct rte_flow_item_vlan *vlan_mask) 5927 { 5928 struct mlx5_priv *priv = dev->data->dev_private; 5929 const struct rte_flow_attr attr = { 5930 .ingress = 1, 5931 .priority = MLX5_FLOW_PRIO_RSVD, 5932 }; 5933 struct rte_flow_item items[] = { 5934 { 5935 .type = RTE_FLOW_ITEM_TYPE_ETH, 5936 .spec = eth_spec, 5937 .last = NULL, 5938 .mask = eth_mask, 5939 }, 5940 { 5941 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 5942 RTE_FLOW_ITEM_TYPE_END, 5943 .spec = vlan_spec, 5944 .last = NULL, 5945 .mask = vlan_mask, 5946 }, 5947 { 5948 .type = RTE_FLOW_ITEM_TYPE_END, 5949 }, 5950 }; 5951 uint16_t queue[priv->reta_idx_n]; 5952 struct rte_flow_action_rss action_rss = { 5953 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 5954 .level = 0, 5955 .types = priv->rss_conf.rss_hf, 5956 .key_len = priv->rss_conf.rss_key_len, 5957 .queue_num = priv->reta_idx_n, 5958 .key = priv->rss_conf.rss_key, 5959 .queue = queue, 5960 }; 5961 struct rte_flow_action actions[] = { 5962 { 5963 .type = RTE_FLOW_ACTION_TYPE_RSS, 5964 .conf = &action_rss, 5965 }, 5966 { 5967 .type = RTE_FLOW_ACTION_TYPE_END, 5968 }, 5969 }; 5970 uint32_t flow_idx; 5971 struct rte_flow_error error; 5972 unsigned int i; 5973 5974 if (!priv->reta_idx_n || !priv->rxqs_n) { 5975 return 0; 5976 } 5977 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 5978 action_rss.types = 0; 5979 for (i = 0; i != priv->reta_idx_n; ++i) 5980 queue[i] = (*priv->reta_idx)[i]; 5981 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5982 &attr, items, actions, false, &error); 5983 if (!flow_idx) 5984 return -rte_errno; 5985 return 0; 5986 } 5987 5988 /** 5989 * Enable a flow control configured from the control plane. 5990 * 5991 * @param dev 5992 * Pointer to Ethernet device. 5993 * @param eth_spec 5994 * An Ethernet flow spec to apply. 5995 * @param eth_mask 5996 * An Ethernet flow mask to apply. 5997 * 5998 * @return 5999 * 0 on success, a negative errno value otherwise and rte_errno is set. 6000 */ 6001 int 6002 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6003 struct rte_flow_item_eth *eth_spec, 6004 struct rte_flow_item_eth *eth_mask) 6005 { 6006 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6007 } 6008 6009 /** 6010 * Create default miss flow rule matching lacp traffic 6011 * 6012 * @param dev 6013 * Pointer to Ethernet device. 6014 * @param eth_spec 6015 * An Ethernet flow spec to apply. 6016 * 6017 * @return 6018 * 0 on success, a negative errno value otherwise and rte_errno is set. 6019 */ 6020 int 6021 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6022 { 6023 struct mlx5_priv *priv = dev->data->dev_private; 6024 /* 6025 * The LACP matching is done by only using ether type since using 6026 * a multicast dst mac causes kernel to give low priority to this flow. 6027 */ 6028 static const struct rte_flow_item_eth lacp_spec = { 6029 .type = RTE_BE16(0x8809), 6030 }; 6031 static const struct rte_flow_item_eth lacp_mask = { 6032 .type = 0xffff, 6033 }; 6034 const struct rte_flow_attr attr = { 6035 .ingress = 1, 6036 }; 6037 struct rte_flow_item items[] = { 6038 { 6039 .type = RTE_FLOW_ITEM_TYPE_ETH, 6040 .spec = &lacp_spec, 6041 .mask = &lacp_mask, 6042 }, 6043 { 6044 .type = RTE_FLOW_ITEM_TYPE_END, 6045 }, 6046 }; 6047 struct rte_flow_action actions[] = { 6048 { 6049 .type = (enum rte_flow_action_type) 6050 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6051 }, 6052 { 6053 .type = RTE_FLOW_ACTION_TYPE_END, 6054 }, 6055 }; 6056 struct rte_flow_error error; 6057 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6058 &attr, items, actions, false, &error); 6059 6060 if (!flow_idx) 6061 return -rte_errno; 6062 return 0; 6063 } 6064 6065 /** 6066 * Destroy a flow. 6067 * 6068 * @see rte_flow_destroy() 6069 * @see rte_flow_ops 6070 */ 6071 int 6072 mlx5_flow_destroy(struct rte_eth_dev *dev, 6073 struct rte_flow *flow, 6074 struct rte_flow_error *error __rte_unused) 6075 { 6076 struct mlx5_priv *priv = dev->data->dev_private; 6077 6078 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6079 return 0; 6080 } 6081 6082 /** 6083 * Destroy all flows. 6084 * 6085 * @see rte_flow_flush() 6086 * @see rte_flow_ops 6087 */ 6088 int 6089 mlx5_flow_flush(struct rte_eth_dev *dev, 6090 struct rte_flow_error *error __rte_unused) 6091 { 6092 struct mlx5_priv *priv = dev->data->dev_private; 6093 6094 mlx5_flow_list_flush(dev, &priv->flows, false); 6095 return 0; 6096 } 6097 6098 /** 6099 * Isolated mode. 6100 * 6101 * @see rte_flow_isolate() 6102 * @see rte_flow_ops 6103 */ 6104 int 6105 mlx5_flow_isolate(struct rte_eth_dev *dev, 6106 int enable, 6107 struct rte_flow_error *error) 6108 { 6109 struct mlx5_priv *priv = dev->data->dev_private; 6110 6111 if (dev->data->dev_started) { 6112 rte_flow_error_set(error, EBUSY, 6113 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6114 NULL, 6115 "port must be stopped first"); 6116 return -rte_errno; 6117 } 6118 priv->isolated = !!enable; 6119 if (enable) 6120 dev->dev_ops = &mlx5_os_dev_ops_isolate; 6121 else 6122 dev->dev_ops = &mlx5_os_dev_ops; 6123 6124 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 6125 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 6126 6127 return 0; 6128 } 6129 6130 /** 6131 * Query a flow. 6132 * 6133 * @see rte_flow_query() 6134 * @see rte_flow_ops 6135 */ 6136 static int 6137 flow_drv_query(struct rte_eth_dev *dev, 6138 uint32_t flow_idx, 6139 const struct rte_flow_action *actions, 6140 void *data, 6141 struct rte_flow_error *error) 6142 { 6143 struct mlx5_priv *priv = dev->data->dev_private; 6144 const struct mlx5_flow_driver_ops *fops; 6145 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6146 [MLX5_IPOOL_RTE_FLOW], 6147 flow_idx); 6148 enum mlx5_flow_drv_type ftype; 6149 6150 if (!flow) { 6151 return rte_flow_error_set(error, ENOENT, 6152 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6153 NULL, 6154 "invalid flow handle"); 6155 } 6156 ftype = flow->drv_type; 6157 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 6158 fops = flow_get_drv_ops(ftype); 6159 6160 return fops->query(dev, flow, actions, data, error); 6161 } 6162 6163 /** 6164 * Query a flow. 6165 * 6166 * @see rte_flow_query() 6167 * @see rte_flow_ops 6168 */ 6169 int 6170 mlx5_flow_query(struct rte_eth_dev *dev, 6171 struct rte_flow *flow, 6172 const struct rte_flow_action *actions, 6173 void *data, 6174 struct rte_flow_error *error) 6175 { 6176 int ret; 6177 6178 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 6179 error); 6180 if (ret < 0) 6181 return ret; 6182 return 0; 6183 } 6184 6185 /** 6186 * Manage filter operations. 6187 * 6188 * @param dev 6189 * Pointer to Ethernet device structure. 6190 * @param filter_type 6191 * Filter type. 6192 * @param filter_op 6193 * Operation to perform. 6194 * @param arg 6195 * Pointer to operation-specific structure. 6196 * 6197 * @return 6198 * 0 on success, a negative errno value otherwise and rte_errno is set. 6199 */ 6200 int 6201 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 6202 enum rte_filter_type filter_type, 6203 enum rte_filter_op filter_op, 6204 void *arg) 6205 { 6206 switch (filter_type) { 6207 case RTE_ETH_FILTER_GENERIC: 6208 if (filter_op != RTE_ETH_FILTER_GET) { 6209 rte_errno = EINVAL; 6210 return -rte_errno; 6211 } 6212 *(const void **)arg = &mlx5_flow_ops; 6213 return 0; 6214 default: 6215 DRV_LOG(ERR, "port %u filter type (%d) not supported", 6216 dev->data->port_id, filter_type); 6217 rte_errno = ENOTSUP; 6218 return -rte_errno; 6219 } 6220 return 0; 6221 } 6222 6223 /** 6224 * Create the needed meter and suffix tables. 6225 * 6226 * @param[in] dev 6227 * Pointer to Ethernet device. 6228 * @param[in] fm 6229 * Pointer to the flow meter. 6230 * 6231 * @return 6232 * Pointer to table set on success, NULL otherwise. 6233 */ 6234 struct mlx5_meter_domains_infos * 6235 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 6236 const struct mlx5_flow_meter *fm) 6237 { 6238 const struct mlx5_flow_driver_ops *fops; 6239 6240 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6241 return fops->create_mtr_tbls(dev, fm); 6242 } 6243 6244 /** 6245 * Destroy the meter table set. 6246 * 6247 * @param[in] dev 6248 * Pointer to Ethernet device. 6249 * @param[in] tbl 6250 * Pointer to the meter table set. 6251 * 6252 * @return 6253 * 0 on success. 6254 */ 6255 int 6256 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 6257 struct mlx5_meter_domains_infos *tbls) 6258 { 6259 const struct mlx5_flow_driver_ops *fops; 6260 6261 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6262 return fops->destroy_mtr_tbls(dev, tbls); 6263 } 6264 6265 /** 6266 * Create policer rules. 6267 * 6268 * @param[in] dev 6269 * Pointer to Ethernet device. 6270 * @param[in] fm 6271 * Pointer to flow meter structure. 6272 * @param[in] attr 6273 * Pointer to flow attributes. 6274 * 6275 * @return 6276 * 0 on success, -1 otherwise. 6277 */ 6278 int 6279 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 6280 struct mlx5_flow_meter *fm, 6281 const struct rte_flow_attr *attr) 6282 { 6283 const struct mlx5_flow_driver_ops *fops; 6284 6285 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6286 return fops->create_policer_rules(dev, fm, attr); 6287 } 6288 6289 /** 6290 * Destroy policer rules. 6291 * 6292 * @param[in] fm 6293 * Pointer to flow meter structure. 6294 * @param[in] attr 6295 * Pointer to flow attributes. 6296 * 6297 * @return 6298 * 0 on success, -1 otherwise. 6299 */ 6300 int 6301 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 6302 struct mlx5_flow_meter *fm, 6303 const struct rte_flow_attr *attr) 6304 { 6305 const struct mlx5_flow_driver_ops *fops; 6306 6307 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6308 return fops->destroy_policer_rules(dev, fm, attr); 6309 } 6310 6311 /** 6312 * Allocate a counter. 6313 * 6314 * @param[in] dev 6315 * Pointer to Ethernet device structure. 6316 * 6317 * @return 6318 * Index to allocated counter on success, 0 otherwise. 6319 */ 6320 uint32_t 6321 mlx5_counter_alloc(struct rte_eth_dev *dev) 6322 { 6323 const struct mlx5_flow_driver_ops *fops; 6324 struct rte_flow_attr attr = { .transfer = 0 }; 6325 6326 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6327 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6328 return fops->counter_alloc(dev); 6329 } 6330 DRV_LOG(ERR, 6331 "port %u counter allocate is not supported.", 6332 dev->data->port_id); 6333 return 0; 6334 } 6335 6336 /** 6337 * Free a counter. 6338 * 6339 * @param[in] dev 6340 * Pointer to Ethernet device structure. 6341 * @param[in] cnt 6342 * Index to counter to be free. 6343 */ 6344 void 6345 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 6346 { 6347 const struct mlx5_flow_driver_ops *fops; 6348 struct rte_flow_attr attr = { .transfer = 0 }; 6349 6350 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6351 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6352 fops->counter_free(dev, cnt); 6353 return; 6354 } 6355 DRV_LOG(ERR, 6356 "port %u counter free is not supported.", 6357 dev->data->port_id); 6358 } 6359 6360 /** 6361 * Query counter statistics. 6362 * 6363 * @param[in] dev 6364 * Pointer to Ethernet device structure. 6365 * @param[in] cnt 6366 * Index to counter to query. 6367 * @param[in] clear 6368 * Set to clear counter statistics. 6369 * @param[out] pkts 6370 * The counter hits packets number to save. 6371 * @param[out] bytes 6372 * The counter hits bytes number to save. 6373 * 6374 * @return 6375 * 0 on success, a negative errno value otherwise. 6376 */ 6377 int 6378 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 6379 bool clear, uint64_t *pkts, uint64_t *bytes) 6380 { 6381 const struct mlx5_flow_driver_ops *fops; 6382 struct rte_flow_attr attr = { .transfer = 0 }; 6383 6384 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6385 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6386 return fops->counter_query(dev, cnt, clear, pkts, bytes); 6387 } 6388 DRV_LOG(ERR, 6389 "port %u counter query is not supported.", 6390 dev->data->port_id); 6391 return -ENOTSUP; 6392 } 6393 6394 /** 6395 * Allocate a new memory for the counter values wrapped by all the needed 6396 * management. 6397 * 6398 * @param[in] sh 6399 * Pointer to mlx5_dev_ctx_shared object. 6400 * 6401 * @return 6402 * 0 on success, a negative errno value otherwise. 6403 */ 6404 static int 6405 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 6406 { 6407 struct mlx5_devx_mkey_attr mkey_attr; 6408 struct mlx5_counter_stats_mem_mng *mem_mng; 6409 volatile struct flow_counter_stats *raw_data; 6410 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 6411 int size = (sizeof(struct flow_counter_stats) * 6412 MLX5_COUNTERS_PER_POOL + 6413 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 6414 sizeof(struct mlx5_counter_stats_mem_mng); 6415 size_t pgsize = rte_mem_page_size(); 6416 uint8_t *mem; 6417 int i; 6418 6419 if (pgsize == (size_t)-1) { 6420 DRV_LOG(ERR, "Failed to get mem page size"); 6421 rte_errno = ENOMEM; 6422 return -ENOMEM; 6423 } 6424 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 6425 if (!mem) { 6426 rte_errno = ENOMEM; 6427 return -ENOMEM; 6428 } 6429 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 6430 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 6431 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 6432 IBV_ACCESS_LOCAL_WRITE); 6433 if (!mem_mng->umem) { 6434 rte_errno = errno; 6435 mlx5_free(mem); 6436 return -rte_errno; 6437 } 6438 mkey_attr.addr = (uintptr_t)mem; 6439 mkey_attr.size = size; 6440 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 6441 mkey_attr.pd = sh->pdn; 6442 mkey_attr.log_entity_size = 0; 6443 mkey_attr.pg_access = 0; 6444 mkey_attr.klm_array = NULL; 6445 mkey_attr.klm_num = 0; 6446 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 6447 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 6448 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 6449 if (!mem_mng->dm) { 6450 mlx5_glue->devx_umem_dereg(mem_mng->umem); 6451 rte_errno = errno; 6452 mlx5_free(mem); 6453 return -rte_errno; 6454 } 6455 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 6456 raw_data = (volatile struct flow_counter_stats *)mem; 6457 for (i = 0; i < raws_n; ++i) { 6458 mem_mng->raws[i].mem_mng = mem_mng; 6459 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 6460 } 6461 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 6462 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 6463 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 6464 next); 6465 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 6466 sh->cmng.mem_mng = mem_mng; 6467 return 0; 6468 } 6469 6470 /** 6471 * Set the statistic memory to the new counter pool. 6472 * 6473 * @param[in] sh 6474 * Pointer to mlx5_dev_ctx_shared object. 6475 * @param[in] pool 6476 * Pointer to the pool to set the statistic memory. 6477 * 6478 * @return 6479 * 0 on success, a negative errno value otherwise. 6480 */ 6481 static int 6482 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 6483 struct mlx5_flow_counter_pool *pool) 6484 { 6485 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6486 /* Resize statistic memory once used out. */ 6487 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 6488 mlx5_flow_create_counter_stat_mem_mng(sh)) { 6489 DRV_LOG(ERR, "Cannot resize counter stat mem."); 6490 return -1; 6491 } 6492 rte_spinlock_lock(&pool->sl); 6493 pool->raw = cmng->mem_mng->raws + pool->index % 6494 MLX5_CNT_CONTAINER_RESIZE; 6495 rte_spinlock_unlock(&pool->sl); 6496 pool->raw_hw = NULL; 6497 return 0; 6498 } 6499 6500 #define MLX5_POOL_QUERY_FREQ_US 1000000 6501 6502 /** 6503 * Set the periodic procedure for triggering asynchronous batch queries for all 6504 * the counter pools. 6505 * 6506 * @param[in] sh 6507 * Pointer to mlx5_dev_ctx_shared object. 6508 */ 6509 void 6510 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 6511 { 6512 uint32_t pools_n, us; 6513 6514 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 6515 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 6516 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 6517 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 6518 sh->cmng.query_thread_on = 0; 6519 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 6520 } else { 6521 sh->cmng.query_thread_on = 1; 6522 } 6523 } 6524 6525 /** 6526 * The periodic procedure for triggering asynchronous batch queries for all the 6527 * counter pools. This function is probably called by the host thread. 6528 * 6529 * @param[in] arg 6530 * The parameter for the alarm process. 6531 */ 6532 void 6533 mlx5_flow_query_alarm(void *arg) 6534 { 6535 struct mlx5_dev_ctx_shared *sh = arg; 6536 int ret; 6537 uint16_t pool_index = sh->cmng.pool_index; 6538 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6539 struct mlx5_flow_counter_pool *pool; 6540 uint16_t n_valid; 6541 6542 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 6543 goto set_alarm; 6544 rte_spinlock_lock(&cmng->pool_update_sl); 6545 pool = cmng->pools[pool_index]; 6546 n_valid = cmng->n_valid; 6547 rte_spinlock_unlock(&cmng->pool_update_sl); 6548 /* Set the statistic memory to the new created pool. */ 6549 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 6550 goto set_alarm; 6551 if (pool->raw_hw) 6552 /* There is a pool query in progress. */ 6553 goto set_alarm; 6554 pool->raw_hw = 6555 LIST_FIRST(&sh->cmng.free_stat_raws); 6556 if (!pool->raw_hw) 6557 /* No free counter statistics raw memory. */ 6558 goto set_alarm; 6559 /* 6560 * Identify the counters released between query trigger and query 6561 * handle more efficiently. The counter released in this gap period 6562 * should wait for a new round of query as the new arrived packets 6563 * will not be taken into account. 6564 */ 6565 pool->query_gen++; 6566 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 6567 MLX5_COUNTERS_PER_POOL, 6568 NULL, NULL, 6569 pool->raw_hw->mem_mng->dm->id, 6570 (void *)(uintptr_t) 6571 pool->raw_hw->data, 6572 sh->devx_comp, 6573 (uint64_t)(uintptr_t)pool); 6574 if (ret) { 6575 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 6576 " %d", pool->min_dcs->id); 6577 pool->raw_hw = NULL; 6578 goto set_alarm; 6579 } 6580 LIST_REMOVE(pool->raw_hw, next); 6581 sh->cmng.pending_queries++; 6582 pool_index++; 6583 if (pool_index >= n_valid) 6584 pool_index = 0; 6585 set_alarm: 6586 sh->cmng.pool_index = pool_index; 6587 mlx5_set_query_alarm(sh); 6588 } 6589 6590 /** 6591 * Check and callback event for new aged flow in the counter pool 6592 * 6593 * @param[in] sh 6594 * Pointer to mlx5_dev_ctx_shared object. 6595 * @param[in] pool 6596 * Pointer to Current counter pool. 6597 */ 6598 static void 6599 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 6600 struct mlx5_flow_counter_pool *pool) 6601 { 6602 struct mlx5_priv *priv; 6603 struct mlx5_flow_counter *cnt; 6604 struct mlx5_age_info *age_info; 6605 struct mlx5_age_param *age_param; 6606 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 6607 struct mlx5_counter_stats_raw *prev = pool->raw; 6608 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 6609 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 6610 uint16_t expected = AGE_CANDIDATE; 6611 uint32_t i; 6612 6613 pool->time_of_last_age_check = curr_time; 6614 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 6615 cnt = MLX5_POOL_GET_CNT(pool, i); 6616 age_param = MLX5_CNT_TO_AGE(cnt); 6617 if (__atomic_load_n(&age_param->state, 6618 __ATOMIC_RELAXED) != AGE_CANDIDATE) 6619 continue; 6620 if (cur->data[i].hits != prev->data[i].hits) { 6621 __atomic_store_n(&age_param->sec_since_last_hit, 0, 6622 __ATOMIC_RELAXED); 6623 continue; 6624 } 6625 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 6626 time_delta, 6627 __ATOMIC_RELAXED) <= age_param->timeout) 6628 continue; 6629 /** 6630 * Hold the lock first, or if between the 6631 * state AGE_TMOUT and tailq operation the 6632 * release happened, the release procedure 6633 * may delete a non-existent tailq node. 6634 */ 6635 priv = rte_eth_devices[age_param->port_id].data->dev_private; 6636 age_info = GET_PORT_AGE_INFO(priv); 6637 rte_spinlock_lock(&age_info->aged_sl); 6638 if (__atomic_compare_exchange_n(&age_param->state, &expected, 6639 AGE_TMOUT, false, 6640 __ATOMIC_RELAXED, 6641 __ATOMIC_RELAXED)) { 6642 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 6643 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 6644 } 6645 rte_spinlock_unlock(&age_info->aged_sl); 6646 } 6647 mlx5_age_event_prepare(sh); 6648 } 6649 6650 /** 6651 * Handler for the HW respond about ready values from an asynchronous batch 6652 * query. This function is probably called by the host thread. 6653 * 6654 * @param[in] sh 6655 * The pointer to the shared device context. 6656 * @param[in] async_id 6657 * The Devx async ID. 6658 * @param[in] status 6659 * The status of the completion. 6660 */ 6661 void 6662 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 6663 uint64_t async_id, int status) 6664 { 6665 struct mlx5_flow_counter_pool *pool = 6666 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 6667 struct mlx5_counter_stats_raw *raw_to_free; 6668 uint8_t query_gen = pool->query_gen ^ 1; 6669 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6670 enum mlx5_counter_type cnt_type = 6671 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 6672 MLX5_COUNTER_TYPE_ORIGIN; 6673 6674 if (unlikely(status)) { 6675 raw_to_free = pool->raw_hw; 6676 } else { 6677 raw_to_free = pool->raw; 6678 if (pool->is_aged) 6679 mlx5_flow_aging_check(sh, pool); 6680 rte_spinlock_lock(&pool->sl); 6681 pool->raw = pool->raw_hw; 6682 rte_spinlock_unlock(&pool->sl); 6683 /* Be sure the new raw counters data is updated in memory. */ 6684 rte_io_wmb(); 6685 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 6686 rte_spinlock_lock(&cmng->csl[cnt_type]); 6687 TAILQ_CONCAT(&cmng->counters[cnt_type], 6688 &pool->counters[query_gen], next); 6689 rte_spinlock_unlock(&cmng->csl[cnt_type]); 6690 } 6691 } 6692 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 6693 pool->raw_hw = NULL; 6694 sh->cmng.pending_queries--; 6695 } 6696 6697 static int 6698 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 6699 const struct flow_grp_info *grp_info, 6700 struct rte_flow_error *error) 6701 { 6702 if (grp_info->transfer && grp_info->external && 6703 grp_info->fdb_def_rule) { 6704 if (group == UINT32_MAX) 6705 return rte_flow_error_set 6706 (error, EINVAL, 6707 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6708 NULL, 6709 "group index not supported"); 6710 *table = group + 1; 6711 } else { 6712 *table = group; 6713 } 6714 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 6715 return 0; 6716 } 6717 6718 /** 6719 * Translate the rte_flow group index to HW table value. 6720 * 6721 * If tunnel offload is disabled, all group ids converted to flow table 6722 * id using the standard method. 6723 * If tunnel offload is enabled, group id can be converted using the 6724 * standard or tunnel conversion method. Group conversion method 6725 * selection depends on flags in `grp_info` parameter: 6726 * - Internal (grp_info.external == 0) groups conversion uses the 6727 * standard method. 6728 * - Group ids in JUMP action converted with the tunnel conversion. 6729 * - Group id in rule attribute conversion depends on a rule type and 6730 * group id value: 6731 * ** non zero group attributes converted with the tunnel method 6732 * ** zero group attribute in non-tunnel rule is converted using the 6733 * standard method - there's only one root table 6734 * ** zero group attribute in steer tunnel rule is converted with the 6735 * standard method - single root table 6736 * ** zero group attribute in match tunnel rule is a special OvS 6737 * case: that value is used for portability reasons. That group 6738 * id is converted with the tunnel conversion method. 6739 * 6740 * @param[in] dev 6741 * Port device 6742 * @param[in] tunnel 6743 * PMD tunnel offload object 6744 * @param[in] group 6745 * rte_flow group index value. 6746 * @param[out] table 6747 * HW table value. 6748 * @param[in] grp_info 6749 * flags used for conversion 6750 * @param[out] error 6751 * Pointer to error structure. 6752 * 6753 * @return 6754 * 0 on success, a negative errno value otherwise and rte_errno is set. 6755 */ 6756 int 6757 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 6758 const struct mlx5_flow_tunnel *tunnel, 6759 uint32_t group, uint32_t *table, 6760 const struct flow_grp_info *grp_info, 6761 struct rte_flow_error *error) 6762 { 6763 int ret; 6764 bool standard_translation; 6765 6766 if (!grp_info->skip_scale && grp_info->external && 6767 group < MLX5_MAX_TABLES_EXTERNAL) 6768 group *= MLX5_FLOW_TABLE_FACTOR; 6769 if (is_tunnel_offload_active(dev)) { 6770 standard_translation = !grp_info->external || 6771 grp_info->std_tbl_fix; 6772 } else { 6773 standard_translation = true; 6774 } 6775 DRV_LOG(DEBUG, 6776 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 6777 dev->data->port_id, group, grp_info->transfer, 6778 grp_info->external, grp_info->fdb_def_rule, 6779 standard_translation ? "STANDARD" : "TUNNEL"); 6780 if (standard_translation) 6781 ret = flow_group_to_table(dev->data->port_id, group, table, 6782 grp_info, error); 6783 else 6784 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 6785 table, error); 6786 6787 return ret; 6788 } 6789 6790 /** 6791 * Discover availability of metadata reg_c's. 6792 * 6793 * Iteratively use test flows to check availability. 6794 * 6795 * @param[in] dev 6796 * Pointer to the Ethernet device structure. 6797 * 6798 * @return 6799 * 0 on success, a negative errno value otherwise and rte_errno is set. 6800 */ 6801 int 6802 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 6803 { 6804 struct mlx5_priv *priv = dev->data->dev_private; 6805 struct mlx5_dev_config *config = &priv->config; 6806 enum modify_reg idx; 6807 int n = 0; 6808 6809 /* reg_c[0] and reg_c[1] are reserved. */ 6810 config->flow_mreg_c[n++] = REG_C_0; 6811 config->flow_mreg_c[n++] = REG_C_1; 6812 /* Discover availability of other reg_c's. */ 6813 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 6814 struct rte_flow_attr attr = { 6815 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6816 .priority = MLX5_FLOW_PRIO_RSVD, 6817 .ingress = 1, 6818 }; 6819 struct rte_flow_item items[] = { 6820 [0] = { 6821 .type = RTE_FLOW_ITEM_TYPE_END, 6822 }, 6823 }; 6824 struct rte_flow_action actions[] = { 6825 [0] = { 6826 .type = (enum rte_flow_action_type) 6827 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6828 .conf = &(struct mlx5_flow_action_copy_mreg){ 6829 .src = REG_C_1, 6830 .dst = idx, 6831 }, 6832 }, 6833 [1] = { 6834 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6835 .conf = &(struct rte_flow_action_jump){ 6836 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6837 }, 6838 }, 6839 [2] = { 6840 .type = RTE_FLOW_ACTION_TYPE_END, 6841 }, 6842 }; 6843 uint32_t flow_idx; 6844 struct rte_flow *flow; 6845 struct rte_flow_error error; 6846 6847 if (!config->dv_flow_en) 6848 break; 6849 /* Create internal flow, validation skips copy action. */ 6850 flow_idx = flow_list_create(dev, NULL, &attr, items, 6851 actions, false, &error); 6852 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 6853 flow_idx); 6854 if (!flow) 6855 continue; 6856 config->flow_mreg_c[n++] = idx; 6857 flow_list_destroy(dev, NULL, flow_idx); 6858 } 6859 for (; n < MLX5_MREG_C_NUM; ++n) 6860 config->flow_mreg_c[n] = REG_NON; 6861 return 0; 6862 } 6863 6864 /** 6865 * Dump flow raw hw data to file 6866 * 6867 * @param[in] dev 6868 * The pointer to Ethernet device. 6869 * @param[in] file 6870 * A pointer to a file for output. 6871 * @param[out] error 6872 * Perform verbose error reporting if not NULL. PMDs initialize this 6873 * structure in case of error only. 6874 * @return 6875 * 0 on success, a nagative value otherwise. 6876 */ 6877 int 6878 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 6879 FILE *file, 6880 struct rte_flow_error *error __rte_unused) 6881 { 6882 struct mlx5_priv *priv = dev->data->dev_private; 6883 struct mlx5_dev_ctx_shared *sh = priv->sh; 6884 6885 if (!priv->config.dv_flow_en) { 6886 if (fputs("device dv flow disabled\n", file) <= 0) 6887 return -errno; 6888 return -ENOTSUP; 6889 } 6890 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 6891 sh->tx_domain, file); 6892 } 6893 6894 /** 6895 * Get aged-out flows. 6896 * 6897 * @param[in] dev 6898 * Pointer to the Ethernet device structure. 6899 * @param[in] context 6900 * The address of an array of pointers to the aged-out flows contexts. 6901 * @param[in] nb_countexts 6902 * The length of context array pointers. 6903 * @param[out] error 6904 * Perform verbose error reporting if not NULL. Initialized in case of 6905 * error only. 6906 * 6907 * @return 6908 * how many contexts get in success, otherwise negative errno value. 6909 * if nb_contexts is 0, return the amount of all aged contexts. 6910 * if nb_contexts is not 0 , return the amount of aged flows reported 6911 * in the context array. 6912 */ 6913 int 6914 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 6915 uint32_t nb_contexts, struct rte_flow_error *error) 6916 { 6917 const struct mlx5_flow_driver_ops *fops; 6918 struct rte_flow_attr attr = { .transfer = 0 }; 6919 6920 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6921 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6922 return fops->get_aged_flows(dev, contexts, nb_contexts, 6923 error); 6924 } 6925 DRV_LOG(ERR, 6926 "port %u get aged flows is not supported.", 6927 dev->data->port_id); 6928 return -ENOTSUP; 6929 } 6930 6931 /* Wrapper for driver action_validate op callback */ 6932 static int 6933 flow_drv_action_validate(struct rte_eth_dev *dev, 6934 const struct rte_flow_shared_action_conf *conf, 6935 const struct rte_flow_action *action, 6936 const struct mlx5_flow_driver_ops *fops, 6937 struct rte_flow_error *error) 6938 { 6939 static const char err_msg[] = "shared action validation unsupported"; 6940 6941 if (!fops->action_validate) { 6942 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 6943 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 6944 NULL, err_msg); 6945 return -rte_errno; 6946 } 6947 return fops->action_validate(dev, conf, action, error); 6948 } 6949 6950 /** 6951 * Destroys the shared action by handle. 6952 * 6953 * @param dev 6954 * Pointer to Ethernet device structure. 6955 * @param[in] action 6956 * Handle for the shared action to be destroyed. 6957 * @param[out] error 6958 * Perform verbose error reporting if not NULL. PMDs initialize this 6959 * structure in case of error only. 6960 * 6961 * @return 6962 * 0 on success, a negative errno value otherwise and rte_errno is set. 6963 * 6964 * @note: wrapper for driver action_create op callback. 6965 */ 6966 static int 6967 mlx5_shared_action_destroy(struct rte_eth_dev *dev, 6968 struct rte_flow_shared_action *action, 6969 struct rte_flow_error *error) 6970 { 6971 static const char err_msg[] = "shared action destruction unsupported"; 6972 struct rte_flow_attr attr = { .transfer = 0 }; 6973 const struct mlx5_flow_driver_ops *fops = 6974 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 6975 6976 if (!fops->action_destroy) { 6977 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 6978 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 6979 NULL, err_msg); 6980 return -rte_errno; 6981 } 6982 return fops->action_destroy(dev, action, error); 6983 } 6984 6985 /* Wrapper for driver action_destroy op callback */ 6986 static int 6987 flow_drv_action_update(struct rte_eth_dev *dev, 6988 struct rte_flow_shared_action *action, 6989 const void *action_conf, 6990 const struct mlx5_flow_driver_ops *fops, 6991 struct rte_flow_error *error) 6992 { 6993 static const char err_msg[] = "shared action update unsupported"; 6994 6995 if (!fops->action_update) { 6996 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 6997 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 6998 NULL, err_msg); 6999 return -rte_errno; 7000 } 7001 return fops->action_update(dev, action, action_conf, error); 7002 } 7003 7004 /* Wrapper for driver action_destroy op callback */ 7005 static int 7006 flow_drv_action_query(struct rte_eth_dev *dev, 7007 const struct rte_flow_shared_action *action, 7008 void *data, 7009 const struct mlx5_flow_driver_ops *fops, 7010 struct rte_flow_error *error) 7011 { 7012 static const char err_msg[] = "shared action query unsupported"; 7013 7014 if (!fops->action_query) { 7015 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7016 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7017 NULL, err_msg); 7018 return -rte_errno; 7019 } 7020 return fops->action_query(dev, action, data, error); 7021 } 7022 7023 /** 7024 * Create shared action for reuse in multiple flow rules. 7025 * 7026 * @param dev 7027 * Pointer to Ethernet device structure. 7028 * @param[in] action 7029 * Action configuration for shared action creation. 7030 * @param[out] error 7031 * Perform verbose error reporting if not NULL. PMDs initialize this 7032 * structure in case of error only. 7033 * @return 7034 * A valid handle in case of success, NULL otherwise and rte_errno is set. 7035 */ 7036 static struct rte_flow_shared_action * 7037 mlx5_shared_action_create(struct rte_eth_dev *dev, 7038 const struct rte_flow_shared_action_conf *conf, 7039 const struct rte_flow_action *action, 7040 struct rte_flow_error *error) 7041 { 7042 static const char err_msg[] = "shared action creation unsupported"; 7043 struct rte_flow_attr attr = { .transfer = 0 }; 7044 const struct mlx5_flow_driver_ops *fops = 7045 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7046 7047 if (flow_drv_action_validate(dev, conf, action, fops, error)) 7048 return NULL; 7049 if (!fops->action_create) { 7050 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7051 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7052 NULL, err_msg); 7053 return NULL; 7054 } 7055 return fops->action_create(dev, conf, action, error); 7056 } 7057 7058 /** 7059 * Updates inplace the shared action configuration pointed by *action* handle 7060 * with the configuration provided as *action* argument. 7061 * The update of the shared action configuration effects all flow rules reusing 7062 * the action via handle. 7063 * 7064 * @param dev 7065 * Pointer to Ethernet device structure. 7066 * @param[in] shared_action 7067 * Handle for the shared action to be updated. 7068 * @param[in] action 7069 * Action specification used to modify the action pointed by handle. 7070 * *action* should be of same type with the action pointed by the *action* 7071 * handle argument, otherwise considered as invalid. 7072 * @param[out] error 7073 * Perform verbose error reporting if not NULL. PMDs initialize this 7074 * structure in case of error only. 7075 * 7076 * @return 7077 * 0 on success, a negative errno value otherwise and rte_errno is set. 7078 */ 7079 static int 7080 mlx5_shared_action_update(struct rte_eth_dev *dev, 7081 struct rte_flow_shared_action *shared_action, 7082 const struct rte_flow_action *action, 7083 struct rte_flow_error *error) 7084 { 7085 struct rte_flow_attr attr = { .transfer = 0 }; 7086 const struct mlx5_flow_driver_ops *fops = 7087 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7088 int ret; 7089 7090 ret = flow_drv_action_validate(dev, NULL, action, fops, error); 7091 if (ret) 7092 return ret; 7093 return flow_drv_action_update(dev, shared_action, action->conf, fops, 7094 error); 7095 } 7096 7097 /** 7098 * Query the shared action by handle. 7099 * 7100 * This function allows retrieving action-specific data such as counters. 7101 * Data is gathered by special action which may be present/referenced in 7102 * more than one flow rule definition. 7103 * 7104 * \see RTE_FLOW_ACTION_TYPE_COUNT 7105 * 7106 * @param dev 7107 * Pointer to Ethernet device structure. 7108 * @param[in] action 7109 * Handle for the shared action to query. 7110 * @param[in, out] data 7111 * Pointer to storage for the associated query data type. 7112 * @param[out] error 7113 * Perform verbose error reporting if not NULL. PMDs initialize this 7114 * structure in case of error only. 7115 * 7116 * @return 7117 * 0 on success, a negative errno value otherwise and rte_errno is set. 7118 */ 7119 static int 7120 mlx5_shared_action_query(struct rte_eth_dev *dev, 7121 const struct rte_flow_shared_action *action, 7122 void *data, 7123 struct rte_flow_error *error) 7124 { 7125 struct rte_flow_attr attr = { .transfer = 0 }; 7126 const struct mlx5_flow_driver_ops *fops = 7127 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7128 7129 return flow_drv_action_query(dev, action, data, fops, error); 7130 } 7131 7132 /** 7133 * Destroy all shared actions. 7134 * 7135 * @param dev 7136 * Pointer to Ethernet device. 7137 * 7138 * @return 7139 * 0 on success, a negative errno value otherwise and rte_errno is set. 7140 */ 7141 int 7142 mlx5_shared_action_flush(struct rte_eth_dev *dev) 7143 { 7144 struct rte_flow_error error; 7145 struct mlx5_priv *priv = dev->data->dev_private; 7146 struct mlx5_shared_action_rss *action; 7147 int ret = 0; 7148 uint32_t idx; 7149 7150 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7151 priv->rss_shared_actions, idx, action, next) { 7152 ret |= mlx5_shared_action_destroy(dev, 7153 (struct rte_flow_shared_action *)(uintptr_t)idx, &error); 7154 } 7155 return ret; 7156 } 7157 7158 #ifndef HAVE_MLX5DV_DR 7159 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 7160 #else 7161 #define MLX5_DOMAIN_SYNC_FLOW \ 7162 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 7163 #endif 7164 7165 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 7166 { 7167 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 7168 const struct mlx5_flow_driver_ops *fops; 7169 int ret; 7170 struct rte_flow_attr attr = { .transfer = 0 }; 7171 7172 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7173 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 7174 if (ret > 0) 7175 ret = -ret; 7176 return ret; 7177 } 7178 7179 /** 7180 * tunnel offload functionalilty is defined for DV environment only 7181 */ 7182 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7183 __extension__ 7184 union tunnel_offload_mark { 7185 uint32_t val; 7186 struct { 7187 uint32_t app_reserve:8; 7188 uint32_t table_id:15; 7189 uint32_t transfer:1; 7190 uint32_t _unused_:8; 7191 }; 7192 }; 7193 7194 static bool 7195 mlx5_access_tunnel_offload_db 7196 (struct rte_eth_dev *dev, 7197 bool (*match)(struct rte_eth_dev *, 7198 struct mlx5_flow_tunnel *, const void *), 7199 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 7200 void (*miss)(struct rte_eth_dev *, void *), 7201 void *ctx, bool lock_op); 7202 7203 static int 7204 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 7205 struct rte_flow *flow, 7206 const struct rte_flow_attr *attr, 7207 const struct rte_flow_action *app_actions, 7208 uint32_t flow_idx, 7209 struct tunnel_default_miss_ctx *ctx, 7210 struct rte_flow_error *error) 7211 { 7212 struct mlx5_priv *priv = dev->data->dev_private; 7213 struct mlx5_flow *dev_flow; 7214 struct rte_flow_attr miss_attr = *attr; 7215 const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf; 7216 const struct rte_flow_item miss_items[2] = { 7217 { 7218 .type = RTE_FLOW_ITEM_TYPE_ETH, 7219 .spec = NULL, 7220 .last = NULL, 7221 .mask = NULL 7222 }, 7223 { 7224 .type = RTE_FLOW_ITEM_TYPE_END, 7225 .spec = NULL, 7226 .last = NULL, 7227 .mask = NULL 7228 } 7229 }; 7230 union tunnel_offload_mark mark_id; 7231 struct rte_flow_action_mark miss_mark; 7232 struct rte_flow_action miss_actions[3] = { 7233 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 7234 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 7235 }; 7236 const struct rte_flow_action_jump *jump_data; 7237 uint32_t i, flow_table = 0; /* prevent compilation warning */ 7238 struct flow_grp_info grp_info = { 7239 .external = 1, 7240 .transfer = attr->transfer, 7241 .fdb_def_rule = !!priv->fdb_def_rule, 7242 .std_tbl_fix = 0, 7243 }; 7244 int ret; 7245 7246 if (!attr->transfer) { 7247 uint32_t q_size; 7248 7249 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 7250 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 7251 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 7252 0, SOCKET_ID_ANY); 7253 if (!ctx->queue) 7254 return rte_flow_error_set 7255 (error, ENOMEM, 7256 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7257 NULL, "invalid default miss RSS"); 7258 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7259 ctx->action_rss.level = 0, 7260 ctx->action_rss.types = priv->rss_conf.rss_hf, 7261 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 7262 ctx->action_rss.queue_num = priv->reta_idx_n, 7263 ctx->action_rss.key = priv->rss_conf.rss_key, 7264 ctx->action_rss.queue = ctx->queue; 7265 if (!priv->reta_idx_n || !priv->rxqs_n) 7266 return rte_flow_error_set 7267 (error, EINVAL, 7268 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7269 NULL, "invalid port configuration"); 7270 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 7271 ctx->action_rss.types = 0; 7272 for (i = 0; i != priv->reta_idx_n; ++i) 7273 ctx->queue[i] = (*priv->reta_idx)[i]; 7274 } else { 7275 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 7276 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 7277 } 7278 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 7279 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 7280 jump_data = app_actions->conf; 7281 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 7282 miss_attr.group = jump_data->group; 7283 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 7284 &flow_table, &grp_info, error); 7285 if (ret) 7286 return rte_flow_error_set(error, EINVAL, 7287 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7288 NULL, "invalid tunnel id"); 7289 mark_id.app_reserve = 0; 7290 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 7291 mark_id.transfer = !!attr->transfer; 7292 mark_id._unused_ = 0; 7293 miss_mark.id = mark_id.val; 7294 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 7295 miss_items, miss_actions, flow_idx, error); 7296 if (!dev_flow) 7297 return -rte_errno; 7298 dev_flow->flow = flow; 7299 dev_flow->external = true; 7300 dev_flow->tunnel = tunnel; 7301 /* Subflow object was created, we must include one in the list. */ 7302 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7303 dev_flow->handle, next); 7304 DRV_LOG(DEBUG, 7305 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 7306 dev->data->port_id, tunnel->app_tunnel.type, 7307 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 7308 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 7309 miss_actions, error); 7310 if (!ret) 7311 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 7312 error); 7313 7314 return ret; 7315 } 7316 7317 static const struct mlx5_flow_tbl_data_entry * 7318 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 7319 { 7320 struct mlx5_priv *priv = dev->data->dev_private; 7321 struct mlx5_dev_ctx_shared *sh = priv->sh; 7322 struct mlx5_hlist_entry *he; 7323 union tunnel_offload_mark mbits = { .val = mark }; 7324 union mlx5_flow_tbl_key table_key = { 7325 { 7326 .table_id = tunnel_id_to_flow_tbl(mbits.table_id), 7327 .dummy = 0, 7328 .domain = !!mbits.transfer, 7329 .direction = 0, 7330 } 7331 }; 7332 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 7333 return he ? 7334 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 7335 } 7336 7337 static void 7338 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 7339 struct mlx5_hlist_entry *entry) 7340 { 7341 struct mlx5_dev_ctx_shared *sh = list->ctx; 7342 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 7343 7344 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7345 tunnel_flow_tbl_to_id(tte->flow_table)); 7346 mlx5_free(tte); 7347 } 7348 7349 static struct mlx5_hlist_entry * 7350 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, 7351 uint64_t key __rte_unused, 7352 void *ctx __rte_unused) 7353 { 7354 struct mlx5_dev_ctx_shared *sh = list->ctx; 7355 struct tunnel_tbl_entry *tte; 7356 7357 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 7358 sizeof(*tte), 0, 7359 SOCKET_ID_ANY); 7360 if (!tte) 7361 goto err; 7362 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7363 &tte->flow_table); 7364 if (tte->flow_table >= MLX5_MAX_TABLES) { 7365 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 7366 tte->flow_table); 7367 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7368 tte->flow_table); 7369 goto err; 7370 } else if (!tte->flow_table) { 7371 goto err; 7372 } 7373 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 7374 return &tte->hash; 7375 err: 7376 if (tte) 7377 mlx5_free(tte); 7378 return NULL; 7379 } 7380 7381 static uint32_t 7382 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 7383 const struct mlx5_flow_tunnel *tunnel, 7384 uint32_t group, uint32_t *table, 7385 struct rte_flow_error *error) 7386 { 7387 struct mlx5_hlist_entry *he; 7388 struct tunnel_tbl_entry *tte; 7389 union tunnel_tbl_key key = { 7390 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 7391 .group = group 7392 }; 7393 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7394 struct mlx5_hlist *group_hash; 7395 7396 group_hash = tunnel ? tunnel->groups : thub->groups; 7397 he = mlx5_hlist_register(group_hash, key.val, NULL); 7398 if (!he) 7399 return rte_flow_error_set(error, EINVAL, 7400 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7401 NULL, 7402 "tunnel group index not supported"); 7403 tte = container_of(he, typeof(*tte), hash); 7404 *table = tte->flow_table; 7405 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 7406 dev->data->port_id, key.tunnel_id, group, *table); 7407 return 0; 7408 } 7409 7410 static void 7411 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 7412 struct mlx5_flow_tunnel *tunnel) 7413 { 7414 struct mlx5_priv *priv = dev->data->dev_private; 7415 struct mlx5_indexed_pool *ipool; 7416 7417 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 7418 dev->data->port_id, tunnel->tunnel_id); 7419 mlx5_hlist_destroy(tunnel->groups); 7420 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 7421 mlx5_ipool_free(ipool, tunnel->tunnel_id); 7422 } 7423 7424 static bool 7425 mlx5_access_tunnel_offload_db 7426 (struct rte_eth_dev *dev, 7427 bool (*match)(struct rte_eth_dev *, 7428 struct mlx5_flow_tunnel *, const void *), 7429 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 7430 void (*miss)(struct rte_eth_dev *, void *), 7431 void *ctx, bool lock_op) 7432 { 7433 bool verdict = false; 7434 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7435 struct mlx5_flow_tunnel *tunnel; 7436 7437 rte_spinlock_lock(&thub->sl); 7438 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 7439 verdict = match(dev, tunnel, (const void *)ctx); 7440 if (verdict) 7441 break; 7442 } 7443 if (!lock_op) 7444 rte_spinlock_unlock(&thub->sl); 7445 if (verdict && hit) 7446 hit(dev, tunnel, ctx); 7447 if (!verdict && miss) 7448 miss(dev, ctx); 7449 if (lock_op) 7450 rte_spinlock_unlock(&thub->sl); 7451 7452 return verdict; 7453 } 7454 7455 struct tunnel_db_find_tunnel_id_ctx { 7456 uint32_t tunnel_id; 7457 struct mlx5_flow_tunnel *tunnel; 7458 }; 7459 7460 static bool 7461 find_tunnel_id_match(struct rte_eth_dev *dev, 7462 struct mlx5_flow_tunnel *tunnel, const void *x) 7463 { 7464 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 7465 7466 RTE_SET_USED(dev); 7467 return tunnel->tunnel_id == ctx->tunnel_id; 7468 } 7469 7470 static void 7471 find_tunnel_id_hit(struct rte_eth_dev *dev, 7472 struct mlx5_flow_tunnel *tunnel, void *x) 7473 { 7474 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 7475 RTE_SET_USED(dev); 7476 ctx->tunnel = tunnel; 7477 } 7478 7479 static struct mlx5_flow_tunnel * 7480 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 7481 { 7482 struct tunnel_db_find_tunnel_id_ctx ctx = { 7483 .tunnel_id = id, 7484 }; 7485 7486 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 7487 find_tunnel_id_hit, NULL, &ctx, true); 7488 7489 return ctx.tunnel; 7490 } 7491 7492 static struct mlx5_flow_tunnel * 7493 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 7494 const struct rte_flow_tunnel *app_tunnel) 7495 { 7496 struct mlx5_priv *priv = dev->data->dev_private; 7497 struct mlx5_indexed_pool *ipool; 7498 struct mlx5_flow_tunnel *tunnel; 7499 uint32_t id; 7500 7501 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 7502 tunnel = mlx5_ipool_zmalloc(ipool, &id); 7503 if (!tunnel) 7504 return NULL; 7505 if (id >= MLX5_MAX_TUNNELS) { 7506 mlx5_ipool_free(ipool, id); 7507 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 7508 return NULL; 7509 } 7510 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 7511 mlx5_flow_tunnel_grp2tbl_create_cb, 7512 NULL, 7513 mlx5_flow_tunnel_grp2tbl_remove_cb); 7514 if (!tunnel->groups) { 7515 mlx5_ipool_free(ipool, id); 7516 return NULL; 7517 } 7518 tunnel->groups->ctx = priv->sh; 7519 /* initiate new PMD tunnel */ 7520 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 7521 tunnel->tunnel_id = id; 7522 tunnel->action.type = (typeof(tunnel->action.type)) 7523 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 7524 tunnel->action.conf = tunnel; 7525 tunnel->item.type = (typeof(tunnel->item.type)) 7526 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 7527 tunnel->item.spec = tunnel; 7528 tunnel->item.last = NULL; 7529 tunnel->item.mask = NULL; 7530 7531 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 7532 dev->data->port_id, tunnel->tunnel_id); 7533 7534 return tunnel; 7535 } 7536 7537 struct tunnel_db_get_tunnel_ctx { 7538 const struct rte_flow_tunnel *app_tunnel; 7539 struct mlx5_flow_tunnel *tunnel; 7540 }; 7541 7542 static bool get_tunnel_match(struct rte_eth_dev *dev, 7543 struct mlx5_flow_tunnel *tunnel, const void *x) 7544 { 7545 const struct tunnel_db_get_tunnel_ctx *ctx = x; 7546 7547 RTE_SET_USED(dev); 7548 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 7549 sizeof(*ctx->app_tunnel)); 7550 } 7551 7552 static void get_tunnel_hit(struct rte_eth_dev *dev, 7553 struct mlx5_flow_tunnel *tunnel, void *x) 7554 { 7555 /* called under tunnel spinlock protection */ 7556 struct tunnel_db_get_tunnel_ctx *ctx = x; 7557 7558 RTE_SET_USED(dev); 7559 tunnel->refctn++; 7560 ctx->tunnel = tunnel; 7561 } 7562 7563 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 7564 { 7565 /* called under tunnel spinlock protection */ 7566 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7567 struct tunnel_db_get_tunnel_ctx *ctx = x; 7568 7569 rte_spinlock_unlock(&thub->sl); 7570 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 7571 ctx->tunnel->refctn = 1; 7572 rte_spinlock_lock(&thub->sl); 7573 if (ctx->tunnel) 7574 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 7575 } 7576 7577 7578 static int 7579 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 7580 const struct rte_flow_tunnel *app_tunnel, 7581 struct mlx5_flow_tunnel **tunnel) 7582 { 7583 struct tunnel_db_get_tunnel_ctx ctx = { 7584 .app_tunnel = app_tunnel, 7585 }; 7586 7587 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 7588 get_tunnel_miss, &ctx, true); 7589 *tunnel = ctx.tunnel; 7590 return ctx.tunnel ? 0 : -ENOMEM; 7591 } 7592 7593 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 7594 { 7595 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 7596 7597 if (!thub) 7598 return; 7599 if (!LIST_EMPTY(&thub->tunnels)) 7600 DRV_LOG(WARNING, "port %u tunnels present\n", port_id); 7601 mlx5_hlist_destroy(thub->groups); 7602 mlx5_free(thub); 7603 } 7604 7605 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 7606 { 7607 int err; 7608 struct mlx5_flow_tunnel_hub *thub; 7609 7610 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 7611 0, SOCKET_ID_ANY); 7612 if (!thub) 7613 return -ENOMEM; 7614 LIST_INIT(&thub->tunnels); 7615 rte_spinlock_init(&thub->sl); 7616 thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0, 7617 0, mlx5_flow_tunnel_grp2tbl_create_cb, 7618 NULL, 7619 mlx5_flow_tunnel_grp2tbl_remove_cb); 7620 if (!thub->groups) { 7621 err = -rte_errno; 7622 goto err; 7623 } 7624 thub->groups->ctx = sh; 7625 sh->tunnel_hub = thub; 7626 7627 return 0; 7628 7629 err: 7630 if (thub->groups) 7631 mlx5_hlist_destroy(thub->groups); 7632 if (thub) 7633 mlx5_free(thub); 7634 return err; 7635 } 7636 7637 static inline bool 7638 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 7639 struct rte_flow_tunnel *tunnel, 7640 const char *err_msg) 7641 { 7642 err_msg = NULL; 7643 if (!is_tunnel_offload_active(dev)) { 7644 err_msg = "tunnel offload was not activated"; 7645 goto out; 7646 } else if (!tunnel) { 7647 err_msg = "no application tunnel"; 7648 goto out; 7649 } 7650 7651 switch (tunnel->type) { 7652 default: 7653 err_msg = "unsupported tunnel type"; 7654 goto out; 7655 case RTE_FLOW_ITEM_TYPE_VXLAN: 7656 break; 7657 } 7658 7659 out: 7660 return !err_msg; 7661 } 7662 7663 static int 7664 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 7665 struct rte_flow_tunnel *app_tunnel, 7666 struct rte_flow_action **actions, 7667 uint32_t *num_of_actions, 7668 struct rte_flow_error *error) 7669 { 7670 int ret; 7671 struct mlx5_flow_tunnel *tunnel; 7672 const char *err_msg = NULL; 7673 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 7674 7675 if (!verdict) 7676 return rte_flow_error_set(error, EINVAL, 7677 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 7678 err_msg); 7679 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 7680 if (ret < 0) { 7681 return rte_flow_error_set(error, ret, 7682 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 7683 "failed to initialize pmd tunnel"); 7684 } 7685 *actions = &tunnel->action; 7686 *num_of_actions = 1; 7687 return 0; 7688 } 7689 7690 static int 7691 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 7692 struct rte_flow_tunnel *app_tunnel, 7693 struct rte_flow_item **items, 7694 uint32_t *num_of_items, 7695 struct rte_flow_error *error) 7696 { 7697 int ret; 7698 struct mlx5_flow_tunnel *tunnel; 7699 const char *err_msg = NULL; 7700 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 7701 7702 if (!verdict) 7703 return rte_flow_error_set(error, EINVAL, 7704 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 7705 err_msg); 7706 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 7707 if (ret < 0) { 7708 return rte_flow_error_set(error, ret, 7709 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 7710 "failed to initialize pmd tunnel"); 7711 } 7712 *items = &tunnel->item; 7713 *num_of_items = 1; 7714 return 0; 7715 } 7716 7717 struct tunnel_db_element_release_ctx { 7718 struct rte_flow_item *items; 7719 struct rte_flow_action *actions; 7720 uint32_t num_elements; 7721 struct rte_flow_error *error; 7722 int ret; 7723 }; 7724 7725 static bool 7726 tunnel_element_release_match(struct rte_eth_dev *dev, 7727 struct mlx5_flow_tunnel *tunnel, const void *x) 7728 { 7729 const struct tunnel_db_element_release_ctx *ctx = x; 7730 7731 RTE_SET_USED(dev); 7732 if (ctx->num_elements != 1) 7733 return false; 7734 else if (ctx->items) 7735 return ctx->items == &tunnel->item; 7736 else if (ctx->actions) 7737 return ctx->actions == &tunnel->action; 7738 7739 return false; 7740 } 7741 7742 static void 7743 tunnel_element_release_hit(struct rte_eth_dev *dev, 7744 struct mlx5_flow_tunnel *tunnel, void *x) 7745 { 7746 struct tunnel_db_element_release_ctx *ctx = x; 7747 ctx->ret = 0; 7748 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7749 mlx5_flow_tunnel_free(dev, tunnel); 7750 } 7751 7752 static void 7753 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 7754 { 7755 struct tunnel_db_element_release_ctx *ctx = x; 7756 RTE_SET_USED(dev); 7757 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 7758 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 7759 "invalid argument"); 7760 } 7761 7762 static int 7763 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 7764 struct rte_flow_item *pmd_items, 7765 uint32_t num_items, struct rte_flow_error *err) 7766 { 7767 struct tunnel_db_element_release_ctx ctx = { 7768 .items = pmd_items, 7769 .actions = NULL, 7770 .num_elements = num_items, 7771 .error = err, 7772 }; 7773 7774 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 7775 tunnel_element_release_hit, 7776 tunnel_element_release_miss, &ctx, false); 7777 7778 return ctx.ret; 7779 } 7780 7781 static int 7782 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 7783 struct rte_flow_action *pmd_actions, 7784 uint32_t num_actions, struct rte_flow_error *err) 7785 { 7786 struct tunnel_db_element_release_ctx ctx = { 7787 .items = NULL, 7788 .actions = pmd_actions, 7789 .num_elements = num_actions, 7790 .error = err, 7791 }; 7792 7793 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 7794 tunnel_element_release_hit, 7795 tunnel_element_release_miss, &ctx, false); 7796 7797 return ctx.ret; 7798 } 7799 7800 static int 7801 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 7802 struct rte_mbuf *m, 7803 struct rte_flow_restore_info *info, 7804 struct rte_flow_error *err) 7805 { 7806 uint64_t ol_flags = m->ol_flags; 7807 const struct mlx5_flow_tbl_data_entry *tble; 7808 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 7809 7810 if ((ol_flags & mask) != mask) 7811 goto err; 7812 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 7813 if (!tble) { 7814 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 7815 dev->data->port_id, m->hash.fdir.hi); 7816 goto err; 7817 } 7818 MLX5_ASSERT(tble->tunnel); 7819 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 7820 info->group_id = tble->group_id; 7821 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 7822 RTE_FLOW_RESTORE_INFO_GROUP_ID | 7823 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 7824 7825 return 0; 7826 7827 err: 7828 return rte_flow_error_set(err, EINVAL, 7829 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7830 "failed to get restore info"); 7831 } 7832 7833 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 7834 static int 7835 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 7836 __rte_unused struct rte_flow_tunnel *app_tunnel, 7837 __rte_unused struct rte_flow_action **actions, 7838 __rte_unused uint32_t *num_of_actions, 7839 __rte_unused struct rte_flow_error *error) 7840 { 7841 return -ENOTSUP; 7842 } 7843 7844 static int 7845 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 7846 __rte_unused struct rte_flow_tunnel *app_tunnel, 7847 __rte_unused struct rte_flow_item **items, 7848 __rte_unused uint32_t *num_of_items, 7849 __rte_unused struct rte_flow_error *error) 7850 { 7851 return -ENOTSUP; 7852 } 7853 7854 static int 7855 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 7856 __rte_unused struct rte_flow_item *pmd_items, 7857 __rte_unused uint32_t num_items, 7858 __rte_unused struct rte_flow_error *err) 7859 { 7860 return -ENOTSUP; 7861 } 7862 7863 static int 7864 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 7865 __rte_unused struct rte_flow_action *pmd_action, 7866 __rte_unused uint32_t num_actions, 7867 __rte_unused struct rte_flow_error *err) 7868 { 7869 return -ENOTSUP; 7870 } 7871 7872 static int 7873 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 7874 __rte_unused struct rte_mbuf *m, 7875 __rte_unused struct rte_flow_restore_info *i, 7876 __rte_unused struct rte_flow_error *err) 7877 { 7878 return -ENOTSUP; 7879 } 7880 7881 static int 7882 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 7883 __rte_unused struct rte_flow *flow, 7884 __rte_unused const struct rte_flow_attr *attr, 7885 __rte_unused const struct rte_flow_action *actions, 7886 __rte_unused uint32_t flow_idx, 7887 __rte_unused struct tunnel_default_miss_ctx *ctx, 7888 __rte_unused struct rte_flow_error *error) 7889 { 7890 return -ENOTSUP; 7891 } 7892 7893 static struct mlx5_flow_tunnel * 7894 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 7895 __rte_unused uint32_t id) 7896 { 7897 return NULL; 7898 } 7899 7900 static void 7901 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 7902 __rte_unused struct mlx5_flow_tunnel *tunnel) 7903 { 7904 } 7905 7906 static uint32_t 7907 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 7908 __rte_unused const struct mlx5_flow_tunnel *t, 7909 __rte_unused uint32_t group, 7910 __rte_unused uint32_t *table, 7911 struct rte_flow_error *error) 7912 { 7913 return rte_flow_error_set(error, ENOTSUP, 7914 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7915 "tunnel offload requires DV support"); 7916 } 7917 7918 void 7919 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 7920 __rte_unused uint16_t port_id) 7921 { 7922 } 7923 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 7924 7925