1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 #include <rte_common.h> 14 #include <rte_ether.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_eal_paging.h> 17 #include <rte_flow.h> 18 #include <rte_cycles.h> 19 #include <rte_flow_driver.h> 20 #include <rte_malloc.h> 21 #include <rte_ip.h> 22 23 #include <mlx5_glue.h> 24 #include <mlx5_devx_cmds.h> 25 #include <mlx5_prm.h> 26 #include <mlx5_malloc.h> 27 28 #include "mlx5_defs.h" 29 #include "mlx5.h" 30 #include "mlx5_flow.h" 31 #include "mlx5_flow_os.h" 32 #include "mlx5_rxtx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 struct tunnel_default_miss_ctx *ctx, 54 struct rte_flow_error *error); 55 static struct mlx5_flow_tunnel * 56 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 57 static void 58 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 59 static uint32_t 60 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 61 const struct mlx5_flow_tunnel *tunnel, 62 uint32_t group, uint32_t *table, 63 struct rte_flow_error *error); 64 65 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 66 static void mlx5_flow_pop_thread_workspace(void); 67 68 69 /** Device flow drivers. */ 70 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 71 72 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 73 74 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 75 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 76 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 77 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 78 #endif 79 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 80 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 81 }; 82 83 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 84 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 85 (const int []){ \ 86 __VA_ARGS__, 0, \ 87 } 88 89 /** Node object of input graph for mlx5_flow_expand_rss(). */ 90 struct mlx5_flow_expand_node { 91 const int *const next; 92 /**< 93 * List of next node indexes. Index 0 is interpreted as a terminator. 94 */ 95 const enum rte_flow_item_type type; 96 /**< Pattern item type of current node. */ 97 uint64_t rss_types; 98 /**< 99 * RSS types bit-field associated with this node 100 * (see ETH_RSS_* definitions). 101 */ 102 }; 103 104 /** Object returned by mlx5_flow_expand_rss(). */ 105 struct mlx5_flow_expand_rss { 106 uint32_t entries; 107 /**< Number of entries @p patterns and @p priorities. */ 108 struct { 109 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 110 uint32_t priority; /**< Priority offset for each expansion. */ 111 } entry[]; 112 }; 113 114 static enum rte_flow_item_type 115 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 116 { 117 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 118 uint16_t ether_type = 0; 119 uint16_t ether_type_m; 120 uint8_t ip_next_proto = 0; 121 uint8_t ip_next_proto_m; 122 123 if (item == NULL || item->spec == NULL) 124 return ret; 125 switch (item->type) { 126 case RTE_FLOW_ITEM_TYPE_ETH: 127 if (item->mask) 128 ether_type_m = ((const struct rte_flow_item_eth *) 129 (item->mask))->type; 130 else 131 ether_type_m = rte_flow_item_eth_mask.type; 132 if (ether_type_m != RTE_BE16(0xFFFF)) 133 break; 134 ether_type = ((const struct rte_flow_item_eth *) 135 (item->spec))->type; 136 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 137 ret = RTE_FLOW_ITEM_TYPE_IPV4; 138 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 139 ret = RTE_FLOW_ITEM_TYPE_IPV6; 140 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 141 ret = RTE_FLOW_ITEM_TYPE_VLAN; 142 else 143 ret = RTE_FLOW_ITEM_TYPE_END; 144 break; 145 case RTE_FLOW_ITEM_TYPE_VLAN: 146 if (item->mask) 147 ether_type_m = ((const struct rte_flow_item_vlan *) 148 (item->mask))->inner_type; 149 else 150 ether_type_m = rte_flow_item_vlan_mask.inner_type; 151 if (ether_type_m != RTE_BE16(0xFFFF)) 152 break; 153 ether_type = ((const struct rte_flow_item_vlan *) 154 (item->spec))->inner_type; 155 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 156 ret = RTE_FLOW_ITEM_TYPE_IPV4; 157 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 158 ret = RTE_FLOW_ITEM_TYPE_IPV6; 159 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 160 ret = RTE_FLOW_ITEM_TYPE_VLAN; 161 else 162 ret = RTE_FLOW_ITEM_TYPE_END; 163 break; 164 case RTE_FLOW_ITEM_TYPE_IPV4: 165 if (item->mask) 166 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 167 (item->mask))->hdr.next_proto_id; 168 else 169 ip_next_proto_m = 170 rte_flow_item_ipv4_mask.hdr.next_proto_id; 171 if (ip_next_proto_m != 0xFF) 172 break; 173 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 174 (item->spec))->hdr.next_proto_id; 175 if (ip_next_proto == IPPROTO_UDP) 176 ret = RTE_FLOW_ITEM_TYPE_UDP; 177 else if (ip_next_proto == IPPROTO_TCP) 178 ret = RTE_FLOW_ITEM_TYPE_TCP; 179 else if (ip_next_proto == IPPROTO_IP) 180 ret = RTE_FLOW_ITEM_TYPE_IPV4; 181 else if (ip_next_proto == IPPROTO_IPV6) 182 ret = RTE_FLOW_ITEM_TYPE_IPV6; 183 else 184 ret = RTE_FLOW_ITEM_TYPE_END; 185 break; 186 case RTE_FLOW_ITEM_TYPE_IPV6: 187 if (item->mask) 188 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 189 (item->mask))->hdr.proto; 190 else 191 ip_next_proto_m = 192 rte_flow_item_ipv6_mask.hdr.proto; 193 if (ip_next_proto_m != 0xFF) 194 break; 195 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 196 (item->spec))->hdr.proto; 197 if (ip_next_proto == IPPROTO_UDP) 198 ret = RTE_FLOW_ITEM_TYPE_UDP; 199 else if (ip_next_proto == IPPROTO_TCP) 200 ret = RTE_FLOW_ITEM_TYPE_TCP; 201 else if (ip_next_proto == IPPROTO_IP) 202 ret = RTE_FLOW_ITEM_TYPE_IPV4; 203 else if (ip_next_proto == IPPROTO_IPV6) 204 ret = RTE_FLOW_ITEM_TYPE_IPV6; 205 else 206 ret = RTE_FLOW_ITEM_TYPE_END; 207 break; 208 default: 209 ret = RTE_FLOW_ITEM_TYPE_VOID; 210 break; 211 } 212 return ret; 213 } 214 215 /** 216 * Expand RSS flows into several possible flows according to the RSS hash 217 * fields requested and the driver capabilities. 218 * 219 * @param[out] buf 220 * Buffer to store the result expansion. 221 * @param[in] size 222 * Buffer size in bytes. If 0, @p buf can be NULL. 223 * @param[in] pattern 224 * User flow pattern. 225 * @param[in] types 226 * RSS types to expand (see ETH_RSS_* definitions). 227 * @param[in] graph 228 * Input graph to expand @p pattern according to @p types. 229 * @param[in] graph_root_index 230 * Index of root node in @p graph, typically 0. 231 * 232 * @return 233 * A positive value representing the size of @p buf in bytes regardless of 234 * @p size on success, a negative errno value otherwise and rte_errno is 235 * set, the following errors are defined: 236 * 237 * -E2BIG: graph-depth @p graph is too deep. 238 */ 239 static int 240 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 241 const struct rte_flow_item *pattern, uint64_t types, 242 const struct mlx5_flow_expand_node graph[], 243 int graph_root_index) 244 { 245 const int elt_n = 8; 246 const struct rte_flow_item *item; 247 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 248 const int *next_node; 249 const int *stack[elt_n]; 250 int stack_pos = 0; 251 struct rte_flow_item flow_items[elt_n]; 252 unsigned int i; 253 size_t lsize; 254 size_t user_pattern_size = 0; 255 void *addr = NULL; 256 const struct mlx5_flow_expand_node *next = NULL; 257 struct rte_flow_item missed_item; 258 int missed = 0; 259 int elt = 0; 260 const struct rte_flow_item *last_item = NULL; 261 262 memset(&missed_item, 0, sizeof(missed_item)); 263 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 264 elt_n * sizeof(buf->entry[0]); 265 if (lsize <= size) { 266 buf->entry[0].priority = 0; 267 buf->entry[0].pattern = (void *)&buf->entry[elt_n]; 268 buf->entries = 0; 269 addr = buf->entry[0].pattern; 270 } 271 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 272 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 273 last_item = item; 274 for (i = 0; node->next && node->next[i]; ++i) { 275 next = &graph[node->next[i]]; 276 if (next->type == item->type) 277 break; 278 } 279 if (next) 280 node = next; 281 user_pattern_size += sizeof(*item); 282 } 283 user_pattern_size += sizeof(*item); /* Handle END item. */ 284 lsize += user_pattern_size; 285 /* Copy the user pattern in the first entry of the buffer. */ 286 if (lsize <= size) { 287 rte_memcpy(addr, pattern, user_pattern_size); 288 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 289 buf->entries = 1; 290 } 291 /* Start expanding. */ 292 memset(flow_items, 0, sizeof(flow_items)); 293 user_pattern_size -= sizeof(*item); 294 /* 295 * Check if the last valid item has spec set, need complete pattern, 296 * and the pattern can be used for expansion. 297 */ 298 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 299 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 300 /* Item type END indicates expansion is not required. */ 301 return lsize; 302 } 303 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 304 next = NULL; 305 missed = 1; 306 for (i = 0; node->next && node->next[i]; ++i) { 307 next = &graph[node->next[i]]; 308 if (next->type == missed_item.type) { 309 flow_items[0].type = missed_item.type; 310 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 311 break; 312 } 313 next = NULL; 314 } 315 } 316 if (next && missed) { 317 elt = 2; /* missed item + item end. */ 318 node = next; 319 lsize += elt * sizeof(*item) + user_pattern_size; 320 if ((node->rss_types & types) && lsize <= size) { 321 buf->entry[buf->entries].priority = 1; 322 buf->entry[buf->entries].pattern = addr; 323 buf->entries++; 324 rte_memcpy(addr, buf->entry[0].pattern, 325 user_pattern_size); 326 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 327 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 328 addr = (void *)(((uintptr_t)addr) + 329 elt * sizeof(*item)); 330 } 331 } 332 memset(flow_items, 0, sizeof(flow_items)); 333 next_node = node->next; 334 stack[stack_pos] = next_node; 335 node = next_node ? &graph[*next_node] : NULL; 336 while (node) { 337 flow_items[stack_pos].type = node->type; 338 if (node->rss_types & types) { 339 /* 340 * compute the number of items to copy from the 341 * expansion and copy it. 342 * When the stack_pos is 0, there are 1 element in it, 343 * plus the addition END item. 344 */ 345 elt = stack_pos + 2; 346 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 347 lsize += elt * sizeof(*item) + user_pattern_size; 348 if (lsize <= size) { 349 size_t n = elt * sizeof(*item); 350 351 buf->entry[buf->entries].priority = 352 stack_pos + 1 + missed; 353 buf->entry[buf->entries].pattern = addr; 354 buf->entries++; 355 rte_memcpy(addr, buf->entry[0].pattern, 356 user_pattern_size); 357 addr = (void *)(((uintptr_t)addr) + 358 user_pattern_size); 359 rte_memcpy(addr, &missed_item, 360 missed * sizeof(*item)); 361 addr = (void *)(((uintptr_t)addr) + 362 missed * sizeof(*item)); 363 rte_memcpy(addr, flow_items, n); 364 addr = (void *)(((uintptr_t)addr) + n); 365 } 366 } 367 /* Go deeper. */ 368 if (node->next) { 369 next_node = node->next; 370 if (stack_pos++ == elt_n) { 371 rte_errno = E2BIG; 372 return -rte_errno; 373 } 374 stack[stack_pos] = next_node; 375 } else if (*(next_node + 1)) { 376 /* Follow up with the next possibility. */ 377 ++next_node; 378 } else { 379 /* Move to the next path. */ 380 if (stack_pos) 381 next_node = stack[--stack_pos]; 382 next_node++; 383 stack[stack_pos] = next_node; 384 } 385 node = *next_node ? &graph[*next_node] : NULL; 386 }; 387 /* no expanded flows but we have missed item, create one rule for it */ 388 if (buf->entries == 1 && missed != 0) { 389 elt = 2; 390 lsize += elt * sizeof(*item) + user_pattern_size; 391 if (lsize <= size) { 392 buf->entry[buf->entries].priority = 1; 393 buf->entry[buf->entries].pattern = addr; 394 buf->entries++; 395 flow_items[0].type = missed_item.type; 396 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 397 rte_memcpy(addr, buf->entry[0].pattern, 398 user_pattern_size); 399 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 400 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 401 } 402 } 403 return lsize; 404 } 405 406 enum mlx5_expansion { 407 MLX5_EXPANSION_ROOT, 408 MLX5_EXPANSION_ROOT_OUTER, 409 MLX5_EXPANSION_ROOT_ETH_VLAN, 410 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 411 MLX5_EXPANSION_OUTER_ETH, 412 MLX5_EXPANSION_OUTER_ETH_VLAN, 413 MLX5_EXPANSION_OUTER_VLAN, 414 MLX5_EXPANSION_OUTER_IPV4, 415 MLX5_EXPANSION_OUTER_IPV4_UDP, 416 MLX5_EXPANSION_OUTER_IPV4_TCP, 417 MLX5_EXPANSION_OUTER_IPV6, 418 MLX5_EXPANSION_OUTER_IPV6_UDP, 419 MLX5_EXPANSION_OUTER_IPV6_TCP, 420 MLX5_EXPANSION_VXLAN, 421 MLX5_EXPANSION_VXLAN_GPE, 422 MLX5_EXPANSION_GRE, 423 MLX5_EXPANSION_MPLS, 424 MLX5_EXPANSION_ETH, 425 MLX5_EXPANSION_ETH_VLAN, 426 MLX5_EXPANSION_VLAN, 427 MLX5_EXPANSION_IPV4, 428 MLX5_EXPANSION_IPV4_UDP, 429 MLX5_EXPANSION_IPV4_TCP, 430 MLX5_EXPANSION_IPV6, 431 MLX5_EXPANSION_IPV6_UDP, 432 MLX5_EXPANSION_IPV6_TCP, 433 }; 434 435 /** Supported expansion of items. */ 436 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 437 [MLX5_EXPANSION_ROOT] = { 438 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 439 MLX5_EXPANSION_IPV4, 440 MLX5_EXPANSION_IPV6), 441 .type = RTE_FLOW_ITEM_TYPE_END, 442 }, 443 [MLX5_EXPANSION_ROOT_OUTER] = { 444 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 445 MLX5_EXPANSION_OUTER_IPV4, 446 MLX5_EXPANSION_OUTER_IPV6), 447 .type = RTE_FLOW_ITEM_TYPE_END, 448 }, 449 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 450 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 451 .type = RTE_FLOW_ITEM_TYPE_END, 452 }, 453 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 454 .next = MLX5_FLOW_EXPAND_RSS_NEXT 455 (MLX5_EXPANSION_OUTER_ETH_VLAN), 456 .type = RTE_FLOW_ITEM_TYPE_END, 457 }, 458 [MLX5_EXPANSION_OUTER_ETH] = { 459 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 460 MLX5_EXPANSION_OUTER_IPV6, 461 MLX5_EXPANSION_MPLS), 462 .type = RTE_FLOW_ITEM_TYPE_ETH, 463 .rss_types = 0, 464 }, 465 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 466 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 467 .type = RTE_FLOW_ITEM_TYPE_ETH, 468 .rss_types = 0, 469 }, 470 [MLX5_EXPANSION_OUTER_VLAN] = { 471 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 472 MLX5_EXPANSION_OUTER_IPV6), 473 .type = RTE_FLOW_ITEM_TYPE_VLAN, 474 }, 475 [MLX5_EXPANSION_OUTER_IPV4] = { 476 .next = MLX5_FLOW_EXPAND_RSS_NEXT 477 (MLX5_EXPANSION_OUTER_IPV4_UDP, 478 MLX5_EXPANSION_OUTER_IPV4_TCP, 479 MLX5_EXPANSION_GRE, 480 MLX5_EXPANSION_IPV4, 481 MLX5_EXPANSION_IPV6), 482 .type = RTE_FLOW_ITEM_TYPE_IPV4, 483 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 484 ETH_RSS_NONFRAG_IPV4_OTHER, 485 }, 486 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 487 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 488 MLX5_EXPANSION_VXLAN_GPE), 489 .type = RTE_FLOW_ITEM_TYPE_UDP, 490 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 491 }, 492 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 493 .type = RTE_FLOW_ITEM_TYPE_TCP, 494 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 495 }, 496 [MLX5_EXPANSION_OUTER_IPV6] = { 497 .next = MLX5_FLOW_EXPAND_RSS_NEXT 498 (MLX5_EXPANSION_OUTER_IPV6_UDP, 499 MLX5_EXPANSION_OUTER_IPV6_TCP, 500 MLX5_EXPANSION_IPV4, 501 MLX5_EXPANSION_IPV6), 502 .type = RTE_FLOW_ITEM_TYPE_IPV6, 503 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 504 ETH_RSS_NONFRAG_IPV6_OTHER, 505 }, 506 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 507 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 508 MLX5_EXPANSION_VXLAN_GPE), 509 .type = RTE_FLOW_ITEM_TYPE_UDP, 510 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 511 }, 512 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 513 .type = RTE_FLOW_ITEM_TYPE_TCP, 514 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 515 }, 516 [MLX5_EXPANSION_VXLAN] = { 517 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 518 MLX5_EXPANSION_IPV4, 519 MLX5_EXPANSION_IPV6), 520 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 521 }, 522 [MLX5_EXPANSION_VXLAN_GPE] = { 523 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 524 MLX5_EXPANSION_IPV4, 525 MLX5_EXPANSION_IPV6), 526 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 527 }, 528 [MLX5_EXPANSION_GRE] = { 529 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 530 .type = RTE_FLOW_ITEM_TYPE_GRE, 531 }, 532 [MLX5_EXPANSION_MPLS] = { 533 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 534 MLX5_EXPANSION_IPV6), 535 .type = RTE_FLOW_ITEM_TYPE_MPLS, 536 }, 537 [MLX5_EXPANSION_ETH] = { 538 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 539 MLX5_EXPANSION_IPV6), 540 .type = RTE_FLOW_ITEM_TYPE_ETH, 541 }, 542 [MLX5_EXPANSION_ETH_VLAN] = { 543 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 544 .type = RTE_FLOW_ITEM_TYPE_ETH, 545 }, 546 [MLX5_EXPANSION_VLAN] = { 547 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 548 MLX5_EXPANSION_IPV6), 549 .type = RTE_FLOW_ITEM_TYPE_VLAN, 550 }, 551 [MLX5_EXPANSION_IPV4] = { 552 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 553 MLX5_EXPANSION_IPV4_TCP), 554 .type = RTE_FLOW_ITEM_TYPE_IPV4, 555 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 556 ETH_RSS_NONFRAG_IPV4_OTHER, 557 }, 558 [MLX5_EXPANSION_IPV4_UDP] = { 559 .type = RTE_FLOW_ITEM_TYPE_UDP, 560 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 561 }, 562 [MLX5_EXPANSION_IPV4_TCP] = { 563 .type = RTE_FLOW_ITEM_TYPE_TCP, 564 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 565 }, 566 [MLX5_EXPANSION_IPV6] = { 567 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 568 MLX5_EXPANSION_IPV6_TCP), 569 .type = RTE_FLOW_ITEM_TYPE_IPV6, 570 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 571 ETH_RSS_NONFRAG_IPV6_OTHER, 572 }, 573 [MLX5_EXPANSION_IPV6_UDP] = { 574 .type = RTE_FLOW_ITEM_TYPE_UDP, 575 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 576 }, 577 [MLX5_EXPANSION_IPV6_TCP] = { 578 .type = RTE_FLOW_ITEM_TYPE_TCP, 579 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 580 }, 581 }; 582 583 static struct rte_flow_shared_action * 584 mlx5_shared_action_create(struct rte_eth_dev *dev, 585 const struct rte_flow_shared_action_conf *conf, 586 const struct rte_flow_action *action, 587 struct rte_flow_error *error); 588 static int mlx5_shared_action_destroy 589 (struct rte_eth_dev *dev, 590 struct rte_flow_shared_action *shared_action, 591 struct rte_flow_error *error); 592 static int mlx5_shared_action_update 593 (struct rte_eth_dev *dev, 594 struct rte_flow_shared_action *shared_action, 595 const struct rte_flow_action *action, 596 struct rte_flow_error *error); 597 static int mlx5_shared_action_query 598 (struct rte_eth_dev *dev, 599 const struct rte_flow_shared_action *action, 600 void *data, 601 struct rte_flow_error *error); 602 static int 603 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 604 struct rte_flow_tunnel *app_tunnel, 605 struct rte_flow_action **actions, 606 uint32_t *num_of_actions, 607 struct rte_flow_error *error); 608 static int 609 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 610 struct rte_flow_tunnel *app_tunnel, 611 struct rte_flow_item **items, 612 uint32_t *num_of_items, 613 struct rte_flow_error *error); 614 static int 615 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 616 struct rte_flow_item *pmd_items, 617 uint32_t num_items, struct rte_flow_error *err); 618 static int 619 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 620 struct rte_flow_action *pmd_actions, 621 uint32_t num_actions, 622 struct rte_flow_error *err); 623 static int 624 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 625 struct rte_mbuf *m, 626 struct rte_flow_restore_info *info, 627 struct rte_flow_error *err); 628 629 static const struct rte_flow_ops mlx5_flow_ops = { 630 .validate = mlx5_flow_validate, 631 .create = mlx5_flow_create, 632 .destroy = mlx5_flow_destroy, 633 .flush = mlx5_flow_flush, 634 .isolate = mlx5_flow_isolate, 635 .query = mlx5_flow_query, 636 .dev_dump = mlx5_flow_dev_dump, 637 .get_aged_flows = mlx5_flow_get_aged_flows, 638 .shared_action_create = mlx5_shared_action_create, 639 .shared_action_destroy = mlx5_shared_action_destroy, 640 .shared_action_update = mlx5_shared_action_update, 641 .shared_action_query = mlx5_shared_action_query, 642 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 643 .tunnel_match = mlx5_flow_tunnel_match, 644 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 645 .tunnel_item_release = mlx5_flow_tunnel_item_release, 646 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 647 }; 648 649 /* Tunnel information. */ 650 struct mlx5_flow_tunnel_info { 651 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 652 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 653 }; 654 655 static struct mlx5_flow_tunnel_info tunnels_info[] = { 656 { 657 .tunnel = MLX5_FLOW_LAYER_VXLAN, 658 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 659 }, 660 { 661 .tunnel = MLX5_FLOW_LAYER_GENEVE, 662 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 663 }, 664 { 665 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 666 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 667 }, 668 { 669 .tunnel = MLX5_FLOW_LAYER_GRE, 670 .ptype = RTE_PTYPE_TUNNEL_GRE, 671 }, 672 { 673 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 674 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 675 }, 676 { 677 .tunnel = MLX5_FLOW_LAYER_MPLS, 678 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 679 }, 680 { 681 .tunnel = MLX5_FLOW_LAYER_NVGRE, 682 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 683 }, 684 { 685 .tunnel = MLX5_FLOW_LAYER_IPIP, 686 .ptype = RTE_PTYPE_TUNNEL_IP, 687 }, 688 { 689 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 690 .ptype = RTE_PTYPE_TUNNEL_IP, 691 }, 692 { 693 .tunnel = MLX5_FLOW_LAYER_GTP, 694 .ptype = RTE_PTYPE_TUNNEL_GTPU, 695 }, 696 }; 697 698 /* Key of thread specific flow workspace data. */ 699 static pthread_key_t key_workspace; 700 701 /* Thread specific flow workspace data once initialization data. */ 702 static pthread_once_t key_workspace_init; 703 704 705 /** 706 * Translate tag ID to register. 707 * 708 * @param[in] dev 709 * Pointer to the Ethernet device structure. 710 * @param[in] feature 711 * The feature that request the register. 712 * @param[in] id 713 * The request register ID. 714 * @param[out] error 715 * Error description in case of any. 716 * 717 * @return 718 * The request register on success, a negative errno 719 * value otherwise and rte_errno is set. 720 */ 721 int 722 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 723 enum mlx5_feature_name feature, 724 uint32_t id, 725 struct rte_flow_error *error) 726 { 727 struct mlx5_priv *priv = dev->data->dev_private; 728 struct mlx5_dev_config *config = &priv->config; 729 enum modify_reg start_reg; 730 bool skip_mtr_reg = false; 731 732 switch (feature) { 733 case MLX5_HAIRPIN_RX: 734 return REG_B; 735 case MLX5_HAIRPIN_TX: 736 return REG_A; 737 case MLX5_METADATA_RX: 738 switch (config->dv_xmeta_en) { 739 case MLX5_XMETA_MODE_LEGACY: 740 return REG_B; 741 case MLX5_XMETA_MODE_META16: 742 return REG_C_0; 743 case MLX5_XMETA_MODE_META32: 744 return REG_C_1; 745 } 746 break; 747 case MLX5_METADATA_TX: 748 return REG_A; 749 case MLX5_METADATA_FDB: 750 switch (config->dv_xmeta_en) { 751 case MLX5_XMETA_MODE_LEGACY: 752 return REG_NON; 753 case MLX5_XMETA_MODE_META16: 754 return REG_C_0; 755 case MLX5_XMETA_MODE_META32: 756 return REG_C_1; 757 } 758 break; 759 case MLX5_FLOW_MARK: 760 switch (config->dv_xmeta_en) { 761 case MLX5_XMETA_MODE_LEGACY: 762 return REG_NON; 763 case MLX5_XMETA_MODE_META16: 764 return REG_C_1; 765 case MLX5_XMETA_MODE_META32: 766 return REG_C_0; 767 } 768 break; 769 case MLX5_MTR_SFX: 770 /* 771 * If meter color and flow match share one register, flow match 772 * should use the meter color register for match. 773 */ 774 if (priv->mtr_reg_share) 775 return priv->mtr_color_reg; 776 else 777 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 778 REG_C_3; 779 case MLX5_MTR_COLOR: 780 case MLX5_ASO_FLOW_HIT: /* Both features use the same REG_C. */ 781 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 782 return priv->mtr_color_reg; 783 case MLX5_COPY_MARK: 784 /* 785 * Metadata COPY_MARK register using is in meter suffix sub 786 * flow while with meter. It's safe to share the same register. 787 */ 788 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 789 case MLX5_APP_TAG: 790 /* 791 * If meter is enable, it will engage the register for color 792 * match and flow match. If meter color match is not using the 793 * REG_C_2, need to skip the REG_C_x be used by meter color 794 * match. 795 * If meter is disable, free to use all available registers. 796 */ 797 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 798 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 799 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 800 if (id > (REG_C_7 - start_reg)) 801 return rte_flow_error_set(error, EINVAL, 802 RTE_FLOW_ERROR_TYPE_ITEM, 803 NULL, "invalid tag id"); 804 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 805 return rte_flow_error_set(error, ENOTSUP, 806 RTE_FLOW_ERROR_TYPE_ITEM, 807 NULL, "unsupported tag id"); 808 /* 809 * This case means meter is using the REG_C_x great than 2. 810 * Take care not to conflict with meter color REG_C_x. 811 * If the available index REG_C_y >= REG_C_x, skip the 812 * color register. 813 */ 814 if (skip_mtr_reg && config->flow_mreg_c 815 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 816 if (id >= (REG_C_7 - start_reg)) 817 return rte_flow_error_set(error, EINVAL, 818 RTE_FLOW_ERROR_TYPE_ITEM, 819 NULL, "invalid tag id"); 820 if (config->flow_mreg_c 821 [id + 1 + start_reg - REG_C_0] != REG_NON) 822 return config->flow_mreg_c 823 [id + 1 + start_reg - REG_C_0]; 824 return rte_flow_error_set(error, ENOTSUP, 825 RTE_FLOW_ERROR_TYPE_ITEM, 826 NULL, "unsupported tag id"); 827 } 828 return config->flow_mreg_c[id + start_reg - REG_C_0]; 829 } 830 MLX5_ASSERT(false); 831 return rte_flow_error_set(error, EINVAL, 832 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 833 NULL, "invalid feature name"); 834 } 835 836 /** 837 * Check extensive flow metadata register support. 838 * 839 * @param dev 840 * Pointer to rte_eth_dev structure. 841 * 842 * @return 843 * True if device supports extensive flow metadata register, otherwise false. 844 */ 845 bool 846 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 847 { 848 struct mlx5_priv *priv = dev->data->dev_private; 849 struct mlx5_dev_config *config = &priv->config; 850 851 /* 852 * Having available reg_c can be regarded inclusively as supporting 853 * extensive flow metadata register, which could mean, 854 * - metadata register copy action by modify header. 855 * - 16 modify header actions is supported. 856 * - reg_c's are preserved across different domain (FDB and NIC) on 857 * packet loopback by flow lookup miss. 858 */ 859 return config->flow_mreg_c[2] != REG_NON; 860 } 861 862 /** 863 * Verify the @p item specifications (spec, last, mask) are compatible with the 864 * NIC capabilities. 865 * 866 * @param[in] item 867 * Item specification. 868 * @param[in] mask 869 * @p item->mask or flow default bit-masks. 870 * @param[in] nic_mask 871 * Bit-masks covering supported fields by the NIC to compare with user mask. 872 * @param[in] size 873 * Bit-masks size in bytes. 874 * @param[in] range_accepted 875 * True if range of values is accepted for specific fields, false otherwise. 876 * @param[out] error 877 * Pointer to error structure. 878 * 879 * @return 880 * 0 on success, a negative errno value otherwise and rte_errno is set. 881 */ 882 int 883 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 884 const uint8_t *mask, 885 const uint8_t *nic_mask, 886 unsigned int size, 887 bool range_accepted, 888 struct rte_flow_error *error) 889 { 890 unsigned int i; 891 892 MLX5_ASSERT(nic_mask); 893 for (i = 0; i < size; ++i) 894 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 895 return rte_flow_error_set(error, ENOTSUP, 896 RTE_FLOW_ERROR_TYPE_ITEM, 897 item, 898 "mask enables non supported" 899 " bits"); 900 if (!item->spec && (item->mask || item->last)) 901 return rte_flow_error_set(error, EINVAL, 902 RTE_FLOW_ERROR_TYPE_ITEM, item, 903 "mask/last without a spec is not" 904 " supported"); 905 if (item->spec && item->last && !range_accepted) { 906 uint8_t spec[size]; 907 uint8_t last[size]; 908 unsigned int i; 909 int ret; 910 911 for (i = 0; i < size; ++i) { 912 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 913 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 914 } 915 ret = memcmp(spec, last, size); 916 if (ret != 0) 917 return rte_flow_error_set(error, EINVAL, 918 RTE_FLOW_ERROR_TYPE_ITEM, 919 item, 920 "range is not valid"); 921 } 922 return 0; 923 } 924 925 /** 926 * Adjust the hash fields according to the @p flow information. 927 * 928 * @param[in] dev_flow. 929 * Pointer to the mlx5_flow. 930 * @param[in] tunnel 931 * 1 when the hash field is for a tunnel item. 932 * @param[in] layer_types 933 * ETH_RSS_* types. 934 * @param[in] hash_fields 935 * Item hash fields. 936 * 937 * @return 938 * The hash fields that should be used. 939 */ 940 uint64_t 941 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 942 int tunnel __rte_unused, uint64_t layer_types, 943 uint64_t hash_fields) 944 { 945 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 946 int rss_request_inner = rss_desc->level >= 2; 947 948 /* Check RSS hash level for tunnel. */ 949 if (tunnel && rss_request_inner) 950 hash_fields |= IBV_RX_HASH_INNER; 951 else if (tunnel || rss_request_inner) 952 return 0; 953 #endif 954 /* Check if requested layer matches RSS hash fields. */ 955 if (!(rss_desc->types & layer_types)) 956 return 0; 957 return hash_fields; 958 } 959 960 /** 961 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 962 * if several tunnel rules are used on this queue, the tunnel ptype will be 963 * cleared. 964 * 965 * @param rxq_ctrl 966 * Rx queue to update. 967 */ 968 static void 969 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 970 { 971 unsigned int i; 972 uint32_t tunnel_ptype = 0; 973 974 /* Look up for the ptype to use. */ 975 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 976 if (!rxq_ctrl->flow_tunnels_n[i]) 977 continue; 978 if (!tunnel_ptype) { 979 tunnel_ptype = tunnels_info[i].ptype; 980 } else { 981 tunnel_ptype = 0; 982 break; 983 } 984 } 985 rxq_ctrl->rxq.tunnel = tunnel_ptype; 986 } 987 988 /** 989 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 990 * flow. 991 * 992 * @param[in] dev 993 * Pointer to the Ethernet device structure. 994 * @param[in] dev_handle 995 * Pointer to device flow handle structure. 996 */ 997 static void 998 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 999 struct mlx5_flow_handle *dev_handle) 1000 { 1001 struct mlx5_priv *priv = dev->data->dev_private; 1002 const int mark = dev_handle->mark; 1003 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1004 struct mlx5_hrxq *hrxq; 1005 unsigned int i; 1006 1007 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1008 return; 1009 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1010 dev_handle->rix_hrxq); 1011 if (!hrxq) 1012 return; 1013 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1014 int idx = hrxq->ind_table->queues[i]; 1015 struct mlx5_rxq_ctrl *rxq_ctrl = 1016 container_of((*priv->rxqs)[idx], 1017 struct mlx5_rxq_ctrl, rxq); 1018 1019 /* 1020 * To support metadata register copy on Tx loopback, 1021 * this must be always enabled (metadata may arive 1022 * from other port - not from local flows only. 1023 */ 1024 if (priv->config.dv_flow_en && 1025 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1026 mlx5_flow_ext_mreg_supported(dev)) { 1027 rxq_ctrl->rxq.mark = 1; 1028 rxq_ctrl->flow_mark_n = 1; 1029 } else if (mark) { 1030 rxq_ctrl->rxq.mark = 1; 1031 rxq_ctrl->flow_mark_n++; 1032 } 1033 if (tunnel) { 1034 unsigned int j; 1035 1036 /* Increase the counter matching the flow. */ 1037 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1038 if ((tunnels_info[j].tunnel & 1039 dev_handle->layers) == 1040 tunnels_info[j].tunnel) { 1041 rxq_ctrl->flow_tunnels_n[j]++; 1042 break; 1043 } 1044 } 1045 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1046 } 1047 } 1048 } 1049 1050 /** 1051 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1052 * 1053 * @param[in] dev 1054 * Pointer to the Ethernet device structure. 1055 * @param[in] flow 1056 * Pointer to flow structure. 1057 */ 1058 static void 1059 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1060 { 1061 struct mlx5_priv *priv = dev->data->dev_private; 1062 uint32_t handle_idx; 1063 struct mlx5_flow_handle *dev_handle; 1064 1065 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1066 handle_idx, dev_handle, next) 1067 flow_drv_rxq_flags_set(dev, dev_handle); 1068 } 1069 1070 /** 1071 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1072 * device flow if no other flow uses it with the same kind of request. 1073 * 1074 * @param dev 1075 * Pointer to Ethernet device. 1076 * @param[in] dev_handle 1077 * Pointer to the device flow handle structure. 1078 */ 1079 static void 1080 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1081 struct mlx5_flow_handle *dev_handle) 1082 { 1083 struct mlx5_priv *priv = dev->data->dev_private; 1084 const int mark = dev_handle->mark; 1085 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1086 struct mlx5_hrxq *hrxq; 1087 unsigned int i; 1088 1089 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1090 return; 1091 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1092 dev_handle->rix_hrxq); 1093 if (!hrxq) 1094 return; 1095 MLX5_ASSERT(dev->data->dev_started); 1096 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1097 int idx = hrxq->ind_table->queues[i]; 1098 struct mlx5_rxq_ctrl *rxq_ctrl = 1099 container_of((*priv->rxqs)[idx], 1100 struct mlx5_rxq_ctrl, rxq); 1101 1102 if (priv->config.dv_flow_en && 1103 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1104 mlx5_flow_ext_mreg_supported(dev)) { 1105 rxq_ctrl->rxq.mark = 1; 1106 rxq_ctrl->flow_mark_n = 1; 1107 } else if (mark) { 1108 rxq_ctrl->flow_mark_n--; 1109 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1110 } 1111 if (tunnel) { 1112 unsigned int j; 1113 1114 /* Decrease the counter matching the flow. */ 1115 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1116 if ((tunnels_info[j].tunnel & 1117 dev_handle->layers) == 1118 tunnels_info[j].tunnel) { 1119 rxq_ctrl->flow_tunnels_n[j]--; 1120 break; 1121 } 1122 } 1123 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1124 } 1125 } 1126 } 1127 1128 /** 1129 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1130 * @p flow if no other flow uses it with the same kind of request. 1131 * 1132 * @param dev 1133 * Pointer to Ethernet device. 1134 * @param[in] flow 1135 * Pointer to the flow. 1136 */ 1137 static void 1138 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1139 { 1140 struct mlx5_priv *priv = dev->data->dev_private; 1141 uint32_t handle_idx; 1142 struct mlx5_flow_handle *dev_handle; 1143 1144 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1145 handle_idx, dev_handle, next) 1146 flow_drv_rxq_flags_trim(dev, dev_handle); 1147 } 1148 1149 /** 1150 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1151 * 1152 * @param dev 1153 * Pointer to Ethernet device. 1154 */ 1155 static void 1156 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1157 { 1158 struct mlx5_priv *priv = dev->data->dev_private; 1159 unsigned int i; 1160 1161 for (i = 0; i != priv->rxqs_n; ++i) { 1162 struct mlx5_rxq_ctrl *rxq_ctrl; 1163 unsigned int j; 1164 1165 if (!(*priv->rxqs)[i]) 1166 continue; 1167 rxq_ctrl = container_of((*priv->rxqs)[i], 1168 struct mlx5_rxq_ctrl, rxq); 1169 rxq_ctrl->flow_mark_n = 0; 1170 rxq_ctrl->rxq.mark = 0; 1171 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1172 rxq_ctrl->flow_tunnels_n[j] = 0; 1173 rxq_ctrl->rxq.tunnel = 0; 1174 } 1175 } 1176 1177 /** 1178 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1179 * 1180 * @param[in] dev 1181 * Pointer to the Ethernet device structure. 1182 */ 1183 void 1184 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1185 { 1186 struct mlx5_priv *priv = dev->data->dev_private; 1187 struct mlx5_rxq_data *data; 1188 unsigned int i; 1189 1190 for (i = 0; i != priv->rxqs_n; ++i) { 1191 if (!(*priv->rxqs)[i]) 1192 continue; 1193 data = (*priv->rxqs)[i]; 1194 if (!rte_flow_dynf_metadata_avail()) { 1195 data->dynf_meta = 0; 1196 data->flow_meta_mask = 0; 1197 data->flow_meta_offset = -1; 1198 } else { 1199 data->dynf_meta = 1; 1200 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1201 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1202 } 1203 } 1204 } 1205 1206 /* 1207 * return a pointer to the desired action in the list of actions. 1208 * 1209 * @param[in] actions 1210 * The list of actions to search the action in. 1211 * @param[in] action 1212 * The action to find. 1213 * 1214 * @return 1215 * Pointer to the action in the list, if found. NULL otherwise. 1216 */ 1217 const struct rte_flow_action * 1218 mlx5_flow_find_action(const struct rte_flow_action *actions, 1219 enum rte_flow_action_type action) 1220 { 1221 if (actions == NULL) 1222 return NULL; 1223 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1224 if (actions->type == action) 1225 return actions; 1226 return NULL; 1227 } 1228 1229 /* 1230 * Validate the flag action. 1231 * 1232 * @param[in] action_flags 1233 * Bit-fields that holds the actions detected until now. 1234 * @param[in] attr 1235 * Attributes of flow that includes this action. 1236 * @param[out] error 1237 * Pointer to error structure. 1238 * 1239 * @return 1240 * 0 on success, a negative errno value otherwise and rte_errno is set. 1241 */ 1242 int 1243 mlx5_flow_validate_action_flag(uint64_t action_flags, 1244 const struct rte_flow_attr *attr, 1245 struct rte_flow_error *error) 1246 { 1247 if (action_flags & MLX5_FLOW_ACTION_MARK) 1248 return rte_flow_error_set(error, EINVAL, 1249 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1250 "can't mark and flag in same flow"); 1251 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1252 return rte_flow_error_set(error, EINVAL, 1253 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1254 "can't have 2 flag" 1255 " actions in same flow"); 1256 if (attr->egress) 1257 return rte_flow_error_set(error, ENOTSUP, 1258 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1259 "flag action not supported for " 1260 "egress"); 1261 return 0; 1262 } 1263 1264 /* 1265 * Validate the mark action. 1266 * 1267 * @param[in] action 1268 * Pointer to the queue action. 1269 * @param[in] action_flags 1270 * Bit-fields that holds the actions detected until now. 1271 * @param[in] attr 1272 * Attributes of flow that includes this action. 1273 * @param[out] error 1274 * Pointer to error structure. 1275 * 1276 * @return 1277 * 0 on success, a negative errno value otherwise and rte_errno is set. 1278 */ 1279 int 1280 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1281 uint64_t action_flags, 1282 const struct rte_flow_attr *attr, 1283 struct rte_flow_error *error) 1284 { 1285 const struct rte_flow_action_mark *mark = action->conf; 1286 1287 if (!mark) 1288 return rte_flow_error_set(error, EINVAL, 1289 RTE_FLOW_ERROR_TYPE_ACTION, 1290 action, 1291 "configuration cannot be null"); 1292 if (mark->id >= MLX5_FLOW_MARK_MAX) 1293 return rte_flow_error_set(error, EINVAL, 1294 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1295 &mark->id, 1296 "mark id must in 0 <= id < " 1297 RTE_STR(MLX5_FLOW_MARK_MAX)); 1298 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1299 return rte_flow_error_set(error, EINVAL, 1300 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1301 "can't flag and mark in same flow"); 1302 if (action_flags & MLX5_FLOW_ACTION_MARK) 1303 return rte_flow_error_set(error, EINVAL, 1304 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1305 "can't have 2 mark actions in same" 1306 " flow"); 1307 if (attr->egress) 1308 return rte_flow_error_set(error, ENOTSUP, 1309 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1310 "mark action not supported for " 1311 "egress"); 1312 return 0; 1313 } 1314 1315 /* 1316 * Validate the drop action. 1317 * 1318 * @param[in] action_flags 1319 * Bit-fields that holds the actions detected until now. 1320 * @param[in] attr 1321 * Attributes of flow that includes this action. 1322 * @param[out] error 1323 * Pointer to error structure. 1324 * 1325 * @return 1326 * 0 on success, a negative errno value otherwise and rte_errno is set. 1327 */ 1328 int 1329 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1330 const struct rte_flow_attr *attr, 1331 struct rte_flow_error *error) 1332 { 1333 if (attr->egress) 1334 return rte_flow_error_set(error, ENOTSUP, 1335 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1336 "drop action not supported for " 1337 "egress"); 1338 return 0; 1339 } 1340 1341 /* 1342 * Validate the queue action. 1343 * 1344 * @param[in] action 1345 * Pointer to the queue action. 1346 * @param[in] action_flags 1347 * Bit-fields that holds the actions detected until now. 1348 * @param[in] dev 1349 * Pointer to the Ethernet device structure. 1350 * @param[in] attr 1351 * Attributes of flow that includes this action. 1352 * @param[out] error 1353 * Pointer to error structure. 1354 * 1355 * @return 1356 * 0 on success, a negative errno value otherwise and rte_errno is set. 1357 */ 1358 int 1359 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1360 uint64_t action_flags, 1361 struct rte_eth_dev *dev, 1362 const struct rte_flow_attr *attr, 1363 struct rte_flow_error *error) 1364 { 1365 struct mlx5_priv *priv = dev->data->dev_private; 1366 const struct rte_flow_action_queue *queue = action->conf; 1367 1368 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1369 return rte_flow_error_set(error, EINVAL, 1370 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1371 "can't have 2 fate actions in" 1372 " same flow"); 1373 if (!priv->rxqs_n) 1374 return rte_flow_error_set(error, EINVAL, 1375 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1376 NULL, "No Rx queues configured"); 1377 if (queue->index >= priv->rxqs_n) 1378 return rte_flow_error_set(error, EINVAL, 1379 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1380 &queue->index, 1381 "queue index out of range"); 1382 if (!(*priv->rxqs)[queue->index]) 1383 return rte_flow_error_set(error, EINVAL, 1384 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1385 &queue->index, 1386 "queue is not configured"); 1387 if (attr->egress) 1388 return rte_flow_error_set(error, ENOTSUP, 1389 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1390 "queue action not supported for " 1391 "egress"); 1392 return 0; 1393 } 1394 1395 /* 1396 * Validate the rss action. 1397 * 1398 * @param[in] dev 1399 * Pointer to the Ethernet device structure. 1400 * @param[in] action 1401 * Pointer to the queue action. 1402 * @param[out] error 1403 * Pointer to error structure. 1404 * 1405 * @return 1406 * 0 on success, a negative errno value otherwise and rte_errno is set. 1407 */ 1408 int 1409 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1410 const struct rte_flow_action *action, 1411 struct rte_flow_error *error) 1412 { 1413 struct mlx5_priv *priv = dev->data->dev_private; 1414 const struct rte_flow_action_rss *rss = action->conf; 1415 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1416 unsigned int i; 1417 1418 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1419 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1420 return rte_flow_error_set(error, ENOTSUP, 1421 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1422 &rss->func, 1423 "RSS hash function not supported"); 1424 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1425 if (rss->level > 2) 1426 #else 1427 if (rss->level > 1) 1428 #endif 1429 return rte_flow_error_set(error, ENOTSUP, 1430 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1431 &rss->level, 1432 "tunnel RSS is not supported"); 1433 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1434 if (rss->key_len == 0 && rss->key != NULL) 1435 return rte_flow_error_set(error, ENOTSUP, 1436 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1437 &rss->key_len, 1438 "RSS hash key length 0"); 1439 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1440 return rte_flow_error_set(error, ENOTSUP, 1441 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1442 &rss->key_len, 1443 "RSS hash key too small"); 1444 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1445 return rte_flow_error_set(error, ENOTSUP, 1446 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1447 &rss->key_len, 1448 "RSS hash key too large"); 1449 if (rss->queue_num > priv->config.ind_table_max_size) 1450 return rte_flow_error_set(error, ENOTSUP, 1451 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1452 &rss->queue_num, 1453 "number of queues too large"); 1454 if (rss->types & MLX5_RSS_HF_MASK) 1455 return rte_flow_error_set(error, ENOTSUP, 1456 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1457 &rss->types, 1458 "some RSS protocols are not" 1459 " supported"); 1460 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1461 !(rss->types & ETH_RSS_IP)) 1462 return rte_flow_error_set(error, EINVAL, 1463 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1464 "L3 partial RSS requested but L3 RSS" 1465 " type not specified"); 1466 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1467 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1468 return rte_flow_error_set(error, EINVAL, 1469 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1470 "L4 partial RSS requested but L4 RSS" 1471 " type not specified"); 1472 if (!priv->rxqs_n) 1473 return rte_flow_error_set(error, EINVAL, 1474 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1475 NULL, "No Rx queues configured"); 1476 if (!rss->queue_num) 1477 return rte_flow_error_set(error, EINVAL, 1478 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1479 NULL, "No queues configured"); 1480 for (i = 0; i != rss->queue_num; ++i) { 1481 struct mlx5_rxq_ctrl *rxq_ctrl; 1482 1483 if (rss->queue[i] >= priv->rxqs_n) 1484 return rte_flow_error_set 1485 (error, EINVAL, 1486 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1487 &rss->queue[i], "queue index out of range"); 1488 if (!(*priv->rxqs)[rss->queue[i]]) 1489 return rte_flow_error_set 1490 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1491 &rss->queue[i], "queue is not configured"); 1492 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1493 struct mlx5_rxq_ctrl, rxq); 1494 if (i == 0) 1495 rxq_type = rxq_ctrl->type; 1496 if (rxq_type != rxq_ctrl->type) 1497 return rte_flow_error_set 1498 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1499 &rss->queue[i], 1500 "combining hairpin and regular RSS queues is not supported"); 1501 } 1502 return 0; 1503 } 1504 1505 /* 1506 * Validate the rss action. 1507 * 1508 * @param[in] action 1509 * Pointer to the queue action. 1510 * @param[in] action_flags 1511 * Bit-fields that holds the actions detected until now. 1512 * @param[in] dev 1513 * Pointer to the Ethernet device structure. 1514 * @param[in] attr 1515 * Attributes of flow that includes this action. 1516 * @param[in] item_flags 1517 * Items that were detected. 1518 * @param[out] error 1519 * Pointer to error structure. 1520 * 1521 * @return 1522 * 0 on success, a negative errno value otherwise and rte_errno is set. 1523 */ 1524 int 1525 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1526 uint64_t action_flags, 1527 struct rte_eth_dev *dev, 1528 const struct rte_flow_attr *attr, 1529 uint64_t item_flags, 1530 struct rte_flow_error *error) 1531 { 1532 const struct rte_flow_action_rss *rss = action->conf; 1533 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1534 int ret; 1535 1536 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1537 return rte_flow_error_set(error, EINVAL, 1538 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1539 "can't have 2 fate actions" 1540 " in same flow"); 1541 ret = mlx5_validate_action_rss(dev, action, error); 1542 if (ret) 1543 return ret; 1544 if (attr->egress) 1545 return rte_flow_error_set(error, ENOTSUP, 1546 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1547 "rss action not supported for " 1548 "egress"); 1549 if (rss->level > 1 && !tunnel) 1550 return rte_flow_error_set(error, EINVAL, 1551 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1552 "inner RSS is not supported for " 1553 "non-tunnel flows"); 1554 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1555 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1556 return rte_flow_error_set(error, EINVAL, 1557 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1558 "RSS on eCPRI is not supported now"); 1559 } 1560 return 0; 1561 } 1562 1563 /* 1564 * Validate the default miss action. 1565 * 1566 * @param[in] action_flags 1567 * Bit-fields that holds the actions detected until now. 1568 * @param[out] error 1569 * Pointer to error structure. 1570 * 1571 * @return 1572 * 0 on success, a negative errno value otherwise and rte_errno is set. 1573 */ 1574 int 1575 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1576 const struct rte_flow_attr *attr, 1577 struct rte_flow_error *error) 1578 { 1579 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1580 return rte_flow_error_set(error, EINVAL, 1581 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1582 "can't have 2 fate actions in" 1583 " same flow"); 1584 if (attr->egress) 1585 return rte_flow_error_set(error, ENOTSUP, 1586 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1587 "default miss action not supported " 1588 "for egress"); 1589 if (attr->group) 1590 return rte_flow_error_set(error, ENOTSUP, 1591 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1592 "only group 0 is supported"); 1593 if (attr->transfer) 1594 return rte_flow_error_set(error, ENOTSUP, 1595 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1596 NULL, "transfer is not supported"); 1597 return 0; 1598 } 1599 1600 /* 1601 * Validate the count action. 1602 * 1603 * @param[in] dev 1604 * Pointer to the Ethernet device structure. 1605 * @param[in] attr 1606 * Attributes of flow that includes this action. 1607 * @param[out] error 1608 * Pointer to error structure. 1609 * 1610 * @return 1611 * 0 on success, a negative errno value otherwise and rte_errno is set. 1612 */ 1613 int 1614 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1615 const struct rte_flow_attr *attr, 1616 struct rte_flow_error *error) 1617 { 1618 if (attr->egress) 1619 return rte_flow_error_set(error, ENOTSUP, 1620 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1621 "count action not supported for " 1622 "egress"); 1623 return 0; 1624 } 1625 1626 /** 1627 * Verify the @p attributes will be correctly understood by the NIC and store 1628 * them in the @p flow if everything is correct. 1629 * 1630 * @param[in] dev 1631 * Pointer to the Ethernet device structure. 1632 * @param[in] attributes 1633 * Pointer to flow attributes 1634 * @param[out] error 1635 * Pointer to error structure. 1636 * 1637 * @return 1638 * 0 on success, a negative errno value otherwise and rte_errno is set. 1639 */ 1640 int 1641 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1642 const struct rte_flow_attr *attributes, 1643 struct rte_flow_error *error) 1644 { 1645 struct mlx5_priv *priv = dev->data->dev_private; 1646 uint32_t priority_max = priv->config.flow_prio - 1; 1647 1648 if (attributes->group) 1649 return rte_flow_error_set(error, ENOTSUP, 1650 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1651 NULL, "groups is not supported"); 1652 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1653 attributes->priority >= priority_max) 1654 return rte_flow_error_set(error, ENOTSUP, 1655 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1656 NULL, "priority out of range"); 1657 if (attributes->egress) 1658 return rte_flow_error_set(error, ENOTSUP, 1659 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1660 "egress is not supported"); 1661 if (attributes->transfer && !priv->config.dv_esw_en) 1662 return rte_flow_error_set(error, ENOTSUP, 1663 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1664 NULL, "transfer is not supported"); 1665 if (!attributes->ingress) 1666 return rte_flow_error_set(error, EINVAL, 1667 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1668 NULL, 1669 "ingress attribute is mandatory"); 1670 return 0; 1671 } 1672 1673 /** 1674 * Validate ICMP6 item. 1675 * 1676 * @param[in] item 1677 * Item specification. 1678 * @param[in] item_flags 1679 * Bit-fields that holds the items detected until now. 1680 * @param[in] ext_vlan_sup 1681 * Whether extended VLAN features are supported or not. 1682 * @param[out] error 1683 * Pointer to error structure. 1684 * 1685 * @return 1686 * 0 on success, a negative errno value otherwise and rte_errno is set. 1687 */ 1688 int 1689 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1690 uint64_t item_flags, 1691 uint8_t target_protocol, 1692 struct rte_flow_error *error) 1693 { 1694 const struct rte_flow_item_icmp6 *mask = item->mask; 1695 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1696 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1697 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1698 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1699 MLX5_FLOW_LAYER_OUTER_L4; 1700 int ret; 1701 1702 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1703 return rte_flow_error_set(error, EINVAL, 1704 RTE_FLOW_ERROR_TYPE_ITEM, item, 1705 "protocol filtering not compatible" 1706 " with ICMP6 layer"); 1707 if (!(item_flags & l3m)) 1708 return rte_flow_error_set(error, EINVAL, 1709 RTE_FLOW_ERROR_TYPE_ITEM, item, 1710 "IPv6 is mandatory to filter on" 1711 " ICMP6"); 1712 if (item_flags & l4m) 1713 return rte_flow_error_set(error, EINVAL, 1714 RTE_FLOW_ERROR_TYPE_ITEM, item, 1715 "multiple L4 layers not supported"); 1716 if (!mask) 1717 mask = &rte_flow_item_icmp6_mask; 1718 ret = mlx5_flow_item_acceptable 1719 (item, (const uint8_t *)mask, 1720 (const uint8_t *)&rte_flow_item_icmp6_mask, 1721 sizeof(struct rte_flow_item_icmp6), 1722 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1723 if (ret < 0) 1724 return ret; 1725 return 0; 1726 } 1727 1728 /** 1729 * Validate ICMP item. 1730 * 1731 * @param[in] item 1732 * Item specification. 1733 * @param[in] item_flags 1734 * Bit-fields that holds the items detected until now. 1735 * @param[out] error 1736 * Pointer to error structure. 1737 * 1738 * @return 1739 * 0 on success, a negative errno value otherwise and rte_errno is set. 1740 */ 1741 int 1742 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1743 uint64_t item_flags, 1744 uint8_t target_protocol, 1745 struct rte_flow_error *error) 1746 { 1747 const struct rte_flow_item_icmp *mask = item->mask; 1748 const struct rte_flow_item_icmp nic_mask = { 1749 .hdr.icmp_type = 0xff, 1750 .hdr.icmp_code = 0xff, 1751 .hdr.icmp_ident = RTE_BE16(0xffff), 1752 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1753 }; 1754 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1755 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1756 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1757 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1758 MLX5_FLOW_LAYER_OUTER_L4; 1759 int ret; 1760 1761 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1762 return rte_flow_error_set(error, EINVAL, 1763 RTE_FLOW_ERROR_TYPE_ITEM, item, 1764 "protocol filtering not compatible" 1765 " with ICMP layer"); 1766 if (!(item_flags & l3m)) 1767 return rte_flow_error_set(error, EINVAL, 1768 RTE_FLOW_ERROR_TYPE_ITEM, item, 1769 "IPv4 is mandatory to filter" 1770 " on ICMP"); 1771 if (item_flags & l4m) 1772 return rte_flow_error_set(error, EINVAL, 1773 RTE_FLOW_ERROR_TYPE_ITEM, item, 1774 "multiple L4 layers not supported"); 1775 if (!mask) 1776 mask = &nic_mask; 1777 ret = mlx5_flow_item_acceptable 1778 (item, (const uint8_t *)mask, 1779 (const uint8_t *)&nic_mask, 1780 sizeof(struct rte_flow_item_icmp), 1781 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1782 if (ret < 0) 1783 return ret; 1784 return 0; 1785 } 1786 1787 /** 1788 * Validate Ethernet item. 1789 * 1790 * @param[in] item 1791 * Item specification. 1792 * @param[in] item_flags 1793 * Bit-fields that holds the items detected until now. 1794 * @param[out] error 1795 * Pointer to error structure. 1796 * 1797 * @return 1798 * 0 on success, a negative errno value otherwise and rte_errno is set. 1799 */ 1800 int 1801 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1802 uint64_t item_flags, bool ext_vlan_sup, 1803 struct rte_flow_error *error) 1804 { 1805 const struct rte_flow_item_eth *mask = item->mask; 1806 const struct rte_flow_item_eth nic_mask = { 1807 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1808 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1809 .type = RTE_BE16(0xffff), 1810 .has_vlan = ext_vlan_sup ? 1 : 0, 1811 }; 1812 int ret; 1813 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1814 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1815 MLX5_FLOW_LAYER_OUTER_L2; 1816 1817 if (item_flags & ethm) 1818 return rte_flow_error_set(error, ENOTSUP, 1819 RTE_FLOW_ERROR_TYPE_ITEM, item, 1820 "multiple L2 layers not supported"); 1821 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1822 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1823 return rte_flow_error_set(error, EINVAL, 1824 RTE_FLOW_ERROR_TYPE_ITEM, item, 1825 "L2 layer should not follow " 1826 "L3 layers"); 1827 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1828 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1829 return rte_flow_error_set(error, EINVAL, 1830 RTE_FLOW_ERROR_TYPE_ITEM, item, 1831 "L2 layer should not follow VLAN"); 1832 if (!mask) 1833 mask = &rte_flow_item_eth_mask; 1834 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1835 (const uint8_t *)&nic_mask, 1836 sizeof(struct rte_flow_item_eth), 1837 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1838 return ret; 1839 } 1840 1841 /** 1842 * Validate VLAN item. 1843 * 1844 * @param[in] item 1845 * Item specification. 1846 * @param[in] item_flags 1847 * Bit-fields that holds the items detected until now. 1848 * @param[in] dev 1849 * Ethernet device flow is being created on. 1850 * @param[out] error 1851 * Pointer to error structure. 1852 * 1853 * @return 1854 * 0 on success, a negative errno value otherwise and rte_errno is set. 1855 */ 1856 int 1857 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1858 uint64_t item_flags, 1859 struct rte_eth_dev *dev, 1860 struct rte_flow_error *error) 1861 { 1862 const struct rte_flow_item_vlan *spec = item->spec; 1863 const struct rte_flow_item_vlan *mask = item->mask; 1864 const struct rte_flow_item_vlan nic_mask = { 1865 .tci = RTE_BE16(UINT16_MAX), 1866 .inner_type = RTE_BE16(UINT16_MAX), 1867 }; 1868 uint16_t vlan_tag = 0; 1869 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1870 int ret; 1871 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1872 MLX5_FLOW_LAYER_INNER_L4) : 1873 (MLX5_FLOW_LAYER_OUTER_L3 | 1874 MLX5_FLOW_LAYER_OUTER_L4); 1875 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1876 MLX5_FLOW_LAYER_OUTER_VLAN; 1877 1878 if (item_flags & vlanm) 1879 return rte_flow_error_set(error, EINVAL, 1880 RTE_FLOW_ERROR_TYPE_ITEM, item, 1881 "multiple VLAN layers not supported"); 1882 else if ((item_flags & l34m) != 0) 1883 return rte_flow_error_set(error, EINVAL, 1884 RTE_FLOW_ERROR_TYPE_ITEM, item, 1885 "VLAN cannot follow L3/L4 layer"); 1886 if (!mask) 1887 mask = &rte_flow_item_vlan_mask; 1888 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1889 (const uint8_t *)&nic_mask, 1890 sizeof(struct rte_flow_item_vlan), 1891 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1892 if (ret) 1893 return ret; 1894 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1895 struct mlx5_priv *priv = dev->data->dev_private; 1896 1897 if (priv->vmwa_context) { 1898 /* 1899 * Non-NULL context means we have a virtual machine 1900 * and SR-IOV enabled, we have to create VLAN interface 1901 * to make hypervisor to setup E-Switch vport 1902 * context correctly. We avoid creating the multiple 1903 * VLAN interfaces, so we cannot support VLAN tag mask. 1904 */ 1905 return rte_flow_error_set(error, EINVAL, 1906 RTE_FLOW_ERROR_TYPE_ITEM, 1907 item, 1908 "VLAN tag mask is not" 1909 " supported in virtual" 1910 " environment"); 1911 } 1912 } 1913 if (spec) { 1914 vlan_tag = spec->tci; 1915 vlan_tag &= mask->tci; 1916 } 1917 /* 1918 * From verbs perspective an empty VLAN is equivalent 1919 * to a packet without VLAN layer. 1920 */ 1921 if (!vlan_tag) 1922 return rte_flow_error_set(error, EINVAL, 1923 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1924 item->spec, 1925 "VLAN cannot be empty"); 1926 return 0; 1927 } 1928 1929 /** 1930 * Validate IPV4 item. 1931 * 1932 * @param[in] item 1933 * Item specification. 1934 * @param[in] item_flags 1935 * Bit-fields that holds the items detected until now. 1936 * @param[in] last_item 1937 * Previous validated item in the pattern items. 1938 * @param[in] ether_type 1939 * Type in the ethernet layer header (including dot1q). 1940 * @param[in] acc_mask 1941 * Acceptable mask, if NULL default internal default mask 1942 * will be used to check whether item fields are supported. 1943 * @param[in] range_accepted 1944 * True if range of values is accepted for specific fields, false otherwise. 1945 * @param[out] error 1946 * Pointer to error structure. 1947 * 1948 * @return 1949 * 0 on success, a negative errno value otherwise and rte_errno is set. 1950 */ 1951 int 1952 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1953 uint64_t item_flags, 1954 uint64_t last_item, 1955 uint16_t ether_type, 1956 const struct rte_flow_item_ipv4 *acc_mask, 1957 bool range_accepted, 1958 struct rte_flow_error *error) 1959 { 1960 const struct rte_flow_item_ipv4 *mask = item->mask; 1961 const struct rte_flow_item_ipv4 *spec = item->spec; 1962 const struct rte_flow_item_ipv4 nic_mask = { 1963 .hdr = { 1964 .src_addr = RTE_BE32(0xffffffff), 1965 .dst_addr = RTE_BE32(0xffffffff), 1966 .type_of_service = 0xff, 1967 .next_proto_id = 0xff, 1968 }, 1969 }; 1970 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1971 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1972 MLX5_FLOW_LAYER_OUTER_L3; 1973 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1974 MLX5_FLOW_LAYER_OUTER_L4; 1975 int ret; 1976 uint8_t next_proto = 0xFF; 1977 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1978 MLX5_FLOW_LAYER_OUTER_VLAN | 1979 MLX5_FLOW_LAYER_INNER_VLAN); 1980 1981 if ((last_item & l2_vlan) && ether_type && 1982 ether_type != RTE_ETHER_TYPE_IPV4) 1983 return rte_flow_error_set(error, EINVAL, 1984 RTE_FLOW_ERROR_TYPE_ITEM, item, 1985 "IPv4 cannot follow L2/VLAN layer " 1986 "which ether type is not IPv4"); 1987 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1988 if (mask && spec) 1989 next_proto = mask->hdr.next_proto_id & 1990 spec->hdr.next_proto_id; 1991 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1992 return rte_flow_error_set(error, EINVAL, 1993 RTE_FLOW_ERROR_TYPE_ITEM, 1994 item, 1995 "multiple tunnel " 1996 "not supported"); 1997 } 1998 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1999 return rte_flow_error_set(error, EINVAL, 2000 RTE_FLOW_ERROR_TYPE_ITEM, item, 2001 "wrong tunnel type - IPv6 specified " 2002 "but IPv4 item provided"); 2003 if (item_flags & l3m) 2004 return rte_flow_error_set(error, ENOTSUP, 2005 RTE_FLOW_ERROR_TYPE_ITEM, item, 2006 "multiple L3 layers not supported"); 2007 else if (item_flags & l4m) 2008 return rte_flow_error_set(error, EINVAL, 2009 RTE_FLOW_ERROR_TYPE_ITEM, item, 2010 "L3 cannot follow an L4 layer."); 2011 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2012 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2013 return rte_flow_error_set(error, EINVAL, 2014 RTE_FLOW_ERROR_TYPE_ITEM, item, 2015 "L3 cannot follow an NVGRE layer."); 2016 if (!mask) 2017 mask = &rte_flow_item_ipv4_mask; 2018 else if (mask->hdr.next_proto_id != 0 && 2019 mask->hdr.next_proto_id != 0xff) 2020 return rte_flow_error_set(error, EINVAL, 2021 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2022 "partial mask is not supported" 2023 " for protocol"); 2024 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2025 acc_mask ? (const uint8_t *)acc_mask 2026 : (const uint8_t *)&nic_mask, 2027 sizeof(struct rte_flow_item_ipv4), 2028 range_accepted, error); 2029 if (ret < 0) 2030 return ret; 2031 return 0; 2032 } 2033 2034 /** 2035 * Validate IPV6 item. 2036 * 2037 * @param[in] item 2038 * Item specification. 2039 * @param[in] item_flags 2040 * Bit-fields that holds the items detected until now. 2041 * @param[in] last_item 2042 * Previous validated item in the pattern items. 2043 * @param[in] ether_type 2044 * Type in the ethernet layer header (including dot1q). 2045 * @param[in] acc_mask 2046 * Acceptable mask, if NULL default internal default mask 2047 * will be used to check whether item fields are supported. 2048 * @param[out] error 2049 * Pointer to error structure. 2050 * 2051 * @return 2052 * 0 on success, a negative errno value otherwise and rte_errno is set. 2053 */ 2054 int 2055 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2056 uint64_t item_flags, 2057 uint64_t last_item, 2058 uint16_t ether_type, 2059 const struct rte_flow_item_ipv6 *acc_mask, 2060 struct rte_flow_error *error) 2061 { 2062 const struct rte_flow_item_ipv6 *mask = item->mask; 2063 const struct rte_flow_item_ipv6 *spec = item->spec; 2064 const struct rte_flow_item_ipv6 nic_mask = { 2065 .hdr = { 2066 .src_addr = 2067 "\xff\xff\xff\xff\xff\xff\xff\xff" 2068 "\xff\xff\xff\xff\xff\xff\xff\xff", 2069 .dst_addr = 2070 "\xff\xff\xff\xff\xff\xff\xff\xff" 2071 "\xff\xff\xff\xff\xff\xff\xff\xff", 2072 .vtc_flow = RTE_BE32(0xffffffff), 2073 .proto = 0xff, 2074 }, 2075 }; 2076 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2077 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2078 MLX5_FLOW_LAYER_OUTER_L3; 2079 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2080 MLX5_FLOW_LAYER_OUTER_L4; 2081 int ret; 2082 uint8_t next_proto = 0xFF; 2083 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2084 MLX5_FLOW_LAYER_OUTER_VLAN | 2085 MLX5_FLOW_LAYER_INNER_VLAN); 2086 2087 if ((last_item & l2_vlan) && ether_type && 2088 ether_type != RTE_ETHER_TYPE_IPV6) 2089 return rte_flow_error_set(error, EINVAL, 2090 RTE_FLOW_ERROR_TYPE_ITEM, item, 2091 "IPv6 cannot follow L2/VLAN layer " 2092 "which ether type is not IPv6"); 2093 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2094 next_proto = spec->hdr.proto; 2095 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 2096 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2097 return rte_flow_error_set(error, EINVAL, 2098 RTE_FLOW_ERROR_TYPE_ITEM, 2099 item, 2100 "multiple tunnel " 2101 "not supported"); 2102 } 2103 if (next_proto == IPPROTO_HOPOPTS || 2104 next_proto == IPPROTO_ROUTING || 2105 next_proto == IPPROTO_FRAGMENT || 2106 next_proto == IPPROTO_ESP || 2107 next_proto == IPPROTO_AH || 2108 next_proto == IPPROTO_DSTOPTS) 2109 return rte_flow_error_set(error, EINVAL, 2110 RTE_FLOW_ERROR_TYPE_ITEM, item, 2111 "IPv6 proto (next header) should " 2112 "not be set as extension header"); 2113 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2114 return rte_flow_error_set(error, EINVAL, 2115 RTE_FLOW_ERROR_TYPE_ITEM, item, 2116 "wrong tunnel type - IPv4 specified " 2117 "but IPv6 item provided"); 2118 if (item_flags & l3m) 2119 return rte_flow_error_set(error, ENOTSUP, 2120 RTE_FLOW_ERROR_TYPE_ITEM, item, 2121 "multiple L3 layers not supported"); 2122 else if (item_flags & l4m) 2123 return rte_flow_error_set(error, EINVAL, 2124 RTE_FLOW_ERROR_TYPE_ITEM, item, 2125 "L3 cannot follow an L4 layer."); 2126 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2127 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2128 return rte_flow_error_set(error, EINVAL, 2129 RTE_FLOW_ERROR_TYPE_ITEM, item, 2130 "L3 cannot follow an NVGRE layer."); 2131 if (!mask) 2132 mask = &rte_flow_item_ipv6_mask; 2133 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2134 acc_mask ? (const uint8_t *)acc_mask 2135 : (const uint8_t *)&nic_mask, 2136 sizeof(struct rte_flow_item_ipv6), 2137 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2138 if (ret < 0) 2139 return ret; 2140 return 0; 2141 } 2142 2143 /** 2144 * Validate UDP item. 2145 * 2146 * @param[in] item 2147 * Item specification. 2148 * @param[in] item_flags 2149 * Bit-fields that holds the items detected until now. 2150 * @param[in] target_protocol 2151 * The next protocol in the previous item. 2152 * @param[in] flow_mask 2153 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2154 * @param[out] error 2155 * Pointer to error structure. 2156 * 2157 * @return 2158 * 0 on success, a negative errno value otherwise and rte_errno is set. 2159 */ 2160 int 2161 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2162 uint64_t item_flags, 2163 uint8_t target_protocol, 2164 struct rte_flow_error *error) 2165 { 2166 const struct rte_flow_item_udp *mask = item->mask; 2167 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2168 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2169 MLX5_FLOW_LAYER_OUTER_L3; 2170 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2171 MLX5_FLOW_LAYER_OUTER_L4; 2172 int ret; 2173 2174 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2175 return rte_flow_error_set(error, EINVAL, 2176 RTE_FLOW_ERROR_TYPE_ITEM, item, 2177 "protocol filtering not compatible" 2178 " with UDP layer"); 2179 if (!(item_flags & l3m)) 2180 return rte_flow_error_set(error, EINVAL, 2181 RTE_FLOW_ERROR_TYPE_ITEM, item, 2182 "L3 is mandatory to filter on L4"); 2183 if (item_flags & l4m) 2184 return rte_flow_error_set(error, EINVAL, 2185 RTE_FLOW_ERROR_TYPE_ITEM, item, 2186 "multiple L4 layers not supported"); 2187 if (!mask) 2188 mask = &rte_flow_item_udp_mask; 2189 ret = mlx5_flow_item_acceptable 2190 (item, (const uint8_t *)mask, 2191 (const uint8_t *)&rte_flow_item_udp_mask, 2192 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2193 error); 2194 if (ret < 0) 2195 return ret; 2196 return 0; 2197 } 2198 2199 /** 2200 * Validate TCP item. 2201 * 2202 * @param[in] item 2203 * Item specification. 2204 * @param[in] item_flags 2205 * Bit-fields that holds the items detected until now. 2206 * @param[in] target_protocol 2207 * The next protocol in the previous item. 2208 * @param[out] error 2209 * Pointer to error structure. 2210 * 2211 * @return 2212 * 0 on success, a negative errno value otherwise and rte_errno is set. 2213 */ 2214 int 2215 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2216 uint64_t item_flags, 2217 uint8_t target_protocol, 2218 const struct rte_flow_item_tcp *flow_mask, 2219 struct rte_flow_error *error) 2220 { 2221 const struct rte_flow_item_tcp *mask = item->mask; 2222 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2223 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2224 MLX5_FLOW_LAYER_OUTER_L3; 2225 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2226 MLX5_FLOW_LAYER_OUTER_L4; 2227 int ret; 2228 2229 MLX5_ASSERT(flow_mask); 2230 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2231 return rte_flow_error_set(error, EINVAL, 2232 RTE_FLOW_ERROR_TYPE_ITEM, item, 2233 "protocol filtering not compatible" 2234 " with TCP layer"); 2235 if (!(item_flags & l3m)) 2236 return rte_flow_error_set(error, EINVAL, 2237 RTE_FLOW_ERROR_TYPE_ITEM, item, 2238 "L3 is mandatory to filter on L4"); 2239 if (item_flags & l4m) 2240 return rte_flow_error_set(error, EINVAL, 2241 RTE_FLOW_ERROR_TYPE_ITEM, item, 2242 "multiple L4 layers not supported"); 2243 if (!mask) 2244 mask = &rte_flow_item_tcp_mask; 2245 ret = mlx5_flow_item_acceptable 2246 (item, (const uint8_t *)mask, 2247 (const uint8_t *)flow_mask, 2248 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2249 error); 2250 if (ret < 0) 2251 return ret; 2252 return 0; 2253 } 2254 2255 /** 2256 * Validate VXLAN item. 2257 * 2258 * @param[in] item 2259 * Item specification. 2260 * @param[in] item_flags 2261 * Bit-fields that holds the items detected until now. 2262 * @param[in] target_protocol 2263 * The next protocol in the previous item. 2264 * @param[out] error 2265 * Pointer to error structure. 2266 * 2267 * @return 2268 * 0 on success, a negative errno value otherwise and rte_errno is set. 2269 */ 2270 int 2271 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2272 uint64_t item_flags, 2273 struct rte_flow_error *error) 2274 { 2275 const struct rte_flow_item_vxlan *spec = item->spec; 2276 const struct rte_flow_item_vxlan *mask = item->mask; 2277 int ret; 2278 union vni { 2279 uint32_t vlan_id; 2280 uint8_t vni[4]; 2281 } id = { .vlan_id = 0, }; 2282 2283 2284 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2285 return rte_flow_error_set(error, ENOTSUP, 2286 RTE_FLOW_ERROR_TYPE_ITEM, item, 2287 "multiple tunnel layers not" 2288 " supported"); 2289 /* 2290 * Verify only UDPv4 is present as defined in 2291 * https://tools.ietf.org/html/rfc7348 2292 */ 2293 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2294 return rte_flow_error_set(error, EINVAL, 2295 RTE_FLOW_ERROR_TYPE_ITEM, item, 2296 "no outer UDP layer found"); 2297 if (!mask) 2298 mask = &rte_flow_item_vxlan_mask; 2299 ret = mlx5_flow_item_acceptable 2300 (item, (const uint8_t *)mask, 2301 (const uint8_t *)&rte_flow_item_vxlan_mask, 2302 sizeof(struct rte_flow_item_vxlan), 2303 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2304 if (ret < 0) 2305 return ret; 2306 if (spec) { 2307 memcpy(&id.vni[1], spec->vni, 3); 2308 memcpy(&id.vni[1], mask->vni, 3); 2309 } 2310 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2311 return rte_flow_error_set(error, ENOTSUP, 2312 RTE_FLOW_ERROR_TYPE_ITEM, item, 2313 "VXLAN tunnel must be fully defined"); 2314 return 0; 2315 } 2316 2317 /** 2318 * Validate VXLAN_GPE item. 2319 * 2320 * @param[in] item 2321 * Item specification. 2322 * @param[in] item_flags 2323 * Bit-fields that holds the items detected until now. 2324 * @param[in] priv 2325 * Pointer to the private data structure. 2326 * @param[in] target_protocol 2327 * The next protocol in the previous item. 2328 * @param[out] error 2329 * Pointer to error structure. 2330 * 2331 * @return 2332 * 0 on success, a negative errno value otherwise and rte_errno is set. 2333 */ 2334 int 2335 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2336 uint64_t item_flags, 2337 struct rte_eth_dev *dev, 2338 struct rte_flow_error *error) 2339 { 2340 struct mlx5_priv *priv = dev->data->dev_private; 2341 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2342 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2343 int ret; 2344 union vni { 2345 uint32_t vlan_id; 2346 uint8_t vni[4]; 2347 } id = { .vlan_id = 0, }; 2348 2349 if (!priv->config.l3_vxlan_en) 2350 return rte_flow_error_set(error, ENOTSUP, 2351 RTE_FLOW_ERROR_TYPE_ITEM, item, 2352 "L3 VXLAN is not enabled by device" 2353 " parameter and/or not configured in" 2354 " firmware"); 2355 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2356 return rte_flow_error_set(error, ENOTSUP, 2357 RTE_FLOW_ERROR_TYPE_ITEM, item, 2358 "multiple tunnel layers not" 2359 " supported"); 2360 /* 2361 * Verify only UDPv4 is present as defined in 2362 * https://tools.ietf.org/html/rfc7348 2363 */ 2364 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2365 return rte_flow_error_set(error, EINVAL, 2366 RTE_FLOW_ERROR_TYPE_ITEM, item, 2367 "no outer UDP layer found"); 2368 if (!mask) 2369 mask = &rte_flow_item_vxlan_gpe_mask; 2370 ret = mlx5_flow_item_acceptable 2371 (item, (const uint8_t *)mask, 2372 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2373 sizeof(struct rte_flow_item_vxlan_gpe), 2374 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2375 if (ret < 0) 2376 return ret; 2377 if (spec) { 2378 if (spec->protocol) 2379 return rte_flow_error_set(error, ENOTSUP, 2380 RTE_FLOW_ERROR_TYPE_ITEM, 2381 item, 2382 "VxLAN-GPE protocol" 2383 " not supported"); 2384 memcpy(&id.vni[1], spec->vni, 3); 2385 memcpy(&id.vni[1], mask->vni, 3); 2386 } 2387 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2388 return rte_flow_error_set(error, ENOTSUP, 2389 RTE_FLOW_ERROR_TYPE_ITEM, item, 2390 "VXLAN-GPE tunnel must be fully" 2391 " defined"); 2392 return 0; 2393 } 2394 /** 2395 * Validate GRE Key item. 2396 * 2397 * @param[in] item 2398 * Item specification. 2399 * @param[in] item_flags 2400 * Bit flags to mark detected items. 2401 * @param[in] gre_item 2402 * Pointer to gre_item 2403 * @param[out] error 2404 * Pointer to error structure. 2405 * 2406 * @return 2407 * 0 on success, a negative errno value otherwise and rte_errno is set. 2408 */ 2409 int 2410 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2411 uint64_t item_flags, 2412 const struct rte_flow_item *gre_item, 2413 struct rte_flow_error *error) 2414 { 2415 const rte_be32_t *mask = item->mask; 2416 int ret = 0; 2417 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2418 const struct rte_flow_item_gre *gre_spec; 2419 const struct rte_flow_item_gre *gre_mask; 2420 2421 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2422 return rte_flow_error_set(error, ENOTSUP, 2423 RTE_FLOW_ERROR_TYPE_ITEM, item, 2424 "Multiple GRE key not support"); 2425 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2426 return rte_flow_error_set(error, ENOTSUP, 2427 RTE_FLOW_ERROR_TYPE_ITEM, item, 2428 "No preceding GRE header"); 2429 if (item_flags & MLX5_FLOW_LAYER_INNER) 2430 return rte_flow_error_set(error, ENOTSUP, 2431 RTE_FLOW_ERROR_TYPE_ITEM, item, 2432 "GRE key following a wrong item"); 2433 gre_mask = gre_item->mask; 2434 if (!gre_mask) 2435 gre_mask = &rte_flow_item_gre_mask; 2436 gre_spec = gre_item->spec; 2437 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2438 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2439 return rte_flow_error_set(error, EINVAL, 2440 RTE_FLOW_ERROR_TYPE_ITEM, item, 2441 "Key bit must be on"); 2442 2443 if (!mask) 2444 mask = &gre_key_default_mask; 2445 ret = mlx5_flow_item_acceptable 2446 (item, (const uint8_t *)mask, 2447 (const uint8_t *)&gre_key_default_mask, 2448 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2449 return ret; 2450 } 2451 2452 /** 2453 * Validate GRE item. 2454 * 2455 * @param[in] item 2456 * Item specification. 2457 * @param[in] item_flags 2458 * Bit flags to mark detected items. 2459 * @param[in] target_protocol 2460 * The next protocol in the previous item. 2461 * @param[out] error 2462 * Pointer to error structure. 2463 * 2464 * @return 2465 * 0 on success, a negative errno value otherwise and rte_errno is set. 2466 */ 2467 int 2468 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2469 uint64_t item_flags, 2470 uint8_t target_protocol, 2471 struct rte_flow_error *error) 2472 { 2473 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2474 const struct rte_flow_item_gre *mask = item->mask; 2475 int ret; 2476 const struct rte_flow_item_gre nic_mask = { 2477 .c_rsvd0_ver = RTE_BE16(0xB000), 2478 .protocol = RTE_BE16(UINT16_MAX), 2479 }; 2480 2481 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2482 return rte_flow_error_set(error, EINVAL, 2483 RTE_FLOW_ERROR_TYPE_ITEM, item, 2484 "protocol filtering not compatible" 2485 " with this GRE layer"); 2486 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2487 return rte_flow_error_set(error, ENOTSUP, 2488 RTE_FLOW_ERROR_TYPE_ITEM, item, 2489 "multiple tunnel layers not" 2490 " supported"); 2491 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2492 return rte_flow_error_set(error, ENOTSUP, 2493 RTE_FLOW_ERROR_TYPE_ITEM, item, 2494 "L3 Layer is missing"); 2495 if (!mask) 2496 mask = &rte_flow_item_gre_mask; 2497 ret = mlx5_flow_item_acceptable 2498 (item, (const uint8_t *)mask, 2499 (const uint8_t *)&nic_mask, 2500 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2501 error); 2502 if (ret < 0) 2503 return ret; 2504 #ifndef HAVE_MLX5DV_DR 2505 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2506 if (spec && (spec->protocol & mask->protocol)) 2507 return rte_flow_error_set(error, ENOTSUP, 2508 RTE_FLOW_ERROR_TYPE_ITEM, item, 2509 "without MPLS support the" 2510 " specification cannot be used for" 2511 " filtering"); 2512 #endif 2513 #endif 2514 return 0; 2515 } 2516 2517 /** 2518 * Validate Geneve item. 2519 * 2520 * @param[in] item 2521 * Item specification. 2522 * @param[in] itemFlags 2523 * Bit-fields that holds the items detected until now. 2524 * @param[in] enPriv 2525 * Pointer to the private data structure. 2526 * @param[out] error 2527 * Pointer to error structure. 2528 * 2529 * @return 2530 * 0 on success, a negative errno value otherwise and rte_errno is set. 2531 */ 2532 2533 int 2534 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2535 uint64_t item_flags, 2536 struct rte_eth_dev *dev, 2537 struct rte_flow_error *error) 2538 { 2539 struct mlx5_priv *priv = dev->data->dev_private; 2540 const struct rte_flow_item_geneve *spec = item->spec; 2541 const struct rte_flow_item_geneve *mask = item->mask; 2542 int ret; 2543 uint16_t gbhdr; 2544 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2545 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2546 const struct rte_flow_item_geneve nic_mask = { 2547 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2548 .vni = "\xff\xff\xff", 2549 .protocol = RTE_BE16(UINT16_MAX), 2550 }; 2551 2552 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2553 return rte_flow_error_set(error, ENOTSUP, 2554 RTE_FLOW_ERROR_TYPE_ITEM, item, 2555 "L3 Geneve is not enabled by device" 2556 " parameter and/or not configured in" 2557 " firmware"); 2558 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2559 return rte_flow_error_set(error, ENOTSUP, 2560 RTE_FLOW_ERROR_TYPE_ITEM, item, 2561 "multiple tunnel layers not" 2562 " supported"); 2563 /* 2564 * Verify only UDPv4 is present as defined in 2565 * https://tools.ietf.org/html/rfc7348 2566 */ 2567 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2568 return rte_flow_error_set(error, EINVAL, 2569 RTE_FLOW_ERROR_TYPE_ITEM, item, 2570 "no outer UDP layer found"); 2571 if (!mask) 2572 mask = &rte_flow_item_geneve_mask; 2573 ret = mlx5_flow_item_acceptable 2574 (item, (const uint8_t *)mask, 2575 (const uint8_t *)&nic_mask, 2576 sizeof(struct rte_flow_item_geneve), 2577 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2578 if (ret) 2579 return ret; 2580 if (spec) { 2581 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2582 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2583 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2584 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2585 return rte_flow_error_set(error, ENOTSUP, 2586 RTE_FLOW_ERROR_TYPE_ITEM, 2587 item, 2588 "Geneve protocol unsupported" 2589 " fields are being used"); 2590 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2591 return rte_flow_error_set 2592 (error, ENOTSUP, 2593 RTE_FLOW_ERROR_TYPE_ITEM, 2594 item, 2595 "Unsupported Geneve options length"); 2596 } 2597 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2598 return rte_flow_error_set 2599 (error, ENOTSUP, 2600 RTE_FLOW_ERROR_TYPE_ITEM, item, 2601 "Geneve tunnel must be fully defined"); 2602 return 0; 2603 } 2604 2605 /** 2606 * Validate MPLS item. 2607 * 2608 * @param[in] dev 2609 * Pointer to the rte_eth_dev structure. 2610 * @param[in] item 2611 * Item specification. 2612 * @param[in] item_flags 2613 * Bit-fields that holds the items detected until now. 2614 * @param[in] prev_layer 2615 * The protocol layer indicated in previous item. 2616 * @param[out] error 2617 * Pointer to error structure. 2618 * 2619 * @return 2620 * 0 on success, a negative errno value otherwise and rte_errno is set. 2621 */ 2622 int 2623 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2624 const struct rte_flow_item *item __rte_unused, 2625 uint64_t item_flags __rte_unused, 2626 uint64_t prev_layer __rte_unused, 2627 struct rte_flow_error *error) 2628 { 2629 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2630 const struct rte_flow_item_mpls *mask = item->mask; 2631 struct mlx5_priv *priv = dev->data->dev_private; 2632 int ret; 2633 2634 if (!priv->config.mpls_en) 2635 return rte_flow_error_set(error, ENOTSUP, 2636 RTE_FLOW_ERROR_TYPE_ITEM, item, 2637 "MPLS not supported or" 2638 " disabled in firmware" 2639 " configuration."); 2640 /* MPLS over IP, UDP, GRE is allowed */ 2641 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2642 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2643 MLX5_FLOW_LAYER_GRE | 2644 MLX5_FLOW_LAYER_GRE_KEY))) 2645 return rte_flow_error_set(error, EINVAL, 2646 RTE_FLOW_ERROR_TYPE_ITEM, item, 2647 "protocol filtering not compatible" 2648 " with MPLS layer"); 2649 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2650 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2651 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2652 return rte_flow_error_set(error, ENOTSUP, 2653 RTE_FLOW_ERROR_TYPE_ITEM, item, 2654 "multiple tunnel layers not" 2655 " supported"); 2656 if (!mask) 2657 mask = &rte_flow_item_mpls_mask; 2658 ret = mlx5_flow_item_acceptable 2659 (item, (const uint8_t *)mask, 2660 (const uint8_t *)&rte_flow_item_mpls_mask, 2661 sizeof(struct rte_flow_item_mpls), 2662 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2663 if (ret < 0) 2664 return ret; 2665 return 0; 2666 #else 2667 return rte_flow_error_set(error, ENOTSUP, 2668 RTE_FLOW_ERROR_TYPE_ITEM, item, 2669 "MPLS is not supported by Verbs, please" 2670 " update."); 2671 #endif 2672 } 2673 2674 /** 2675 * Validate NVGRE item. 2676 * 2677 * @param[in] item 2678 * Item specification. 2679 * @param[in] item_flags 2680 * Bit flags to mark detected items. 2681 * @param[in] target_protocol 2682 * The next protocol in the previous item. 2683 * @param[out] error 2684 * Pointer to error structure. 2685 * 2686 * @return 2687 * 0 on success, a negative errno value otherwise and rte_errno is set. 2688 */ 2689 int 2690 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2691 uint64_t item_flags, 2692 uint8_t target_protocol, 2693 struct rte_flow_error *error) 2694 { 2695 const struct rte_flow_item_nvgre *mask = item->mask; 2696 int ret; 2697 2698 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2699 return rte_flow_error_set(error, EINVAL, 2700 RTE_FLOW_ERROR_TYPE_ITEM, item, 2701 "protocol filtering not compatible" 2702 " with this GRE layer"); 2703 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2704 return rte_flow_error_set(error, ENOTSUP, 2705 RTE_FLOW_ERROR_TYPE_ITEM, item, 2706 "multiple tunnel layers not" 2707 " supported"); 2708 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2709 return rte_flow_error_set(error, ENOTSUP, 2710 RTE_FLOW_ERROR_TYPE_ITEM, item, 2711 "L3 Layer is missing"); 2712 if (!mask) 2713 mask = &rte_flow_item_nvgre_mask; 2714 ret = mlx5_flow_item_acceptable 2715 (item, (const uint8_t *)mask, 2716 (const uint8_t *)&rte_flow_item_nvgre_mask, 2717 sizeof(struct rte_flow_item_nvgre), 2718 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2719 if (ret < 0) 2720 return ret; 2721 return 0; 2722 } 2723 2724 /** 2725 * Validate eCPRI item. 2726 * 2727 * @param[in] item 2728 * Item specification. 2729 * @param[in] item_flags 2730 * Bit-fields that holds the items detected until now. 2731 * @param[in] last_item 2732 * Previous validated item in the pattern items. 2733 * @param[in] ether_type 2734 * Type in the ethernet layer header (including dot1q). 2735 * @param[in] acc_mask 2736 * Acceptable mask, if NULL default internal default mask 2737 * will be used to check whether item fields are supported. 2738 * @param[out] error 2739 * Pointer to error structure. 2740 * 2741 * @return 2742 * 0 on success, a negative errno value otherwise and rte_errno is set. 2743 */ 2744 int 2745 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 2746 uint64_t item_flags, 2747 uint64_t last_item, 2748 uint16_t ether_type, 2749 const struct rte_flow_item_ecpri *acc_mask, 2750 struct rte_flow_error *error) 2751 { 2752 const struct rte_flow_item_ecpri *mask = item->mask; 2753 const struct rte_flow_item_ecpri nic_mask = { 2754 .hdr = { 2755 .common = { 2756 .u32 = 2757 RTE_BE32(((const struct rte_ecpri_common_hdr) { 2758 .type = 0xFF, 2759 }).u32), 2760 }, 2761 .dummy[0] = 0xFFFFFFFF, 2762 }, 2763 }; 2764 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 2765 MLX5_FLOW_LAYER_OUTER_VLAN); 2766 struct rte_flow_item_ecpri mask_lo; 2767 2768 if (!(last_item & outer_l2_vlan) && 2769 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 2770 return rte_flow_error_set(error, EINVAL, 2771 RTE_FLOW_ERROR_TYPE_ITEM, item, 2772 "eCPRI can only follow L2/VLAN layer or UDP layer"); 2773 if ((last_item & outer_l2_vlan) && ether_type && 2774 ether_type != RTE_ETHER_TYPE_ECPRI) 2775 return rte_flow_error_set(error, EINVAL, 2776 RTE_FLOW_ERROR_TYPE_ITEM, item, 2777 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 2778 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2779 return rte_flow_error_set(error, EINVAL, 2780 RTE_FLOW_ERROR_TYPE_ITEM, item, 2781 "eCPRI with tunnel is not supported right now"); 2782 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 2783 return rte_flow_error_set(error, ENOTSUP, 2784 RTE_FLOW_ERROR_TYPE_ITEM, item, 2785 "multiple L3 layers not supported"); 2786 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 2787 return rte_flow_error_set(error, EINVAL, 2788 RTE_FLOW_ERROR_TYPE_ITEM, item, 2789 "eCPRI cannot coexist with a TCP layer"); 2790 /* In specification, eCPRI could be over UDP layer. */ 2791 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 2792 return rte_flow_error_set(error, EINVAL, 2793 RTE_FLOW_ERROR_TYPE_ITEM, item, 2794 "eCPRI over UDP layer is not yet supported right now"); 2795 /* Mask for type field in common header could be zero. */ 2796 if (!mask) 2797 mask = &rte_flow_item_ecpri_mask; 2798 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 2799 /* Input mask is in big-endian format. */ 2800 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 2801 return rte_flow_error_set(error, EINVAL, 2802 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2803 "partial mask is not supported for protocol"); 2804 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 2805 return rte_flow_error_set(error, EINVAL, 2806 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2807 "message header mask must be after a type mask"); 2808 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2809 acc_mask ? (const uint8_t *)acc_mask 2810 : (const uint8_t *)&nic_mask, 2811 sizeof(struct rte_flow_item_ecpri), 2812 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2813 } 2814 2815 /** 2816 * Release resource related QUEUE/RSS action split. 2817 * 2818 * @param dev 2819 * Pointer to Ethernet device. 2820 * @param flow 2821 * Flow to release id's from. 2822 */ 2823 static void 2824 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2825 struct rte_flow *flow) 2826 { 2827 struct mlx5_priv *priv = dev->data->dev_private; 2828 uint32_t handle_idx; 2829 struct mlx5_flow_handle *dev_handle; 2830 2831 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2832 handle_idx, dev_handle, next) 2833 if (dev_handle->split_flow_id) 2834 mlx5_ipool_free(priv->sh->ipool 2835 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 2836 dev_handle->split_flow_id); 2837 } 2838 2839 static int 2840 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2841 const struct rte_flow_attr *attr __rte_unused, 2842 const struct rte_flow_item items[] __rte_unused, 2843 const struct rte_flow_action actions[] __rte_unused, 2844 bool external __rte_unused, 2845 int hairpin __rte_unused, 2846 struct rte_flow_error *error) 2847 { 2848 return rte_flow_error_set(error, ENOTSUP, 2849 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2850 } 2851 2852 static struct mlx5_flow * 2853 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2854 const struct rte_flow_attr *attr __rte_unused, 2855 const struct rte_flow_item items[] __rte_unused, 2856 const struct rte_flow_action actions[] __rte_unused, 2857 struct rte_flow_error *error) 2858 { 2859 rte_flow_error_set(error, ENOTSUP, 2860 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2861 return NULL; 2862 } 2863 2864 static int 2865 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2866 struct mlx5_flow *dev_flow __rte_unused, 2867 const struct rte_flow_attr *attr __rte_unused, 2868 const struct rte_flow_item items[] __rte_unused, 2869 const struct rte_flow_action actions[] __rte_unused, 2870 struct rte_flow_error *error) 2871 { 2872 return rte_flow_error_set(error, ENOTSUP, 2873 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2874 } 2875 2876 static int 2877 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2878 struct rte_flow *flow __rte_unused, 2879 struct rte_flow_error *error) 2880 { 2881 return rte_flow_error_set(error, ENOTSUP, 2882 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2883 } 2884 2885 static void 2886 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2887 struct rte_flow *flow __rte_unused) 2888 { 2889 } 2890 2891 static void 2892 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2893 struct rte_flow *flow __rte_unused) 2894 { 2895 } 2896 2897 static int 2898 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2899 struct rte_flow *flow __rte_unused, 2900 const struct rte_flow_action *actions __rte_unused, 2901 void *data __rte_unused, 2902 struct rte_flow_error *error) 2903 { 2904 return rte_flow_error_set(error, ENOTSUP, 2905 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2906 } 2907 2908 static int 2909 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 2910 uint32_t domains __rte_unused, 2911 uint32_t flags __rte_unused) 2912 { 2913 return 0; 2914 } 2915 2916 /* Void driver to protect from null pointer reference. */ 2917 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2918 .validate = flow_null_validate, 2919 .prepare = flow_null_prepare, 2920 .translate = flow_null_translate, 2921 .apply = flow_null_apply, 2922 .remove = flow_null_remove, 2923 .destroy = flow_null_destroy, 2924 .query = flow_null_query, 2925 .sync_domain = flow_null_sync_domain, 2926 }; 2927 2928 /** 2929 * Select flow driver type according to flow attributes and device 2930 * configuration. 2931 * 2932 * @param[in] dev 2933 * Pointer to the dev structure. 2934 * @param[in] attr 2935 * Pointer to the flow attributes. 2936 * 2937 * @return 2938 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2939 */ 2940 static enum mlx5_flow_drv_type 2941 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2942 { 2943 struct mlx5_priv *priv = dev->data->dev_private; 2944 /* The OS can determine first a specific flow type (DV, VERBS) */ 2945 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 2946 2947 if (type != MLX5_FLOW_TYPE_MAX) 2948 return type; 2949 /* If no OS specific type - continue with DV/VERBS selection */ 2950 if (attr->transfer && priv->config.dv_esw_en) 2951 type = MLX5_FLOW_TYPE_DV; 2952 if (!attr->transfer) 2953 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2954 MLX5_FLOW_TYPE_VERBS; 2955 return type; 2956 } 2957 2958 #define flow_get_drv_ops(type) flow_drv_ops[type] 2959 2960 /** 2961 * Flow driver validation API. This abstracts calling driver specific functions. 2962 * The type of flow driver is determined according to flow attributes. 2963 * 2964 * @param[in] dev 2965 * Pointer to the dev structure. 2966 * @param[in] attr 2967 * Pointer to the flow attributes. 2968 * @param[in] items 2969 * Pointer to the list of items. 2970 * @param[in] actions 2971 * Pointer to the list of actions. 2972 * @param[in] external 2973 * This flow rule is created by request external to PMD. 2974 * @param[in] hairpin 2975 * Number of hairpin TX actions, 0 means classic flow. 2976 * @param[out] error 2977 * Pointer to the error structure. 2978 * 2979 * @return 2980 * 0 on success, a negative errno value otherwise and rte_errno is set. 2981 */ 2982 static inline int 2983 flow_drv_validate(struct rte_eth_dev *dev, 2984 const struct rte_flow_attr *attr, 2985 const struct rte_flow_item items[], 2986 const struct rte_flow_action actions[], 2987 bool external, int hairpin, struct rte_flow_error *error) 2988 { 2989 const struct mlx5_flow_driver_ops *fops; 2990 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2991 2992 fops = flow_get_drv_ops(type); 2993 return fops->validate(dev, attr, items, actions, external, 2994 hairpin, error); 2995 } 2996 2997 /** 2998 * Flow driver preparation API. This abstracts calling driver specific 2999 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3000 * calculates the size of memory required for device flow, allocates the memory, 3001 * initializes the device flow and returns the pointer. 3002 * 3003 * @note 3004 * This function initializes device flow structure such as dv or verbs in 3005 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3006 * rest. For example, adding returning device flow to flow->dev_flow list and 3007 * setting backward reference to the flow should be done out of this function. 3008 * layers field is not filled either. 3009 * 3010 * @param[in] dev 3011 * Pointer to the dev structure. 3012 * @param[in] attr 3013 * Pointer to the flow attributes. 3014 * @param[in] items 3015 * Pointer to the list of items. 3016 * @param[in] actions 3017 * Pointer to the list of actions. 3018 * @param[in] flow_idx 3019 * This memory pool index to the flow. 3020 * @param[out] error 3021 * Pointer to the error structure. 3022 * 3023 * @return 3024 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3025 */ 3026 static inline struct mlx5_flow * 3027 flow_drv_prepare(struct rte_eth_dev *dev, 3028 const struct rte_flow *flow, 3029 const struct rte_flow_attr *attr, 3030 const struct rte_flow_item items[], 3031 const struct rte_flow_action actions[], 3032 uint32_t flow_idx, 3033 struct rte_flow_error *error) 3034 { 3035 const struct mlx5_flow_driver_ops *fops; 3036 enum mlx5_flow_drv_type type = flow->drv_type; 3037 struct mlx5_flow *mlx5_flow = NULL; 3038 3039 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3040 fops = flow_get_drv_ops(type); 3041 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3042 if (mlx5_flow) 3043 mlx5_flow->flow_idx = flow_idx; 3044 return mlx5_flow; 3045 } 3046 3047 /** 3048 * Flow driver translation API. This abstracts calling driver specific 3049 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3050 * translates a generic flow into a driver flow. flow_drv_prepare() must 3051 * precede. 3052 * 3053 * @note 3054 * dev_flow->layers could be filled as a result of parsing during translation 3055 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3056 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3057 * flow->actions could be overwritten even though all the expanded dev_flows 3058 * have the same actions. 3059 * 3060 * @param[in] dev 3061 * Pointer to the rte dev structure. 3062 * @param[in, out] dev_flow 3063 * Pointer to the mlx5 flow. 3064 * @param[in] attr 3065 * Pointer to the flow attributes. 3066 * @param[in] items 3067 * Pointer to the list of items. 3068 * @param[in] actions 3069 * Pointer to the list of actions. 3070 * @param[out] error 3071 * Pointer to the error structure. 3072 * 3073 * @return 3074 * 0 on success, a negative errno value otherwise and rte_errno is set. 3075 */ 3076 static inline int 3077 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3078 const struct rte_flow_attr *attr, 3079 const struct rte_flow_item items[], 3080 const struct rte_flow_action actions[], 3081 struct rte_flow_error *error) 3082 { 3083 const struct mlx5_flow_driver_ops *fops; 3084 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3085 3086 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3087 fops = flow_get_drv_ops(type); 3088 return fops->translate(dev, dev_flow, attr, items, actions, error); 3089 } 3090 3091 /** 3092 * Flow driver apply API. This abstracts calling driver specific functions. 3093 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3094 * translated driver flows on to device. flow_drv_translate() must precede. 3095 * 3096 * @param[in] dev 3097 * Pointer to Ethernet device structure. 3098 * @param[in, out] flow 3099 * Pointer to flow structure. 3100 * @param[out] error 3101 * Pointer to error structure. 3102 * 3103 * @return 3104 * 0 on success, a negative errno value otherwise and rte_errno is set. 3105 */ 3106 static inline int 3107 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3108 struct rte_flow_error *error) 3109 { 3110 const struct mlx5_flow_driver_ops *fops; 3111 enum mlx5_flow_drv_type type = flow->drv_type; 3112 3113 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3114 fops = flow_get_drv_ops(type); 3115 return fops->apply(dev, flow, error); 3116 } 3117 3118 /** 3119 * Flow driver destroy API. This abstracts calling driver specific functions. 3120 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3121 * on device and releases resources of the flow. 3122 * 3123 * @param[in] dev 3124 * Pointer to Ethernet device. 3125 * @param[in, out] flow 3126 * Pointer to flow structure. 3127 */ 3128 static inline void 3129 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3130 { 3131 const struct mlx5_flow_driver_ops *fops; 3132 enum mlx5_flow_drv_type type = flow->drv_type; 3133 3134 flow_mreg_split_qrss_release(dev, flow); 3135 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3136 fops = flow_get_drv_ops(type); 3137 fops->destroy(dev, flow); 3138 } 3139 3140 /** 3141 * Get RSS action from the action list. 3142 * 3143 * @param[in] actions 3144 * Pointer to the list of actions. 3145 * 3146 * @return 3147 * Pointer to the RSS action if exist, else return NULL. 3148 */ 3149 static const struct rte_flow_action_rss* 3150 flow_get_rss_action(const struct rte_flow_action actions[]) 3151 { 3152 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3153 switch (actions->type) { 3154 case RTE_FLOW_ACTION_TYPE_RSS: 3155 return (const struct rte_flow_action_rss *) 3156 actions->conf; 3157 default: 3158 break; 3159 } 3160 } 3161 return NULL; 3162 } 3163 3164 /** 3165 * Get ASO age action by index. 3166 * 3167 * @param[in] dev 3168 * Pointer to the Ethernet device structure. 3169 * @param[in] age_idx 3170 * Index to the ASO age action. 3171 * 3172 * @return 3173 * The specified ASO age action. 3174 */ 3175 struct mlx5_aso_age_action* 3176 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3177 { 3178 uint16_t pool_idx = age_idx & UINT16_MAX; 3179 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3180 struct mlx5_priv *priv = dev->data->dev_private; 3181 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3182 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3183 3184 return &pool->actions[offset - 1]; 3185 } 3186 3187 /* maps shared action to translated non shared in some actions array */ 3188 struct mlx5_translated_shared_action { 3189 struct rte_flow_shared_action *action; /**< Shared action */ 3190 int index; /**< Index in related array of rte_flow_action */ 3191 }; 3192 3193 /** 3194 * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related 3195 * non shared action if translation possible. 3196 * This functionality used to run same execution path for both shared & non 3197 * shared actions on flow create. All necessary preparations for shared 3198 * action handling should be preformed on *shared* actions list returned 3199 * from this call. 3200 * 3201 * @param[in] dev 3202 * Pointer to Ethernet device. 3203 * @param[in] actions 3204 * List of actions to translate. 3205 * @param[out] shared 3206 * List to store translated shared actions. 3207 * @param[in, out] shared_n 3208 * Size of *shared* array. On return should be updated with number of shared 3209 * actions retrieved from the *actions* list. 3210 * @param[out] translated_actions 3211 * List of actions where all shared actions were translated to non shared 3212 * if possible. NULL if no translation took place. 3213 * @param[out] error 3214 * Pointer to the error structure. 3215 * 3216 * @return 3217 * 0 on success, a negative errno value otherwise and rte_errno is set. 3218 */ 3219 static int 3220 flow_shared_actions_translate(struct rte_eth_dev *dev, 3221 const struct rte_flow_action actions[], 3222 struct mlx5_translated_shared_action *shared, 3223 int *shared_n, 3224 struct rte_flow_action **translated_actions, 3225 struct rte_flow_error *error) 3226 { 3227 struct mlx5_priv *priv = dev->data->dev_private; 3228 struct rte_flow_action *translated = NULL; 3229 size_t actions_size; 3230 int n; 3231 int copied_n = 0; 3232 struct mlx5_translated_shared_action *shared_end = NULL; 3233 3234 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3235 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED) 3236 continue; 3237 if (copied_n == *shared_n) { 3238 return rte_flow_error_set 3239 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3240 NULL, "too many shared actions"); 3241 } 3242 rte_memcpy(&shared[copied_n].action, &actions[n].conf, 3243 sizeof(actions[n].conf)); 3244 shared[copied_n].index = n; 3245 copied_n++; 3246 } 3247 n++; 3248 *shared_n = copied_n; 3249 if (!copied_n) 3250 return 0; 3251 actions_size = sizeof(struct rte_flow_action) * n; 3252 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3253 if (!translated) { 3254 rte_errno = ENOMEM; 3255 return -ENOMEM; 3256 } 3257 memcpy(translated, actions, actions_size); 3258 for (shared_end = shared + copied_n; shared < shared_end; shared++) { 3259 struct mlx5_shared_action_rss *shared_rss; 3260 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; 3261 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; 3262 uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) 3263 - 1); 3264 3265 switch (type) { 3266 case MLX5_SHARED_ACTION_TYPE_RSS: 3267 shared_rss = mlx5_ipool_get 3268 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3269 translated[shared->index].type = 3270 RTE_FLOW_ACTION_TYPE_RSS; 3271 translated[shared->index].conf = 3272 &shared_rss->origin; 3273 break; 3274 case MLX5_SHARED_ACTION_TYPE_AGE: 3275 if (priv->sh->flow_hit_aso_en) { 3276 translated[shared->index].type = 3277 (enum rte_flow_action_type) 3278 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3279 translated[shared->index].conf = 3280 (void *)(uintptr_t)idx; 3281 break; 3282 } 3283 /* Fall-through */ 3284 default: 3285 mlx5_free(translated); 3286 return rte_flow_error_set 3287 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3288 NULL, "invalid shared action type"); 3289 } 3290 } 3291 *translated_actions = translated; 3292 return 0; 3293 } 3294 3295 /** 3296 * Get Shared RSS action from the action list. 3297 * 3298 * @param[in] dev 3299 * Pointer to Ethernet device. 3300 * @param[in] shared 3301 * Pointer to the list of actions. 3302 * @param[in] shared_n 3303 * Actions list length. 3304 * 3305 * @return 3306 * The MLX5 RSS action ID if exists, otherwise return 0. 3307 */ 3308 static uint32_t 3309 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3310 struct mlx5_translated_shared_action *shared, 3311 int shared_n) 3312 { 3313 struct mlx5_translated_shared_action *shared_end; 3314 struct mlx5_priv *priv = dev->data->dev_private; 3315 struct mlx5_shared_action_rss *shared_rss; 3316 3317 3318 for (shared_end = shared + shared_n; shared < shared_end; shared++) { 3319 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; 3320 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; 3321 uint32_t idx = act_idx & 3322 ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); 3323 switch (type) { 3324 case MLX5_SHARED_ACTION_TYPE_RSS: 3325 shared_rss = mlx5_ipool_get 3326 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3327 idx); 3328 __atomic_add_fetch(&shared_rss->refcnt, 1, 3329 __ATOMIC_RELAXED); 3330 return idx; 3331 default: 3332 break; 3333 } 3334 } 3335 return 0; 3336 } 3337 3338 static unsigned int 3339 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3340 { 3341 const struct rte_flow_item *item; 3342 unsigned int has_vlan = 0; 3343 3344 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3345 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3346 has_vlan = 1; 3347 break; 3348 } 3349 } 3350 if (has_vlan) 3351 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3352 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3353 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3354 MLX5_EXPANSION_ROOT_OUTER; 3355 } 3356 3357 /** 3358 * Get layer flags from the prefix flow. 3359 * 3360 * Some flows may be split to several subflows, the prefix subflow gets the 3361 * match items and the suffix sub flow gets the actions. 3362 * Some actions need the user defined match item flags to get the detail for 3363 * the action. 3364 * This function helps the suffix flow to get the item layer flags from prefix 3365 * subflow. 3366 * 3367 * @param[in] dev_flow 3368 * Pointer the created preifx subflow. 3369 * 3370 * @return 3371 * The layers get from prefix subflow. 3372 */ 3373 static inline uint64_t 3374 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3375 { 3376 uint64_t layers = 0; 3377 3378 /* 3379 * Layers bits could be localization, but usually the compiler will 3380 * help to do the optimization work for source code. 3381 * If no decap actions, use the layers directly. 3382 */ 3383 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3384 return dev_flow->handle->layers; 3385 /* Convert L3 layers with decap action. */ 3386 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3387 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3388 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3389 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3390 /* Convert L4 layers with decap action. */ 3391 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3392 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3393 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3394 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3395 return layers; 3396 } 3397 3398 /** 3399 * Get metadata split action information. 3400 * 3401 * @param[in] actions 3402 * Pointer to the list of actions. 3403 * @param[out] qrss 3404 * Pointer to the return pointer. 3405 * @param[out] qrss_type 3406 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3407 * if no QUEUE/RSS is found. 3408 * @param[out] encap_idx 3409 * Pointer to the index of the encap action if exists, otherwise the last 3410 * action index. 3411 * 3412 * @return 3413 * Total number of actions. 3414 */ 3415 static int 3416 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3417 const struct rte_flow_action **qrss, 3418 int *encap_idx) 3419 { 3420 const struct rte_flow_action_raw_encap *raw_encap; 3421 int actions_n = 0; 3422 int raw_decap_idx = -1; 3423 3424 *encap_idx = -1; 3425 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3426 switch (actions->type) { 3427 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3428 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3429 *encap_idx = actions_n; 3430 break; 3431 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3432 raw_decap_idx = actions_n; 3433 break; 3434 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3435 raw_encap = actions->conf; 3436 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3437 *encap_idx = raw_decap_idx != -1 ? 3438 raw_decap_idx : actions_n; 3439 break; 3440 case RTE_FLOW_ACTION_TYPE_QUEUE: 3441 case RTE_FLOW_ACTION_TYPE_RSS: 3442 *qrss = actions; 3443 break; 3444 default: 3445 break; 3446 } 3447 actions_n++; 3448 } 3449 if (*encap_idx == -1) 3450 *encap_idx = actions_n; 3451 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3452 return actions_n + 1; 3453 } 3454 3455 /** 3456 * Check meter action from the action list. 3457 * 3458 * @param[in] actions 3459 * Pointer to the list of actions. 3460 * @param[out] mtr 3461 * Pointer to the meter exist flag. 3462 * 3463 * @return 3464 * Total number of actions. 3465 */ 3466 static int 3467 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 3468 { 3469 int actions_n = 0; 3470 3471 MLX5_ASSERT(mtr); 3472 *mtr = 0; 3473 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3474 switch (actions->type) { 3475 case RTE_FLOW_ACTION_TYPE_METER: 3476 *mtr = 1; 3477 break; 3478 default: 3479 break; 3480 } 3481 actions_n++; 3482 } 3483 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3484 return actions_n + 1; 3485 } 3486 3487 /** 3488 * Check if the flow should be split due to hairpin. 3489 * The reason for the split is that in current HW we can't 3490 * support encap and push-vlan on Rx, so if a flow contains 3491 * these actions we move it to Tx. 3492 * 3493 * @param dev 3494 * Pointer to Ethernet device. 3495 * @param[in] attr 3496 * Flow rule attributes. 3497 * @param[in] actions 3498 * Associated actions (list terminated by the END action). 3499 * 3500 * @return 3501 * > 0 the number of actions and the flow should be split, 3502 * 0 when no split required. 3503 */ 3504 static int 3505 flow_check_hairpin_split(struct rte_eth_dev *dev, 3506 const struct rte_flow_attr *attr, 3507 const struct rte_flow_action actions[]) 3508 { 3509 int queue_action = 0; 3510 int action_n = 0; 3511 int split = 0; 3512 const struct rte_flow_action_queue *queue; 3513 const struct rte_flow_action_rss *rss; 3514 const struct rte_flow_action_raw_encap *raw_encap; 3515 const struct rte_eth_hairpin_conf *conf; 3516 3517 if (!attr->ingress) 3518 return 0; 3519 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3520 switch (actions->type) { 3521 case RTE_FLOW_ACTION_TYPE_QUEUE: 3522 queue = actions->conf; 3523 if (queue == NULL) 3524 return 0; 3525 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3526 if (conf != NULL && !!conf->tx_explicit) 3527 return 0; 3528 queue_action = 1; 3529 action_n++; 3530 break; 3531 case RTE_FLOW_ACTION_TYPE_RSS: 3532 rss = actions->conf; 3533 if (rss == NULL || rss->queue_num == 0) 3534 return 0; 3535 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3536 if (conf != NULL && !!conf->tx_explicit) 3537 return 0; 3538 queue_action = 1; 3539 action_n++; 3540 break; 3541 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3542 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3543 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3544 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3545 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3546 split++; 3547 action_n++; 3548 break; 3549 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3550 raw_encap = actions->conf; 3551 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3552 split++; 3553 action_n++; 3554 break; 3555 default: 3556 action_n++; 3557 break; 3558 } 3559 } 3560 if (split && queue_action) 3561 return action_n; 3562 return 0; 3563 } 3564 3565 /* Declare flow create/destroy prototype in advance. */ 3566 static uint32_t 3567 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3568 const struct rte_flow_attr *attr, 3569 const struct rte_flow_item items[], 3570 const struct rte_flow_action actions[], 3571 bool external, struct rte_flow_error *error); 3572 3573 static void 3574 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3575 uint32_t flow_idx); 3576 3577 struct mlx5_hlist_entry * 3578 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 3579 void *cb_ctx) 3580 { 3581 struct rte_eth_dev *dev = list->ctx; 3582 struct mlx5_priv *priv = dev->data->dev_private; 3583 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3584 struct mlx5_flow_mreg_copy_resource *mcp_res; 3585 struct rte_flow_error *error = ctx->error; 3586 uint32_t idx = 0; 3587 int ret; 3588 uint32_t mark_id = key; 3589 struct rte_flow_attr attr = { 3590 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3591 .ingress = 1, 3592 }; 3593 struct mlx5_rte_flow_item_tag tag_spec = { 3594 .data = mark_id, 3595 }; 3596 struct rte_flow_item items[] = { 3597 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 3598 }; 3599 struct rte_flow_action_mark ftag = { 3600 .id = mark_id, 3601 }; 3602 struct mlx5_flow_action_copy_mreg cp_mreg = { 3603 .dst = REG_B, 3604 .src = REG_NON, 3605 }; 3606 struct rte_flow_action_jump jump = { 3607 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3608 }; 3609 struct rte_flow_action actions[] = { 3610 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 3611 }; 3612 3613 /* Fill the register fileds in the flow. */ 3614 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3615 if (ret < 0) 3616 return NULL; 3617 tag_spec.id = ret; 3618 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3619 if (ret < 0) 3620 return NULL; 3621 cp_mreg.src = ret; 3622 /* Provide the full width of FLAG specific value. */ 3623 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3624 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3625 /* Build a new flow. */ 3626 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3627 items[0] = (struct rte_flow_item){ 3628 .type = (enum rte_flow_item_type) 3629 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3630 .spec = &tag_spec, 3631 }; 3632 items[1] = (struct rte_flow_item){ 3633 .type = RTE_FLOW_ITEM_TYPE_END, 3634 }; 3635 actions[0] = (struct rte_flow_action){ 3636 .type = (enum rte_flow_action_type) 3637 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3638 .conf = &ftag, 3639 }; 3640 actions[1] = (struct rte_flow_action){ 3641 .type = (enum rte_flow_action_type) 3642 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3643 .conf = &cp_mreg, 3644 }; 3645 actions[2] = (struct rte_flow_action){ 3646 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3647 .conf = &jump, 3648 }; 3649 actions[3] = (struct rte_flow_action){ 3650 .type = RTE_FLOW_ACTION_TYPE_END, 3651 }; 3652 } else { 3653 /* Default rule, wildcard match. */ 3654 attr.priority = MLX5_FLOW_PRIO_RSVD; 3655 items[0] = (struct rte_flow_item){ 3656 .type = RTE_FLOW_ITEM_TYPE_END, 3657 }; 3658 actions[0] = (struct rte_flow_action){ 3659 .type = (enum rte_flow_action_type) 3660 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3661 .conf = &cp_mreg, 3662 }; 3663 actions[1] = (struct rte_flow_action){ 3664 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3665 .conf = &jump, 3666 }; 3667 actions[2] = (struct rte_flow_action){ 3668 .type = RTE_FLOW_ACTION_TYPE_END, 3669 }; 3670 } 3671 /* Build a new entry. */ 3672 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3673 if (!mcp_res) { 3674 rte_errno = ENOMEM; 3675 return NULL; 3676 } 3677 mcp_res->idx = idx; 3678 /* 3679 * The copy Flows are not included in any list. There 3680 * ones are referenced from other Flows and can not 3681 * be applied, removed, deleted in ardbitrary order 3682 * by list traversing. 3683 */ 3684 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3685 actions, false, error); 3686 if (!mcp_res->rix_flow) { 3687 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 3688 return NULL; 3689 } 3690 return &mcp_res->hlist_ent; 3691 } 3692 3693 /** 3694 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3695 * 3696 * As mark_id is unique, if there's already a registered flow for the mark_id, 3697 * return by increasing the reference counter of the resource. Otherwise, create 3698 * the resource (mcp_res) and flow. 3699 * 3700 * Flow looks like, 3701 * - If ingress port is ANY and reg_c[1] is mark_id, 3702 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3703 * 3704 * For default flow (zero mark_id), flow is like, 3705 * - If ingress port is ANY, 3706 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 3707 * 3708 * @param dev 3709 * Pointer to Ethernet device. 3710 * @param mark_id 3711 * ID of MARK action, zero means default flow for META. 3712 * @param[out] error 3713 * Perform verbose error reporting if not NULL. 3714 * 3715 * @return 3716 * Associated resource on success, NULL otherwise and rte_errno is set. 3717 */ 3718 static struct mlx5_flow_mreg_copy_resource * 3719 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 3720 struct rte_flow_error *error) 3721 { 3722 struct mlx5_priv *priv = dev->data->dev_private; 3723 struct mlx5_hlist_entry *entry; 3724 struct mlx5_flow_cb_ctx ctx = { 3725 .dev = dev, 3726 .error = error, 3727 }; 3728 3729 /* Check if already registered. */ 3730 MLX5_ASSERT(priv->mreg_cp_tbl); 3731 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 3732 if (!entry) 3733 return NULL; 3734 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 3735 hlist_ent); 3736 } 3737 3738 void 3739 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 3740 { 3741 struct mlx5_flow_mreg_copy_resource *mcp_res = 3742 container_of(entry, typeof(*mcp_res), hlist_ent); 3743 struct rte_eth_dev *dev = list->ctx; 3744 struct mlx5_priv *priv = dev->data->dev_private; 3745 3746 MLX5_ASSERT(mcp_res->rix_flow); 3747 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3748 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3749 } 3750 3751 /** 3752 * Release flow in RX_CP_TBL. 3753 * 3754 * @param dev 3755 * Pointer to Ethernet device. 3756 * @flow 3757 * Parent flow for wich copying is provided. 3758 */ 3759 static void 3760 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3761 struct rte_flow *flow) 3762 { 3763 struct mlx5_flow_mreg_copy_resource *mcp_res; 3764 struct mlx5_priv *priv = dev->data->dev_private; 3765 3766 if (!flow->rix_mreg_copy) 3767 return; 3768 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3769 flow->rix_mreg_copy); 3770 if (!mcp_res || !priv->mreg_cp_tbl) 3771 return; 3772 MLX5_ASSERT(mcp_res->rix_flow); 3773 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3774 flow->rix_mreg_copy = 0; 3775 } 3776 3777 /** 3778 * Remove the default copy action from RX_CP_TBL. 3779 * 3780 * This functions is called in the mlx5_dev_start(). No thread safe 3781 * is guaranteed. 3782 * 3783 * @param dev 3784 * Pointer to Ethernet device. 3785 */ 3786 static void 3787 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3788 { 3789 struct mlx5_hlist_entry *entry; 3790 struct mlx5_priv *priv = dev->data->dev_private; 3791 3792 /* Check if default flow is registered. */ 3793 if (!priv->mreg_cp_tbl) 3794 return; 3795 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 3796 MLX5_DEFAULT_COPY_ID, NULL); 3797 if (!entry) 3798 return; 3799 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 3800 } 3801 3802 /** 3803 * Add the default copy action in in RX_CP_TBL. 3804 * 3805 * This functions is called in the mlx5_dev_start(). No thread safe 3806 * is guaranteed. 3807 * 3808 * @param dev 3809 * Pointer to Ethernet device. 3810 * @param[out] error 3811 * Perform verbose error reporting if not NULL. 3812 * 3813 * @return 3814 * 0 for success, negative value otherwise and rte_errno is set. 3815 */ 3816 static int 3817 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3818 struct rte_flow_error *error) 3819 { 3820 struct mlx5_priv *priv = dev->data->dev_private; 3821 struct mlx5_flow_mreg_copy_resource *mcp_res; 3822 3823 /* Check whether extensive metadata feature is engaged. */ 3824 if (!priv->config.dv_flow_en || 3825 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3826 !mlx5_flow_ext_mreg_supported(dev) || 3827 !priv->sh->dv_regc0_mask) 3828 return 0; 3829 /* 3830 * Add default mreg copy flow may be called multiple time, but 3831 * only be called once in stop. Avoid register it twice. 3832 */ 3833 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 3834 return 0; 3835 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3836 if (!mcp_res) 3837 return -rte_errno; 3838 return 0; 3839 } 3840 3841 /** 3842 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3843 * 3844 * All the flow having Q/RSS action should be split by 3845 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3846 * performs the following, 3847 * - CQE->flow_tag := reg_c[1] (MARK) 3848 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3849 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3850 * but there should be a flow per each MARK ID set by MARK action. 3851 * 3852 * For the aforementioned reason, if there's a MARK action in flow's action 3853 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3854 * the MARK ID to CQE's flow_tag like, 3855 * - If reg_c[1] is mark_id, 3856 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3857 * 3858 * For SET_META action which stores value in reg_c[0], as the destination is 3859 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3860 * MARK ID means the default flow. The default flow looks like, 3861 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3862 * 3863 * @param dev 3864 * Pointer to Ethernet device. 3865 * @param flow 3866 * Pointer to flow structure. 3867 * @param[in] actions 3868 * Pointer to the list of actions. 3869 * @param[out] error 3870 * Perform verbose error reporting if not NULL. 3871 * 3872 * @return 3873 * 0 on success, negative value otherwise and rte_errno is set. 3874 */ 3875 static int 3876 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3877 struct rte_flow *flow, 3878 const struct rte_flow_action *actions, 3879 struct rte_flow_error *error) 3880 { 3881 struct mlx5_priv *priv = dev->data->dev_private; 3882 struct mlx5_dev_config *config = &priv->config; 3883 struct mlx5_flow_mreg_copy_resource *mcp_res; 3884 const struct rte_flow_action_mark *mark; 3885 3886 /* Check whether extensive metadata feature is engaged. */ 3887 if (!config->dv_flow_en || 3888 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3889 !mlx5_flow_ext_mreg_supported(dev) || 3890 !priv->sh->dv_regc0_mask) 3891 return 0; 3892 /* Find MARK action. */ 3893 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3894 switch (actions->type) { 3895 case RTE_FLOW_ACTION_TYPE_FLAG: 3896 mcp_res = flow_mreg_add_copy_action 3897 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3898 if (!mcp_res) 3899 return -rte_errno; 3900 flow->rix_mreg_copy = mcp_res->idx; 3901 return 0; 3902 case RTE_FLOW_ACTION_TYPE_MARK: 3903 mark = (const struct rte_flow_action_mark *) 3904 actions->conf; 3905 mcp_res = 3906 flow_mreg_add_copy_action(dev, mark->id, error); 3907 if (!mcp_res) 3908 return -rte_errno; 3909 flow->rix_mreg_copy = mcp_res->idx; 3910 return 0; 3911 default: 3912 break; 3913 } 3914 } 3915 return 0; 3916 } 3917 3918 #define MLX5_MAX_SPLIT_ACTIONS 24 3919 #define MLX5_MAX_SPLIT_ITEMS 24 3920 3921 /** 3922 * Split the hairpin flow. 3923 * Since HW can't support encap and push-vlan on Rx, we move these 3924 * actions to Tx. 3925 * If the count action is after the encap then we also 3926 * move the count action. in this case the count will also measure 3927 * the outer bytes. 3928 * 3929 * @param dev 3930 * Pointer to Ethernet device. 3931 * @param[in] actions 3932 * Associated actions (list terminated by the END action). 3933 * @param[out] actions_rx 3934 * Rx flow actions. 3935 * @param[out] actions_tx 3936 * Tx flow actions.. 3937 * @param[out] pattern_tx 3938 * The pattern items for the Tx flow. 3939 * @param[out] flow_id 3940 * The flow ID connected to this flow. 3941 * 3942 * @return 3943 * 0 on success. 3944 */ 3945 static int 3946 flow_hairpin_split(struct rte_eth_dev *dev, 3947 const struct rte_flow_action actions[], 3948 struct rte_flow_action actions_rx[], 3949 struct rte_flow_action actions_tx[], 3950 struct rte_flow_item pattern_tx[], 3951 uint32_t flow_id) 3952 { 3953 const struct rte_flow_action_raw_encap *raw_encap; 3954 const struct rte_flow_action_raw_decap *raw_decap; 3955 struct mlx5_rte_flow_action_set_tag *set_tag; 3956 struct rte_flow_action *tag_action; 3957 struct mlx5_rte_flow_item_tag *tag_item; 3958 struct rte_flow_item *item; 3959 char *addr; 3960 int encap = 0; 3961 3962 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3963 switch (actions->type) { 3964 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3965 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3966 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3967 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3968 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3969 rte_memcpy(actions_tx, actions, 3970 sizeof(struct rte_flow_action)); 3971 actions_tx++; 3972 break; 3973 case RTE_FLOW_ACTION_TYPE_COUNT: 3974 if (encap) { 3975 rte_memcpy(actions_tx, actions, 3976 sizeof(struct rte_flow_action)); 3977 actions_tx++; 3978 } else { 3979 rte_memcpy(actions_rx, actions, 3980 sizeof(struct rte_flow_action)); 3981 actions_rx++; 3982 } 3983 break; 3984 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3985 raw_encap = actions->conf; 3986 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 3987 memcpy(actions_tx, actions, 3988 sizeof(struct rte_flow_action)); 3989 actions_tx++; 3990 encap = 1; 3991 } else { 3992 rte_memcpy(actions_rx, actions, 3993 sizeof(struct rte_flow_action)); 3994 actions_rx++; 3995 } 3996 break; 3997 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3998 raw_decap = actions->conf; 3999 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4000 memcpy(actions_tx, actions, 4001 sizeof(struct rte_flow_action)); 4002 actions_tx++; 4003 } else { 4004 rte_memcpy(actions_rx, actions, 4005 sizeof(struct rte_flow_action)); 4006 actions_rx++; 4007 } 4008 break; 4009 default: 4010 rte_memcpy(actions_rx, actions, 4011 sizeof(struct rte_flow_action)); 4012 actions_rx++; 4013 break; 4014 } 4015 } 4016 /* Add set meta action and end action for the Rx flow. */ 4017 tag_action = actions_rx; 4018 tag_action->type = (enum rte_flow_action_type) 4019 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4020 actions_rx++; 4021 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4022 actions_rx++; 4023 set_tag = (void *)actions_rx; 4024 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 4025 MLX5_ASSERT(set_tag->id > REG_NON); 4026 set_tag->data = flow_id; 4027 tag_action->conf = set_tag; 4028 /* Create Tx item list. */ 4029 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4030 addr = (void *)&pattern_tx[2]; 4031 item = pattern_tx; 4032 item->type = (enum rte_flow_item_type) 4033 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4034 tag_item = (void *)addr; 4035 tag_item->data = flow_id; 4036 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4037 MLX5_ASSERT(set_tag->id > REG_NON); 4038 item->spec = tag_item; 4039 addr += sizeof(struct mlx5_rte_flow_item_tag); 4040 tag_item = (void *)addr; 4041 tag_item->data = UINT32_MAX; 4042 tag_item->id = UINT16_MAX; 4043 item->mask = tag_item; 4044 item->last = NULL; 4045 item++; 4046 item->type = RTE_FLOW_ITEM_TYPE_END; 4047 return 0; 4048 } 4049 4050 /** 4051 * The last stage of splitting chain, just creates the subflow 4052 * without any modification. 4053 * 4054 * @param[in] dev 4055 * Pointer to Ethernet device. 4056 * @param[in] flow 4057 * Parent flow structure pointer. 4058 * @param[in, out] sub_flow 4059 * Pointer to return the created subflow, may be NULL. 4060 * @param[in] attr 4061 * Flow rule attributes. 4062 * @param[in] items 4063 * Pattern specification (list terminated by the END pattern item). 4064 * @param[in] actions 4065 * Associated actions (list terminated by the END action). 4066 * @param[in] flow_split_info 4067 * Pointer to flow split info structure. 4068 * @param[out] error 4069 * Perform verbose error reporting if not NULL. 4070 * @return 4071 * 0 on success, negative value otherwise 4072 */ 4073 static int 4074 flow_create_split_inner(struct rte_eth_dev *dev, 4075 struct rte_flow *flow, 4076 struct mlx5_flow **sub_flow, 4077 const struct rte_flow_attr *attr, 4078 const struct rte_flow_item items[], 4079 const struct rte_flow_action actions[], 4080 struct mlx5_flow_split_info *flow_split_info, 4081 struct rte_flow_error *error) 4082 { 4083 struct mlx5_flow *dev_flow; 4084 4085 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4086 flow_split_info->flow_idx, error); 4087 if (!dev_flow) 4088 return -rte_errno; 4089 dev_flow->flow = flow; 4090 dev_flow->external = flow_split_info->external; 4091 dev_flow->skip_scale = flow_split_info->skip_scale; 4092 /* Subflow object was created, we must include one in the list. */ 4093 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4094 dev_flow->handle, next); 4095 /* 4096 * If dev_flow is as one of the suffix flow, some actions in suffix 4097 * flow may need some user defined item layer flags, and pass the 4098 * Metadate rxq mark flag to suffix flow as well. 4099 */ 4100 if (flow_split_info->prefix_layers) 4101 dev_flow->handle->layers = flow_split_info->prefix_layers; 4102 if (flow_split_info->prefix_mark) 4103 dev_flow->handle->mark = 1; 4104 if (sub_flow) 4105 *sub_flow = dev_flow; 4106 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4107 } 4108 4109 /** 4110 * Split the meter flow. 4111 * 4112 * As meter flow will split to three sub flow, other than meter 4113 * action, the other actions make sense to only meter accepts 4114 * the packet. If it need to be dropped, no other additional 4115 * actions should be take. 4116 * 4117 * One kind of special action which decapsulates the L3 tunnel 4118 * header will be in the prefix sub flow, as not to take the 4119 * L3 tunnel header into account. 4120 * 4121 * @param dev 4122 * Pointer to Ethernet device. 4123 * @param[in] items 4124 * Pattern specification (list terminated by the END pattern item). 4125 * @param[out] sfx_items 4126 * Suffix flow match items (list terminated by the END pattern item). 4127 * @param[in] actions 4128 * Associated actions (list terminated by the END action). 4129 * @param[out] actions_sfx 4130 * Suffix flow actions. 4131 * @param[out] actions_pre 4132 * Prefix flow actions. 4133 * @param[out] pattern_sfx 4134 * The pattern items for the suffix flow. 4135 * @param[out] tag_sfx 4136 * Pointer to suffix flow tag. 4137 * 4138 * @return 4139 * 0 on success. 4140 */ 4141 static int 4142 flow_meter_split_prep(struct rte_eth_dev *dev, 4143 const struct rte_flow_item items[], 4144 struct rte_flow_item sfx_items[], 4145 const struct rte_flow_action actions[], 4146 struct rte_flow_action actions_sfx[], 4147 struct rte_flow_action actions_pre[]) 4148 { 4149 struct mlx5_priv *priv = dev->data->dev_private; 4150 struct rte_flow_action *tag_action = NULL; 4151 struct rte_flow_item *tag_item; 4152 struct mlx5_rte_flow_action_set_tag *set_tag; 4153 struct rte_flow_error error; 4154 const struct rte_flow_action_raw_encap *raw_encap; 4155 const struct rte_flow_action_raw_decap *raw_decap; 4156 struct mlx5_rte_flow_item_tag *tag_spec; 4157 struct mlx5_rte_flow_item_tag *tag_mask; 4158 uint32_t tag_id = 0; 4159 bool copy_vlan = false; 4160 4161 /* Prepare the actions for prefix and suffix flow. */ 4162 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4163 struct rte_flow_action **action_cur = NULL; 4164 4165 switch (actions->type) { 4166 case RTE_FLOW_ACTION_TYPE_METER: 4167 /* Add the extra tag action first. */ 4168 tag_action = actions_pre; 4169 tag_action->type = (enum rte_flow_action_type) 4170 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4171 actions_pre++; 4172 action_cur = &actions_pre; 4173 break; 4174 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4175 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4176 action_cur = &actions_pre; 4177 break; 4178 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4179 raw_encap = actions->conf; 4180 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4181 action_cur = &actions_pre; 4182 break; 4183 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4184 raw_decap = actions->conf; 4185 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4186 action_cur = &actions_pre; 4187 break; 4188 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4189 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4190 copy_vlan = true; 4191 break; 4192 default: 4193 break; 4194 } 4195 if (!action_cur) 4196 action_cur = &actions_sfx; 4197 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 4198 (*action_cur)++; 4199 } 4200 /* Add end action to the actions. */ 4201 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4202 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4203 actions_pre++; 4204 /* Set the tag. */ 4205 set_tag = (void *)actions_pre; 4206 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4207 mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 4208 &tag_id); 4209 if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { 4210 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", 4211 dev->data->port_id); 4212 mlx5_ipool_free(priv->sh->ipool 4213 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); 4214 return 0; 4215 } else if (!tag_id) { 4216 return 0; 4217 } 4218 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 4219 assert(tag_action); 4220 tag_action->conf = set_tag; 4221 /* Prepare the suffix subflow items. */ 4222 tag_item = sfx_items++; 4223 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4224 int item_type = items->type; 4225 4226 switch (item_type) { 4227 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4228 memcpy(sfx_items, items, sizeof(*sfx_items)); 4229 sfx_items++; 4230 break; 4231 case RTE_FLOW_ITEM_TYPE_VLAN: 4232 if (copy_vlan) { 4233 memcpy(sfx_items, items, sizeof(*sfx_items)); 4234 /* 4235 * Convert to internal match item, it is used 4236 * for vlan push and set vid. 4237 */ 4238 sfx_items->type = (enum rte_flow_item_type) 4239 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4240 sfx_items++; 4241 } 4242 break; 4243 default: 4244 break; 4245 } 4246 } 4247 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4248 sfx_items++; 4249 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4250 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 4251 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4252 tag_mask = tag_spec + 1; 4253 tag_mask->data = 0xffffff00; 4254 tag_item->type = (enum rte_flow_item_type) 4255 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4256 tag_item->spec = tag_spec; 4257 tag_item->last = NULL; 4258 tag_item->mask = tag_mask; 4259 return tag_id; 4260 } 4261 4262 /** 4263 * Split action list having QUEUE/RSS for metadata register copy. 4264 * 4265 * Once Q/RSS action is detected in user's action list, the flow action 4266 * should be split in order to copy metadata registers, which will happen in 4267 * RX_CP_TBL like, 4268 * - CQE->flow_tag := reg_c[1] (MARK) 4269 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4270 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4271 * This is because the last action of each flow must be a terminal action 4272 * (QUEUE, RSS or DROP). 4273 * 4274 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4275 * stored and kept in the mlx5_flow structure per each sub_flow. 4276 * 4277 * The Q/RSS action is replaced with, 4278 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4279 * And the following JUMP action is added at the end, 4280 * - JUMP, to RX_CP_TBL. 4281 * 4282 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4283 * flow_create_split_metadata() routine. The flow will look like, 4284 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4285 * 4286 * @param dev 4287 * Pointer to Ethernet device. 4288 * @param[out] split_actions 4289 * Pointer to store split actions to jump to CP_TBL. 4290 * @param[in] actions 4291 * Pointer to the list of original flow actions. 4292 * @param[in] qrss 4293 * Pointer to the Q/RSS action. 4294 * @param[in] actions_n 4295 * Number of original actions. 4296 * @param[out] error 4297 * Perform verbose error reporting if not NULL. 4298 * 4299 * @return 4300 * non-zero unique flow_id on success, otherwise 0 and 4301 * error/rte_error are set. 4302 */ 4303 static uint32_t 4304 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4305 struct rte_flow_action *split_actions, 4306 const struct rte_flow_action *actions, 4307 const struct rte_flow_action *qrss, 4308 int actions_n, struct rte_flow_error *error) 4309 { 4310 struct mlx5_priv *priv = dev->data->dev_private; 4311 struct mlx5_rte_flow_action_set_tag *set_tag; 4312 struct rte_flow_action_jump *jump; 4313 const int qrss_idx = qrss - actions; 4314 uint32_t flow_id = 0; 4315 int ret = 0; 4316 4317 /* 4318 * Given actions will be split 4319 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4320 * - Add jump to mreg CP_TBL. 4321 * As a result, there will be one more action. 4322 */ 4323 ++actions_n; 4324 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4325 set_tag = (void *)(split_actions + actions_n); 4326 /* 4327 * If tag action is not set to void(it means we are not the meter 4328 * suffix flow), add the tag action. Since meter suffix flow already 4329 * has the tag added. 4330 */ 4331 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4332 /* 4333 * Allocate the new subflow ID. This one is unique within 4334 * device and not shared with representors. Otherwise, 4335 * we would have to resolve multi-thread access synch 4336 * issue. Each flow on the shared device is appended 4337 * with source vport identifier, so the resulting 4338 * flows will be unique in the shared (by master and 4339 * representors) domain even if they have coinciding 4340 * IDs. 4341 */ 4342 mlx5_ipool_malloc(priv->sh->ipool 4343 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 4344 if (!flow_id) 4345 return rte_flow_error_set(error, ENOMEM, 4346 RTE_FLOW_ERROR_TYPE_ACTION, 4347 NULL, "can't allocate id " 4348 "for split Q/RSS subflow"); 4349 /* Internal SET_TAG action to set flow ID. */ 4350 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4351 .data = flow_id, 4352 }; 4353 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4354 if (ret < 0) 4355 return ret; 4356 set_tag->id = ret; 4357 /* Construct new actions array. */ 4358 /* Replace QUEUE/RSS action. */ 4359 split_actions[qrss_idx] = (struct rte_flow_action){ 4360 .type = (enum rte_flow_action_type) 4361 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4362 .conf = set_tag, 4363 }; 4364 } 4365 /* JUMP action to jump to mreg copy table (CP_TBL). */ 4366 jump = (void *)(set_tag + 1); 4367 *jump = (struct rte_flow_action_jump){ 4368 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4369 }; 4370 split_actions[actions_n - 2] = (struct rte_flow_action){ 4371 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4372 .conf = jump, 4373 }; 4374 split_actions[actions_n - 1] = (struct rte_flow_action){ 4375 .type = RTE_FLOW_ACTION_TYPE_END, 4376 }; 4377 return flow_id; 4378 } 4379 4380 /** 4381 * Extend the given action list for Tx metadata copy. 4382 * 4383 * Copy the given action list to the ext_actions and add flow metadata register 4384 * copy action in order to copy reg_a set by WQE to reg_c[0]. 4385 * 4386 * @param[out] ext_actions 4387 * Pointer to the extended action list. 4388 * @param[in] actions 4389 * Pointer to the list of actions. 4390 * @param[in] actions_n 4391 * Number of actions in the list. 4392 * @param[out] error 4393 * Perform verbose error reporting if not NULL. 4394 * @param[in] encap_idx 4395 * The encap action inndex. 4396 * 4397 * @return 4398 * 0 on success, negative value otherwise 4399 */ 4400 static int 4401 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 4402 struct rte_flow_action *ext_actions, 4403 const struct rte_flow_action *actions, 4404 int actions_n, struct rte_flow_error *error, 4405 int encap_idx) 4406 { 4407 struct mlx5_flow_action_copy_mreg *cp_mreg = 4408 (struct mlx5_flow_action_copy_mreg *) 4409 (ext_actions + actions_n + 1); 4410 int ret; 4411 4412 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4413 if (ret < 0) 4414 return ret; 4415 cp_mreg->dst = ret; 4416 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 4417 if (ret < 0) 4418 return ret; 4419 cp_mreg->src = ret; 4420 if (encap_idx != 0) 4421 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 4422 if (encap_idx == actions_n - 1) { 4423 ext_actions[actions_n - 1] = (struct rte_flow_action){ 4424 .type = (enum rte_flow_action_type) 4425 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4426 .conf = cp_mreg, 4427 }; 4428 ext_actions[actions_n] = (struct rte_flow_action){ 4429 .type = RTE_FLOW_ACTION_TYPE_END, 4430 }; 4431 } else { 4432 ext_actions[encap_idx] = (struct rte_flow_action){ 4433 .type = (enum rte_flow_action_type) 4434 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4435 .conf = cp_mreg, 4436 }; 4437 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 4438 sizeof(*ext_actions) * (actions_n - encap_idx)); 4439 } 4440 return 0; 4441 } 4442 4443 /** 4444 * Check the match action from the action list. 4445 * 4446 * @param[in] actions 4447 * Pointer to the list of actions. 4448 * @param[in] attr 4449 * Flow rule attributes. 4450 * @param[in] action 4451 * The action to be check if exist. 4452 * @param[out] match_action_pos 4453 * Pointer to the position of the matched action if exists, otherwise is -1. 4454 * @param[out] qrss_action_pos 4455 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 4456 * 4457 * @return 4458 * > 0 the total number of actions. 4459 * 0 if not found match action in action list. 4460 */ 4461 static int 4462 flow_check_match_action(const struct rte_flow_action actions[], 4463 const struct rte_flow_attr *attr, 4464 enum rte_flow_action_type action, 4465 int *match_action_pos, int *qrss_action_pos) 4466 { 4467 const struct rte_flow_action_sample *sample; 4468 int actions_n = 0; 4469 int jump_flag = 0; 4470 uint32_t ratio = 0; 4471 int sub_type = 0; 4472 int flag = 0; 4473 4474 *match_action_pos = -1; 4475 *qrss_action_pos = -1; 4476 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4477 if (actions->type == action) { 4478 flag = 1; 4479 *match_action_pos = actions_n; 4480 } 4481 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE || 4482 actions->type == RTE_FLOW_ACTION_TYPE_RSS) 4483 *qrss_action_pos = actions_n; 4484 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP) 4485 jump_flag = 1; 4486 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) { 4487 sample = actions->conf; 4488 ratio = sample->ratio; 4489 sub_type = ((const struct rte_flow_action *) 4490 (sample->actions))->type; 4491 } 4492 actions_n++; 4493 } 4494 if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) { 4495 if (ratio == 1) { 4496 /* JUMP Action not support for Mirroring; 4497 * Mirroring support multi-destination; 4498 */ 4499 if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END) 4500 flag = 0; 4501 } 4502 } 4503 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4504 return flag ? actions_n + 1 : 0; 4505 } 4506 4507 #define SAMPLE_SUFFIX_ITEM 2 4508 4509 /** 4510 * Split the sample flow. 4511 * 4512 * As sample flow will split to two sub flow, sample flow with 4513 * sample action, the other actions will move to new suffix flow. 4514 * 4515 * Also add unique tag id with tag action in the sample flow, 4516 * the same tag id will be as match in the suffix flow. 4517 * 4518 * @param dev 4519 * Pointer to Ethernet device. 4520 * @param[in] fdb_tx 4521 * FDB egress flow flag. 4522 * @param[out] sfx_items 4523 * Suffix flow match items (list terminated by the END pattern item). 4524 * @param[in] actions 4525 * Associated actions (list terminated by the END action). 4526 * @param[out] actions_sfx 4527 * Suffix flow actions. 4528 * @param[out] actions_pre 4529 * Prefix flow actions. 4530 * @param[in] actions_n 4531 * The total number of actions. 4532 * @param[in] sample_action_pos 4533 * The sample action position. 4534 * @param[in] qrss_action_pos 4535 * The Queue/RSS action position. 4536 * @param[out] error 4537 * Perform verbose error reporting if not NULL. 4538 * 4539 * @return 4540 * 0 on success, or unique flow_id, a negative errno value 4541 * otherwise and rte_errno is set. 4542 */ 4543 static int 4544 flow_sample_split_prep(struct rte_eth_dev *dev, 4545 uint32_t fdb_tx, 4546 struct rte_flow_item sfx_items[], 4547 const struct rte_flow_action actions[], 4548 struct rte_flow_action actions_sfx[], 4549 struct rte_flow_action actions_pre[], 4550 int actions_n, 4551 int sample_action_pos, 4552 int qrss_action_pos, 4553 struct rte_flow_error *error) 4554 { 4555 struct mlx5_priv *priv = dev->data->dev_private; 4556 struct mlx5_rte_flow_action_set_tag *set_tag; 4557 struct mlx5_rte_flow_item_tag *tag_spec; 4558 struct mlx5_rte_flow_item_tag *tag_mask; 4559 uint32_t tag_id = 0; 4560 int index; 4561 int ret; 4562 4563 if (sample_action_pos < 0) 4564 return rte_flow_error_set(error, EINVAL, 4565 RTE_FLOW_ERROR_TYPE_ACTION, 4566 NULL, "invalid position of sample " 4567 "action in list"); 4568 if (!fdb_tx) { 4569 /* Prepare the prefix tag action. */ 4570 set_tag = (void *)(actions_pre + actions_n + 1); 4571 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 4572 if (ret < 0) 4573 return ret; 4574 set_tag->id = ret; 4575 mlx5_ipool_malloc(priv->sh->ipool 4576 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 4577 set_tag->data = tag_id; 4578 /* Prepare the suffix subflow items. */ 4579 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 4580 tag_spec->data = tag_id; 4581 tag_spec->id = set_tag->id; 4582 tag_mask = tag_spec + 1; 4583 tag_mask->data = UINT32_MAX; 4584 sfx_items[0] = (struct rte_flow_item){ 4585 .type = (enum rte_flow_item_type) 4586 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4587 .spec = tag_spec, 4588 .last = NULL, 4589 .mask = tag_mask, 4590 }; 4591 sfx_items[1] = (struct rte_flow_item){ 4592 .type = (enum rte_flow_item_type) 4593 RTE_FLOW_ITEM_TYPE_END, 4594 }; 4595 } 4596 /* Prepare the actions for prefix and suffix flow. */ 4597 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 4598 index = qrss_action_pos; 4599 /* Put the preceding the Queue/RSS action into prefix flow. */ 4600 if (index != 0) 4601 memcpy(actions_pre, actions, 4602 sizeof(struct rte_flow_action) * index); 4603 /* Put others preceding the sample action into prefix flow. */ 4604 if (sample_action_pos > index + 1) 4605 memcpy(actions_pre + index, actions + index + 1, 4606 sizeof(struct rte_flow_action) * 4607 (sample_action_pos - index - 1)); 4608 index = sample_action_pos - 1; 4609 /* Put Queue/RSS action into Suffix flow. */ 4610 memcpy(actions_sfx, actions + qrss_action_pos, 4611 sizeof(struct rte_flow_action)); 4612 actions_sfx++; 4613 } else { 4614 index = sample_action_pos; 4615 if (index != 0) 4616 memcpy(actions_pre, actions, 4617 sizeof(struct rte_flow_action) * index); 4618 } 4619 /* Add the extra tag action for NIC-RX and E-Switch ingress. */ 4620 if (!fdb_tx) { 4621 actions_pre[index++] = 4622 (struct rte_flow_action){ 4623 .type = (enum rte_flow_action_type) 4624 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4625 .conf = set_tag, 4626 }; 4627 } 4628 memcpy(actions_pre + index, actions + sample_action_pos, 4629 sizeof(struct rte_flow_action)); 4630 index += 1; 4631 actions_pre[index] = (struct rte_flow_action){ 4632 .type = (enum rte_flow_action_type) 4633 RTE_FLOW_ACTION_TYPE_END, 4634 }; 4635 /* Put the actions after sample into Suffix flow. */ 4636 memcpy(actions_sfx, actions + sample_action_pos + 1, 4637 sizeof(struct rte_flow_action) * 4638 (actions_n - sample_action_pos - 1)); 4639 return tag_id; 4640 } 4641 4642 /** 4643 * The splitting for metadata feature. 4644 * 4645 * - Q/RSS action on NIC Rx should be split in order to pass by 4646 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4647 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4648 * 4649 * - All the actions on NIC Tx should have a mreg copy action to 4650 * copy reg_a from WQE to reg_c[0]. 4651 * 4652 * @param dev 4653 * Pointer to Ethernet device. 4654 * @param[in] flow 4655 * Parent flow structure pointer. 4656 * @param[in] attr 4657 * Flow rule attributes. 4658 * @param[in] items 4659 * Pattern specification (list terminated by the END pattern item). 4660 * @param[in] actions 4661 * Associated actions (list terminated by the END action). 4662 * @param[in] flow_split_info 4663 * Pointer to flow split info structure. 4664 * @param[out] error 4665 * Perform verbose error reporting if not NULL. 4666 * @return 4667 * 0 on success, negative value otherwise 4668 */ 4669 static int 4670 flow_create_split_metadata(struct rte_eth_dev *dev, 4671 struct rte_flow *flow, 4672 const struct rte_flow_attr *attr, 4673 const struct rte_flow_item items[], 4674 const struct rte_flow_action actions[], 4675 struct mlx5_flow_split_info *flow_split_info, 4676 struct rte_flow_error *error) 4677 { 4678 struct mlx5_priv *priv = dev->data->dev_private; 4679 struct mlx5_dev_config *config = &priv->config; 4680 const struct rte_flow_action *qrss = NULL; 4681 struct rte_flow_action *ext_actions = NULL; 4682 struct mlx5_flow *dev_flow = NULL; 4683 uint32_t qrss_id = 0; 4684 int mtr_sfx = 0; 4685 size_t act_size; 4686 int actions_n; 4687 int encap_idx; 4688 int ret; 4689 4690 /* Check whether extensive metadata feature is engaged. */ 4691 if (!config->dv_flow_en || 4692 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4693 !mlx5_flow_ext_mreg_supported(dev)) 4694 return flow_create_split_inner(dev, flow, NULL, attr, items, 4695 actions, flow_split_info, error); 4696 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 4697 &encap_idx); 4698 if (qrss) { 4699 /* Exclude hairpin flows from splitting. */ 4700 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 4701 const struct rte_flow_action_queue *queue; 4702 4703 queue = qrss->conf; 4704 if (mlx5_rxq_get_type(dev, queue->index) == 4705 MLX5_RXQ_TYPE_HAIRPIN) 4706 qrss = NULL; 4707 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 4708 const struct rte_flow_action_rss *rss; 4709 4710 rss = qrss->conf; 4711 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 4712 MLX5_RXQ_TYPE_HAIRPIN) 4713 qrss = NULL; 4714 } 4715 } 4716 if (qrss) { 4717 /* Check if it is in meter suffix table. */ 4718 mtr_sfx = attr->group == (attr->transfer ? 4719 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4720 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 4721 /* 4722 * Q/RSS action on NIC Rx should be split in order to pass by 4723 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4724 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4725 */ 4726 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4727 sizeof(struct rte_flow_action_set_tag) + 4728 sizeof(struct rte_flow_action_jump); 4729 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4730 SOCKET_ID_ANY); 4731 if (!ext_actions) 4732 return rte_flow_error_set(error, ENOMEM, 4733 RTE_FLOW_ERROR_TYPE_ACTION, 4734 NULL, "no memory to split " 4735 "metadata flow"); 4736 /* 4737 * If we are the suffix flow of meter, tag already exist. 4738 * Set the tag action to void. 4739 */ 4740 if (mtr_sfx) 4741 ext_actions[qrss - actions].type = 4742 RTE_FLOW_ACTION_TYPE_VOID; 4743 else 4744 ext_actions[qrss - actions].type = 4745 (enum rte_flow_action_type) 4746 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4747 /* 4748 * Create the new actions list with removed Q/RSS action 4749 * and appended set tag and jump to register copy table 4750 * (RX_CP_TBL). We should preallocate unique tag ID here 4751 * in advance, because it is needed for set tag action. 4752 */ 4753 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 4754 qrss, actions_n, error); 4755 if (!mtr_sfx && !qrss_id) { 4756 ret = -rte_errno; 4757 goto exit; 4758 } 4759 } else if (attr->egress && !attr->transfer) { 4760 /* 4761 * All the actions on NIC Tx should have a metadata register 4762 * copy action to copy reg_a from WQE to reg_c[meta] 4763 */ 4764 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4765 sizeof(struct mlx5_flow_action_copy_mreg); 4766 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4767 SOCKET_ID_ANY); 4768 if (!ext_actions) 4769 return rte_flow_error_set(error, ENOMEM, 4770 RTE_FLOW_ERROR_TYPE_ACTION, 4771 NULL, "no memory to split " 4772 "metadata flow"); 4773 /* Create the action list appended with copy register. */ 4774 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 4775 actions_n, error, encap_idx); 4776 if (ret < 0) 4777 goto exit; 4778 } 4779 /* Add the unmodified original or prefix subflow. */ 4780 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 4781 items, ext_actions ? ext_actions : 4782 actions, flow_split_info, error); 4783 if (ret < 0) 4784 goto exit; 4785 MLX5_ASSERT(dev_flow); 4786 if (qrss) { 4787 const struct rte_flow_attr q_attr = { 4788 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4789 .ingress = 1, 4790 }; 4791 /* Internal PMD action to set register. */ 4792 struct mlx5_rte_flow_item_tag q_tag_spec = { 4793 .data = qrss_id, 4794 .id = REG_NON, 4795 }; 4796 struct rte_flow_item q_items[] = { 4797 { 4798 .type = (enum rte_flow_item_type) 4799 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4800 .spec = &q_tag_spec, 4801 .last = NULL, 4802 .mask = NULL, 4803 }, 4804 { 4805 .type = RTE_FLOW_ITEM_TYPE_END, 4806 }, 4807 }; 4808 struct rte_flow_action q_actions[] = { 4809 { 4810 .type = qrss->type, 4811 .conf = qrss->conf, 4812 }, 4813 { 4814 .type = RTE_FLOW_ACTION_TYPE_END, 4815 }, 4816 }; 4817 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4818 4819 /* 4820 * Configure the tag item only if there is no meter subflow. 4821 * Since tag is already marked in the meter suffix subflow 4822 * we can just use the meter suffix items as is. 4823 */ 4824 if (qrss_id) { 4825 /* Not meter subflow. */ 4826 MLX5_ASSERT(!mtr_sfx); 4827 /* 4828 * Put unique id in prefix flow due to it is destroyed 4829 * after suffix flow and id will be freed after there 4830 * is no actual flows with this id and identifier 4831 * reallocation becomes possible (for example, for 4832 * other flows in other threads). 4833 */ 4834 dev_flow->handle->split_flow_id = qrss_id; 4835 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4836 error); 4837 if (ret < 0) 4838 goto exit; 4839 q_tag_spec.id = ret; 4840 } 4841 dev_flow = NULL; 4842 /* Add suffix subflow to execute Q/RSS. */ 4843 flow_split_info->prefix_layers = layers; 4844 flow_split_info->prefix_mark = 0; 4845 ret = flow_create_split_inner(dev, flow, &dev_flow, 4846 &q_attr, mtr_sfx ? items : 4847 q_items, q_actions, 4848 flow_split_info, error); 4849 if (ret < 0) 4850 goto exit; 4851 /* qrss ID should be freed if failed. */ 4852 qrss_id = 0; 4853 MLX5_ASSERT(dev_flow); 4854 } 4855 4856 exit: 4857 /* 4858 * We do not destroy the partially created sub_flows in case of error. 4859 * These ones are included into parent flow list and will be destroyed 4860 * by flow_drv_destroy. 4861 */ 4862 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 4863 qrss_id); 4864 mlx5_free(ext_actions); 4865 return ret; 4866 } 4867 4868 /** 4869 * The splitting for meter feature. 4870 * 4871 * - The meter flow will be split to two flows as prefix and 4872 * suffix flow. The packets make sense only it pass the prefix 4873 * meter action. 4874 * 4875 * - Reg_C_5 is used for the packet to match betweend prefix and 4876 * suffix flow. 4877 * 4878 * @param dev 4879 * Pointer to Ethernet device. 4880 * @param[in] flow 4881 * Parent flow structure pointer. 4882 * @param[in] attr 4883 * Flow rule attributes. 4884 * @param[in] items 4885 * Pattern specification (list terminated by the END pattern item). 4886 * @param[in] actions 4887 * Associated actions (list terminated by the END action). 4888 * @param[in] flow_split_info 4889 * Pointer to flow split info structure. 4890 * @param[out] error 4891 * Perform verbose error reporting if not NULL. 4892 * @return 4893 * 0 on success, negative value otherwise 4894 */ 4895 static int 4896 flow_create_split_meter(struct rte_eth_dev *dev, 4897 struct rte_flow *flow, 4898 const struct rte_flow_attr *attr, 4899 const struct rte_flow_item items[], 4900 const struct rte_flow_action actions[], 4901 struct mlx5_flow_split_info *flow_split_info, 4902 struct rte_flow_error *error) 4903 { 4904 struct mlx5_priv *priv = dev->data->dev_private; 4905 struct rte_flow_action *sfx_actions = NULL; 4906 struct rte_flow_action *pre_actions = NULL; 4907 struct rte_flow_item *sfx_items = NULL; 4908 struct mlx5_flow *dev_flow = NULL; 4909 struct rte_flow_attr sfx_attr = *attr; 4910 uint32_t mtr = 0; 4911 uint32_t mtr_tag_id = 0; 4912 size_t act_size; 4913 size_t item_size; 4914 int actions_n = 0; 4915 int ret; 4916 4917 if (priv->mtr_en) 4918 actions_n = flow_check_meter_action(actions, &mtr); 4919 if (mtr) { 4920 /* The five prefix actions: meter, decap, encap, tag, end. */ 4921 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4922 sizeof(struct mlx5_rte_flow_action_set_tag); 4923 /* tag, vlan, port id, end. */ 4924 #define METER_SUFFIX_ITEM 4 4925 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4926 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4927 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 4928 0, SOCKET_ID_ANY); 4929 if (!sfx_actions) 4930 return rte_flow_error_set(error, ENOMEM, 4931 RTE_FLOW_ERROR_TYPE_ACTION, 4932 NULL, "no memory to split " 4933 "meter flow"); 4934 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4935 act_size); 4936 pre_actions = sfx_actions + actions_n; 4937 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4938 actions, sfx_actions, 4939 pre_actions); 4940 if (!mtr_tag_id) { 4941 ret = -rte_errno; 4942 goto exit; 4943 } 4944 /* Add the prefix subflow. */ 4945 flow_split_info->prefix_mark = 0; 4946 ret = flow_create_split_inner(dev, flow, &dev_flow, 4947 attr, items, pre_actions, 4948 flow_split_info, error); 4949 if (ret) { 4950 ret = -rte_errno; 4951 goto exit; 4952 } 4953 dev_flow->handle->split_flow_id = mtr_tag_id; 4954 /* Setting the sfx group atrr. */ 4955 sfx_attr.group = sfx_attr.transfer ? 4956 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4957 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4958 flow_split_info->prefix_layers = 4959 flow_get_prefix_layer_flags(dev_flow); 4960 flow_split_info->prefix_mark = dev_flow->handle->mark; 4961 } 4962 /* Add the prefix subflow. */ 4963 ret = flow_create_split_metadata(dev, flow, 4964 &sfx_attr, sfx_items ? 4965 sfx_items : items, 4966 sfx_actions ? sfx_actions : actions, 4967 flow_split_info, error); 4968 exit: 4969 if (sfx_actions) 4970 mlx5_free(sfx_actions); 4971 return ret; 4972 } 4973 4974 /** 4975 * The splitting for sample feature. 4976 * 4977 * Once Sample action is detected in the action list, the flow actions should 4978 * be split into prefix sub flow and suffix sub flow. 4979 * 4980 * The original items remain in the prefix sub flow, all actions preceding the 4981 * sample action and the sample action itself will be copied to the prefix 4982 * sub flow, the actions following the sample action will be copied to the 4983 * suffix sub flow, Queue action always be located in the suffix sub flow. 4984 * 4985 * In order to make the packet from prefix sub flow matches with suffix sub 4986 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 4987 * flow uses tag item with the unique flow id. 4988 * 4989 * @param dev 4990 * Pointer to Ethernet device. 4991 * @param[in] flow 4992 * Parent flow structure pointer. 4993 * @param[in] attr 4994 * Flow rule attributes. 4995 * @param[in] items 4996 * Pattern specification (list terminated by the END pattern item). 4997 * @param[in] actions 4998 * Associated actions (list terminated by the END action). 4999 * @param[in] flow_split_info 5000 * Pointer to flow split info structure. 5001 * @param[out] error 5002 * Perform verbose error reporting if not NULL. 5003 * @return 5004 * 0 on success, negative value otherwise 5005 */ 5006 static int 5007 flow_create_split_sample(struct rte_eth_dev *dev, 5008 struct rte_flow *flow, 5009 const struct rte_flow_attr *attr, 5010 const struct rte_flow_item items[], 5011 const struct rte_flow_action actions[], 5012 struct mlx5_flow_split_info *flow_split_info, 5013 struct rte_flow_error *error) 5014 { 5015 struct mlx5_priv *priv = dev->data->dev_private; 5016 struct rte_flow_action *sfx_actions = NULL; 5017 struct rte_flow_action *pre_actions = NULL; 5018 struct rte_flow_item *sfx_items = NULL; 5019 struct mlx5_flow *dev_flow = NULL; 5020 struct rte_flow_attr sfx_attr = *attr; 5021 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5022 struct mlx5_flow_dv_sample_resource *sample_res; 5023 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5024 struct mlx5_flow_tbl_resource *sfx_tbl; 5025 union mlx5_flow_tbl_key sfx_table_key; 5026 #endif 5027 size_t act_size; 5028 size_t item_size; 5029 uint32_t fdb_tx = 0; 5030 int32_t tag_id = 0; 5031 int actions_n = 0; 5032 int sample_action_pos; 5033 int qrss_action_pos; 5034 int ret = 0; 5035 5036 if (priv->sampler_en) 5037 actions_n = flow_check_match_action(actions, attr, 5038 RTE_FLOW_ACTION_TYPE_SAMPLE, 5039 &sample_action_pos, &qrss_action_pos); 5040 if (actions_n) { 5041 /* The prefix actions must includes sample, tag, end. */ 5042 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5043 + sizeof(struct mlx5_rte_flow_action_set_tag); 5044 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5045 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5046 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5047 item_size), 0, SOCKET_ID_ANY); 5048 if (!sfx_actions) 5049 return rte_flow_error_set(error, ENOMEM, 5050 RTE_FLOW_ERROR_TYPE_ACTION, 5051 NULL, "no memory to split " 5052 "sample flow"); 5053 /* The representor_id is -1 for uplink. */ 5054 fdb_tx = (attr->transfer && priv->representor_id != -1); 5055 if (!fdb_tx) 5056 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5057 + act_size); 5058 pre_actions = sfx_actions + actions_n; 5059 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items, 5060 actions, sfx_actions, 5061 pre_actions, actions_n, 5062 sample_action_pos, 5063 qrss_action_pos, error); 5064 if (tag_id < 0 || (!fdb_tx && !tag_id)) { 5065 ret = -rte_errno; 5066 goto exit; 5067 } 5068 /* Add the prefix subflow. */ 5069 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5070 items, pre_actions, 5071 flow_split_info, error); 5072 if (ret) { 5073 ret = -rte_errno; 5074 goto exit; 5075 } 5076 dev_flow->handle->split_flow_id = tag_id; 5077 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5078 /* Set the sfx group attr. */ 5079 sample_res = (struct mlx5_flow_dv_sample_resource *) 5080 dev_flow->dv.sample_res; 5081 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5082 sample_res->normal_path_tbl; 5083 sfx_tbl_data = container_of(sfx_tbl, 5084 struct mlx5_flow_tbl_data_entry, tbl); 5085 sfx_table_key.v64 = sfx_tbl_data->entry.key; 5086 sfx_attr.group = sfx_attr.transfer ? 5087 (sfx_table_key.table_id - 1) : 5088 sfx_table_key.table_id; 5089 flow_split_info->prefix_layers = 5090 flow_get_prefix_layer_flags(dev_flow); 5091 flow_split_info->prefix_mark = dev_flow->handle->mark; 5092 /* Suffix group level already be scaled with factor, set 5093 * skip_scale to 1 to avoid scale again in translation. 5094 */ 5095 flow_split_info->skip_scale = 1; 5096 #endif 5097 } 5098 /* Add the suffix subflow. */ 5099 ret = flow_create_split_meter(dev, flow, &sfx_attr, 5100 sfx_items ? sfx_items : items, 5101 sfx_actions ? sfx_actions : actions, 5102 flow_split_info, error); 5103 exit: 5104 if (sfx_actions) 5105 mlx5_free(sfx_actions); 5106 return ret; 5107 } 5108 5109 /** 5110 * Split the flow to subflow set. The splitters might be linked 5111 * in the chain, like this: 5112 * flow_create_split_outer() calls: 5113 * flow_create_split_meter() calls: 5114 * flow_create_split_metadata(meter_subflow_0) calls: 5115 * flow_create_split_inner(metadata_subflow_0) 5116 * flow_create_split_inner(metadata_subflow_1) 5117 * flow_create_split_inner(metadata_subflow_2) 5118 * flow_create_split_metadata(meter_subflow_1) calls: 5119 * flow_create_split_inner(metadata_subflow_0) 5120 * flow_create_split_inner(metadata_subflow_1) 5121 * flow_create_split_inner(metadata_subflow_2) 5122 * 5123 * This provide flexible way to add new levels of flow splitting. 5124 * The all of successfully created subflows are included to the 5125 * parent flow dev_flow list. 5126 * 5127 * @param dev 5128 * Pointer to Ethernet device. 5129 * @param[in] flow 5130 * Parent flow structure pointer. 5131 * @param[in] attr 5132 * Flow rule attributes. 5133 * @param[in] items 5134 * Pattern specification (list terminated by the END pattern item). 5135 * @param[in] actions 5136 * Associated actions (list terminated by the END action). 5137 * @param[in] flow_split_info 5138 * Pointer to flow split info structure. 5139 * @param[out] error 5140 * Perform verbose error reporting if not NULL. 5141 * @return 5142 * 0 on success, negative value otherwise 5143 */ 5144 static int 5145 flow_create_split_outer(struct rte_eth_dev *dev, 5146 struct rte_flow *flow, 5147 const struct rte_flow_attr *attr, 5148 const struct rte_flow_item items[], 5149 const struct rte_flow_action actions[], 5150 struct mlx5_flow_split_info *flow_split_info, 5151 struct rte_flow_error *error) 5152 { 5153 int ret; 5154 5155 ret = flow_create_split_sample(dev, flow, attr, items, 5156 actions, flow_split_info, error); 5157 MLX5_ASSERT(ret <= 0); 5158 return ret; 5159 } 5160 5161 static struct mlx5_flow_tunnel * 5162 flow_tunnel_from_rule(struct rte_eth_dev *dev, 5163 const struct rte_flow_attr *attr, 5164 const struct rte_flow_item items[], 5165 const struct rte_flow_action actions[]) 5166 { 5167 struct mlx5_flow_tunnel *tunnel; 5168 5169 #pragma GCC diagnostic push 5170 #pragma GCC diagnostic ignored "-Wcast-qual" 5171 if (is_flow_tunnel_match_rule(dev, attr, items, actions)) 5172 tunnel = (struct mlx5_flow_tunnel *)items[0].spec; 5173 else if (is_flow_tunnel_steer_rule(dev, attr, items, actions)) 5174 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf; 5175 else 5176 tunnel = NULL; 5177 #pragma GCC diagnostic pop 5178 5179 return tunnel; 5180 } 5181 5182 /** 5183 * Adjust flow RSS workspace if needed. 5184 * 5185 * @param wks 5186 * Pointer to thread flow work space. 5187 * @param rss_desc 5188 * Pointer to RSS descriptor. 5189 * @param[in] nrssq_num 5190 * New RSS queue number. 5191 * 5192 * @return 5193 * 0 on success, -1 otherwise and rte_errno is set. 5194 */ 5195 static int 5196 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 5197 struct mlx5_flow_rss_desc *rss_desc, 5198 uint32_t nrssq_num) 5199 { 5200 if (likely(nrssq_num <= wks->rssq_num)) 5201 return 0; 5202 rss_desc->queue = realloc(rss_desc->queue, 5203 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 5204 if (!rss_desc->queue) { 5205 rte_errno = ENOMEM; 5206 return -1; 5207 } 5208 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 5209 return 0; 5210 } 5211 5212 /** 5213 * Create a flow and add it to @p list. 5214 * 5215 * @param dev 5216 * Pointer to Ethernet device. 5217 * @param list 5218 * Pointer to a TAILQ flow list. If this parameter NULL, 5219 * no list insertion occurred, flow is just created, 5220 * this is caller's responsibility to track the 5221 * created flow. 5222 * @param[in] attr 5223 * Flow rule attributes. 5224 * @param[in] items 5225 * Pattern specification (list terminated by the END pattern item). 5226 * @param[in] actions 5227 * Associated actions (list terminated by the END action). 5228 * @param[in] external 5229 * This flow rule is created by request external to PMD. 5230 * @param[out] error 5231 * Perform verbose error reporting if not NULL. 5232 * 5233 * @return 5234 * A flow index on success, 0 otherwise and rte_errno is set. 5235 */ 5236 static uint32_t 5237 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 5238 const struct rte_flow_attr *attr, 5239 const struct rte_flow_item items[], 5240 const struct rte_flow_action original_actions[], 5241 bool external, struct rte_flow_error *error) 5242 { 5243 struct mlx5_priv *priv = dev->data->dev_private; 5244 struct rte_flow *flow = NULL; 5245 struct mlx5_flow *dev_flow; 5246 const struct rte_flow_action_rss *rss; 5247 struct mlx5_translated_shared_action 5248 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5249 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5250 union { 5251 struct mlx5_flow_expand_rss buf; 5252 uint8_t buffer[2048]; 5253 } expand_buffer; 5254 union { 5255 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5256 uint8_t buffer[2048]; 5257 } actions_rx; 5258 union { 5259 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5260 uint8_t buffer[2048]; 5261 } actions_hairpin_tx; 5262 union { 5263 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 5264 uint8_t buffer[2048]; 5265 } items_tx; 5266 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 5267 struct mlx5_flow_rss_desc *rss_desc; 5268 const struct rte_flow_action *p_actions_rx; 5269 uint32_t i; 5270 uint32_t idx = 0; 5271 int hairpin_flow; 5272 struct rte_flow_attr attr_tx = { .priority = 0 }; 5273 const struct rte_flow_action *actions; 5274 struct rte_flow_action *translated_actions = NULL; 5275 struct mlx5_flow_tunnel *tunnel; 5276 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 5277 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 5278 struct mlx5_flow_split_info flow_split_info = { 5279 .external = !!external, 5280 .skip_scale = 0, 5281 .flow_idx = 0, 5282 .prefix_mark = 0, 5283 .prefix_layers = 0 5284 }; 5285 int ret; 5286 5287 MLX5_ASSERT(wks); 5288 rss_desc = &wks->rss_desc; 5289 ret = flow_shared_actions_translate(dev, original_actions, 5290 shared_actions, 5291 &shared_actions_n, 5292 &translated_actions, error); 5293 if (ret < 0) { 5294 MLX5_ASSERT(translated_actions == NULL); 5295 return 0; 5296 } 5297 actions = translated_actions ? translated_actions : original_actions; 5298 p_actions_rx = actions; 5299 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5300 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 5301 external, hairpin_flow, error); 5302 if (ret < 0) 5303 goto error_before_hairpin_split; 5304 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 5305 if (!flow) { 5306 rte_errno = ENOMEM; 5307 goto error_before_hairpin_split; 5308 } 5309 if (hairpin_flow > 0) { 5310 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 5311 rte_errno = EINVAL; 5312 goto error_before_hairpin_split; 5313 } 5314 flow_hairpin_split(dev, actions, actions_rx.actions, 5315 actions_hairpin_tx.actions, items_tx.items, 5316 idx); 5317 p_actions_rx = actions_rx.actions; 5318 } 5319 flow_split_info.flow_idx = idx; 5320 flow->drv_type = flow_get_drv_type(dev, attr); 5321 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 5322 flow->drv_type < MLX5_FLOW_TYPE_MAX); 5323 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 5324 rss = flow_get_rss_action(p_actions_rx); 5325 if (rss) { 5326 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 5327 return 0; 5328 /* 5329 * The following information is required by 5330 * mlx5_flow_hashfields_adjust() in advance. 5331 */ 5332 rss_desc->level = rss->level; 5333 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 5334 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 5335 } 5336 flow->dev_handles = 0; 5337 if (rss && rss->types) { 5338 unsigned int graph_root; 5339 5340 graph_root = find_graph_root(items, rss->level); 5341 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 5342 items, rss->types, 5343 mlx5_support_expansion, graph_root); 5344 MLX5_ASSERT(ret > 0 && 5345 (unsigned int)ret < sizeof(expand_buffer.buffer)); 5346 } else { 5347 buf->entries = 1; 5348 buf->entry[0].pattern = (void *)(uintptr_t)items; 5349 } 5350 rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions, 5351 shared_actions_n); 5352 for (i = 0; i < buf->entries; ++i) { 5353 /* Initialize flow split data. */ 5354 flow_split_info.prefix_layers = 0; 5355 flow_split_info.prefix_mark = 0; 5356 flow_split_info.skip_scale = 0; 5357 /* 5358 * The splitter may create multiple dev_flows, 5359 * depending on configuration. In the simplest 5360 * case it just creates unmodified original flow. 5361 */ 5362 ret = flow_create_split_outer(dev, flow, attr, 5363 buf->entry[i].pattern, 5364 p_actions_rx, &flow_split_info, 5365 error); 5366 if (ret < 0) 5367 goto error; 5368 if (is_flow_tunnel_steer_rule(dev, attr, 5369 buf->entry[i].pattern, 5370 p_actions_rx)) { 5371 ret = flow_tunnel_add_default_miss(dev, flow, attr, 5372 p_actions_rx, 5373 idx, 5374 &default_miss_ctx, 5375 error); 5376 if (ret < 0) { 5377 mlx5_free(default_miss_ctx.queue); 5378 goto error; 5379 } 5380 } 5381 } 5382 /* Create the tx flow. */ 5383 if (hairpin_flow) { 5384 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 5385 attr_tx.ingress = 0; 5386 attr_tx.egress = 1; 5387 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 5388 actions_hairpin_tx.actions, 5389 idx, error); 5390 if (!dev_flow) 5391 goto error; 5392 dev_flow->flow = flow; 5393 dev_flow->external = 0; 5394 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5395 dev_flow->handle, next); 5396 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 5397 items_tx.items, 5398 actions_hairpin_tx.actions, error); 5399 if (ret < 0) 5400 goto error; 5401 } 5402 /* 5403 * Update the metadata register copy table. If extensive 5404 * metadata feature is enabled and registers are supported 5405 * we might create the extra rte_flow for each unique 5406 * MARK/FLAG action ID. 5407 * 5408 * The table is updated for ingress Flows only, because 5409 * the egress Flows belong to the different device and 5410 * copy table should be updated in peer NIC Rx domain. 5411 */ 5412 if (attr->ingress && 5413 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 5414 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 5415 if (ret) 5416 goto error; 5417 } 5418 /* 5419 * If the flow is external (from application) OR device is started, 5420 * OR mreg discover, then apply immediately. 5421 */ 5422 if (external || dev->data->dev_started || 5423 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 5424 attr->priority == MLX5_FLOW_PRIO_RSVD)) { 5425 ret = flow_drv_apply(dev, flow, error); 5426 if (ret < 0) 5427 goto error; 5428 } 5429 if (list) { 5430 rte_spinlock_lock(&priv->flow_list_lock); 5431 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 5432 flow, next); 5433 rte_spinlock_unlock(&priv->flow_list_lock); 5434 } 5435 flow_rxq_flags_set(dev, flow); 5436 rte_free(translated_actions); 5437 tunnel = flow_tunnel_from_rule(dev, attr, items, actions); 5438 if (tunnel) { 5439 flow->tunnel = 1; 5440 flow->tunnel_id = tunnel->tunnel_id; 5441 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 5442 mlx5_free(default_miss_ctx.queue); 5443 } 5444 mlx5_flow_pop_thread_workspace(); 5445 return idx; 5446 error: 5447 MLX5_ASSERT(flow); 5448 ret = rte_errno; /* Save rte_errno before cleanup. */ 5449 flow_mreg_del_copy_action(dev, flow); 5450 flow_drv_destroy(dev, flow); 5451 if (rss_desc->shared_rss) 5452 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 5453 mlx5_ipool_get 5454 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 5455 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 5456 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 5457 rte_errno = ret; /* Restore rte_errno. */ 5458 ret = rte_errno; 5459 rte_errno = ret; 5460 mlx5_flow_pop_thread_workspace(); 5461 error_before_hairpin_split: 5462 rte_free(translated_actions); 5463 return 0; 5464 } 5465 5466 /** 5467 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 5468 * incoming packets to table 1. 5469 * 5470 * Other flow rules, requested for group n, will be created in 5471 * e-switch table n+1. 5472 * Jump action to e-switch group n will be created to group n+1. 5473 * 5474 * Used when working in switchdev mode, to utilise advantages of table 1 5475 * and above. 5476 * 5477 * @param dev 5478 * Pointer to Ethernet device. 5479 * 5480 * @return 5481 * Pointer to flow on success, NULL otherwise and rte_errno is set. 5482 */ 5483 struct rte_flow * 5484 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 5485 { 5486 const struct rte_flow_attr attr = { 5487 .group = 0, 5488 .priority = 0, 5489 .ingress = 1, 5490 .egress = 0, 5491 .transfer = 1, 5492 }; 5493 const struct rte_flow_item pattern = { 5494 .type = RTE_FLOW_ITEM_TYPE_END, 5495 }; 5496 struct rte_flow_action_jump jump = { 5497 .group = 1, 5498 }; 5499 const struct rte_flow_action actions[] = { 5500 { 5501 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5502 .conf = &jump, 5503 }, 5504 { 5505 .type = RTE_FLOW_ACTION_TYPE_END, 5506 }, 5507 }; 5508 struct mlx5_priv *priv = dev->data->dev_private; 5509 struct rte_flow_error error; 5510 5511 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 5512 &attr, &pattern, 5513 actions, false, &error); 5514 } 5515 5516 /** 5517 * Validate a flow supported by the NIC. 5518 * 5519 * @see rte_flow_validate() 5520 * @see rte_flow_ops 5521 */ 5522 int 5523 mlx5_flow_validate(struct rte_eth_dev *dev, 5524 const struct rte_flow_attr *attr, 5525 const struct rte_flow_item items[], 5526 const struct rte_flow_action original_actions[], 5527 struct rte_flow_error *error) 5528 { 5529 int hairpin_flow; 5530 struct mlx5_translated_shared_action 5531 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5532 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5533 const struct rte_flow_action *actions; 5534 struct rte_flow_action *translated_actions = NULL; 5535 int ret = flow_shared_actions_translate(dev, original_actions, 5536 shared_actions, 5537 &shared_actions_n, 5538 &translated_actions, error); 5539 5540 if (ret) 5541 return ret; 5542 actions = translated_actions ? translated_actions : original_actions; 5543 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5544 ret = flow_drv_validate(dev, attr, items, actions, 5545 true, hairpin_flow, error); 5546 rte_free(translated_actions); 5547 return ret; 5548 } 5549 5550 /** 5551 * Create a flow. 5552 * 5553 * @see rte_flow_create() 5554 * @see rte_flow_ops 5555 */ 5556 struct rte_flow * 5557 mlx5_flow_create(struct rte_eth_dev *dev, 5558 const struct rte_flow_attr *attr, 5559 const struct rte_flow_item items[], 5560 const struct rte_flow_action actions[], 5561 struct rte_flow_error *error) 5562 { 5563 struct mlx5_priv *priv = dev->data->dev_private; 5564 5565 /* 5566 * If the device is not started yet, it is not allowed to created a 5567 * flow from application. PMD default flows and traffic control flows 5568 * are not affected. 5569 */ 5570 if (unlikely(!dev->data->dev_started)) { 5571 DRV_LOG(DEBUG, "port %u is not started when " 5572 "inserting a flow", dev->data->port_id); 5573 rte_flow_error_set(error, ENODEV, 5574 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5575 NULL, 5576 "port not started"); 5577 return NULL; 5578 } 5579 5580 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 5581 attr, items, actions, true, error); 5582 } 5583 5584 /** 5585 * Destroy a flow in a list. 5586 * 5587 * @param dev 5588 * Pointer to Ethernet device. 5589 * @param list 5590 * Pointer to the Indexed flow list. If this parameter NULL, 5591 * there is no flow removal from the list. Be noted that as 5592 * flow is add to the indexed list, memory of the indexed 5593 * list points to maybe changed as flow destroyed. 5594 * @param[in] flow_idx 5595 * Index of flow to destroy. 5596 */ 5597 static void 5598 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 5599 uint32_t flow_idx) 5600 { 5601 struct mlx5_priv *priv = dev->data->dev_private; 5602 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5603 [MLX5_IPOOL_RTE_FLOW], flow_idx); 5604 5605 if (!flow) 5606 return; 5607 /* 5608 * Update RX queue flags only if port is started, otherwise it is 5609 * already clean. 5610 */ 5611 if (dev->data->dev_started) 5612 flow_rxq_flags_trim(dev, flow); 5613 flow_drv_destroy(dev, flow); 5614 if (list) { 5615 rte_spinlock_lock(&priv->flow_list_lock); 5616 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 5617 flow_idx, flow, next); 5618 rte_spinlock_unlock(&priv->flow_list_lock); 5619 } 5620 if (flow->tunnel) { 5621 struct mlx5_flow_tunnel *tunnel; 5622 5623 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 5624 RTE_VERIFY(tunnel); 5625 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 5626 mlx5_flow_tunnel_free(dev, tunnel); 5627 } 5628 flow_mreg_del_copy_action(dev, flow); 5629 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5630 } 5631 5632 /** 5633 * Destroy all flows. 5634 * 5635 * @param dev 5636 * Pointer to Ethernet device. 5637 * @param list 5638 * Pointer to the Indexed flow list. 5639 * @param active 5640 * If flushing is called avtively. 5641 */ 5642 void 5643 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 5644 { 5645 uint32_t num_flushed = 0; 5646 5647 while (*list) { 5648 flow_list_destroy(dev, list, *list); 5649 num_flushed++; 5650 } 5651 if (active) { 5652 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 5653 dev->data->port_id, num_flushed); 5654 } 5655 } 5656 5657 /** 5658 * Stop all default actions for flows. 5659 * 5660 * @param dev 5661 * Pointer to Ethernet device. 5662 */ 5663 void 5664 mlx5_flow_stop_default(struct rte_eth_dev *dev) 5665 { 5666 flow_mreg_del_default_copy_action(dev); 5667 flow_rxq_flags_clear(dev); 5668 } 5669 5670 /** 5671 * Start all default actions for flows. 5672 * 5673 * @param dev 5674 * Pointer to Ethernet device. 5675 * @return 5676 * 0 on success, a negative errno value otherwise and rte_errno is set. 5677 */ 5678 int 5679 mlx5_flow_start_default(struct rte_eth_dev *dev) 5680 { 5681 struct rte_flow_error error; 5682 5683 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 5684 return flow_mreg_add_default_copy_action(dev, &error); 5685 } 5686 5687 /** 5688 * Release key of thread specific flow workspace data. 5689 */ 5690 static void 5691 flow_release_workspace(void *data) 5692 { 5693 struct mlx5_flow_workspace *wks = data; 5694 struct mlx5_flow_workspace *next; 5695 5696 while (wks) { 5697 next = wks->next; 5698 free(wks->rss_desc.queue); 5699 free(wks); 5700 wks = next; 5701 } 5702 } 5703 5704 /** 5705 * Initialize key of thread specific flow workspace data. 5706 */ 5707 static void 5708 flow_alloc_workspace(void) 5709 { 5710 if (pthread_key_create(&key_workspace, flow_release_workspace)) 5711 DRV_LOG(ERR, "Can't create flow workspace data thread key."); 5712 } 5713 5714 /** 5715 * Get thread specific current flow workspace. 5716 * 5717 * @return pointer to thread specific flow workspace data, NULL on error. 5718 */ 5719 struct mlx5_flow_workspace* 5720 mlx5_flow_get_thread_workspace(void) 5721 { 5722 struct mlx5_flow_workspace *data; 5723 5724 data = pthread_getspecific(key_workspace); 5725 MLX5_ASSERT(data && data->inuse); 5726 if (!data || !data->inuse) 5727 DRV_LOG(ERR, "flow workspace not initialized."); 5728 return data; 5729 } 5730 5731 /** 5732 * Allocate and init new flow workspace. 5733 * 5734 * @return pointer to flow workspace data, NULL on error. 5735 */ 5736 static struct mlx5_flow_workspace* 5737 flow_alloc_thread_workspace(void) 5738 { 5739 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 5740 5741 if (!data) { 5742 DRV_LOG(ERR, "Failed to allocate flow workspace " 5743 "memory."); 5744 return NULL; 5745 } 5746 data->rss_desc.queue = calloc(1, 5747 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 5748 if (!data->rss_desc.queue) 5749 goto err; 5750 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 5751 return data; 5752 err: 5753 if (data->rss_desc.queue) 5754 free(data->rss_desc.queue); 5755 free(data); 5756 return NULL; 5757 } 5758 5759 /** 5760 * Get new thread specific flow workspace. 5761 * 5762 * If current workspace inuse, create new one and set as current. 5763 * 5764 * @return pointer to thread specific flow workspace data, NULL on error. 5765 */ 5766 static struct mlx5_flow_workspace* 5767 mlx5_flow_push_thread_workspace(void) 5768 { 5769 struct mlx5_flow_workspace *curr; 5770 struct mlx5_flow_workspace *data; 5771 5772 if (pthread_once(&key_workspace_init, flow_alloc_workspace)) { 5773 DRV_LOG(ERR, "Failed to init flow workspace data thread key."); 5774 return NULL; 5775 } 5776 curr = pthread_getspecific(key_workspace); 5777 if (!curr) { 5778 data = flow_alloc_thread_workspace(); 5779 if (!data) 5780 return NULL; 5781 } else if (!curr->inuse) { 5782 data = curr; 5783 } else if (curr->next) { 5784 data = curr->next; 5785 } else { 5786 data = flow_alloc_thread_workspace(); 5787 if (!data) 5788 return NULL; 5789 curr->next = data; 5790 data->prev = curr; 5791 } 5792 data->inuse = 1; 5793 data->flow_idx = 0; 5794 /* Set as current workspace */ 5795 if (pthread_setspecific(key_workspace, data)) 5796 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 5797 return data; 5798 } 5799 5800 /** 5801 * Close current thread specific flow workspace. 5802 * 5803 * If previous workspace available, set it as current. 5804 * 5805 * @return pointer to thread specific flow workspace data, NULL on error. 5806 */ 5807 static void 5808 mlx5_flow_pop_thread_workspace(void) 5809 { 5810 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 5811 5812 if (!data) 5813 return; 5814 if (!data->inuse) { 5815 DRV_LOG(ERR, "Failed to close unused flow workspace."); 5816 return; 5817 } 5818 data->inuse = 0; 5819 if (!data->prev) 5820 return; 5821 if (pthread_setspecific(key_workspace, data->prev)) 5822 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 5823 } 5824 5825 /** 5826 * Verify the flow list is empty 5827 * 5828 * @param dev 5829 * Pointer to Ethernet device. 5830 * 5831 * @return the number of flows not released. 5832 */ 5833 int 5834 mlx5_flow_verify(struct rte_eth_dev *dev) 5835 { 5836 struct mlx5_priv *priv = dev->data->dev_private; 5837 struct rte_flow *flow; 5838 uint32_t idx; 5839 int ret = 0; 5840 5841 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 5842 flow, next) { 5843 DRV_LOG(DEBUG, "port %u flow %p still referenced", 5844 dev->data->port_id, (void *)flow); 5845 ++ret; 5846 } 5847 return ret; 5848 } 5849 5850 /** 5851 * Enable default hairpin egress flow. 5852 * 5853 * @param dev 5854 * Pointer to Ethernet device. 5855 * @param queue 5856 * The queue index. 5857 * 5858 * @return 5859 * 0 on success, a negative errno value otherwise and rte_errno is set. 5860 */ 5861 int 5862 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 5863 uint32_t queue) 5864 { 5865 struct mlx5_priv *priv = dev->data->dev_private; 5866 const struct rte_flow_attr attr = { 5867 .egress = 1, 5868 .priority = 0, 5869 }; 5870 struct mlx5_rte_flow_item_tx_queue queue_spec = { 5871 .queue = queue, 5872 }; 5873 struct mlx5_rte_flow_item_tx_queue queue_mask = { 5874 .queue = UINT32_MAX, 5875 }; 5876 struct rte_flow_item items[] = { 5877 { 5878 .type = (enum rte_flow_item_type) 5879 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 5880 .spec = &queue_spec, 5881 .last = NULL, 5882 .mask = &queue_mask, 5883 }, 5884 { 5885 .type = RTE_FLOW_ITEM_TYPE_END, 5886 }, 5887 }; 5888 struct rte_flow_action_jump jump = { 5889 .group = MLX5_HAIRPIN_TX_TABLE, 5890 }; 5891 struct rte_flow_action actions[2]; 5892 uint32_t flow_idx; 5893 struct rte_flow_error error; 5894 5895 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 5896 actions[0].conf = &jump; 5897 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 5898 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5899 &attr, items, actions, false, &error); 5900 if (!flow_idx) { 5901 DRV_LOG(DEBUG, 5902 "Failed to create ctrl flow: rte_errno(%d)," 5903 " type(%d), message(%s)", 5904 rte_errno, error.type, 5905 error.message ? error.message : " (no stated reason)"); 5906 return -rte_errno; 5907 } 5908 return 0; 5909 } 5910 5911 /** 5912 * Enable a control flow configured from the control plane. 5913 * 5914 * @param dev 5915 * Pointer to Ethernet device. 5916 * @param eth_spec 5917 * An Ethernet flow spec to apply. 5918 * @param eth_mask 5919 * An Ethernet flow mask to apply. 5920 * @param vlan_spec 5921 * A VLAN flow spec to apply. 5922 * @param vlan_mask 5923 * A VLAN flow mask to apply. 5924 * 5925 * @return 5926 * 0 on success, a negative errno value otherwise and rte_errno is set. 5927 */ 5928 int 5929 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 5930 struct rte_flow_item_eth *eth_spec, 5931 struct rte_flow_item_eth *eth_mask, 5932 struct rte_flow_item_vlan *vlan_spec, 5933 struct rte_flow_item_vlan *vlan_mask) 5934 { 5935 struct mlx5_priv *priv = dev->data->dev_private; 5936 const struct rte_flow_attr attr = { 5937 .ingress = 1, 5938 .priority = MLX5_FLOW_PRIO_RSVD, 5939 }; 5940 struct rte_flow_item items[] = { 5941 { 5942 .type = RTE_FLOW_ITEM_TYPE_ETH, 5943 .spec = eth_spec, 5944 .last = NULL, 5945 .mask = eth_mask, 5946 }, 5947 { 5948 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 5949 RTE_FLOW_ITEM_TYPE_END, 5950 .spec = vlan_spec, 5951 .last = NULL, 5952 .mask = vlan_mask, 5953 }, 5954 { 5955 .type = RTE_FLOW_ITEM_TYPE_END, 5956 }, 5957 }; 5958 uint16_t queue[priv->reta_idx_n]; 5959 struct rte_flow_action_rss action_rss = { 5960 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 5961 .level = 0, 5962 .types = priv->rss_conf.rss_hf, 5963 .key_len = priv->rss_conf.rss_key_len, 5964 .queue_num = priv->reta_idx_n, 5965 .key = priv->rss_conf.rss_key, 5966 .queue = queue, 5967 }; 5968 struct rte_flow_action actions[] = { 5969 { 5970 .type = RTE_FLOW_ACTION_TYPE_RSS, 5971 .conf = &action_rss, 5972 }, 5973 { 5974 .type = RTE_FLOW_ACTION_TYPE_END, 5975 }, 5976 }; 5977 uint32_t flow_idx; 5978 struct rte_flow_error error; 5979 unsigned int i; 5980 5981 if (!priv->reta_idx_n || !priv->rxqs_n) { 5982 return 0; 5983 } 5984 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 5985 action_rss.types = 0; 5986 for (i = 0; i != priv->reta_idx_n; ++i) 5987 queue[i] = (*priv->reta_idx)[i]; 5988 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5989 &attr, items, actions, false, &error); 5990 if (!flow_idx) 5991 return -rte_errno; 5992 return 0; 5993 } 5994 5995 /** 5996 * Enable a flow control configured from the control plane. 5997 * 5998 * @param dev 5999 * Pointer to Ethernet device. 6000 * @param eth_spec 6001 * An Ethernet flow spec to apply. 6002 * @param eth_mask 6003 * An Ethernet flow mask to apply. 6004 * 6005 * @return 6006 * 0 on success, a negative errno value otherwise and rte_errno is set. 6007 */ 6008 int 6009 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6010 struct rte_flow_item_eth *eth_spec, 6011 struct rte_flow_item_eth *eth_mask) 6012 { 6013 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6014 } 6015 6016 /** 6017 * Create default miss flow rule matching lacp traffic 6018 * 6019 * @param dev 6020 * Pointer to Ethernet device. 6021 * @param eth_spec 6022 * An Ethernet flow spec to apply. 6023 * 6024 * @return 6025 * 0 on success, a negative errno value otherwise and rte_errno is set. 6026 */ 6027 int 6028 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6029 { 6030 struct mlx5_priv *priv = dev->data->dev_private; 6031 /* 6032 * The LACP matching is done by only using ether type since using 6033 * a multicast dst mac causes kernel to give low priority to this flow. 6034 */ 6035 static const struct rte_flow_item_eth lacp_spec = { 6036 .type = RTE_BE16(0x8809), 6037 }; 6038 static const struct rte_flow_item_eth lacp_mask = { 6039 .type = 0xffff, 6040 }; 6041 const struct rte_flow_attr attr = { 6042 .ingress = 1, 6043 }; 6044 struct rte_flow_item items[] = { 6045 { 6046 .type = RTE_FLOW_ITEM_TYPE_ETH, 6047 .spec = &lacp_spec, 6048 .mask = &lacp_mask, 6049 }, 6050 { 6051 .type = RTE_FLOW_ITEM_TYPE_END, 6052 }, 6053 }; 6054 struct rte_flow_action actions[] = { 6055 { 6056 .type = (enum rte_flow_action_type) 6057 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6058 }, 6059 { 6060 .type = RTE_FLOW_ACTION_TYPE_END, 6061 }, 6062 }; 6063 struct rte_flow_error error; 6064 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6065 &attr, items, actions, false, &error); 6066 6067 if (!flow_idx) 6068 return -rte_errno; 6069 return 0; 6070 } 6071 6072 /** 6073 * Destroy a flow. 6074 * 6075 * @see rte_flow_destroy() 6076 * @see rte_flow_ops 6077 */ 6078 int 6079 mlx5_flow_destroy(struct rte_eth_dev *dev, 6080 struct rte_flow *flow, 6081 struct rte_flow_error *error __rte_unused) 6082 { 6083 struct mlx5_priv *priv = dev->data->dev_private; 6084 6085 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6086 return 0; 6087 } 6088 6089 /** 6090 * Destroy all flows. 6091 * 6092 * @see rte_flow_flush() 6093 * @see rte_flow_ops 6094 */ 6095 int 6096 mlx5_flow_flush(struct rte_eth_dev *dev, 6097 struct rte_flow_error *error __rte_unused) 6098 { 6099 struct mlx5_priv *priv = dev->data->dev_private; 6100 6101 mlx5_flow_list_flush(dev, &priv->flows, false); 6102 return 0; 6103 } 6104 6105 /** 6106 * Isolated mode. 6107 * 6108 * @see rte_flow_isolate() 6109 * @see rte_flow_ops 6110 */ 6111 int 6112 mlx5_flow_isolate(struct rte_eth_dev *dev, 6113 int enable, 6114 struct rte_flow_error *error) 6115 { 6116 struct mlx5_priv *priv = dev->data->dev_private; 6117 6118 if (dev->data->dev_started) { 6119 rte_flow_error_set(error, EBUSY, 6120 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6121 NULL, 6122 "port must be stopped first"); 6123 return -rte_errno; 6124 } 6125 priv->isolated = !!enable; 6126 if (enable) 6127 dev->dev_ops = &mlx5_os_dev_ops_isolate; 6128 else 6129 dev->dev_ops = &mlx5_os_dev_ops; 6130 6131 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 6132 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 6133 6134 return 0; 6135 } 6136 6137 /** 6138 * Query a flow. 6139 * 6140 * @see rte_flow_query() 6141 * @see rte_flow_ops 6142 */ 6143 static int 6144 flow_drv_query(struct rte_eth_dev *dev, 6145 uint32_t flow_idx, 6146 const struct rte_flow_action *actions, 6147 void *data, 6148 struct rte_flow_error *error) 6149 { 6150 struct mlx5_priv *priv = dev->data->dev_private; 6151 const struct mlx5_flow_driver_ops *fops; 6152 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6153 [MLX5_IPOOL_RTE_FLOW], 6154 flow_idx); 6155 enum mlx5_flow_drv_type ftype; 6156 6157 if (!flow) { 6158 return rte_flow_error_set(error, ENOENT, 6159 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6160 NULL, 6161 "invalid flow handle"); 6162 } 6163 ftype = flow->drv_type; 6164 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 6165 fops = flow_get_drv_ops(ftype); 6166 6167 return fops->query(dev, flow, actions, data, error); 6168 } 6169 6170 /** 6171 * Query a flow. 6172 * 6173 * @see rte_flow_query() 6174 * @see rte_flow_ops 6175 */ 6176 int 6177 mlx5_flow_query(struct rte_eth_dev *dev, 6178 struct rte_flow *flow, 6179 const struct rte_flow_action *actions, 6180 void *data, 6181 struct rte_flow_error *error) 6182 { 6183 int ret; 6184 6185 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 6186 error); 6187 if (ret < 0) 6188 return ret; 6189 return 0; 6190 } 6191 6192 /** 6193 * Manage filter operations. 6194 * 6195 * @param dev 6196 * Pointer to Ethernet device structure. 6197 * @param filter_type 6198 * Filter type. 6199 * @param filter_op 6200 * Operation to perform. 6201 * @param arg 6202 * Pointer to operation-specific structure. 6203 * 6204 * @return 6205 * 0 on success, a negative errno value otherwise and rte_errno is set. 6206 */ 6207 int 6208 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 6209 enum rte_filter_type filter_type, 6210 enum rte_filter_op filter_op, 6211 void *arg) 6212 { 6213 switch (filter_type) { 6214 case RTE_ETH_FILTER_GENERIC: 6215 if (filter_op != RTE_ETH_FILTER_GET) { 6216 rte_errno = EINVAL; 6217 return -rte_errno; 6218 } 6219 *(const void **)arg = &mlx5_flow_ops; 6220 return 0; 6221 default: 6222 DRV_LOG(ERR, "port %u filter type (%d) not supported", 6223 dev->data->port_id, filter_type); 6224 rte_errno = ENOTSUP; 6225 return -rte_errno; 6226 } 6227 return 0; 6228 } 6229 6230 /** 6231 * Create the needed meter and suffix tables. 6232 * 6233 * @param[in] dev 6234 * Pointer to Ethernet device. 6235 * @param[in] fm 6236 * Pointer to the flow meter. 6237 * 6238 * @return 6239 * Pointer to table set on success, NULL otherwise. 6240 */ 6241 struct mlx5_meter_domains_infos * 6242 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 6243 const struct mlx5_flow_meter *fm) 6244 { 6245 const struct mlx5_flow_driver_ops *fops; 6246 6247 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6248 return fops->create_mtr_tbls(dev, fm); 6249 } 6250 6251 /** 6252 * Destroy the meter table set. 6253 * 6254 * @param[in] dev 6255 * Pointer to Ethernet device. 6256 * @param[in] tbl 6257 * Pointer to the meter table set. 6258 * 6259 * @return 6260 * 0 on success. 6261 */ 6262 int 6263 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 6264 struct mlx5_meter_domains_infos *tbls) 6265 { 6266 const struct mlx5_flow_driver_ops *fops; 6267 6268 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6269 return fops->destroy_mtr_tbls(dev, tbls); 6270 } 6271 6272 /** 6273 * Create policer rules. 6274 * 6275 * @param[in] dev 6276 * Pointer to Ethernet device. 6277 * @param[in] fm 6278 * Pointer to flow meter structure. 6279 * @param[in] attr 6280 * Pointer to flow attributes. 6281 * 6282 * @return 6283 * 0 on success, -1 otherwise. 6284 */ 6285 int 6286 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 6287 struct mlx5_flow_meter *fm, 6288 const struct rte_flow_attr *attr) 6289 { 6290 const struct mlx5_flow_driver_ops *fops; 6291 6292 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6293 return fops->create_policer_rules(dev, fm, attr); 6294 } 6295 6296 /** 6297 * Destroy policer rules. 6298 * 6299 * @param[in] fm 6300 * Pointer to flow meter structure. 6301 * @param[in] attr 6302 * Pointer to flow attributes. 6303 * 6304 * @return 6305 * 0 on success, -1 otherwise. 6306 */ 6307 int 6308 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 6309 struct mlx5_flow_meter *fm, 6310 const struct rte_flow_attr *attr) 6311 { 6312 const struct mlx5_flow_driver_ops *fops; 6313 6314 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6315 return fops->destroy_policer_rules(dev, fm, attr); 6316 } 6317 6318 /** 6319 * Allocate a counter. 6320 * 6321 * @param[in] dev 6322 * Pointer to Ethernet device structure. 6323 * 6324 * @return 6325 * Index to allocated counter on success, 0 otherwise. 6326 */ 6327 uint32_t 6328 mlx5_counter_alloc(struct rte_eth_dev *dev) 6329 { 6330 const struct mlx5_flow_driver_ops *fops; 6331 struct rte_flow_attr attr = { .transfer = 0 }; 6332 6333 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6334 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6335 return fops->counter_alloc(dev); 6336 } 6337 DRV_LOG(ERR, 6338 "port %u counter allocate is not supported.", 6339 dev->data->port_id); 6340 return 0; 6341 } 6342 6343 /** 6344 * Free a counter. 6345 * 6346 * @param[in] dev 6347 * Pointer to Ethernet device structure. 6348 * @param[in] cnt 6349 * Index to counter to be free. 6350 */ 6351 void 6352 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 6353 { 6354 const struct mlx5_flow_driver_ops *fops; 6355 struct rte_flow_attr attr = { .transfer = 0 }; 6356 6357 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6358 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6359 fops->counter_free(dev, cnt); 6360 return; 6361 } 6362 DRV_LOG(ERR, 6363 "port %u counter free is not supported.", 6364 dev->data->port_id); 6365 } 6366 6367 /** 6368 * Query counter statistics. 6369 * 6370 * @param[in] dev 6371 * Pointer to Ethernet device structure. 6372 * @param[in] cnt 6373 * Index to counter to query. 6374 * @param[in] clear 6375 * Set to clear counter statistics. 6376 * @param[out] pkts 6377 * The counter hits packets number to save. 6378 * @param[out] bytes 6379 * The counter hits bytes number to save. 6380 * 6381 * @return 6382 * 0 on success, a negative errno value otherwise. 6383 */ 6384 int 6385 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 6386 bool clear, uint64_t *pkts, uint64_t *bytes) 6387 { 6388 const struct mlx5_flow_driver_ops *fops; 6389 struct rte_flow_attr attr = { .transfer = 0 }; 6390 6391 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6392 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6393 return fops->counter_query(dev, cnt, clear, pkts, bytes); 6394 } 6395 DRV_LOG(ERR, 6396 "port %u counter query is not supported.", 6397 dev->data->port_id); 6398 return -ENOTSUP; 6399 } 6400 6401 /** 6402 * Allocate a new memory for the counter values wrapped by all the needed 6403 * management. 6404 * 6405 * @param[in] sh 6406 * Pointer to mlx5_dev_ctx_shared object. 6407 * 6408 * @return 6409 * 0 on success, a negative errno value otherwise. 6410 */ 6411 static int 6412 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 6413 { 6414 struct mlx5_devx_mkey_attr mkey_attr; 6415 struct mlx5_counter_stats_mem_mng *mem_mng; 6416 volatile struct flow_counter_stats *raw_data; 6417 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 6418 int size = (sizeof(struct flow_counter_stats) * 6419 MLX5_COUNTERS_PER_POOL + 6420 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 6421 sizeof(struct mlx5_counter_stats_mem_mng); 6422 size_t pgsize = rte_mem_page_size(); 6423 uint8_t *mem; 6424 int i; 6425 6426 if (pgsize == (size_t)-1) { 6427 DRV_LOG(ERR, "Failed to get mem page size"); 6428 rte_errno = ENOMEM; 6429 return -ENOMEM; 6430 } 6431 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 6432 if (!mem) { 6433 rte_errno = ENOMEM; 6434 return -ENOMEM; 6435 } 6436 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 6437 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 6438 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 6439 IBV_ACCESS_LOCAL_WRITE); 6440 if (!mem_mng->umem) { 6441 rte_errno = errno; 6442 mlx5_free(mem); 6443 return -rte_errno; 6444 } 6445 mkey_attr.addr = (uintptr_t)mem; 6446 mkey_attr.size = size; 6447 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 6448 mkey_attr.pd = sh->pdn; 6449 mkey_attr.log_entity_size = 0; 6450 mkey_attr.pg_access = 0; 6451 mkey_attr.klm_array = NULL; 6452 mkey_attr.klm_num = 0; 6453 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 6454 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 6455 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 6456 if (!mem_mng->dm) { 6457 mlx5_glue->devx_umem_dereg(mem_mng->umem); 6458 rte_errno = errno; 6459 mlx5_free(mem); 6460 return -rte_errno; 6461 } 6462 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 6463 raw_data = (volatile struct flow_counter_stats *)mem; 6464 for (i = 0; i < raws_n; ++i) { 6465 mem_mng->raws[i].mem_mng = mem_mng; 6466 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 6467 } 6468 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 6469 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 6470 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 6471 next); 6472 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 6473 sh->cmng.mem_mng = mem_mng; 6474 return 0; 6475 } 6476 6477 /** 6478 * Set the statistic memory to the new counter pool. 6479 * 6480 * @param[in] sh 6481 * Pointer to mlx5_dev_ctx_shared object. 6482 * @param[in] pool 6483 * Pointer to the pool to set the statistic memory. 6484 * 6485 * @return 6486 * 0 on success, a negative errno value otherwise. 6487 */ 6488 static int 6489 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 6490 struct mlx5_flow_counter_pool *pool) 6491 { 6492 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6493 /* Resize statistic memory once used out. */ 6494 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 6495 mlx5_flow_create_counter_stat_mem_mng(sh)) { 6496 DRV_LOG(ERR, "Cannot resize counter stat mem."); 6497 return -1; 6498 } 6499 rte_spinlock_lock(&pool->sl); 6500 pool->raw = cmng->mem_mng->raws + pool->index % 6501 MLX5_CNT_CONTAINER_RESIZE; 6502 rte_spinlock_unlock(&pool->sl); 6503 pool->raw_hw = NULL; 6504 return 0; 6505 } 6506 6507 #define MLX5_POOL_QUERY_FREQ_US 1000000 6508 6509 /** 6510 * Set the periodic procedure for triggering asynchronous batch queries for all 6511 * the counter pools. 6512 * 6513 * @param[in] sh 6514 * Pointer to mlx5_dev_ctx_shared object. 6515 */ 6516 void 6517 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 6518 { 6519 uint32_t pools_n, us; 6520 6521 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 6522 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 6523 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 6524 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 6525 sh->cmng.query_thread_on = 0; 6526 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 6527 } else { 6528 sh->cmng.query_thread_on = 1; 6529 } 6530 } 6531 6532 /** 6533 * The periodic procedure for triggering asynchronous batch queries for all the 6534 * counter pools. This function is probably called by the host thread. 6535 * 6536 * @param[in] arg 6537 * The parameter for the alarm process. 6538 */ 6539 void 6540 mlx5_flow_query_alarm(void *arg) 6541 { 6542 struct mlx5_dev_ctx_shared *sh = arg; 6543 int ret; 6544 uint16_t pool_index = sh->cmng.pool_index; 6545 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6546 struct mlx5_flow_counter_pool *pool; 6547 uint16_t n_valid; 6548 6549 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 6550 goto set_alarm; 6551 rte_spinlock_lock(&cmng->pool_update_sl); 6552 pool = cmng->pools[pool_index]; 6553 n_valid = cmng->n_valid; 6554 rte_spinlock_unlock(&cmng->pool_update_sl); 6555 /* Set the statistic memory to the new created pool. */ 6556 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 6557 goto set_alarm; 6558 if (pool->raw_hw) 6559 /* There is a pool query in progress. */ 6560 goto set_alarm; 6561 pool->raw_hw = 6562 LIST_FIRST(&sh->cmng.free_stat_raws); 6563 if (!pool->raw_hw) 6564 /* No free counter statistics raw memory. */ 6565 goto set_alarm; 6566 /* 6567 * Identify the counters released between query trigger and query 6568 * handle more efficiently. The counter released in this gap period 6569 * should wait for a new round of query as the new arrived packets 6570 * will not be taken into account. 6571 */ 6572 pool->query_gen++; 6573 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 6574 MLX5_COUNTERS_PER_POOL, 6575 NULL, NULL, 6576 pool->raw_hw->mem_mng->dm->id, 6577 (void *)(uintptr_t) 6578 pool->raw_hw->data, 6579 sh->devx_comp, 6580 (uint64_t)(uintptr_t)pool); 6581 if (ret) { 6582 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 6583 " %d", pool->min_dcs->id); 6584 pool->raw_hw = NULL; 6585 goto set_alarm; 6586 } 6587 LIST_REMOVE(pool->raw_hw, next); 6588 sh->cmng.pending_queries++; 6589 pool_index++; 6590 if (pool_index >= n_valid) 6591 pool_index = 0; 6592 set_alarm: 6593 sh->cmng.pool_index = pool_index; 6594 mlx5_set_query_alarm(sh); 6595 } 6596 6597 /** 6598 * Check and callback event for new aged flow in the counter pool 6599 * 6600 * @param[in] sh 6601 * Pointer to mlx5_dev_ctx_shared object. 6602 * @param[in] pool 6603 * Pointer to Current counter pool. 6604 */ 6605 static void 6606 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 6607 struct mlx5_flow_counter_pool *pool) 6608 { 6609 struct mlx5_priv *priv; 6610 struct mlx5_flow_counter *cnt; 6611 struct mlx5_age_info *age_info; 6612 struct mlx5_age_param *age_param; 6613 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 6614 struct mlx5_counter_stats_raw *prev = pool->raw; 6615 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 6616 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 6617 uint16_t expected = AGE_CANDIDATE; 6618 uint32_t i; 6619 6620 pool->time_of_last_age_check = curr_time; 6621 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 6622 cnt = MLX5_POOL_GET_CNT(pool, i); 6623 age_param = MLX5_CNT_TO_AGE(cnt); 6624 if (__atomic_load_n(&age_param->state, 6625 __ATOMIC_RELAXED) != AGE_CANDIDATE) 6626 continue; 6627 if (cur->data[i].hits != prev->data[i].hits) { 6628 __atomic_store_n(&age_param->sec_since_last_hit, 0, 6629 __ATOMIC_RELAXED); 6630 continue; 6631 } 6632 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 6633 time_delta, 6634 __ATOMIC_RELAXED) <= age_param->timeout) 6635 continue; 6636 /** 6637 * Hold the lock first, or if between the 6638 * state AGE_TMOUT and tailq operation the 6639 * release happened, the release procedure 6640 * may delete a non-existent tailq node. 6641 */ 6642 priv = rte_eth_devices[age_param->port_id].data->dev_private; 6643 age_info = GET_PORT_AGE_INFO(priv); 6644 rte_spinlock_lock(&age_info->aged_sl); 6645 if (__atomic_compare_exchange_n(&age_param->state, &expected, 6646 AGE_TMOUT, false, 6647 __ATOMIC_RELAXED, 6648 __ATOMIC_RELAXED)) { 6649 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 6650 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 6651 } 6652 rte_spinlock_unlock(&age_info->aged_sl); 6653 } 6654 mlx5_age_event_prepare(sh); 6655 } 6656 6657 /** 6658 * Handler for the HW respond about ready values from an asynchronous batch 6659 * query. This function is probably called by the host thread. 6660 * 6661 * @param[in] sh 6662 * The pointer to the shared device context. 6663 * @param[in] async_id 6664 * The Devx async ID. 6665 * @param[in] status 6666 * The status of the completion. 6667 */ 6668 void 6669 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 6670 uint64_t async_id, int status) 6671 { 6672 struct mlx5_flow_counter_pool *pool = 6673 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 6674 struct mlx5_counter_stats_raw *raw_to_free; 6675 uint8_t query_gen = pool->query_gen ^ 1; 6676 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6677 enum mlx5_counter_type cnt_type = 6678 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 6679 MLX5_COUNTER_TYPE_ORIGIN; 6680 6681 if (unlikely(status)) { 6682 raw_to_free = pool->raw_hw; 6683 } else { 6684 raw_to_free = pool->raw; 6685 if (pool->is_aged) 6686 mlx5_flow_aging_check(sh, pool); 6687 rte_spinlock_lock(&pool->sl); 6688 pool->raw = pool->raw_hw; 6689 rte_spinlock_unlock(&pool->sl); 6690 /* Be sure the new raw counters data is updated in memory. */ 6691 rte_io_wmb(); 6692 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 6693 rte_spinlock_lock(&cmng->csl[cnt_type]); 6694 TAILQ_CONCAT(&cmng->counters[cnt_type], 6695 &pool->counters[query_gen], next); 6696 rte_spinlock_unlock(&cmng->csl[cnt_type]); 6697 } 6698 } 6699 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 6700 pool->raw_hw = NULL; 6701 sh->cmng.pending_queries--; 6702 } 6703 6704 static int 6705 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 6706 const struct flow_grp_info *grp_info, 6707 struct rte_flow_error *error) 6708 { 6709 if (grp_info->transfer && grp_info->external && 6710 grp_info->fdb_def_rule) { 6711 if (group == UINT32_MAX) 6712 return rte_flow_error_set 6713 (error, EINVAL, 6714 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6715 NULL, 6716 "group index not supported"); 6717 *table = group + 1; 6718 } else { 6719 *table = group; 6720 } 6721 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 6722 return 0; 6723 } 6724 6725 /** 6726 * Translate the rte_flow group index to HW table value. 6727 * 6728 * If tunnel offload is disabled, all group ids converted to flow table 6729 * id using the standard method. 6730 * If tunnel offload is enabled, group id can be converted using the 6731 * standard or tunnel conversion method. Group conversion method 6732 * selection depends on flags in `grp_info` parameter: 6733 * - Internal (grp_info.external == 0) groups conversion uses the 6734 * standard method. 6735 * - Group ids in JUMP action converted with the tunnel conversion. 6736 * - Group id in rule attribute conversion depends on a rule type and 6737 * group id value: 6738 * ** non zero group attributes converted with the tunnel method 6739 * ** zero group attribute in non-tunnel rule is converted using the 6740 * standard method - there's only one root table 6741 * ** zero group attribute in steer tunnel rule is converted with the 6742 * standard method - single root table 6743 * ** zero group attribute in match tunnel rule is a special OvS 6744 * case: that value is used for portability reasons. That group 6745 * id is converted with the tunnel conversion method. 6746 * 6747 * @param[in] dev 6748 * Port device 6749 * @param[in] tunnel 6750 * PMD tunnel offload object 6751 * @param[in] group 6752 * rte_flow group index value. 6753 * @param[out] table 6754 * HW table value. 6755 * @param[in] grp_info 6756 * flags used for conversion 6757 * @param[out] error 6758 * Pointer to error structure. 6759 * 6760 * @return 6761 * 0 on success, a negative errno value otherwise and rte_errno is set. 6762 */ 6763 int 6764 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 6765 const struct mlx5_flow_tunnel *tunnel, 6766 uint32_t group, uint32_t *table, 6767 const struct flow_grp_info *grp_info, 6768 struct rte_flow_error *error) 6769 { 6770 int ret; 6771 bool standard_translation; 6772 6773 if (!grp_info->skip_scale && grp_info->external && 6774 group < MLX5_MAX_TABLES_EXTERNAL) 6775 group *= MLX5_FLOW_TABLE_FACTOR; 6776 if (is_tunnel_offload_active(dev)) { 6777 standard_translation = !grp_info->external || 6778 grp_info->std_tbl_fix; 6779 } else { 6780 standard_translation = true; 6781 } 6782 DRV_LOG(DEBUG, 6783 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 6784 dev->data->port_id, group, grp_info->transfer, 6785 grp_info->external, grp_info->fdb_def_rule, 6786 standard_translation ? "STANDARD" : "TUNNEL"); 6787 if (standard_translation) 6788 ret = flow_group_to_table(dev->data->port_id, group, table, 6789 grp_info, error); 6790 else 6791 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 6792 table, error); 6793 6794 return ret; 6795 } 6796 6797 /** 6798 * Discover availability of metadata reg_c's. 6799 * 6800 * Iteratively use test flows to check availability. 6801 * 6802 * @param[in] dev 6803 * Pointer to the Ethernet device structure. 6804 * 6805 * @return 6806 * 0 on success, a negative errno value otherwise and rte_errno is set. 6807 */ 6808 int 6809 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 6810 { 6811 struct mlx5_priv *priv = dev->data->dev_private; 6812 struct mlx5_dev_config *config = &priv->config; 6813 enum modify_reg idx; 6814 int n = 0; 6815 6816 /* reg_c[0] and reg_c[1] are reserved. */ 6817 config->flow_mreg_c[n++] = REG_C_0; 6818 config->flow_mreg_c[n++] = REG_C_1; 6819 /* Discover availability of other reg_c's. */ 6820 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 6821 struct rte_flow_attr attr = { 6822 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6823 .priority = MLX5_FLOW_PRIO_RSVD, 6824 .ingress = 1, 6825 }; 6826 struct rte_flow_item items[] = { 6827 [0] = { 6828 .type = RTE_FLOW_ITEM_TYPE_END, 6829 }, 6830 }; 6831 struct rte_flow_action actions[] = { 6832 [0] = { 6833 .type = (enum rte_flow_action_type) 6834 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6835 .conf = &(struct mlx5_flow_action_copy_mreg){ 6836 .src = REG_C_1, 6837 .dst = idx, 6838 }, 6839 }, 6840 [1] = { 6841 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6842 .conf = &(struct rte_flow_action_jump){ 6843 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6844 }, 6845 }, 6846 [2] = { 6847 .type = RTE_FLOW_ACTION_TYPE_END, 6848 }, 6849 }; 6850 uint32_t flow_idx; 6851 struct rte_flow *flow; 6852 struct rte_flow_error error; 6853 6854 if (!config->dv_flow_en) 6855 break; 6856 /* Create internal flow, validation skips copy action. */ 6857 flow_idx = flow_list_create(dev, NULL, &attr, items, 6858 actions, false, &error); 6859 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 6860 flow_idx); 6861 if (!flow) 6862 continue; 6863 config->flow_mreg_c[n++] = idx; 6864 flow_list_destroy(dev, NULL, flow_idx); 6865 } 6866 for (; n < MLX5_MREG_C_NUM; ++n) 6867 config->flow_mreg_c[n] = REG_NON; 6868 return 0; 6869 } 6870 6871 /** 6872 * Dump flow raw hw data to file 6873 * 6874 * @param[in] dev 6875 * The pointer to Ethernet device. 6876 * @param[in] file 6877 * A pointer to a file for output. 6878 * @param[out] error 6879 * Perform verbose error reporting if not NULL. PMDs initialize this 6880 * structure in case of error only. 6881 * @return 6882 * 0 on success, a nagative value otherwise. 6883 */ 6884 int 6885 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 6886 FILE *file, 6887 struct rte_flow_error *error __rte_unused) 6888 { 6889 struct mlx5_priv *priv = dev->data->dev_private; 6890 struct mlx5_dev_ctx_shared *sh = priv->sh; 6891 6892 if (!priv->config.dv_flow_en) { 6893 if (fputs("device dv flow disabled\n", file) <= 0) 6894 return -errno; 6895 return -ENOTSUP; 6896 } 6897 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 6898 sh->tx_domain, file); 6899 } 6900 6901 /** 6902 * Get aged-out flows. 6903 * 6904 * @param[in] dev 6905 * Pointer to the Ethernet device structure. 6906 * @param[in] context 6907 * The address of an array of pointers to the aged-out flows contexts. 6908 * @param[in] nb_countexts 6909 * The length of context array pointers. 6910 * @param[out] error 6911 * Perform verbose error reporting if not NULL. Initialized in case of 6912 * error only. 6913 * 6914 * @return 6915 * how many contexts get in success, otherwise negative errno value. 6916 * if nb_contexts is 0, return the amount of all aged contexts. 6917 * if nb_contexts is not 0 , return the amount of aged flows reported 6918 * in the context array. 6919 */ 6920 int 6921 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 6922 uint32_t nb_contexts, struct rte_flow_error *error) 6923 { 6924 const struct mlx5_flow_driver_ops *fops; 6925 struct rte_flow_attr attr = { .transfer = 0 }; 6926 6927 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6928 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6929 return fops->get_aged_flows(dev, contexts, nb_contexts, 6930 error); 6931 } 6932 DRV_LOG(ERR, 6933 "port %u get aged flows is not supported.", 6934 dev->data->port_id); 6935 return -ENOTSUP; 6936 } 6937 6938 /* Wrapper for driver action_validate op callback */ 6939 static int 6940 flow_drv_action_validate(struct rte_eth_dev *dev, 6941 const struct rte_flow_shared_action_conf *conf, 6942 const struct rte_flow_action *action, 6943 const struct mlx5_flow_driver_ops *fops, 6944 struct rte_flow_error *error) 6945 { 6946 static const char err_msg[] = "shared action validation unsupported"; 6947 6948 if (!fops->action_validate) { 6949 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 6950 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 6951 NULL, err_msg); 6952 return -rte_errno; 6953 } 6954 return fops->action_validate(dev, conf, action, error); 6955 } 6956 6957 /** 6958 * Destroys the shared action by handle. 6959 * 6960 * @param dev 6961 * Pointer to Ethernet device structure. 6962 * @param[in] action 6963 * Handle for the shared action to be destroyed. 6964 * @param[out] error 6965 * Perform verbose error reporting if not NULL. PMDs initialize this 6966 * structure in case of error only. 6967 * 6968 * @return 6969 * 0 on success, a negative errno value otherwise and rte_errno is set. 6970 * 6971 * @note: wrapper for driver action_create op callback. 6972 */ 6973 static int 6974 mlx5_shared_action_destroy(struct rte_eth_dev *dev, 6975 struct rte_flow_shared_action *action, 6976 struct rte_flow_error *error) 6977 { 6978 static const char err_msg[] = "shared action destruction unsupported"; 6979 struct rte_flow_attr attr = { .transfer = 0 }; 6980 const struct mlx5_flow_driver_ops *fops = 6981 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 6982 6983 if (!fops->action_destroy) { 6984 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 6985 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 6986 NULL, err_msg); 6987 return -rte_errno; 6988 } 6989 return fops->action_destroy(dev, action, error); 6990 } 6991 6992 /* Wrapper for driver action_destroy op callback */ 6993 static int 6994 flow_drv_action_update(struct rte_eth_dev *dev, 6995 struct rte_flow_shared_action *action, 6996 const void *action_conf, 6997 const struct mlx5_flow_driver_ops *fops, 6998 struct rte_flow_error *error) 6999 { 7000 static const char err_msg[] = "shared action update unsupported"; 7001 7002 if (!fops->action_update) { 7003 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7004 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7005 NULL, err_msg); 7006 return -rte_errno; 7007 } 7008 return fops->action_update(dev, action, action_conf, error); 7009 } 7010 7011 /* Wrapper for driver action_destroy op callback */ 7012 static int 7013 flow_drv_action_query(struct rte_eth_dev *dev, 7014 const struct rte_flow_shared_action *action, 7015 void *data, 7016 const struct mlx5_flow_driver_ops *fops, 7017 struct rte_flow_error *error) 7018 { 7019 static const char err_msg[] = "shared action query unsupported"; 7020 7021 if (!fops->action_query) { 7022 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7023 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7024 NULL, err_msg); 7025 return -rte_errno; 7026 } 7027 return fops->action_query(dev, action, data, error); 7028 } 7029 7030 /** 7031 * Create shared action for reuse in multiple flow rules. 7032 * 7033 * @param dev 7034 * Pointer to Ethernet device structure. 7035 * @param[in] action 7036 * Action configuration for shared action creation. 7037 * @param[out] error 7038 * Perform verbose error reporting if not NULL. PMDs initialize this 7039 * structure in case of error only. 7040 * @return 7041 * A valid handle in case of success, NULL otherwise and rte_errno is set. 7042 */ 7043 static struct rte_flow_shared_action * 7044 mlx5_shared_action_create(struct rte_eth_dev *dev, 7045 const struct rte_flow_shared_action_conf *conf, 7046 const struct rte_flow_action *action, 7047 struct rte_flow_error *error) 7048 { 7049 static const char err_msg[] = "shared action creation unsupported"; 7050 struct rte_flow_attr attr = { .transfer = 0 }; 7051 const struct mlx5_flow_driver_ops *fops = 7052 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7053 7054 if (flow_drv_action_validate(dev, conf, action, fops, error)) 7055 return NULL; 7056 if (!fops->action_create) { 7057 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7058 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7059 NULL, err_msg); 7060 return NULL; 7061 } 7062 return fops->action_create(dev, conf, action, error); 7063 } 7064 7065 /** 7066 * Updates inplace the shared action configuration pointed by *action* handle 7067 * with the configuration provided as *action* argument. 7068 * The update of the shared action configuration effects all flow rules reusing 7069 * the action via handle. 7070 * 7071 * @param dev 7072 * Pointer to Ethernet device structure. 7073 * @param[in] shared_action 7074 * Handle for the shared action to be updated. 7075 * @param[in] action 7076 * Action specification used to modify the action pointed by handle. 7077 * *action* should be of same type with the action pointed by the *action* 7078 * handle argument, otherwise considered as invalid. 7079 * @param[out] error 7080 * Perform verbose error reporting if not NULL. PMDs initialize this 7081 * structure in case of error only. 7082 * 7083 * @return 7084 * 0 on success, a negative errno value otherwise and rte_errno is set. 7085 */ 7086 static int 7087 mlx5_shared_action_update(struct rte_eth_dev *dev, 7088 struct rte_flow_shared_action *shared_action, 7089 const struct rte_flow_action *action, 7090 struct rte_flow_error *error) 7091 { 7092 struct rte_flow_attr attr = { .transfer = 0 }; 7093 const struct mlx5_flow_driver_ops *fops = 7094 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7095 int ret; 7096 7097 ret = flow_drv_action_validate(dev, NULL, action, fops, error); 7098 if (ret) 7099 return ret; 7100 return flow_drv_action_update(dev, shared_action, action->conf, fops, 7101 error); 7102 } 7103 7104 /** 7105 * Query the shared action by handle. 7106 * 7107 * This function allows retrieving action-specific data such as counters. 7108 * Data is gathered by special action which may be present/referenced in 7109 * more than one flow rule definition. 7110 * 7111 * \see RTE_FLOW_ACTION_TYPE_COUNT 7112 * 7113 * @param dev 7114 * Pointer to Ethernet device structure. 7115 * @param[in] action 7116 * Handle for the shared action to query. 7117 * @param[in, out] data 7118 * Pointer to storage for the associated query data type. 7119 * @param[out] error 7120 * Perform verbose error reporting if not NULL. PMDs initialize this 7121 * structure in case of error only. 7122 * 7123 * @return 7124 * 0 on success, a negative errno value otherwise and rte_errno is set. 7125 */ 7126 static int 7127 mlx5_shared_action_query(struct rte_eth_dev *dev, 7128 const struct rte_flow_shared_action *action, 7129 void *data, 7130 struct rte_flow_error *error) 7131 { 7132 struct rte_flow_attr attr = { .transfer = 0 }; 7133 const struct mlx5_flow_driver_ops *fops = 7134 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7135 7136 return flow_drv_action_query(dev, action, data, fops, error); 7137 } 7138 7139 /** 7140 * Destroy all shared actions. 7141 * 7142 * @param dev 7143 * Pointer to Ethernet device. 7144 * 7145 * @return 7146 * 0 on success, a negative errno value otherwise and rte_errno is set. 7147 */ 7148 int 7149 mlx5_shared_action_flush(struct rte_eth_dev *dev) 7150 { 7151 struct rte_flow_error error; 7152 struct mlx5_priv *priv = dev->data->dev_private; 7153 struct mlx5_shared_action_rss *action; 7154 int ret = 0; 7155 uint32_t idx; 7156 7157 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7158 priv->rss_shared_actions, idx, action, next) { 7159 ret |= mlx5_shared_action_destroy(dev, 7160 (struct rte_flow_shared_action *)(uintptr_t)idx, &error); 7161 } 7162 return ret; 7163 } 7164 7165 #ifndef HAVE_MLX5DV_DR 7166 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 7167 #else 7168 #define MLX5_DOMAIN_SYNC_FLOW \ 7169 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 7170 #endif 7171 7172 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 7173 { 7174 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 7175 const struct mlx5_flow_driver_ops *fops; 7176 int ret; 7177 struct rte_flow_attr attr = { .transfer = 0 }; 7178 7179 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7180 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 7181 if (ret > 0) 7182 ret = -ret; 7183 return ret; 7184 } 7185 7186 /** 7187 * tunnel offload functionalilty is defined for DV environment only 7188 */ 7189 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7190 __extension__ 7191 union tunnel_offload_mark { 7192 uint32_t val; 7193 struct { 7194 uint32_t app_reserve:8; 7195 uint32_t table_id:15; 7196 uint32_t transfer:1; 7197 uint32_t _unused_:8; 7198 }; 7199 }; 7200 7201 static bool 7202 mlx5_access_tunnel_offload_db 7203 (struct rte_eth_dev *dev, 7204 bool (*match)(struct rte_eth_dev *, 7205 struct mlx5_flow_tunnel *, const void *), 7206 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 7207 void (*miss)(struct rte_eth_dev *, void *), 7208 void *ctx, bool lock_op); 7209 7210 static int 7211 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 7212 struct rte_flow *flow, 7213 const struct rte_flow_attr *attr, 7214 const struct rte_flow_action *app_actions, 7215 uint32_t flow_idx, 7216 struct tunnel_default_miss_ctx *ctx, 7217 struct rte_flow_error *error) 7218 { 7219 struct mlx5_priv *priv = dev->data->dev_private; 7220 struct mlx5_flow *dev_flow; 7221 struct rte_flow_attr miss_attr = *attr; 7222 const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf; 7223 const struct rte_flow_item miss_items[2] = { 7224 { 7225 .type = RTE_FLOW_ITEM_TYPE_ETH, 7226 .spec = NULL, 7227 .last = NULL, 7228 .mask = NULL 7229 }, 7230 { 7231 .type = RTE_FLOW_ITEM_TYPE_END, 7232 .spec = NULL, 7233 .last = NULL, 7234 .mask = NULL 7235 } 7236 }; 7237 union tunnel_offload_mark mark_id; 7238 struct rte_flow_action_mark miss_mark; 7239 struct rte_flow_action miss_actions[3] = { 7240 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 7241 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 7242 }; 7243 const struct rte_flow_action_jump *jump_data; 7244 uint32_t i, flow_table = 0; /* prevent compilation warning */ 7245 struct flow_grp_info grp_info = { 7246 .external = 1, 7247 .transfer = attr->transfer, 7248 .fdb_def_rule = !!priv->fdb_def_rule, 7249 .std_tbl_fix = 0, 7250 }; 7251 int ret; 7252 7253 if (!attr->transfer) { 7254 uint32_t q_size; 7255 7256 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 7257 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 7258 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 7259 0, SOCKET_ID_ANY); 7260 if (!ctx->queue) 7261 return rte_flow_error_set 7262 (error, ENOMEM, 7263 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7264 NULL, "invalid default miss RSS"); 7265 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7266 ctx->action_rss.level = 0, 7267 ctx->action_rss.types = priv->rss_conf.rss_hf, 7268 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 7269 ctx->action_rss.queue_num = priv->reta_idx_n, 7270 ctx->action_rss.key = priv->rss_conf.rss_key, 7271 ctx->action_rss.queue = ctx->queue; 7272 if (!priv->reta_idx_n || !priv->rxqs_n) 7273 return rte_flow_error_set 7274 (error, EINVAL, 7275 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7276 NULL, "invalid port configuration"); 7277 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 7278 ctx->action_rss.types = 0; 7279 for (i = 0; i != priv->reta_idx_n; ++i) 7280 ctx->queue[i] = (*priv->reta_idx)[i]; 7281 } else { 7282 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 7283 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 7284 } 7285 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 7286 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 7287 jump_data = app_actions->conf; 7288 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 7289 miss_attr.group = jump_data->group; 7290 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 7291 &flow_table, &grp_info, error); 7292 if (ret) 7293 return rte_flow_error_set(error, EINVAL, 7294 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7295 NULL, "invalid tunnel id"); 7296 mark_id.app_reserve = 0; 7297 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 7298 mark_id.transfer = !!attr->transfer; 7299 mark_id._unused_ = 0; 7300 miss_mark.id = mark_id.val; 7301 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 7302 miss_items, miss_actions, flow_idx, error); 7303 if (!dev_flow) 7304 return -rte_errno; 7305 dev_flow->flow = flow; 7306 dev_flow->external = true; 7307 dev_flow->tunnel = tunnel; 7308 /* Subflow object was created, we must include one in the list. */ 7309 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7310 dev_flow->handle, next); 7311 DRV_LOG(DEBUG, 7312 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 7313 dev->data->port_id, tunnel->app_tunnel.type, 7314 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 7315 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 7316 miss_actions, error); 7317 if (!ret) 7318 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 7319 error); 7320 7321 return ret; 7322 } 7323 7324 static const struct mlx5_flow_tbl_data_entry * 7325 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 7326 { 7327 struct mlx5_priv *priv = dev->data->dev_private; 7328 struct mlx5_dev_ctx_shared *sh = priv->sh; 7329 struct mlx5_hlist_entry *he; 7330 union tunnel_offload_mark mbits = { .val = mark }; 7331 union mlx5_flow_tbl_key table_key = { 7332 { 7333 .table_id = tunnel_id_to_flow_tbl(mbits.table_id), 7334 .dummy = 0, 7335 .domain = !!mbits.transfer, 7336 .direction = 0, 7337 } 7338 }; 7339 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 7340 return he ? 7341 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 7342 } 7343 7344 static void 7345 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 7346 struct mlx5_hlist_entry *entry) 7347 { 7348 struct mlx5_dev_ctx_shared *sh = list->ctx; 7349 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 7350 7351 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7352 tunnel_flow_tbl_to_id(tte->flow_table)); 7353 mlx5_free(tte); 7354 } 7355 7356 static struct mlx5_hlist_entry * 7357 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, 7358 uint64_t key __rte_unused, 7359 void *ctx __rte_unused) 7360 { 7361 struct mlx5_dev_ctx_shared *sh = list->ctx; 7362 struct tunnel_tbl_entry *tte; 7363 7364 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 7365 sizeof(*tte), 0, 7366 SOCKET_ID_ANY); 7367 if (!tte) 7368 goto err; 7369 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7370 &tte->flow_table); 7371 if (tte->flow_table >= MLX5_MAX_TABLES) { 7372 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 7373 tte->flow_table); 7374 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7375 tte->flow_table); 7376 goto err; 7377 } else if (!tte->flow_table) { 7378 goto err; 7379 } 7380 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 7381 return &tte->hash; 7382 err: 7383 if (tte) 7384 mlx5_free(tte); 7385 return NULL; 7386 } 7387 7388 static uint32_t 7389 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 7390 const struct mlx5_flow_tunnel *tunnel, 7391 uint32_t group, uint32_t *table, 7392 struct rte_flow_error *error) 7393 { 7394 struct mlx5_hlist_entry *he; 7395 struct tunnel_tbl_entry *tte; 7396 union tunnel_tbl_key key = { 7397 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 7398 .group = group 7399 }; 7400 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7401 struct mlx5_hlist *group_hash; 7402 7403 group_hash = tunnel ? tunnel->groups : thub->groups; 7404 he = mlx5_hlist_register(group_hash, key.val, NULL); 7405 if (!he) 7406 return rte_flow_error_set(error, EINVAL, 7407 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7408 NULL, 7409 "tunnel group index not supported"); 7410 tte = container_of(he, typeof(*tte), hash); 7411 *table = tte->flow_table; 7412 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 7413 dev->data->port_id, key.tunnel_id, group, *table); 7414 return 0; 7415 } 7416 7417 static void 7418 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 7419 struct mlx5_flow_tunnel *tunnel) 7420 { 7421 struct mlx5_priv *priv = dev->data->dev_private; 7422 struct mlx5_indexed_pool *ipool; 7423 7424 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 7425 dev->data->port_id, tunnel->tunnel_id); 7426 mlx5_hlist_destroy(tunnel->groups); 7427 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 7428 mlx5_ipool_free(ipool, tunnel->tunnel_id); 7429 } 7430 7431 static bool 7432 mlx5_access_tunnel_offload_db 7433 (struct rte_eth_dev *dev, 7434 bool (*match)(struct rte_eth_dev *, 7435 struct mlx5_flow_tunnel *, const void *), 7436 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 7437 void (*miss)(struct rte_eth_dev *, void *), 7438 void *ctx, bool lock_op) 7439 { 7440 bool verdict = false; 7441 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7442 struct mlx5_flow_tunnel *tunnel; 7443 7444 rte_spinlock_lock(&thub->sl); 7445 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 7446 verdict = match(dev, tunnel, (const void *)ctx); 7447 if (verdict) 7448 break; 7449 } 7450 if (!lock_op) 7451 rte_spinlock_unlock(&thub->sl); 7452 if (verdict && hit) 7453 hit(dev, tunnel, ctx); 7454 if (!verdict && miss) 7455 miss(dev, ctx); 7456 if (lock_op) 7457 rte_spinlock_unlock(&thub->sl); 7458 7459 return verdict; 7460 } 7461 7462 struct tunnel_db_find_tunnel_id_ctx { 7463 uint32_t tunnel_id; 7464 struct mlx5_flow_tunnel *tunnel; 7465 }; 7466 7467 static bool 7468 find_tunnel_id_match(struct rte_eth_dev *dev, 7469 struct mlx5_flow_tunnel *tunnel, const void *x) 7470 { 7471 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 7472 7473 RTE_SET_USED(dev); 7474 return tunnel->tunnel_id == ctx->tunnel_id; 7475 } 7476 7477 static void 7478 find_tunnel_id_hit(struct rte_eth_dev *dev, 7479 struct mlx5_flow_tunnel *tunnel, void *x) 7480 { 7481 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 7482 RTE_SET_USED(dev); 7483 ctx->tunnel = tunnel; 7484 } 7485 7486 static struct mlx5_flow_tunnel * 7487 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 7488 { 7489 struct tunnel_db_find_tunnel_id_ctx ctx = { 7490 .tunnel_id = id, 7491 }; 7492 7493 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 7494 find_tunnel_id_hit, NULL, &ctx, true); 7495 7496 return ctx.tunnel; 7497 } 7498 7499 static struct mlx5_flow_tunnel * 7500 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 7501 const struct rte_flow_tunnel *app_tunnel) 7502 { 7503 struct mlx5_priv *priv = dev->data->dev_private; 7504 struct mlx5_indexed_pool *ipool; 7505 struct mlx5_flow_tunnel *tunnel; 7506 uint32_t id; 7507 7508 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 7509 tunnel = mlx5_ipool_zmalloc(ipool, &id); 7510 if (!tunnel) 7511 return NULL; 7512 if (id >= MLX5_MAX_TUNNELS) { 7513 mlx5_ipool_free(ipool, id); 7514 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 7515 return NULL; 7516 } 7517 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 7518 mlx5_flow_tunnel_grp2tbl_create_cb, 7519 NULL, 7520 mlx5_flow_tunnel_grp2tbl_remove_cb); 7521 if (!tunnel->groups) { 7522 mlx5_ipool_free(ipool, id); 7523 return NULL; 7524 } 7525 tunnel->groups->ctx = priv->sh; 7526 /* initiate new PMD tunnel */ 7527 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 7528 tunnel->tunnel_id = id; 7529 tunnel->action.type = (typeof(tunnel->action.type)) 7530 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 7531 tunnel->action.conf = tunnel; 7532 tunnel->item.type = (typeof(tunnel->item.type)) 7533 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 7534 tunnel->item.spec = tunnel; 7535 tunnel->item.last = NULL; 7536 tunnel->item.mask = NULL; 7537 7538 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 7539 dev->data->port_id, tunnel->tunnel_id); 7540 7541 return tunnel; 7542 } 7543 7544 struct tunnel_db_get_tunnel_ctx { 7545 const struct rte_flow_tunnel *app_tunnel; 7546 struct mlx5_flow_tunnel *tunnel; 7547 }; 7548 7549 static bool get_tunnel_match(struct rte_eth_dev *dev, 7550 struct mlx5_flow_tunnel *tunnel, const void *x) 7551 { 7552 const struct tunnel_db_get_tunnel_ctx *ctx = x; 7553 7554 RTE_SET_USED(dev); 7555 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 7556 sizeof(*ctx->app_tunnel)); 7557 } 7558 7559 static void get_tunnel_hit(struct rte_eth_dev *dev, 7560 struct mlx5_flow_tunnel *tunnel, void *x) 7561 { 7562 /* called under tunnel spinlock protection */ 7563 struct tunnel_db_get_tunnel_ctx *ctx = x; 7564 7565 RTE_SET_USED(dev); 7566 tunnel->refctn++; 7567 ctx->tunnel = tunnel; 7568 } 7569 7570 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 7571 { 7572 /* called under tunnel spinlock protection */ 7573 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7574 struct tunnel_db_get_tunnel_ctx *ctx = x; 7575 7576 rte_spinlock_unlock(&thub->sl); 7577 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 7578 ctx->tunnel->refctn = 1; 7579 rte_spinlock_lock(&thub->sl); 7580 if (ctx->tunnel) 7581 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 7582 } 7583 7584 7585 static int 7586 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 7587 const struct rte_flow_tunnel *app_tunnel, 7588 struct mlx5_flow_tunnel **tunnel) 7589 { 7590 struct tunnel_db_get_tunnel_ctx ctx = { 7591 .app_tunnel = app_tunnel, 7592 }; 7593 7594 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 7595 get_tunnel_miss, &ctx, true); 7596 *tunnel = ctx.tunnel; 7597 return ctx.tunnel ? 0 : -ENOMEM; 7598 } 7599 7600 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 7601 { 7602 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 7603 7604 if (!thub) 7605 return; 7606 if (!LIST_EMPTY(&thub->tunnels)) 7607 DRV_LOG(WARNING, "port %u tunnels present\n", port_id); 7608 mlx5_hlist_destroy(thub->groups); 7609 mlx5_free(thub); 7610 } 7611 7612 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 7613 { 7614 int err; 7615 struct mlx5_flow_tunnel_hub *thub; 7616 7617 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 7618 0, SOCKET_ID_ANY); 7619 if (!thub) 7620 return -ENOMEM; 7621 LIST_INIT(&thub->tunnels); 7622 rte_spinlock_init(&thub->sl); 7623 thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0, 7624 0, mlx5_flow_tunnel_grp2tbl_create_cb, 7625 NULL, 7626 mlx5_flow_tunnel_grp2tbl_remove_cb); 7627 if (!thub->groups) { 7628 err = -rte_errno; 7629 goto err; 7630 } 7631 thub->groups->ctx = sh; 7632 sh->tunnel_hub = thub; 7633 7634 return 0; 7635 7636 err: 7637 if (thub->groups) 7638 mlx5_hlist_destroy(thub->groups); 7639 if (thub) 7640 mlx5_free(thub); 7641 return err; 7642 } 7643 7644 static inline bool 7645 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 7646 struct rte_flow_tunnel *tunnel, 7647 const char *err_msg) 7648 { 7649 err_msg = NULL; 7650 if (!is_tunnel_offload_active(dev)) { 7651 err_msg = "tunnel offload was not activated"; 7652 goto out; 7653 } else if (!tunnel) { 7654 err_msg = "no application tunnel"; 7655 goto out; 7656 } 7657 7658 switch (tunnel->type) { 7659 default: 7660 err_msg = "unsupported tunnel type"; 7661 goto out; 7662 case RTE_FLOW_ITEM_TYPE_VXLAN: 7663 break; 7664 } 7665 7666 out: 7667 return !err_msg; 7668 } 7669 7670 static int 7671 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 7672 struct rte_flow_tunnel *app_tunnel, 7673 struct rte_flow_action **actions, 7674 uint32_t *num_of_actions, 7675 struct rte_flow_error *error) 7676 { 7677 int ret; 7678 struct mlx5_flow_tunnel *tunnel; 7679 const char *err_msg = NULL; 7680 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 7681 7682 if (!verdict) 7683 return rte_flow_error_set(error, EINVAL, 7684 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 7685 err_msg); 7686 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 7687 if (ret < 0) { 7688 return rte_flow_error_set(error, ret, 7689 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 7690 "failed to initialize pmd tunnel"); 7691 } 7692 *actions = &tunnel->action; 7693 *num_of_actions = 1; 7694 return 0; 7695 } 7696 7697 static int 7698 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 7699 struct rte_flow_tunnel *app_tunnel, 7700 struct rte_flow_item **items, 7701 uint32_t *num_of_items, 7702 struct rte_flow_error *error) 7703 { 7704 int ret; 7705 struct mlx5_flow_tunnel *tunnel; 7706 const char *err_msg = NULL; 7707 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 7708 7709 if (!verdict) 7710 return rte_flow_error_set(error, EINVAL, 7711 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 7712 err_msg); 7713 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 7714 if (ret < 0) { 7715 return rte_flow_error_set(error, ret, 7716 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 7717 "failed to initialize pmd tunnel"); 7718 } 7719 *items = &tunnel->item; 7720 *num_of_items = 1; 7721 return 0; 7722 } 7723 7724 struct tunnel_db_element_release_ctx { 7725 struct rte_flow_item *items; 7726 struct rte_flow_action *actions; 7727 uint32_t num_elements; 7728 struct rte_flow_error *error; 7729 int ret; 7730 }; 7731 7732 static bool 7733 tunnel_element_release_match(struct rte_eth_dev *dev, 7734 struct mlx5_flow_tunnel *tunnel, const void *x) 7735 { 7736 const struct tunnel_db_element_release_ctx *ctx = x; 7737 7738 RTE_SET_USED(dev); 7739 if (ctx->num_elements != 1) 7740 return false; 7741 else if (ctx->items) 7742 return ctx->items == &tunnel->item; 7743 else if (ctx->actions) 7744 return ctx->actions == &tunnel->action; 7745 7746 return false; 7747 } 7748 7749 static void 7750 tunnel_element_release_hit(struct rte_eth_dev *dev, 7751 struct mlx5_flow_tunnel *tunnel, void *x) 7752 { 7753 struct tunnel_db_element_release_ctx *ctx = x; 7754 ctx->ret = 0; 7755 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7756 mlx5_flow_tunnel_free(dev, tunnel); 7757 } 7758 7759 static void 7760 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 7761 { 7762 struct tunnel_db_element_release_ctx *ctx = x; 7763 RTE_SET_USED(dev); 7764 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 7765 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 7766 "invalid argument"); 7767 } 7768 7769 static int 7770 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 7771 struct rte_flow_item *pmd_items, 7772 uint32_t num_items, struct rte_flow_error *err) 7773 { 7774 struct tunnel_db_element_release_ctx ctx = { 7775 .items = pmd_items, 7776 .actions = NULL, 7777 .num_elements = num_items, 7778 .error = err, 7779 }; 7780 7781 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 7782 tunnel_element_release_hit, 7783 tunnel_element_release_miss, &ctx, false); 7784 7785 return ctx.ret; 7786 } 7787 7788 static int 7789 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 7790 struct rte_flow_action *pmd_actions, 7791 uint32_t num_actions, struct rte_flow_error *err) 7792 { 7793 struct tunnel_db_element_release_ctx ctx = { 7794 .items = NULL, 7795 .actions = pmd_actions, 7796 .num_elements = num_actions, 7797 .error = err, 7798 }; 7799 7800 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 7801 tunnel_element_release_hit, 7802 tunnel_element_release_miss, &ctx, false); 7803 7804 return ctx.ret; 7805 } 7806 7807 static int 7808 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 7809 struct rte_mbuf *m, 7810 struct rte_flow_restore_info *info, 7811 struct rte_flow_error *err) 7812 { 7813 uint64_t ol_flags = m->ol_flags; 7814 const struct mlx5_flow_tbl_data_entry *tble; 7815 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 7816 7817 if (!is_tunnel_offload_active(dev)) { 7818 info->flags = 0; 7819 return 0; 7820 } 7821 7822 if ((ol_flags & mask) != mask) 7823 goto err; 7824 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 7825 if (!tble) { 7826 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 7827 dev->data->port_id, m->hash.fdir.hi); 7828 goto err; 7829 } 7830 MLX5_ASSERT(tble->tunnel); 7831 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 7832 info->group_id = tble->group_id; 7833 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 7834 RTE_FLOW_RESTORE_INFO_GROUP_ID | 7835 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 7836 7837 return 0; 7838 7839 err: 7840 return rte_flow_error_set(err, EINVAL, 7841 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7842 "failed to get restore info"); 7843 } 7844 7845 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 7846 static int 7847 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 7848 __rte_unused struct rte_flow_tunnel *app_tunnel, 7849 __rte_unused struct rte_flow_action **actions, 7850 __rte_unused uint32_t *num_of_actions, 7851 __rte_unused struct rte_flow_error *error) 7852 { 7853 return -ENOTSUP; 7854 } 7855 7856 static int 7857 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 7858 __rte_unused struct rte_flow_tunnel *app_tunnel, 7859 __rte_unused struct rte_flow_item **items, 7860 __rte_unused uint32_t *num_of_items, 7861 __rte_unused struct rte_flow_error *error) 7862 { 7863 return -ENOTSUP; 7864 } 7865 7866 static int 7867 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 7868 __rte_unused struct rte_flow_item *pmd_items, 7869 __rte_unused uint32_t num_items, 7870 __rte_unused struct rte_flow_error *err) 7871 { 7872 return -ENOTSUP; 7873 } 7874 7875 static int 7876 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 7877 __rte_unused struct rte_flow_action *pmd_action, 7878 __rte_unused uint32_t num_actions, 7879 __rte_unused struct rte_flow_error *err) 7880 { 7881 return -ENOTSUP; 7882 } 7883 7884 static int 7885 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 7886 __rte_unused struct rte_mbuf *m, 7887 __rte_unused struct rte_flow_restore_info *i, 7888 __rte_unused struct rte_flow_error *err) 7889 { 7890 return -ENOTSUP; 7891 } 7892 7893 static int 7894 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 7895 __rte_unused struct rte_flow *flow, 7896 __rte_unused const struct rte_flow_attr *attr, 7897 __rte_unused const struct rte_flow_action *actions, 7898 __rte_unused uint32_t flow_idx, 7899 __rte_unused struct tunnel_default_miss_ctx *ctx, 7900 __rte_unused struct rte_flow_error *error) 7901 { 7902 return -ENOTSUP; 7903 } 7904 7905 static struct mlx5_flow_tunnel * 7906 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 7907 __rte_unused uint32_t id) 7908 { 7909 return NULL; 7910 } 7911 7912 static void 7913 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 7914 __rte_unused struct mlx5_flow_tunnel *tunnel) 7915 { 7916 } 7917 7918 static uint32_t 7919 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 7920 __rte_unused const struct mlx5_flow_tunnel *t, 7921 __rte_unused uint32_t group, 7922 __rte_unused uint32_t *table, 7923 struct rte_flow_error *error) 7924 { 7925 return rte_flow_error_set(error, ENOTSUP, 7926 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7927 "tunnel offload requires DV support"); 7928 } 7929 7930 void 7931 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 7932 __rte_unused uint16_t port_id) 7933 { 7934 } 7935 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 7936 7937