1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 #include <rte_common.h> 14 #include <rte_ether.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_eal_paging.h> 17 #include <rte_flow.h> 18 #include <rte_cycles.h> 19 #include <rte_flow_driver.h> 20 #include <rte_malloc.h> 21 #include <rte_ip.h> 22 23 #include <mlx5_glue.h> 24 #include <mlx5_devx_cmds.h> 25 #include <mlx5_prm.h> 26 #include <mlx5_malloc.h> 27 28 #include "mlx5_defs.h" 29 #include "mlx5.h" 30 #include "mlx5_flow.h" 31 #include "mlx5_flow_os.h" 32 #include "mlx5_rxtx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 struct tunnel_default_miss_ctx *ctx, 54 struct rte_flow_error *error); 55 static struct mlx5_flow_tunnel * 56 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 57 static void 58 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 59 static uint32_t 60 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 61 const struct mlx5_flow_tunnel *tunnel, 62 uint32_t group, uint32_t *table, 63 struct rte_flow_error *error); 64 65 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 66 static void mlx5_flow_pop_thread_workspace(void); 67 68 69 /** Device flow drivers. */ 70 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 71 72 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 73 74 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 75 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 76 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 77 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 78 #endif 79 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 80 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 81 }; 82 83 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 84 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 85 (const int []){ \ 86 __VA_ARGS__, 0, \ 87 } 88 89 /** Node object of input graph for mlx5_flow_expand_rss(). */ 90 struct mlx5_flow_expand_node { 91 const int *const next; 92 /**< 93 * List of next node indexes. Index 0 is interpreted as a terminator. 94 */ 95 const enum rte_flow_item_type type; 96 /**< Pattern item type of current node. */ 97 uint64_t rss_types; 98 /**< 99 * RSS types bit-field associated with this node 100 * (see ETH_RSS_* definitions). 101 */ 102 }; 103 104 /** Object returned by mlx5_flow_expand_rss(). */ 105 struct mlx5_flow_expand_rss { 106 uint32_t entries; 107 /**< Number of entries @p patterns and @p priorities. */ 108 struct { 109 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 110 uint32_t priority; /**< Priority offset for each expansion. */ 111 } entry[]; 112 }; 113 114 static enum rte_flow_item_type 115 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 116 { 117 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 118 uint16_t ether_type = 0; 119 uint16_t ether_type_m; 120 uint8_t ip_next_proto = 0; 121 uint8_t ip_next_proto_m; 122 123 if (item == NULL || item->spec == NULL) 124 return ret; 125 switch (item->type) { 126 case RTE_FLOW_ITEM_TYPE_ETH: 127 if (item->mask) 128 ether_type_m = ((const struct rte_flow_item_eth *) 129 (item->mask))->type; 130 else 131 ether_type_m = rte_flow_item_eth_mask.type; 132 if (ether_type_m != RTE_BE16(0xFFFF)) 133 break; 134 ether_type = ((const struct rte_flow_item_eth *) 135 (item->spec))->type; 136 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 137 ret = RTE_FLOW_ITEM_TYPE_IPV4; 138 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 139 ret = RTE_FLOW_ITEM_TYPE_IPV6; 140 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 141 ret = RTE_FLOW_ITEM_TYPE_VLAN; 142 else 143 ret = RTE_FLOW_ITEM_TYPE_END; 144 break; 145 case RTE_FLOW_ITEM_TYPE_VLAN: 146 if (item->mask) 147 ether_type_m = ((const struct rte_flow_item_vlan *) 148 (item->mask))->inner_type; 149 else 150 ether_type_m = rte_flow_item_vlan_mask.inner_type; 151 if (ether_type_m != RTE_BE16(0xFFFF)) 152 break; 153 ether_type = ((const struct rte_flow_item_vlan *) 154 (item->spec))->inner_type; 155 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 156 ret = RTE_FLOW_ITEM_TYPE_IPV4; 157 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 158 ret = RTE_FLOW_ITEM_TYPE_IPV6; 159 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 160 ret = RTE_FLOW_ITEM_TYPE_VLAN; 161 else 162 ret = RTE_FLOW_ITEM_TYPE_END; 163 break; 164 case RTE_FLOW_ITEM_TYPE_IPV4: 165 if (item->mask) 166 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 167 (item->mask))->hdr.next_proto_id; 168 else 169 ip_next_proto_m = 170 rte_flow_item_ipv4_mask.hdr.next_proto_id; 171 if (ip_next_proto_m != 0xFF) 172 break; 173 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 174 (item->spec))->hdr.next_proto_id; 175 if (ip_next_proto == IPPROTO_UDP) 176 ret = RTE_FLOW_ITEM_TYPE_UDP; 177 else if (ip_next_proto == IPPROTO_TCP) 178 ret = RTE_FLOW_ITEM_TYPE_TCP; 179 else if (ip_next_proto == IPPROTO_IP) 180 ret = RTE_FLOW_ITEM_TYPE_IPV4; 181 else if (ip_next_proto == IPPROTO_IPV6) 182 ret = RTE_FLOW_ITEM_TYPE_IPV6; 183 else 184 ret = RTE_FLOW_ITEM_TYPE_END; 185 break; 186 case RTE_FLOW_ITEM_TYPE_IPV6: 187 if (item->mask) 188 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 189 (item->mask))->hdr.proto; 190 else 191 ip_next_proto_m = 192 rte_flow_item_ipv6_mask.hdr.proto; 193 if (ip_next_proto_m != 0xFF) 194 break; 195 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 196 (item->spec))->hdr.proto; 197 if (ip_next_proto == IPPROTO_UDP) 198 ret = RTE_FLOW_ITEM_TYPE_UDP; 199 else if (ip_next_proto == IPPROTO_TCP) 200 ret = RTE_FLOW_ITEM_TYPE_TCP; 201 else if (ip_next_proto == IPPROTO_IP) 202 ret = RTE_FLOW_ITEM_TYPE_IPV4; 203 else if (ip_next_proto == IPPROTO_IPV6) 204 ret = RTE_FLOW_ITEM_TYPE_IPV6; 205 else 206 ret = RTE_FLOW_ITEM_TYPE_END; 207 break; 208 default: 209 ret = RTE_FLOW_ITEM_TYPE_VOID; 210 break; 211 } 212 return ret; 213 } 214 215 /** 216 * Expand RSS flows into several possible flows according to the RSS hash 217 * fields requested and the driver capabilities. 218 * 219 * @param[out] buf 220 * Buffer to store the result expansion. 221 * @param[in] size 222 * Buffer size in bytes. If 0, @p buf can be NULL. 223 * @param[in] pattern 224 * User flow pattern. 225 * @param[in] types 226 * RSS types to expand (see ETH_RSS_* definitions). 227 * @param[in] graph 228 * Input graph to expand @p pattern according to @p types. 229 * @param[in] graph_root_index 230 * Index of root node in @p graph, typically 0. 231 * 232 * @return 233 * A positive value representing the size of @p buf in bytes regardless of 234 * @p size on success, a negative errno value otherwise and rte_errno is 235 * set, the following errors are defined: 236 * 237 * -E2BIG: graph-depth @p graph is too deep. 238 */ 239 static int 240 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 241 const struct rte_flow_item *pattern, uint64_t types, 242 const struct mlx5_flow_expand_node graph[], 243 int graph_root_index) 244 { 245 const int elt_n = 8; 246 const struct rte_flow_item *item; 247 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 248 const int *next_node; 249 const int *stack[elt_n]; 250 int stack_pos = 0; 251 struct rte_flow_item flow_items[elt_n]; 252 unsigned int i; 253 size_t lsize; 254 size_t user_pattern_size = 0; 255 void *addr = NULL; 256 const struct mlx5_flow_expand_node *next = NULL; 257 struct rte_flow_item missed_item; 258 int missed = 0; 259 int elt = 0; 260 const struct rte_flow_item *last_item = NULL; 261 262 memset(&missed_item, 0, sizeof(missed_item)); 263 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 264 elt_n * sizeof(buf->entry[0]); 265 if (lsize <= size) { 266 buf->entry[0].priority = 0; 267 buf->entry[0].pattern = (void *)&buf->entry[elt_n]; 268 buf->entries = 0; 269 addr = buf->entry[0].pattern; 270 } 271 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 272 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 273 last_item = item; 274 for (i = 0; node->next && node->next[i]; ++i) { 275 next = &graph[node->next[i]]; 276 if (next->type == item->type) 277 break; 278 } 279 if (next) 280 node = next; 281 user_pattern_size += sizeof(*item); 282 } 283 user_pattern_size += sizeof(*item); /* Handle END item. */ 284 lsize += user_pattern_size; 285 /* Copy the user pattern in the first entry of the buffer. */ 286 if (lsize <= size) { 287 rte_memcpy(addr, pattern, user_pattern_size); 288 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 289 buf->entries = 1; 290 } 291 /* Start expanding. */ 292 memset(flow_items, 0, sizeof(flow_items)); 293 user_pattern_size -= sizeof(*item); 294 /* 295 * Check if the last valid item has spec set, need complete pattern, 296 * and the pattern can be used for expansion. 297 */ 298 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 299 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 300 /* Item type END indicates expansion is not required. */ 301 return lsize; 302 } 303 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 304 next = NULL; 305 missed = 1; 306 for (i = 0; node->next && node->next[i]; ++i) { 307 next = &graph[node->next[i]]; 308 if (next->type == missed_item.type) { 309 flow_items[0].type = missed_item.type; 310 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 311 break; 312 } 313 next = NULL; 314 } 315 } 316 if (next && missed) { 317 elt = 2; /* missed item + item end. */ 318 node = next; 319 lsize += elt * sizeof(*item) + user_pattern_size; 320 if ((node->rss_types & types) && lsize <= size) { 321 buf->entry[buf->entries].priority = 1; 322 buf->entry[buf->entries].pattern = addr; 323 buf->entries++; 324 rte_memcpy(addr, buf->entry[0].pattern, 325 user_pattern_size); 326 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 327 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 328 addr = (void *)(((uintptr_t)addr) + 329 elt * sizeof(*item)); 330 } 331 } 332 memset(flow_items, 0, sizeof(flow_items)); 333 next_node = node->next; 334 stack[stack_pos] = next_node; 335 node = next_node ? &graph[*next_node] : NULL; 336 while (node) { 337 flow_items[stack_pos].type = node->type; 338 if (node->rss_types & types) { 339 /* 340 * compute the number of items to copy from the 341 * expansion and copy it. 342 * When the stack_pos is 0, there are 1 element in it, 343 * plus the addition END item. 344 */ 345 elt = stack_pos + 2; 346 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 347 lsize += elt * sizeof(*item) + user_pattern_size; 348 if (lsize <= size) { 349 size_t n = elt * sizeof(*item); 350 351 buf->entry[buf->entries].priority = 352 stack_pos + 1 + missed; 353 buf->entry[buf->entries].pattern = addr; 354 buf->entries++; 355 rte_memcpy(addr, buf->entry[0].pattern, 356 user_pattern_size); 357 addr = (void *)(((uintptr_t)addr) + 358 user_pattern_size); 359 rte_memcpy(addr, &missed_item, 360 missed * sizeof(*item)); 361 addr = (void *)(((uintptr_t)addr) + 362 missed * sizeof(*item)); 363 rte_memcpy(addr, flow_items, n); 364 addr = (void *)(((uintptr_t)addr) + n); 365 } 366 } 367 /* Go deeper. */ 368 if (node->next) { 369 next_node = node->next; 370 if (stack_pos++ == elt_n) { 371 rte_errno = E2BIG; 372 return -rte_errno; 373 } 374 stack[stack_pos] = next_node; 375 } else if (*(next_node + 1)) { 376 /* Follow up with the next possibility. */ 377 ++next_node; 378 } else { 379 /* Move to the next path. */ 380 if (stack_pos) 381 next_node = stack[--stack_pos]; 382 next_node++; 383 stack[stack_pos] = next_node; 384 } 385 node = *next_node ? &graph[*next_node] : NULL; 386 }; 387 /* no expanded flows but we have missed item, create one rule for it */ 388 if (buf->entries == 1 && missed != 0) { 389 elt = 2; 390 lsize += elt * sizeof(*item) + user_pattern_size; 391 if (lsize <= size) { 392 buf->entry[buf->entries].priority = 1; 393 buf->entry[buf->entries].pattern = addr; 394 buf->entries++; 395 flow_items[0].type = missed_item.type; 396 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 397 rte_memcpy(addr, buf->entry[0].pattern, 398 user_pattern_size); 399 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 400 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 401 } 402 } 403 return lsize; 404 } 405 406 enum mlx5_expansion { 407 MLX5_EXPANSION_ROOT, 408 MLX5_EXPANSION_ROOT_OUTER, 409 MLX5_EXPANSION_ROOT_ETH_VLAN, 410 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 411 MLX5_EXPANSION_OUTER_ETH, 412 MLX5_EXPANSION_OUTER_ETH_VLAN, 413 MLX5_EXPANSION_OUTER_VLAN, 414 MLX5_EXPANSION_OUTER_IPV4, 415 MLX5_EXPANSION_OUTER_IPV4_UDP, 416 MLX5_EXPANSION_OUTER_IPV4_TCP, 417 MLX5_EXPANSION_OUTER_IPV6, 418 MLX5_EXPANSION_OUTER_IPV6_UDP, 419 MLX5_EXPANSION_OUTER_IPV6_TCP, 420 MLX5_EXPANSION_VXLAN, 421 MLX5_EXPANSION_VXLAN_GPE, 422 MLX5_EXPANSION_GRE, 423 MLX5_EXPANSION_MPLS, 424 MLX5_EXPANSION_ETH, 425 MLX5_EXPANSION_ETH_VLAN, 426 MLX5_EXPANSION_VLAN, 427 MLX5_EXPANSION_IPV4, 428 MLX5_EXPANSION_IPV4_UDP, 429 MLX5_EXPANSION_IPV4_TCP, 430 MLX5_EXPANSION_IPV6, 431 MLX5_EXPANSION_IPV6_UDP, 432 MLX5_EXPANSION_IPV6_TCP, 433 }; 434 435 /** Supported expansion of items. */ 436 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 437 [MLX5_EXPANSION_ROOT] = { 438 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 439 MLX5_EXPANSION_IPV4, 440 MLX5_EXPANSION_IPV6), 441 .type = RTE_FLOW_ITEM_TYPE_END, 442 }, 443 [MLX5_EXPANSION_ROOT_OUTER] = { 444 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 445 MLX5_EXPANSION_OUTER_IPV4, 446 MLX5_EXPANSION_OUTER_IPV6), 447 .type = RTE_FLOW_ITEM_TYPE_END, 448 }, 449 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 450 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 451 .type = RTE_FLOW_ITEM_TYPE_END, 452 }, 453 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 454 .next = MLX5_FLOW_EXPAND_RSS_NEXT 455 (MLX5_EXPANSION_OUTER_ETH_VLAN), 456 .type = RTE_FLOW_ITEM_TYPE_END, 457 }, 458 [MLX5_EXPANSION_OUTER_ETH] = { 459 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 460 MLX5_EXPANSION_OUTER_IPV6, 461 MLX5_EXPANSION_MPLS), 462 .type = RTE_FLOW_ITEM_TYPE_ETH, 463 .rss_types = 0, 464 }, 465 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 466 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 467 .type = RTE_FLOW_ITEM_TYPE_ETH, 468 .rss_types = 0, 469 }, 470 [MLX5_EXPANSION_OUTER_VLAN] = { 471 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 472 MLX5_EXPANSION_OUTER_IPV6), 473 .type = RTE_FLOW_ITEM_TYPE_VLAN, 474 }, 475 [MLX5_EXPANSION_OUTER_IPV4] = { 476 .next = MLX5_FLOW_EXPAND_RSS_NEXT 477 (MLX5_EXPANSION_OUTER_IPV4_UDP, 478 MLX5_EXPANSION_OUTER_IPV4_TCP, 479 MLX5_EXPANSION_GRE, 480 MLX5_EXPANSION_IPV4, 481 MLX5_EXPANSION_IPV6), 482 .type = RTE_FLOW_ITEM_TYPE_IPV4, 483 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 484 ETH_RSS_NONFRAG_IPV4_OTHER, 485 }, 486 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 487 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 488 MLX5_EXPANSION_VXLAN_GPE), 489 .type = RTE_FLOW_ITEM_TYPE_UDP, 490 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 491 }, 492 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 493 .type = RTE_FLOW_ITEM_TYPE_TCP, 494 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 495 }, 496 [MLX5_EXPANSION_OUTER_IPV6] = { 497 .next = MLX5_FLOW_EXPAND_RSS_NEXT 498 (MLX5_EXPANSION_OUTER_IPV6_UDP, 499 MLX5_EXPANSION_OUTER_IPV6_TCP, 500 MLX5_EXPANSION_IPV4, 501 MLX5_EXPANSION_IPV6), 502 .type = RTE_FLOW_ITEM_TYPE_IPV6, 503 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 504 ETH_RSS_NONFRAG_IPV6_OTHER, 505 }, 506 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 507 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 508 MLX5_EXPANSION_VXLAN_GPE), 509 .type = RTE_FLOW_ITEM_TYPE_UDP, 510 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 511 }, 512 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 513 .type = RTE_FLOW_ITEM_TYPE_TCP, 514 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 515 }, 516 [MLX5_EXPANSION_VXLAN] = { 517 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 518 MLX5_EXPANSION_IPV4, 519 MLX5_EXPANSION_IPV6), 520 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 521 }, 522 [MLX5_EXPANSION_VXLAN_GPE] = { 523 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 524 MLX5_EXPANSION_IPV4, 525 MLX5_EXPANSION_IPV6), 526 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 527 }, 528 [MLX5_EXPANSION_GRE] = { 529 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 530 .type = RTE_FLOW_ITEM_TYPE_GRE, 531 }, 532 [MLX5_EXPANSION_MPLS] = { 533 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 534 MLX5_EXPANSION_IPV6), 535 .type = RTE_FLOW_ITEM_TYPE_MPLS, 536 }, 537 [MLX5_EXPANSION_ETH] = { 538 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 539 MLX5_EXPANSION_IPV6), 540 .type = RTE_FLOW_ITEM_TYPE_ETH, 541 }, 542 [MLX5_EXPANSION_ETH_VLAN] = { 543 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 544 .type = RTE_FLOW_ITEM_TYPE_ETH, 545 }, 546 [MLX5_EXPANSION_VLAN] = { 547 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 548 MLX5_EXPANSION_IPV6), 549 .type = RTE_FLOW_ITEM_TYPE_VLAN, 550 }, 551 [MLX5_EXPANSION_IPV4] = { 552 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 553 MLX5_EXPANSION_IPV4_TCP), 554 .type = RTE_FLOW_ITEM_TYPE_IPV4, 555 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 556 ETH_RSS_NONFRAG_IPV4_OTHER, 557 }, 558 [MLX5_EXPANSION_IPV4_UDP] = { 559 .type = RTE_FLOW_ITEM_TYPE_UDP, 560 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 561 }, 562 [MLX5_EXPANSION_IPV4_TCP] = { 563 .type = RTE_FLOW_ITEM_TYPE_TCP, 564 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 565 }, 566 [MLX5_EXPANSION_IPV6] = { 567 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 568 MLX5_EXPANSION_IPV6_TCP), 569 .type = RTE_FLOW_ITEM_TYPE_IPV6, 570 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 571 ETH_RSS_NONFRAG_IPV6_OTHER, 572 }, 573 [MLX5_EXPANSION_IPV6_UDP] = { 574 .type = RTE_FLOW_ITEM_TYPE_UDP, 575 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 576 }, 577 [MLX5_EXPANSION_IPV6_TCP] = { 578 .type = RTE_FLOW_ITEM_TYPE_TCP, 579 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 580 }, 581 }; 582 583 static struct rte_flow_shared_action * 584 mlx5_shared_action_create(struct rte_eth_dev *dev, 585 const struct rte_flow_shared_action_conf *conf, 586 const struct rte_flow_action *action, 587 struct rte_flow_error *error); 588 static int mlx5_shared_action_destroy 589 (struct rte_eth_dev *dev, 590 struct rte_flow_shared_action *shared_action, 591 struct rte_flow_error *error); 592 static int mlx5_shared_action_update 593 (struct rte_eth_dev *dev, 594 struct rte_flow_shared_action *shared_action, 595 const struct rte_flow_action *action, 596 struct rte_flow_error *error); 597 static int mlx5_shared_action_query 598 (struct rte_eth_dev *dev, 599 const struct rte_flow_shared_action *action, 600 void *data, 601 struct rte_flow_error *error); 602 static int 603 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 604 struct rte_flow_tunnel *app_tunnel, 605 struct rte_flow_action **actions, 606 uint32_t *num_of_actions, 607 struct rte_flow_error *error); 608 static int 609 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 610 struct rte_flow_tunnel *app_tunnel, 611 struct rte_flow_item **items, 612 uint32_t *num_of_items, 613 struct rte_flow_error *error); 614 static int 615 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 616 struct rte_flow_item *pmd_items, 617 uint32_t num_items, struct rte_flow_error *err); 618 static int 619 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 620 struct rte_flow_action *pmd_actions, 621 uint32_t num_actions, 622 struct rte_flow_error *err); 623 static int 624 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 625 struct rte_mbuf *m, 626 struct rte_flow_restore_info *info, 627 struct rte_flow_error *err); 628 629 static const struct rte_flow_ops mlx5_flow_ops = { 630 .validate = mlx5_flow_validate, 631 .create = mlx5_flow_create, 632 .destroy = mlx5_flow_destroy, 633 .flush = mlx5_flow_flush, 634 .isolate = mlx5_flow_isolate, 635 .query = mlx5_flow_query, 636 .dev_dump = mlx5_flow_dev_dump, 637 .get_aged_flows = mlx5_flow_get_aged_flows, 638 .shared_action_create = mlx5_shared_action_create, 639 .shared_action_destroy = mlx5_shared_action_destroy, 640 .shared_action_update = mlx5_shared_action_update, 641 .shared_action_query = mlx5_shared_action_query, 642 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 643 .tunnel_match = mlx5_flow_tunnel_match, 644 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 645 .tunnel_item_release = mlx5_flow_tunnel_item_release, 646 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 647 }; 648 649 /* Tunnel information. */ 650 struct mlx5_flow_tunnel_info { 651 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 652 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 653 }; 654 655 static struct mlx5_flow_tunnel_info tunnels_info[] = { 656 { 657 .tunnel = MLX5_FLOW_LAYER_VXLAN, 658 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 659 }, 660 { 661 .tunnel = MLX5_FLOW_LAYER_GENEVE, 662 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 663 }, 664 { 665 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 666 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 667 }, 668 { 669 .tunnel = MLX5_FLOW_LAYER_GRE, 670 .ptype = RTE_PTYPE_TUNNEL_GRE, 671 }, 672 { 673 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 674 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 675 }, 676 { 677 .tunnel = MLX5_FLOW_LAYER_MPLS, 678 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 679 }, 680 { 681 .tunnel = MLX5_FLOW_LAYER_NVGRE, 682 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 683 }, 684 { 685 .tunnel = MLX5_FLOW_LAYER_IPIP, 686 .ptype = RTE_PTYPE_TUNNEL_IP, 687 }, 688 { 689 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 690 .ptype = RTE_PTYPE_TUNNEL_IP, 691 }, 692 { 693 .tunnel = MLX5_FLOW_LAYER_GTP, 694 .ptype = RTE_PTYPE_TUNNEL_GTPU, 695 }, 696 }; 697 698 /* Key of thread specific flow workspace data. */ 699 static pthread_key_t key_workspace; 700 701 /* Thread specific flow workspace data once initialization data. */ 702 static pthread_once_t key_workspace_init; 703 704 705 /** 706 * Translate tag ID to register. 707 * 708 * @param[in] dev 709 * Pointer to the Ethernet device structure. 710 * @param[in] feature 711 * The feature that request the register. 712 * @param[in] id 713 * The request register ID. 714 * @param[out] error 715 * Error description in case of any. 716 * 717 * @return 718 * The request register on success, a negative errno 719 * value otherwise and rte_errno is set. 720 */ 721 int 722 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 723 enum mlx5_feature_name feature, 724 uint32_t id, 725 struct rte_flow_error *error) 726 { 727 struct mlx5_priv *priv = dev->data->dev_private; 728 struct mlx5_dev_config *config = &priv->config; 729 enum modify_reg start_reg; 730 bool skip_mtr_reg = false; 731 732 switch (feature) { 733 case MLX5_HAIRPIN_RX: 734 return REG_B; 735 case MLX5_HAIRPIN_TX: 736 return REG_A; 737 case MLX5_METADATA_RX: 738 switch (config->dv_xmeta_en) { 739 case MLX5_XMETA_MODE_LEGACY: 740 return REG_B; 741 case MLX5_XMETA_MODE_META16: 742 return REG_C_0; 743 case MLX5_XMETA_MODE_META32: 744 return REG_C_1; 745 } 746 break; 747 case MLX5_METADATA_TX: 748 return REG_A; 749 case MLX5_METADATA_FDB: 750 switch (config->dv_xmeta_en) { 751 case MLX5_XMETA_MODE_LEGACY: 752 return REG_NON; 753 case MLX5_XMETA_MODE_META16: 754 return REG_C_0; 755 case MLX5_XMETA_MODE_META32: 756 return REG_C_1; 757 } 758 break; 759 case MLX5_FLOW_MARK: 760 switch (config->dv_xmeta_en) { 761 case MLX5_XMETA_MODE_LEGACY: 762 return REG_NON; 763 case MLX5_XMETA_MODE_META16: 764 return REG_C_1; 765 case MLX5_XMETA_MODE_META32: 766 return REG_C_0; 767 } 768 break; 769 case MLX5_MTR_SFX: 770 /* 771 * If meter color and flow match share one register, flow match 772 * should use the meter color register for match. 773 */ 774 if (priv->mtr_reg_share) 775 return priv->mtr_color_reg; 776 else 777 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 778 REG_C_3; 779 case MLX5_MTR_COLOR: 780 case MLX5_ASO_FLOW_HIT: /* Both features use the same REG_C. */ 781 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 782 return priv->mtr_color_reg; 783 case MLX5_COPY_MARK: 784 /* 785 * Metadata COPY_MARK register using is in meter suffix sub 786 * flow while with meter. It's safe to share the same register. 787 */ 788 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 789 case MLX5_APP_TAG: 790 /* 791 * If meter is enable, it will engage the register for color 792 * match and flow match. If meter color match is not using the 793 * REG_C_2, need to skip the REG_C_x be used by meter color 794 * match. 795 * If meter is disable, free to use all available registers. 796 */ 797 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 798 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 799 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 800 if (id > (REG_C_7 - start_reg)) 801 return rte_flow_error_set(error, EINVAL, 802 RTE_FLOW_ERROR_TYPE_ITEM, 803 NULL, "invalid tag id"); 804 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 805 return rte_flow_error_set(error, ENOTSUP, 806 RTE_FLOW_ERROR_TYPE_ITEM, 807 NULL, "unsupported tag id"); 808 /* 809 * This case means meter is using the REG_C_x great than 2. 810 * Take care not to conflict with meter color REG_C_x. 811 * If the available index REG_C_y >= REG_C_x, skip the 812 * color register. 813 */ 814 if (skip_mtr_reg && config->flow_mreg_c 815 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 816 if (id >= (REG_C_7 - start_reg)) 817 return rte_flow_error_set(error, EINVAL, 818 RTE_FLOW_ERROR_TYPE_ITEM, 819 NULL, "invalid tag id"); 820 if (config->flow_mreg_c 821 [id + 1 + start_reg - REG_C_0] != REG_NON) 822 return config->flow_mreg_c 823 [id + 1 + start_reg - REG_C_0]; 824 return rte_flow_error_set(error, ENOTSUP, 825 RTE_FLOW_ERROR_TYPE_ITEM, 826 NULL, "unsupported tag id"); 827 } 828 return config->flow_mreg_c[id + start_reg - REG_C_0]; 829 } 830 MLX5_ASSERT(false); 831 return rte_flow_error_set(error, EINVAL, 832 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 833 NULL, "invalid feature name"); 834 } 835 836 /** 837 * Check extensive flow metadata register support. 838 * 839 * @param dev 840 * Pointer to rte_eth_dev structure. 841 * 842 * @return 843 * True if device supports extensive flow metadata register, otherwise false. 844 */ 845 bool 846 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 847 { 848 struct mlx5_priv *priv = dev->data->dev_private; 849 struct mlx5_dev_config *config = &priv->config; 850 851 /* 852 * Having available reg_c can be regarded inclusively as supporting 853 * extensive flow metadata register, which could mean, 854 * - metadata register copy action by modify header. 855 * - 16 modify header actions is supported. 856 * - reg_c's are preserved across different domain (FDB and NIC) on 857 * packet loopback by flow lookup miss. 858 */ 859 return config->flow_mreg_c[2] != REG_NON; 860 } 861 862 /** 863 * Verify the @p item specifications (spec, last, mask) are compatible with the 864 * NIC capabilities. 865 * 866 * @param[in] item 867 * Item specification. 868 * @param[in] mask 869 * @p item->mask or flow default bit-masks. 870 * @param[in] nic_mask 871 * Bit-masks covering supported fields by the NIC to compare with user mask. 872 * @param[in] size 873 * Bit-masks size in bytes. 874 * @param[in] range_accepted 875 * True if range of values is accepted for specific fields, false otherwise. 876 * @param[out] error 877 * Pointer to error structure. 878 * 879 * @return 880 * 0 on success, a negative errno value otherwise and rte_errno is set. 881 */ 882 int 883 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 884 const uint8_t *mask, 885 const uint8_t *nic_mask, 886 unsigned int size, 887 bool range_accepted, 888 struct rte_flow_error *error) 889 { 890 unsigned int i; 891 892 MLX5_ASSERT(nic_mask); 893 for (i = 0; i < size; ++i) 894 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 895 return rte_flow_error_set(error, ENOTSUP, 896 RTE_FLOW_ERROR_TYPE_ITEM, 897 item, 898 "mask enables non supported" 899 " bits"); 900 if (!item->spec && (item->mask || item->last)) 901 return rte_flow_error_set(error, EINVAL, 902 RTE_FLOW_ERROR_TYPE_ITEM, item, 903 "mask/last without a spec is not" 904 " supported"); 905 if (item->spec && item->last && !range_accepted) { 906 uint8_t spec[size]; 907 uint8_t last[size]; 908 unsigned int i; 909 int ret; 910 911 for (i = 0; i < size; ++i) { 912 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 913 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 914 } 915 ret = memcmp(spec, last, size); 916 if (ret != 0) 917 return rte_flow_error_set(error, EINVAL, 918 RTE_FLOW_ERROR_TYPE_ITEM, 919 item, 920 "range is not valid"); 921 } 922 return 0; 923 } 924 925 /** 926 * Adjust the hash fields according to the @p flow information. 927 * 928 * @param[in] dev_flow. 929 * Pointer to the mlx5_flow. 930 * @param[in] tunnel 931 * 1 when the hash field is for a tunnel item. 932 * @param[in] layer_types 933 * ETH_RSS_* types. 934 * @param[in] hash_fields 935 * Item hash fields. 936 * 937 * @return 938 * The hash fields that should be used. 939 */ 940 uint64_t 941 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 942 int tunnel __rte_unused, uint64_t layer_types, 943 uint64_t hash_fields) 944 { 945 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 946 int rss_request_inner = rss_desc->level >= 2; 947 948 /* Check RSS hash level for tunnel. */ 949 if (tunnel && rss_request_inner) 950 hash_fields |= IBV_RX_HASH_INNER; 951 else if (tunnel || rss_request_inner) 952 return 0; 953 #endif 954 /* Check if requested layer matches RSS hash fields. */ 955 if (!(rss_desc->types & layer_types)) 956 return 0; 957 return hash_fields; 958 } 959 960 /** 961 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 962 * if several tunnel rules are used on this queue, the tunnel ptype will be 963 * cleared. 964 * 965 * @param rxq_ctrl 966 * Rx queue to update. 967 */ 968 static void 969 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 970 { 971 unsigned int i; 972 uint32_t tunnel_ptype = 0; 973 974 /* Look up for the ptype to use. */ 975 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 976 if (!rxq_ctrl->flow_tunnels_n[i]) 977 continue; 978 if (!tunnel_ptype) { 979 tunnel_ptype = tunnels_info[i].ptype; 980 } else { 981 tunnel_ptype = 0; 982 break; 983 } 984 } 985 rxq_ctrl->rxq.tunnel = tunnel_ptype; 986 } 987 988 /** 989 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 990 * flow. 991 * 992 * @param[in] dev 993 * Pointer to the Ethernet device structure. 994 * @param[in] dev_handle 995 * Pointer to device flow handle structure. 996 */ 997 static void 998 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 999 struct mlx5_flow_handle *dev_handle) 1000 { 1001 struct mlx5_priv *priv = dev->data->dev_private; 1002 const int mark = dev_handle->mark; 1003 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1004 struct mlx5_hrxq *hrxq; 1005 unsigned int i; 1006 1007 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1008 return; 1009 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1010 dev_handle->rix_hrxq); 1011 if (!hrxq) 1012 return; 1013 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1014 int idx = hrxq->ind_table->queues[i]; 1015 struct mlx5_rxq_ctrl *rxq_ctrl = 1016 container_of((*priv->rxqs)[idx], 1017 struct mlx5_rxq_ctrl, rxq); 1018 1019 /* 1020 * To support metadata register copy on Tx loopback, 1021 * this must be always enabled (metadata may arive 1022 * from other port - not from local flows only. 1023 */ 1024 if (priv->config.dv_flow_en && 1025 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1026 mlx5_flow_ext_mreg_supported(dev)) { 1027 rxq_ctrl->rxq.mark = 1; 1028 rxq_ctrl->flow_mark_n = 1; 1029 } else if (mark) { 1030 rxq_ctrl->rxq.mark = 1; 1031 rxq_ctrl->flow_mark_n++; 1032 } 1033 if (tunnel) { 1034 unsigned int j; 1035 1036 /* Increase the counter matching the flow. */ 1037 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1038 if ((tunnels_info[j].tunnel & 1039 dev_handle->layers) == 1040 tunnels_info[j].tunnel) { 1041 rxq_ctrl->flow_tunnels_n[j]++; 1042 break; 1043 } 1044 } 1045 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1046 } 1047 } 1048 } 1049 1050 /** 1051 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1052 * 1053 * @param[in] dev 1054 * Pointer to the Ethernet device structure. 1055 * @param[in] flow 1056 * Pointer to flow structure. 1057 */ 1058 static void 1059 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1060 { 1061 struct mlx5_priv *priv = dev->data->dev_private; 1062 uint32_t handle_idx; 1063 struct mlx5_flow_handle *dev_handle; 1064 1065 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1066 handle_idx, dev_handle, next) 1067 flow_drv_rxq_flags_set(dev, dev_handle); 1068 } 1069 1070 /** 1071 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1072 * device flow if no other flow uses it with the same kind of request. 1073 * 1074 * @param dev 1075 * Pointer to Ethernet device. 1076 * @param[in] dev_handle 1077 * Pointer to the device flow handle structure. 1078 */ 1079 static void 1080 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1081 struct mlx5_flow_handle *dev_handle) 1082 { 1083 struct mlx5_priv *priv = dev->data->dev_private; 1084 const int mark = dev_handle->mark; 1085 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1086 struct mlx5_hrxq *hrxq; 1087 unsigned int i; 1088 1089 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1090 return; 1091 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1092 dev_handle->rix_hrxq); 1093 if (!hrxq) 1094 return; 1095 MLX5_ASSERT(dev->data->dev_started); 1096 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1097 int idx = hrxq->ind_table->queues[i]; 1098 struct mlx5_rxq_ctrl *rxq_ctrl = 1099 container_of((*priv->rxqs)[idx], 1100 struct mlx5_rxq_ctrl, rxq); 1101 1102 if (priv->config.dv_flow_en && 1103 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1104 mlx5_flow_ext_mreg_supported(dev)) { 1105 rxq_ctrl->rxq.mark = 1; 1106 rxq_ctrl->flow_mark_n = 1; 1107 } else if (mark) { 1108 rxq_ctrl->flow_mark_n--; 1109 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1110 } 1111 if (tunnel) { 1112 unsigned int j; 1113 1114 /* Decrease the counter matching the flow. */ 1115 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1116 if ((tunnels_info[j].tunnel & 1117 dev_handle->layers) == 1118 tunnels_info[j].tunnel) { 1119 rxq_ctrl->flow_tunnels_n[j]--; 1120 break; 1121 } 1122 } 1123 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1124 } 1125 } 1126 } 1127 1128 /** 1129 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1130 * @p flow if no other flow uses it with the same kind of request. 1131 * 1132 * @param dev 1133 * Pointer to Ethernet device. 1134 * @param[in] flow 1135 * Pointer to the flow. 1136 */ 1137 static void 1138 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1139 { 1140 struct mlx5_priv *priv = dev->data->dev_private; 1141 uint32_t handle_idx; 1142 struct mlx5_flow_handle *dev_handle; 1143 1144 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1145 handle_idx, dev_handle, next) 1146 flow_drv_rxq_flags_trim(dev, dev_handle); 1147 } 1148 1149 /** 1150 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1151 * 1152 * @param dev 1153 * Pointer to Ethernet device. 1154 */ 1155 static void 1156 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1157 { 1158 struct mlx5_priv *priv = dev->data->dev_private; 1159 unsigned int i; 1160 1161 for (i = 0; i != priv->rxqs_n; ++i) { 1162 struct mlx5_rxq_ctrl *rxq_ctrl; 1163 unsigned int j; 1164 1165 if (!(*priv->rxqs)[i]) 1166 continue; 1167 rxq_ctrl = container_of((*priv->rxqs)[i], 1168 struct mlx5_rxq_ctrl, rxq); 1169 rxq_ctrl->flow_mark_n = 0; 1170 rxq_ctrl->rxq.mark = 0; 1171 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1172 rxq_ctrl->flow_tunnels_n[j] = 0; 1173 rxq_ctrl->rxq.tunnel = 0; 1174 } 1175 } 1176 1177 /** 1178 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1179 * 1180 * @param[in] dev 1181 * Pointer to the Ethernet device structure. 1182 */ 1183 void 1184 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1185 { 1186 struct mlx5_priv *priv = dev->data->dev_private; 1187 struct mlx5_rxq_data *data; 1188 unsigned int i; 1189 1190 for (i = 0; i != priv->rxqs_n; ++i) { 1191 if (!(*priv->rxqs)[i]) 1192 continue; 1193 data = (*priv->rxqs)[i]; 1194 if (!rte_flow_dynf_metadata_avail()) { 1195 data->dynf_meta = 0; 1196 data->flow_meta_mask = 0; 1197 data->flow_meta_offset = -1; 1198 } else { 1199 data->dynf_meta = 1; 1200 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1201 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1202 } 1203 } 1204 } 1205 1206 /* 1207 * return a pointer to the desired action in the list of actions. 1208 * 1209 * @param[in] actions 1210 * The list of actions to search the action in. 1211 * @param[in] action 1212 * The action to find. 1213 * 1214 * @return 1215 * Pointer to the action in the list, if found. NULL otherwise. 1216 */ 1217 const struct rte_flow_action * 1218 mlx5_flow_find_action(const struct rte_flow_action *actions, 1219 enum rte_flow_action_type action) 1220 { 1221 if (actions == NULL) 1222 return NULL; 1223 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1224 if (actions->type == action) 1225 return actions; 1226 return NULL; 1227 } 1228 1229 /* 1230 * Validate the flag action. 1231 * 1232 * @param[in] action_flags 1233 * Bit-fields that holds the actions detected until now. 1234 * @param[in] attr 1235 * Attributes of flow that includes this action. 1236 * @param[out] error 1237 * Pointer to error structure. 1238 * 1239 * @return 1240 * 0 on success, a negative errno value otherwise and rte_errno is set. 1241 */ 1242 int 1243 mlx5_flow_validate_action_flag(uint64_t action_flags, 1244 const struct rte_flow_attr *attr, 1245 struct rte_flow_error *error) 1246 { 1247 if (action_flags & MLX5_FLOW_ACTION_MARK) 1248 return rte_flow_error_set(error, EINVAL, 1249 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1250 "can't mark and flag in same flow"); 1251 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1252 return rte_flow_error_set(error, EINVAL, 1253 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1254 "can't have 2 flag" 1255 " actions in same flow"); 1256 if (attr->egress) 1257 return rte_flow_error_set(error, ENOTSUP, 1258 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1259 "flag action not supported for " 1260 "egress"); 1261 return 0; 1262 } 1263 1264 /* 1265 * Validate the mark action. 1266 * 1267 * @param[in] action 1268 * Pointer to the queue action. 1269 * @param[in] action_flags 1270 * Bit-fields that holds the actions detected until now. 1271 * @param[in] attr 1272 * Attributes of flow that includes this action. 1273 * @param[out] error 1274 * Pointer to error structure. 1275 * 1276 * @return 1277 * 0 on success, a negative errno value otherwise and rte_errno is set. 1278 */ 1279 int 1280 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1281 uint64_t action_flags, 1282 const struct rte_flow_attr *attr, 1283 struct rte_flow_error *error) 1284 { 1285 const struct rte_flow_action_mark *mark = action->conf; 1286 1287 if (!mark) 1288 return rte_flow_error_set(error, EINVAL, 1289 RTE_FLOW_ERROR_TYPE_ACTION, 1290 action, 1291 "configuration cannot be null"); 1292 if (mark->id >= MLX5_FLOW_MARK_MAX) 1293 return rte_flow_error_set(error, EINVAL, 1294 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1295 &mark->id, 1296 "mark id must in 0 <= id < " 1297 RTE_STR(MLX5_FLOW_MARK_MAX)); 1298 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1299 return rte_flow_error_set(error, EINVAL, 1300 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1301 "can't flag and mark in same flow"); 1302 if (action_flags & MLX5_FLOW_ACTION_MARK) 1303 return rte_flow_error_set(error, EINVAL, 1304 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1305 "can't have 2 mark actions in same" 1306 " flow"); 1307 if (attr->egress) 1308 return rte_flow_error_set(error, ENOTSUP, 1309 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1310 "mark action not supported for " 1311 "egress"); 1312 return 0; 1313 } 1314 1315 /* 1316 * Validate the drop action. 1317 * 1318 * @param[in] action_flags 1319 * Bit-fields that holds the actions detected until now. 1320 * @param[in] attr 1321 * Attributes of flow that includes this action. 1322 * @param[out] error 1323 * Pointer to error structure. 1324 * 1325 * @return 1326 * 0 on success, a negative errno value otherwise and rte_errno is set. 1327 */ 1328 int 1329 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1330 const struct rte_flow_attr *attr, 1331 struct rte_flow_error *error) 1332 { 1333 if (attr->egress) 1334 return rte_flow_error_set(error, ENOTSUP, 1335 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1336 "drop action not supported for " 1337 "egress"); 1338 return 0; 1339 } 1340 1341 /* 1342 * Validate the queue action. 1343 * 1344 * @param[in] action 1345 * Pointer to the queue action. 1346 * @param[in] action_flags 1347 * Bit-fields that holds the actions detected until now. 1348 * @param[in] dev 1349 * Pointer to the Ethernet device structure. 1350 * @param[in] attr 1351 * Attributes of flow that includes this action. 1352 * @param[out] error 1353 * Pointer to error structure. 1354 * 1355 * @return 1356 * 0 on success, a negative errno value otherwise and rte_errno is set. 1357 */ 1358 int 1359 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1360 uint64_t action_flags, 1361 struct rte_eth_dev *dev, 1362 const struct rte_flow_attr *attr, 1363 struct rte_flow_error *error) 1364 { 1365 struct mlx5_priv *priv = dev->data->dev_private; 1366 const struct rte_flow_action_queue *queue = action->conf; 1367 1368 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1369 return rte_flow_error_set(error, EINVAL, 1370 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1371 "can't have 2 fate actions in" 1372 " same flow"); 1373 if (!priv->rxqs_n) 1374 return rte_flow_error_set(error, EINVAL, 1375 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1376 NULL, "No Rx queues configured"); 1377 if (queue->index >= priv->rxqs_n) 1378 return rte_flow_error_set(error, EINVAL, 1379 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1380 &queue->index, 1381 "queue index out of range"); 1382 if (!(*priv->rxqs)[queue->index]) 1383 return rte_flow_error_set(error, EINVAL, 1384 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1385 &queue->index, 1386 "queue is not configured"); 1387 if (attr->egress) 1388 return rte_flow_error_set(error, ENOTSUP, 1389 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1390 "queue action not supported for " 1391 "egress"); 1392 return 0; 1393 } 1394 1395 /* 1396 * Validate the rss action. 1397 * 1398 * @param[in] dev 1399 * Pointer to the Ethernet device structure. 1400 * @param[in] action 1401 * Pointer to the queue action. 1402 * @param[out] error 1403 * Pointer to error structure. 1404 * 1405 * @return 1406 * 0 on success, a negative errno value otherwise and rte_errno is set. 1407 */ 1408 int 1409 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1410 const struct rte_flow_action *action, 1411 struct rte_flow_error *error) 1412 { 1413 struct mlx5_priv *priv = dev->data->dev_private; 1414 const struct rte_flow_action_rss *rss = action->conf; 1415 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1416 unsigned int i; 1417 1418 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1419 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1420 return rte_flow_error_set(error, ENOTSUP, 1421 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1422 &rss->func, 1423 "RSS hash function not supported"); 1424 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1425 if (rss->level > 2) 1426 #else 1427 if (rss->level > 1) 1428 #endif 1429 return rte_flow_error_set(error, ENOTSUP, 1430 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1431 &rss->level, 1432 "tunnel RSS is not supported"); 1433 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1434 if (rss->key_len == 0 && rss->key != NULL) 1435 return rte_flow_error_set(error, ENOTSUP, 1436 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1437 &rss->key_len, 1438 "RSS hash key length 0"); 1439 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1440 return rte_flow_error_set(error, ENOTSUP, 1441 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1442 &rss->key_len, 1443 "RSS hash key too small"); 1444 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1445 return rte_flow_error_set(error, ENOTSUP, 1446 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1447 &rss->key_len, 1448 "RSS hash key too large"); 1449 if (rss->queue_num > priv->config.ind_table_max_size) 1450 return rte_flow_error_set(error, ENOTSUP, 1451 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1452 &rss->queue_num, 1453 "number of queues too large"); 1454 if (rss->types & MLX5_RSS_HF_MASK) 1455 return rte_flow_error_set(error, ENOTSUP, 1456 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1457 &rss->types, 1458 "some RSS protocols are not" 1459 " supported"); 1460 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1461 !(rss->types & ETH_RSS_IP)) 1462 return rte_flow_error_set(error, EINVAL, 1463 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1464 "L3 partial RSS requested but L3 RSS" 1465 " type not specified"); 1466 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1467 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1468 return rte_flow_error_set(error, EINVAL, 1469 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1470 "L4 partial RSS requested but L4 RSS" 1471 " type not specified"); 1472 if (!priv->rxqs_n) 1473 return rte_flow_error_set(error, EINVAL, 1474 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1475 NULL, "No Rx queues configured"); 1476 if (!rss->queue_num) 1477 return rte_flow_error_set(error, EINVAL, 1478 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1479 NULL, "No queues configured"); 1480 for (i = 0; i != rss->queue_num; ++i) { 1481 struct mlx5_rxq_ctrl *rxq_ctrl; 1482 1483 if (rss->queue[i] >= priv->rxqs_n) 1484 return rte_flow_error_set 1485 (error, EINVAL, 1486 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1487 &rss->queue[i], "queue index out of range"); 1488 if (!(*priv->rxqs)[rss->queue[i]]) 1489 return rte_flow_error_set 1490 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1491 &rss->queue[i], "queue is not configured"); 1492 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1493 struct mlx5_rxq_ctrl, rxq); 1494 if (i == 0) 1495 rxq_type = rxq_ctrl->type; 1496 if (rxq_type != rxq_ctrl->type) 1497 return rte_flow_error_set 1498 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1499 &rss->queue[i], 1500 "combining hairpin and regular RSS queues is not supported"); 1501 } 1502 return 0; 1503 } 1504 1505 /* 1506 * Validate the rss action. 1507 * 1508 * @param[in] action 1509 * Pointer to the queue action. 1510 * @param[in] action_flags 1511 * Bit-fields that holds the actions detected until now. 1512 * @param[in] dev 1513 * Pointer to the Ethernet device structure. 1514 * @param[in] attr 1515 * Attributes of flow that includes this action. 1516 * @param[in] item_flags 1517 * Items that were detected. 1518 * @param[out] error 1519 * Pointer to error structure. 1520 * 1521 * @return 1522 * 0 on success, a negative errno value otherwise and rte_errno is set. 1523 */ 1524 int 1525 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1526 uint64_t action_flags, 1527 struct rte_eth_dev *dev, 1528 const struct rte_flow_attr *attr, 1529 uint64_t item_flags, 1530 struct rte_flow_error *error) 1531 { 1532 const struct rte_flow_action_rss *rss = action->conf; 1533 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1534 int ret; 1535 1536 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1537 return rte_flow_error_set(error, EINVAL, 1538 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1539 "can't have 2 fate actions" 1540 " in same flow"); 1541 ret = mlx5_validate_action_rss(dev, action, error); 1542 if (ret) 1543 return ret; 1544 if (attr->egress) 1545 return rte_flow_error_set(error, ENOTSUP, 1546 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1547 "rss action not supported for " 1548 "egress"); 1549 if (rss->level > 1 && !tunnel) 1550 return rte_flow_error_set(error, EINVAL, 1551 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1552 "inner RSS is not supported for " 1553 "non-tunnel flows"); 1554 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1555 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1556 return rte_flow_error_set(error, EINVAL, 1557 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1558 "RSS on eCPRI is not supported now"); 1559 } 1560 return 0; 1561 } 1562 1563 /* 1564 * Validate the default miss action. 1565 * 1566 * @param[in] action_flags 1567 * Bit-fields that holds the actions detected until now. 1568 * @param[out] error 1569 * Pointer to error structure. 1570 * 1571 * @return 1572 * 0 on success, a negative errno value otherwise and rte_errno is set. 1573 */ 1574 int 1575 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1576 const struct rte_flow_attr *attr, 1577 struct rte_flow_error *error) 1578 { 1579 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1580 return rte_flow_error_set(error, EINVAL, 1581 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1582 "can't have 2 fate actions in" 1583 " same flow"); 1584 if (attr->egress) 1585 return rte_flow_error_set(error, ENOTSUP, 1586 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1587 "default miss action not supported " 1588 "for egress"); 1589 if (attr->group) 1590 return rte_flow_error_set(error, ENOTSUP, 1591 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1592 "only group 0 is supported"); 1593 if (attr->transfer) 1594 return rte_flow_error_set(error, ENOTSUP, 1595 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1596 NULL, "transfer is not supported"); 1597 return 0; 1598 } 1599 1600 /* 1601 * Validate the count action. 1602 * 1603 * @param[in] dev 1604 * Pointer to the Ethernet device structure. 1605 * @param[in] attr 1606 * Attributes of flow that includes this action. 1607 * @param[out] error 1608 * Pointer to error structure. 1609 * 1610 * @return 1611 * 0 on success, a negative errno value otherwise and rte_errno is set. 1612 */ 1613 int 1614 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1615 const struct rte_flow_attr *attr, 1616 struct rte_flow_error *error) 1617 { 1618 if (attr->egress) 1619 return rte_flow_error_set(error, ENOTSUP, 1620 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1621 "count action not supported for " 1622 "egress"); 1623 return 0; 1624 } 1625 1626 /** 1627 * Verify the @p attributes will be correctly understood by the NIC and store 1628 * them in the @p flow if everything is correct. 1629 * 1630 * @param[in] dev 1631 * Pointer to the Ethernet device structure. 1632 * @param[in] attributes 1633 * Pointer to flow attributes 1634 * @param[out] error 1635 * Pointer to error structure. 1636 * 1637 * @return 1638 * 0 on success, a negative errno value otherwise and rte_errno is set. 1639 */ 1640 int 1641 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1642 const struct rte_flow_attr *attributes, 1643 struct rte_flow_error *error) 1644 { 1645 struct mlx5_priv *priv = dev->data->dev_private; 1646 uint32_t priority_max = priv->config.flow_prio - 1; 1647 1648 if (attributes->group) 1649 return rte_flow_error_set(error, ENOTSUP, 1650 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1651 NULL, "groups is not supported"); 1652 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1653 attributes->priority >= priority_max) 1654 return rte_flow_error_set(error, ENOTSUP, 1655 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1656 NULL, "priority out of range"); 1657 if (attributes->egress) 1658 return rte_flow_error_set(error, ENOTSUP, 1659 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1660 "egress is not supported"); 1661 if (attributes->transfer && !priv->config.dv_esw_en) 1662 return rte_flow_error_set(error, ENOTSUP, 1663 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1664 NULL, "transfer is not supported"); 1665 if (!attributes->ingress) 1666 return rte_flow_error_set(error, EINVAL, 1667 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1668 NULL, 1669 "ingress attribute is mandatory"); 1670 return 0; 1671 } 1672 1673 /** 1674 * Validate ICMP6 item. 1675 * 1676 * @param[in] item 1677 * Item specification. 1678 * @param[in] item_flags 1679 * Bit-fields that holds the items detected until now. 1680 * @param[in] ext_vlan_sup 1681 * Whether extended VLAN features are supported or not. 1682 * @param[out] error 1683 * Pointer to error structure. 1684 * 1685 * @return 1686 * 0 on success, a negative errno value otherwise and rte_errno is set. 1687 */ 1688 int 1689 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1690 uint64_t item_flags, 1691 uint8_t target_protocol, 1692 struct rte_flow_error *error) 1693 { 1694 const struct rte_flow_item_icmp6 *mask = item->mask; 1695 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1696 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1697 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1698 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1699 MLX5_FLOW_LAYER_OUTER_L4; 1700 int ret; 1701 1702 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1703 return rte_flow_error_set(error, EINVAL, 1704 RTE_FLOW_ERROR_TYPE_ITEM, item, 1705 "protocol filtering not compatible" 1706 " with ICMP6 layer"); 1707 if (!(item_flags & l3m)) 1708 return rte_flow_error_set(error, EINVAL, 1709 RTE_FLOW_ERROR_TYPE_ITEM, item, 1710 "IPv6 is mandatory to filter on" 1711 " ICMP6"); 1712 if (item_flags & l4m) 1713 return rte_flow_error_set(error, EINVAL, 1714 RTE_FLOW_ERROR_TYPE_ITEM, item, 1715 "multiple L4 layers not supported"); 1716 if (!mask) 1717 mask = &rte_flow_item_icmp6_mask; 1718 ret = mlx5_flow_item_acceptable 1719 (item, (const uint8_t *)mask, 1720 (const uint8_t *)&rte_flow_item_icmp6_mask, 1721 sizeof(struct rte_flow_item_icmp6), 1722 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1723 if (ret < 0) 1724 return ret; 1725 return 0; 1726 } 1727 1728 /** 1729 * Validate ICMP item. 1730 * 1731 * @param[in] item 1732 * Item specification. 1733 * @param[in] item_flags 1734 * Bit-fields that holds the items detected until now. 1735 * @param[out] error 1736 * Pointer to error structure. 1737 * 1738 * @return 1739 * 0 on success, a negative errno value otherwise and rte_errno is set. 1740 */ 1741 int 1742 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1743 uint64_t item_flags, 1744 uint8_t target_protocol, 1745 struct rte_flow_error *error) 1746 { 1747 const struct rte_flow_item_icmp *mask = item->mask; 1748 const struct rte_flow_item_icmp nic_mask = { 1749 .hdr.icmp_type = 0xff, 1750 .hdr.icmp_code = 0xff, 1751 .hdr.icmp_ident = RTE_BE16(0xffff), 1752 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1753 }; 1754 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1755 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1756 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1757 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1758 MLX5_FLOW_LAYER_OUTER_L4; 1759 int ret; 1760 1761 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1762 return rte_flow_error_set(error, EINVAL, 1763 RTE_FLOW_ERROR_TYPE_ITEM, item, 1764 "protocol filtering not compatible" 1765 " with ICMP layer"); 1766 if (!(item_flags & l3m)) 1767 return rte_flow_error_set(error, EINVAL, 1768 RTE_FLOW_ERROR_TYPE_ITEM, item, 1769 "IPv4 is mandatory to filter" 1770 " on ICMP"); 1771 if (item_flags & l4m) 1772 return rte_flow_error_set(error, EINVAL, 1773 RTE_FLOW_ERROR_TYPE_ITEM, item, 1774 "multiple L4 layers not supported"); 1775 if (!mask) 1776 mask = &nic_mask; 1777 ret = mlx5_flow_item_acceptable 1778 (item, (const uint8_t *)mask, 1779 (const uint8_t *)&nic_mask, 1780 sizeof(struct rte_flow_item_icmp), 1781 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1782 if (ret < 0) 1783 return ret; 1784 return 0; 1785 } 1786 1787 /** 1788 * Validate Ethernet item. 1789 * 1790 * @param[in] item 1791 * Item specification. 1792 * @param[in] item_flags 1793 * Bit-fields that holds the items detected until now. 1794 * @param[out] error 1795 * Pointer to error structure. 1796 * 1797 * @return 1798 * 0 on success, a negative errno value otherwise and rte_errno is set. 1799 */ 1800 int 1801 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1802 uint64_t item_flags, bool ext_vlan_sup, 1803 struct rte_flow_error *error) 1804 { 1805 const struct rte_flow_item_eth *mask = item->mask; 1806 const struct rte_flow_item_eth nic_mask = { 1807 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1808 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1809 .type = RTE_BE16(0xffff), 1810 .has_vlan = ext_vlan_sup ? 1 : 0, 1811 }; 1812 int ret; 1813 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1814 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1815 MLX5_FLOW_LAYER_OUTER_L2; 1816 1817 if (item_flags & ethm) 1818 return rte_flow_error_set(error, ENOTSUP, 1819 RTE_FLOW_ERROR_TYPE_ITEM, item, 1820 "multiple L2 layers not supported"); 1821 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1822 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1823 return rte_flow_error_set(error, EINVAL, 1824 RTE_FLOW_ERROR_TYPE_ITEM, item, 1825 "L2 layer should not follow " 1826 "L3 layers"); 1827 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1828 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1829 return rte_flow_error_set(error, EINVAL, 1830 RTE_FLOW_ERROR_TYPE_ITEM, item, 1831 "L2 layer should not follow VLAN"); 1832 if (!mask) 1833 mask = &rte_flow_item_eth_mask; 1834 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1835 (const uint8_t *)&nic_mask, 1836 sizeof(struct rte_flow_item_eth), 1837 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1838 return ret; 1839 } 1840 1841 /** 1842 * Validate VLAN item. 1843 * 1844 * @param[in] item 1845 * Item specification. 1846 * @param[in] item_flags 1847 * Bit-fields that holds the items detected until now. 1848 * @param[in] dev 1849 * Ethernet device flow is being created on. 1850 * @param[out] error 1851 * Pointer to error structure. 1852 * 1853 * @return 1854 * 0 on success, a negative errno value otherwise and rte_errno is set. 1855 */ 1856 int 1857 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1858 uint64_t item_flags, 1859 struct rte_eth_dev *dev, 1860 struct rte_flow_error *error) 1861 { 1862 const struct rte_flow_item_vlan *spec = item->spec; 1863 const struct rte_flow_item_vlan *mask = item->mask; 1864 const struct rte_flow_item_vlan nic_mask = { 1865 .tci = RTE_BE16(UINT16_MAX), 1866 .inner_type = RTE_BE16(UINT16_MAX), 1867 }; 1868 uint16_t vlan_tag = 0; 1869 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1870 int ret; 1871 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1872 MLX5_FLOW_LAYER_INNER_L4) : 1873 (MLX5_FLOW_LAYER_OUTER_L3 | 1874 MLX5_FLOW_LAYER_OUTER_L4); 1875 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1876 MLX5_FLOW_LAYER_OUTER_VLAN; 1877 1878 if (item_flags & vlanm) 1879 return rte_flow_error_set(error, EINVAL, 1880 RTE_FLOW_ERROR_TYPE_ITEM, item, 1881 "multiple VLAN layers not supported"); 1882 else if ((item_flags & l34m) != 0) 1883 return rte_flow_error_set(error, EINVAL, 1884 RTE_FLOW_ERROR_TYPE_ITEM, item, 1885 "VLAN cannot follow L3/L4 layer"); 1886 if (!mask) 1887 mask = &rte_flow_item_vlan_mask; 1888 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1889 (const uint8_t *)&nic_mask, 1890 sizeof(struct rte_flow_item_vlan), 1891 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1892 if (ret) 1893 return ret; 1894 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1895 struct mlx5_priv *priv = dev->data->dev_private; 1896 1897 if (priv->vmwa_context) { 1898 /* 1899 * Non-NULL context means we have a virtual machine 1900 * and SR-IOV enabled, we have to create VLAN interface 1901 * to make hypervisor to setup E-Switch vport 1902 * context correctly. We avoid creating the multiple 1903 * VLAN interfaces, so we cannot support VLAN tag mask. 1904 */ 1905 return rte_flow_error_set(error, EINVAL, 1906 RTE_FLOW_ERROR_TYPE_ITEM, 1907 item, 1908 "VLAN tag mask is not" 1909 " supported in virtual" 1910 " environment"); 1911 } 1912 } 1913 if (spec) { 1914 vlan_tag = spec->tci; 1915 vlan_tag &= mask->tci; 1916 } 1917 /* 1918 * From verbs perspective an empty VLAN is equivalent 1919 * to a packet without VLAN layer. 1920 */ 1921 if (!vlan_tag) 1922 return rte_flow_error_set(error, EINVAL, 1923 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1924 item->spec, 1925 "VLAN cannot be empty"); 1926 return 0; 1927 } 1928 1929 /** 1930 * Validate IPV4 item. 1931 * 1932 * @param[in] item 1933 * Item specification. 1934 * @param[in] item_flags 1935 * Bit-fields that holds the items detected until now. 1936 * @param[in] last_item 1937 * Previous validated item in the pattern items. 1938 * @param[in] ether_type 1939 * Type in the ethernet layer header (including dot1q). 1940 * @param[in] acc_mask 1941 * Acceptable mask, if NULL default internal default mask 1942 * will be used to check whether item fields are supported. 1943 * @param[in] range_accepted 1944 * True if range of values is accepted for specific fields, false otherwise. 1945 * @param[out] error 1946 * Pointer to error structure. 1947 * 1948 * @return 1949 * 0 on success, a negative errno value otherwise and rte_errno is set. 1950 */ 1951 int 1952 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1953 uint64_t item_flags, 1954 uint64_t last_item, 1955 uint16_t ether_type, 1956 const struct rte_flow_item_ipv4 *acc_mask, 1957 bool range_accepted, 1958 struct rte_flow_error *error) 1959 { 1960 const struct rte_flow_item_ipv4 *mask = item->mask; 1961 const struct rte_flow_item_ipv4 *spec = item->spec; 1962 const struct rte_flow_item_ipv4 nic_mask = { 1963 .hdr = { 1964 .src_addr = RTE_BE32(0xffffffff), 1965 .dst_addr = RTE_BE32(0xffffffff), 1966 .type_of_service = 0xff, 1967 .next_proto_id = 0xff, 1968 }, 1969 }; 1970 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1971 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1972 MLX5_FLOW_LAYER_OUTER_L3; 1973 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1974 MLX5_FLOW_LAYER_OUTER_L4; 1975 int ret; 1976 uint8_t next_proto = 0xFF; 1977 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1978 MLX5_FLOW_LAYER_OUTER_VLAN | 1979 MLX5_FLOW_LAYER_INNER_VLAN); 1980 1981 if ((last_item & l2_vlan) && ether_type && 1982 ether_type != RTE_ETHER_TYPE_IPV4) 1983 return rte_flow_error_set(error, EINVAL, 1984 RTE_FLOW_ERROR_TYPE_ITEM, item, 1985 "IPv4 cannot follow L2/VLAN layer " 1986 "which ether type is not IPv4"); 1987 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1988 if (mask && spec) 1989 next_proto = mask->hdr.next_proto_id & 1990 spec->hdr.next_proto_id; 1991 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1992 return rte_flow_error_set(error, EINVAL, 1993 RTE_FLOW_ERROR_TYPE_ITEM, 1994 item, 1995 "multiple tunnel " 1996 "not supported"); 1997 } 1998 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1999 return rte_flow_error_set(error, EINVAL, 2000 RTE_FLOW_ERROR_TYPE_ITEM, item, 2001 "wrong tunnel type - IPv6 specified " 2002 "but IPv4 item provided"); 2003 if (item_flags & l3m) 2004 return rte_flow_error_set(error, ENOTSUP, 2005 RTE_FLOW_ERROR_TYPE_ITEM, item, 2006 "multiple L3 layers not supported"); 2007 else if (item_flags & l4m) 2008 return rte_flow_error_set(error, EINVAL, 2009 RTE_FLOW_ERROR_TYPE_ITEM, item, 2010 "L3 cannot follow an L4 layer."); 2011 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2012 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2013 return rte_flow_error_set(error, EINVAL, 2014 RTE_FLOW_ERROR_TYPE_ITEM, item, 2015 "L3 cannot follow an NVGRE layer."); 2016 if (!mask) 2017 mask = &rte_flow_item_ipv4_mask; 2018 else if (mask->hdr.next_proto_id != 0 && 2019 mask->hdr.next_proto_id != 0xff) 2020 return rte_flow_error_set(error, EINVAL, 2021 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2022 "partial mask is not supported" 2023 " for protocol"); 2024 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2025 acc_mask ? (const uint8_t *)acc_mask 2026 : (const uint8_t *)&nic_mask, 2027 sizeof(struct rte_flow_item_ipv4), 2028 range_accepted, error); 2029 if (ret < 0) 2030 return ret; 2031 return 0; 2032 } 2033 2034 /** 2035 * Validate IPV6 item. 2036 * 2037 * @param[in] item 2038 * Item specification. 2039 * @param[in] item_flags 2040 * Bit-fields that holds the items detected until now. 2041 * @param[in] last_item 2042 * Previous validated item in the pattern items. 2043 * @param[in] ether_type 2044 * Type in the ethernet layer header (including dot1q). 2045 * @param[in] acc_mask 2046 * Acceptable mask, if NULL default internal default mask 2047 * will be used to check whether item fields are supported. 2048 * @param[out] error 2049 * Pointer to error structure. 2050 * 2051 * @return 2052 * 0 on success, a negative errno value otherwise and rte_errno is set. 2053 */ 2054 int 2055 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2056 uint64_t item_flags, 2057 uint64_t last_item, 2058 uint16_t ether_type, 2059 const struct rte_flow_item_ipv6 *acc_mask, 2060 struct rte_flow_error *error) 2061 { 2062 const struct rte_flow_item_ipv6 *mask = item->mask; 2063 const struct rte_flow_item_ipv6 *spec = item->spec; 2064 const struct rte_flow_item_ipv6 nic_mask = { 2065 .hdr = { 2066 .src_addr = 2067 "\xff\xff\xff\xff\xff\xff\xff\xff" 2068 "\xff\xff\xff\xff\xff\xff\xff\xff", 2069 .dst_addr = 2070 "\xff\xff\xff\xff\xff\xff\xff\xff" 2071 "\xff\xff\xff\xff\xff\xff\xff\xff", 2072 .vtc_flow = RTE_BE32(0xffffffff), 2073 .proto = 0xff, 2074 }, 2075 }; 2076 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2077 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2078 MLX5_FLOW_LAYER_OUTER_L3; 2079 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2080 MLX5_FLOW_LAYER_OUTER_L4; 2081 int ret; 2082 uint8_t next_proto = 0xFF; 2083 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2084 MLX5_FLOW_LAYER_OUTER_VLAN | 2085 MLX5_FLOW_LAYER_INNER_VLAN); 2086 2087 if ((last_item & l2_vlan) && ether_type && 2088 ether_type != RTE_ETHER_TYPE_IPV6) 2089 return rte_flow_error_set(error, EINVAL, 2090 RTE_FLOW_ERROR_TYPE_ITEM, item, 2091 "IPv6 cannot follow L2/VLAN layer " 2092 "which ether type is not IPv6"); 2093 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2094 next_proto = spec->hdr.proto; 2095 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 2096 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2097 return rte_flow_error_set(error, EINVAL, 2098 RTE_FLOW_ERROR_TYPE_ITEM, 2099 item, 2100 "multiple tunnel " 2101 "not supported"); 2102 } 2103 if (next_proto == IPPROTO_HOPOPTS || 2104 next_proto == IPPROTO_ROUTING || 2105 next_proto == IPPROTO_FRAGMENT || 2106 next_proto == IPPROTO_ESP || 2107 next_proto == IPPROTO_AH || 2108 next_proto == IPPROTO_DSTOPTS) 2109 return rte_flow_error_set(error, EINVAL, 2110 RTE_FLOW_ERROR_TYPE_ITEM, item, 2111 "IPv6 proto (next header) should " 2112 "not be set as extension header"); 2113 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2114 return rte_flow_error_set(error, EINVAL, 2115 RTE_FLOW_ERROR_TYPE_ITEM, item, 2116 "wrong tunnel type - IPv4 specified " 2117 "but IPv6 item provided"); 2118 if (item_flags & l3m) 2119 return rte_flow_error_set(error, ENOTSUP, 2120 RTE_FLOW_ERROR_TYPE_ITEM, item, 2121 "multiple L3 layers not supported"); 2122 else if (item_flags & l4m) 2123 return rte_flow_error_set(error, EINVAL, 2124 RTE_FLOW_ERROR_TYPE_ITEM, item, 2125 "L3 cannot follow an L4 layer."); 2126 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2127 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2128 return rte_flow_error_set(error, EINVAL, 2129 RTE_FLOW_ERROR_TYPE_ITEM, item, 2130 "L3 cannot follow an NVGRE layer."); 2131 if (!mask) 2132 mask = &rte_flow_item_ipv6_mask; 2133 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2134 acc_mask ? (const uint8_t *)acc_mask 2135 : (const uint8_t *)&nic_mask, 2136 sizeof(struct rte_flow_item_ipv6), 2137 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2138 if (ret < 0) 2139 return ret; 2140 return 0; 2141 } 2142 2143 /** 2144 * Validate UDP item. 2145 * 2146 * @param[in] item 2147 * Item specification. 2148 * @param[in] item_flags 2149 * Bit-fields that holds the items detected until now. 2150 * @param[in] target_protocol 2151 * The next protocol in the previous item. 2152 * @param[in] flow_mask 2153 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2154 * @param[out] error 2155 * Pointer to error structure. 2156 * 2157 * @return 2158 * 0 on success, a negative errno value otherwise and rte_errno is set. 2159 */ 2160 int 2161 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2162 uint64_t item_flags, 2163 uint8_t target_protocol, 2164 struct rte_flow_error *error) 2165 { 2166 const struct rte_flow_item_udp *mask = item->mask; 2167 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2168 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2169 MLX5_FLOW_LAYER_OUTER_L3; 2170 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2171 MLX5_FLOW_LAYER_OUTER_L4; 2172 int ret; 2173 2174 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2175 return rte_flow_error_set(error, EINVAL, 2176 RTE_FLOW_ERROR_TYPE_ITEM, item, 2177 "protocol filtering not compatible" 2178 " with UDP layer"); 2179 if (!(item_flags & l3m)) 2180 return rte_flow_error_set(error, EINVAL, 2181 RTE_FLOW_ERROR_TYPE_ITEM, item, 2182 "L3 is mandatory to filter on L4"); 2183 if (item_flags & l4m) 2184 return rte_flow_error_set(error, EINVAL, 2185 RTE_FLOW_ERROR_TYPE_ITEM, item, 2186 "multiple L4 layers not supported"); 2187 if (!mask) 2188 mask = &rte_flow_item_udp_mask; 2189 ret = mlx5_flow_item_acceptable 2190 (item, (const uint8_t *)mask, 2191 (const uint8_t *)&rte_flow_item_udp_mask, 2192 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2193 error); 2194 if (ret < 0) 2195 return ret; 2196 return 0; 2197 } 2198 2199 /** 2200 * Validate TCP item. 2201 * 2202 * @param[in] item 2203 * Item specification. 2204 * @param[in] item_flags 2205 * Bit-fields that holds the items detected until now. 2206 * @param[in] target_protocol 2207 * The next protocol in the previous item. 2208 * @param[out] error 2209 * Pointer to error structure. 2210 * 2211 * @return 2212 * 0 on success, a negative errno value otherwise and rte_errno is set. 2213 */ 2214 int 2215 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2216 uint64_t item_flags, 2217 uint8_t target_protocol, 2218 const struct rte_flow_item_tcp *flow_mask, 2219 struct rte_flow_error *error) 2220 { 2221 const struct rte_flow_item_tcp *mask = item->mask; 2222 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2223 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2224 MLX5_FLOW_LAYER_OUTER_L3; 2225 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2226 MLX5_FLOW_LAYER_OUTER_L4; 2227 int ret; 2228 2229 MLX5_ASSERT(flow_mask); 2230 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2231 return rte_flow_error_set(error, EINVAL, 2232 RTE_FLOW_ERROR_TYPE_ITEM, item, 2233 "protocol filtering not compatible" 2234 " with TCP layer"); 2235 if (!(item_flags & l3m)) 2236 return rte_flow_error_set(error, EINVAL, 2237 RTE_FLOW_ERROR_TYPE_ITEM, item, 2238 "L3 is mandatory to filter on L4"); 2239 if (item_flags & l4m) 2240 return rte_flow_error_set(error, EINVAL, 2241 RTE_FLOW_ERROR_TYPE_ITEM, item, 2242 "multiple L4 layers not supported"); 2243 if (!mask) 2244 mask = &rte_flow_item_tcp_mask; 2245 ret = mlx5_flow_item_acceptable 2246 (item, (const uint8_t *)mask, 2247 (const uint8_t *)flow_mask, 2248 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2249 error); 2250 if (ret < 0) 2251 return ret; 2252 return 0; 2253 } 2254 2255 /** 2256 * Validate VXLAN item. 2257 * 2258 * @param[in] item 2259 * Item specification. 2260 * @param[in] item_flags 2261 * Bit-fields that holds the items detected until now. 2262 * @param[in] target_protocol 2263 * The next protocol in the previous item. 2264 * @param[out] error 2265 * Pointer to error structure. 2266 * 2267 * @return 2268 * 0 on success, a negative errno value otherwise and rte_errno is set. 2269 */ 2270 int 2271 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2272 uint64_t item_flags, 2273 struct rte_flow_error *error) 2274 { 2275 const struct rte_flow_item_vxlan *spec = item->spec; 2276 const struct rte_flow_item_vxlan *mask = item->mask; 2277 int ret; 2278 union vni { 2279 uint32_t vlan_id; 2280 uint8_t vni[4]; 2281 } id = { .vlan_id = 0, }; 2282 2283 2284 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2285 return rte_flow_error_set(error, ENOTSUP, 2286 RTE_FLOW_ERROR_TYPE_ITEM, item, 2287 "multiple tunnel layers not" 2288 " supported"); 2289 /* 2290 * Verify only UDPv4 is present as defined in 2291 * https://tools.ietf.org/html/rfc7348 2292 */ 2293 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2294 return rte_flow_error_set(error, EINVAL, 2295 RTE_FLOW_ERROR_TYPE_ITEM, item, 2296 "no outer UDP layer found"); 2297 if (!mask) 2298 mask = &rte_flow_item_vxlan_mask; 2299 ret = mlx5_flow_item_acceptable 2300 (item, (const uint8_t *)mask, 2301 (const uint8_t *)&rte_flow_item_vxlan_mask, 2302 sizeof(struct rte_flow_item_vxlan), 2303 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2304 if (ret < 0) 2305 return ret; 2306 if (spec) { 2307 memcpy(&id.vni[1], spec->vni, 3); 2308 memcpy(&id.vni[1], mask->vni, 3); 2309 } 2310 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2311 return rte_flow_error_set(error, ENOTSUP, 2312 RTE_FLOW_ERROR_TYPE_ITEM, item, 2313 "VXLAN tunnel must be fully defined"); 2314 return 0; 2315 } 2316 2317 /** 2318 * Validate VXLAN_GPE item. 2319 * 2320 * @param[in] item 2321 * Item specification. 2322 * @param[in] item_flags 2323 * Bit-fields that holds the items detected until now. 2324 * @param[in] priv 2325 * Pointer to the private data structure. 2326 * @param[in] target_protocol 2327 * The next protocol in the previous item. 2328 * @param[out] error 2329 * Pointer to error structure. 2330 * 2331 * @return 2332 * 0 on success, a negative errno value otherwise and rte_errno is set. 2333 */ 2334 int 2335 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2336 uint64_t item_flags, 2337 struct rte_eth_dev *dev, 2338 struct rte_flow_error *error) 2339 { 2340 struct mlx5_priv *priv = dev->data->dev_private; 2341 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2342 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2343 int ret; 2344 union vni { 2345 uint32_t vlan_id; 2346 uint8_t vni[4]; 2347 } id = { .vlan_id = 0, }; 2348 2349 if (!priv->config.l3_vxlan_en) 2350 return rte_flow_error_set(error, ENOTSUP, 2351 RTE_FLOW_ERROR_TYPE_ITEM, item, 2352 "L3 VXLAN is not enabled by device" 2353 " parameter and/or not configured in" 2354 " firmware"); 2355 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2356 return rte_flow_error_set(error, ENOTSUP, 2357 RTE_FLOW_ERROR_TYPE_ITEM, item, 2358 "multiple tunnel layers not" 2359 " supported"); 2360 /* 2361 * Verify only UDPv4 is present as defined in 2362 * https://tools.ietf.org/html/rfc7348 2363 */ 2364 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2365 return rte_flow_error_set(error, EINVAL, 2366 RTE_FLOW_ERROR_TYPE_ITEM, item, 2367 "no outer UDP layer found"); 2368 if (!mask) 2369 mask = &rte_flow_item_vxlan_gpe_mask; 2370 ret = mlx5_flow_item_acceptable 2371 (item, (const uint8_t *)mask, 2372 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2373 sizeof(struct rte_flow_item_vxlan_gpe), 2374 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2375 if (ret < 0) 2376 return ret; 2377 if (spec) { 2378 if (spec->protocol) 2379 return rte_flow_error_set(error, ENOTSUP, 2380 RTE_FLOW_ERROR_TYPE_ITEM, 2381 item, 2382 "VxLAN-GPE protocol" 2383 " not supported"); 2384 memcpy(&id.vni[1], spec->vni, 3); 2385 memcpy(&id.vni[1], mask->vni, 3); 2386 } 2387 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2388 return rte_flow_error_set(error, ENOTSUP, 2389 RTE_FLOW_ERROR_TYPE_ITEM, item, 2390 "VXLAN-GPE tunnel must be fully" 2391 " defined"); 2392 return 0; 2393 } 2394 /** 2395 * Validate GRE Key item. 2396 * 2397 * @param[in] item 2398 * Item specification. 2399 * @param[in] item_flags 2400 * Bit flags to mark detected items. 2401 * @param[in] gre_item 2402 * Pointer to gre_item 2403 * @param[out] error 2404 * Pointer to error structure. 2405 * 2406 * @return 2407 * 0 on success, a negative errno value otherwise and rte_errno is set. 2408 */ 2409 int 2410 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2411 uint64_t item_flags, 2412 const struct rte_flow_item *gre_item, 2413 struct rte_flow_error *error) 2414 { 2415 const rte_be32_t *mask = item->mask; 2416 int ret = 0; 2417 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2418 const struct rte_flow_item_gre *gre_spec; 2419 const struct rte_flow_item_gre *gre_mask; 2420 2421 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2422 return rte_flow_error_set(error, ENOTSUP, 2423 RTE_FLOW_ERROR_TYPE_ITEM, item, 2424 "Multiple GRE key not support"); 2425 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2426 return rte_flow_error_set(error, ENOTSUP, 2427 RTE_FLOW_ERROR_TYPE_ITEM, item, 2428 "No preceding GRE header"); 2429 if (item_flags & MLX5_FLOW_LAYER_INNER) 2430 return rte_flow_error_set(error, ENOTSUP, 2431 RTE_FLOW_ERROR_TYPE_ITEM, item, 2432 "GRE key following a wrong item"); 2433 gre_mask = gre_item->mask; 2434 if (!gre_mask) 2435 gre_mask = &rte_flow_item_gre_mask; 2436 gre_spec = gre_item->spec; 2437 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2438 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2439 return rte_flow_error_set(error, EINVAL, 2440 RTE_FLOW_ERROR_TYPE_ITEM, item, 2441 "Key bit must be on"); 2442 2443 if (!mask) 2444 mask = &gre_key_default_mask; 2445 ret = mlx5_flow_item_acceptable 2446 (item, (const uint8_t *)mask, 2447 (const uint8_t *)&gre_key_default_mask, 2448 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2449 return ret; 2450 } 2451 2452 /** 2453 * Validate GRE item. 2454 * 2455 * @param[in] item 2456 * Item specification. 2457 * @param[in] item_flags 2458 * Bit flags to mark detected items. 2459 * @param[in] target_protocol 2460 * The next protocol in the previous item. 2461 * @param[out] error 2462 * Pointer to error structure. 2463 * 2464 * @return 2465 * 0 on success, a negative errno value otherwise and rte_errno is set. 2466 */ 2467 int 2468 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2469 uint64_t item_flags, 2470 uint8_t target_protocol, 2471 struct rte_flow_error *error) 2472 { 2473 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2474 const struct rte_flow_item_gre *mask = item->mask; 2475 int ret; 2476 const struct rte_flow_item_gre nic_mask = { 2477 .c_rsvd0_ver = RTE_BE16(0xB000), 2478 .protocol = RTE_BE16(UINT16_MAX), 2479 }; 2480 2481 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2482 return rte_flow_error_set(error, EINVAL, 2483 RTE_FLOW_ERROR_TYPE_ITEM, item, 2484 "protocol filtering not compatible" 2485 " with this GRE layer"); 2486 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2487 return rte_flow_error_set(error, ENOTSUP, 2488 RTE_FLOW_ERROR_TYPE_ITEM, item, 2489 "multiple tunnel layers not" 2490 " supported"); 2491 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2492 return rte_flow_error_set(error, ENOTSUP, 2493 RTE_FLOW_ERROR_TYPE_ITEM, item, 2494 "L3 Layer is missing"); 2495 if (!mask) 2496 mask = &rte_flow_item_gre_mask; 2497 ret = mlx5_flow_item_acceptable 2498 (item, (const uint8_t *)mask, 2499 (const uint8_t *)&nic_mask, 2500 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2501 error); 2502 if (ret < 0) 2503 return ret; 2504 #ifndef HAVE_MLX5DV_DR 2505 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2506 if (spec && (spec->protocol & mask->protocol)) 2507 return rte_flow_error_set(error, ENOTSUP, 2508 RTE_FLOW_ERROR_TYPE_ITEM, item, 2509 "without MPLS support the" 2510 " specification cannot be used for" 2511 " filtering"); 2512 #endif 2513 #endif 2514 return 0; 2515 } 2516 2517 /** 2518 * Validate Geneve item. 2519 * 2520 * @param[in] item 2521 * Item specification. 2522 * @param[in] itemFlags 2523 * Bit-fields that holds the items detected until now. 2524 * @param[in] enPriv 2525 * Pointer to the private data structure. 2526 * @param[out] error 2527 * Pointer to error structure. 2528 * 2529 * @return 2530 * 0 on success, a negative errno value otherwise and rte_errno is set. 2531 */ 2532 2533 int 2534 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2535 uint64_t item_flags, 2536 struct rte_eth_dev *dev, 2537 struct rte_flow_error *error) 2538 { 2539 struct mlx5_priv *priv = dev->data->dev_private; 2540 const struct rte_flow_item_geneve *spec = item->spec; 2541 const struct rte_flow_item_geneve *mask = item->mask; 2542 int ret; 2543 uint16_t gbhdr; 2544 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2545 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2546 const struct rte_flow_item_geneve nic_mask = { 2547 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2548 .vni = "\xff\xff\xff", 2549 .protocol = RTE_BE16(UINT16_MAX), 2550 }; 2551 2552 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2553 return rte_flow_error_set(error, ENOTSUP, 2554 RTE_FLOW_ERROR_TYPE_ITEM, item, 2555 "L3 Geneve is not enabled by device" 2556 " parameter and/or not configured in" 2557 " firmware"); 2558 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2559 return rte_flow_error_set(error, ENOTSUP, 2560 RTE_FLOW_ERROR_TYPE_ITEM, item, 2561 "multiple tunnel layers not" 2562 " supported"); 2563 /* 2564 * Verify only UDPv4 is present as defined in 2565 * https://tools.ietf.org/html/rfc7348 2566 */ 2567 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2568 return rte_flow_error_set(error, EINVAL, 2569 RTE_FLOW_ERROR_TYPE_ITEM, item, 2570 "no outer UDP layer found"); 2571 if (!mask) 2572 mask = &rte_flow_item_geneve_mask; 2573 ret = mlx5_flow_item_acceptable 2574 (item, (const uint8_t *)mask, 2575 (const uint8_t *)&nic_mask, 2576 sizeof(struct rte_flow_item_geneve), 2577 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2578 if (ret) 2579 return ret; 2580 if (spec) { 2581 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2582 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2583 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2584 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2585 return rte_flow_error_set(error, ENOTSUP, 2586 RTE_FLOW_ERROR_TYPE_ITEM, 2587 item, 2588 "Geneve protocol unsupported" 2589 " fields are being used"); 2590 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2591 return rte_flow_error_set 2592 (error, ENOTSUP, 2593 RTE_FLOW_ERROR_TYPE_ITEM, 2594 item, 2595 "Unsupported Geneve options length"); 2596 } 2597 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2598 return rte_flow_error_set 2599 (error, ENOTSUP, 2600 RTE_FLOW_ERROR_TYPE_ITEM, item, 2601 "Geneve tunnel must be fully defined"); 2602 return 0; 2603 } 2604 2605 /** 2606 * Validate MPLS item. 2607 * 2608 * @param[in] dev 2609 * Pointer to the rte_eth_dev structure. 2610 * @param[in] item 2611 * Item specification. 2612 * @param[in] item_flags 2613 * Bit-fields that holds the items detected until now. 2614 * @param[in] prev_layer 2615 * The protocol layer indicated in previous item. 2616 * @param[out] error 2617 * Pointer to error structure. 2618 * 2619 * @return 2620 * 0 on success, a negative errno value otherwise and rte_errno is set. 2621 */ 2622 int 2623 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2624 const struct rte_flow_item *item __rte_unused, 2625 uint64_t item_flags __rte_unused, 2626 uint64_t prev_layer __rte_unused, 2627 struct rte_flow_error *error) 2628 { 2629 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2630 const struct rte_flow_item_mpls *mask = item->mask; 2631 struct mlx5_priv *priv = dev->data->dev_private; 2632 int ret; 2633 2634 if (!priv->config.mpls_en) 2635 return rte_flow_error_set(error, ENOTSUP, 2636 RTE_FLOW_ERROR_TYPE_ITEM, item, 2637 "MPLS not supported or" 2638 " disabled in firmware" 2639 " configuration."); 2640 /* MPLS over IP, UDP, GRE is allowed */ 2641 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2642 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2643 MLX5_FLOW_LAYER_GRE | 2644 MLX5_FLOW_LAYER_GRE_KEY))) 2645 return rte_flow_error_set(error, EINVAL, 2646 RTE_FLOW_ERROR_TYPE_ITEM, item, 2647 "protocol filtering not compatible" 2648 " with MPLS layer"); 2649 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2650 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2651 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2652 return rte_flow_error_set(error, ENOTSUP, 2653 RTE_FLOW_ERROR_TYPE_ITEM, item, 2654 "multiple tunnel layers not" 2655 " supported"); 2656 if (!mask) 2657 mask = &rte_flow_item_mpls_mask; 2658 ret = mlx5_flow_item_acceptable 2659 (item, (const uint8_t *)mask, 2660 (const uint8_t *)&rte_flow_item_mpls_mask, 2661 sizeof(struct rte_flow_item_mpls), 2662 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2663 if (ret < 0) 2664 return ret; 2665 return 0; 2666 #else 2667 return rte_flow_error_set(error, ENOTSUP, 2668 RTE_FLOW_ERROR_TYPE_ITEM, item, 2669 "MPLS is not supported by Verbs, please" 2670 " update."); 2671 #endif 2672 } 2673 2674 /** 2675 * Validate NVGRE item. 2676 * 2677 * @param[in] item 2678 * Item specification. 2679 * @param[in] item_flags 2680 * Bit flags to mark detected items. 2681 * @param[in] target_protocol 2682 * The next protocol in the previous item. 2683 * @param[out] error 2684 * Pointer to error structure. 2685 * 2686 * @return 2687 * 0 on success, a negative errno value otherwise and rte_errno is set. 2688 */ 2689 int 2690 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2691 uint64_t item_flags, 2692 uint8_t target_protocol, 2693 struct rte_flow_error *error) 2694 { 2695 const struct rte_flow_item_nvgre *mask = item->mask; 2696 int ret; 2697 2698 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2699 return rte_flow_error_set(error, EINVAL, 2700 RTE_FLOW_ERROR_TYPE_ITEM, item, 2701 "protocol filtering not compatible" 2702 " with this GRE layer"); 2703 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2704 return rte_flow_error_set(error, ENOTSUP, 2705 RTE_FLOW_ERROR_TYPE_ITEM, item, 2706 "multiple tunnel layers not" 2707 " supported"); 2708 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2709 return rte_flow_error_set(error, ENOTSUP, 2710 RTE_FLOW_ERROR_TYPE_ITEM, item, 2711 "L3 Layer is missing"); 2712 if (!mask) 2713 mask = &rte_flow_item_nvgre_mask; 2714 ret = mlx5_flow_item_acceptable 2715 (item, (const uint8_t *)mask, 2716 (const uint8_t *)&rte_flow_item_nvgre_mask, 2717 sizeof(struct rte_flow_item_nvgre), 2718 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2719 if (ret < 0) 2720 return ret; 2721 return 0; 2722 } 2723 2724 /** 2725 * Validate eCPRI item. 2726 * 2727 * @param[in] item 2728 * Item specification. 2729 * @param[in] item_flags 2730 * Bit-fields that holds the items detected until now. 2731 * @param[in] last_item 2732 * Previous validated item in the pattern items. 2733 * @param[in] ether_type 2734 * Type in the ethernet layer header (including dot1q). 2735 * @param[in] acc_mask 2736 * Acceptable mask, if NULL default internal default mask 2737 * will be used to check whether item fields are supported. 2738 * @param[out] error 2739 * Pointer to error structure. 2740 * 2741 * @return 2742 * 0 on success, a negative errno value otherwise and rte_errno is set. 2743 */ 2744 int 2745 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 2746 uint64_t item_flags, 2747 uint64_t last_item, 2748 uint16_t ether_type, 2749 const struct rte_flow_item_ecpri *acc_mask, 2750 struct rte_flow_error *error) 2751 { 2752 const struct rte_flow_item_ecpri *mask = item->mask; 2753 const struct rte_flow_item_ecpri nic_mask = { 2754 .hdr = { 2755 .common = { 2756 .u32 = 2757 RTE_BE32(((const struct rte_ecpri_common_hdr) { 2758 .type = 0xFF, 2759 }).u32), 2760 }, 2761 .dummy[0] = 0xFFFFFFFF, 2762 }, 2763 }; 2764 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 2765 MLX5_FLOW_LAYER_OUTER_VLAN); 2766 struct rte_flow_item_ecpri mask_lo; 2767 2768 if (!(last_item & outer_l2_vlan) && 2769 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 2770 return rte_flow_error_set(error, EINVAL, 2771 RTE_FLOW_ERROR_TYPE_ITEM, item, 2772 "eCPRI can only follow L2/VLAN layer or UDP layer"); 2773 if ((last_item & outer_l2_vlan) && ether_type && 2774 ether_type != RTE_ETHER_TYPE_ECPRI) 2775 return rte_flow_error_set(error, EINVAL, 2776 RTE_FLOW_ERROR_TYPE_ITEM, item, 2777 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 2778 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2779 return rte_flow_error_set(error, EINVAL, 2780 RTE_FLOW_ERROR_TYPE_ITEM, item, 2781 "eCPRI with tunnel is not supported right now"); 2782 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 2783 return rte_flow_error_set(error, ENOTSUP, 2784 RTE_FLOW_ERROR_TYPE_ITEM, item, 2785 "multiple L3 layers not supported"); 2786 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 2787 return rte_flow_error_set(error, EINVAL, 2788 RTE_FLOW_ERROR_TYPE_ITEM, item, 2789 "eCPRI cannot coexist with a TCP layer"); 2790 /* In specification, eCPRI could be over UDP layer. */ 2791 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 2792 return rte_flow_error_set(error, EINVAL, 2793 RTE_FLOW_ERROR_TYPE_ITEM, item, 2794 "eCPRI over UDP layer is not yet supported right now"); 2795 /* Mask for type field in common header could be zero. */ 2796 if (!mask) 2797 mask = &rte_flow_item_ecpri_mask; 2798 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 2799 /* Input mask is in big-endian format. */ 2800 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 2801 return rte_flow_error_set(error, EINVAL, 2802 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2803 "partial mask is not supported for protocol"); 2804 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 2805 return rte_flow_error_set(error, EINVAL, 2806 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2807 "message header mask must be after a type mask"); 2808 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2809 acc_mask ? (const uint8_t *)acc_mask 2810 : (const uint8_t *)&nic_mask, 2811 sizeof(struct rte_flow_item_ecpri), 2812 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2813 } 2814 2815 /** 2816 * Release resource related QUEUE/RSS action split. 2817 * 2818 * @param dev 2819 * Pointer to Ethernet device. 2820 * @param flow 2821 * Flow to release id's from. 2822 */ 2823 static void 2824 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2825 struct rte_flow *flow) 2826 { 2827 struct mlx5_priv *priv = dev->data->dev_private; 2828 uint32_t handle_idx; 2829 struct mlx5_flow_handle *dev_handle; 2830 2831 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2832 handle_idx, dev_handle, next) 2833 if (dev_handle->split_flow_id) 2834 mlx5_ipool_free(priv->sh->ipool 2835 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 2836 dev_handle->split_flow_id); 2837 } 2838 2839 static int 2840 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2841 const struct rte_flow_attr *attr __rte_unused, 2842 const struct rte_flow_item items[] __rte_unused, 2843 const struct rte_flow_action actions[] __rte_unused, 2844 bool external __rte_unused, 2845 int hairpin __rte_unused, 2846 struct rte_flow_error *error) 2847 { 2848 return rte_flow_error_set(error, ENOTSUP, 2849 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2850 } 2851 2852 static struct mlx5_flow * 2853 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2854 const struct rte_flow_attr *attr __rte_unused, 2855 const struct rte_flow_item items[] __rte_unused, 2856 const struct rte_flow_action actions[] __rte_unused, 2857 struct rte_flow_error *error) 2858 { 2859 rte_flow_error_set(error, ENOTSUP, 2860 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2861 return NULL; 2862 } 2863 2864 static int 2865 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2866 struct mlx5_flow *dev_flow __rte_unused, 2867 const struct rte_flow_attr *attr __rte_unused, 2868 const struct rte_flow_item items[] __rte_unused, 2869 const struct rte_flow_action actions[] __rte_unused, 2870 struct rte_flow_error *error) 2871 { 2872 return rte_flow_error_set(error, ENOTSUP, 2873 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2874 } 2875 2876 static int 2877 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2878 struct rte_flow *flow __rte_unused, 2879 struct rte_flow_error *error) 2880 { 2881 return rte_flow_error_set(error, ENOTSUP, 2882 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2883 } 2884 2885 static void 2886 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2887 struct rte_flow *flow __rte_unused) 2888 { 2889 } 2890 2891 static void 2892 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2893 struct rte_flow *flow __rte_unused) 2894 { 2895 } 2896 2897 static int 2898 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2899 struct rte_flow *flow __rte_unused, 2900 const struct rte_flow_action *actions __rte_unused, 2901 void *data __rte_unused, 2902 struct rte_flow_error *error) 2903 { 2904 return rte_flow_error_set(error, ENOTSUP, 2905 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2906 } 2907 2908 static int 2909 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 2910 uint32_t domains __rte_unused, 2911 uint32_t flags __rte_unused) 2912 { 2913 return 0; 2914 } 2915 2916 /* Void driver to protect from null pointer reference. */ 2917 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2918 .validate = flow_null_validate, 2919 .prepare = flow_null_prepare, 2920 .translate = flow_null_translate, 2921 .apply = flow_null_apply, 2922 .remove = flow_null_remove, 2923 .destroy = flow_null_destroy, 2924 .query = flow_null_query, 2925 .sync_domain = flow_null_sync_domain, 2926 }; 2927 2928 /** 2929 * Select flow driver type according to flow attributes and device 2930 * configuration. 2931 * 2932 * @param[in] dev 2933 * Pointer to the dev structure. 2934 * @param[in] attr 2935 * Pointer to the flow attributes. 2936 * 2937 * @return 2938 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2939 */ 2940 static enum mlx5_flow_drv_type 2941 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2942 { 2943 struct mlx5_priv *priv = dev->data->dev_private; 2944 /* The OS can determine first a specific flow type (DV, VERBS) */ 2945 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 2946 2947 if (type != MLX5_FLOW_TYPE_MAX) 2948 return type; 2949 /* If no OS specific type - continue with DV/VERBS selection */ 2950 if (attr->transfer && priv->config.dv_esw_en) 2951 type = MLX5_FLOW_TYPE_DV; 2952 if (!attr->transfer) 2953 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2954 MLX5_FLOW_TYPE_VERBS; 2955 return type; 2956 } 2957 2958 #define flow_get_drv_ops(type) flow_drv_ops[type] 2959 2960 /** 2961 * Flow driver validation API. This abstracts calling driver specific functions. 2962 * The type of flow driver is determined according to flow attributes. 2963 * 2964 * @param[in] dev 2965 * Pointer to the dev structure. 2966 * @param[in] attr 2967 * Pointer to the flow attributes. 2968 * @param[in] items 2969 * Pointer to the list of items. 2970 * @param[in] actions 2971 * Pointer to the list of actions. 2972 * @param[in] external 2973 * This flow rule is created by request external to PMD. 2974 * @param[in] hairpin 2975 * Number of hairpin TX actions, 0 means classic flow. 2976 * @param[out] error 2977 * Pointer to the error structure. 2978 * 2979 * @return 2980 * 0 on success, a negative errno value otherwise and rte_errno is set. 2981 */ 2982 static inline int 2983 flow_drv_validate(struct rte_eth_dev *dev, 2984 const struct rte_flow_attr *attr, 2985 const struct rte_flow_item items[], 2986 const struct rte_flow_action actions[], 2987 bool external, int hairpin, struct rte_flow_error *error) 2988 { 2989 const struct mlx5_flow_driver_ops *fops; 2990 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2991 2992 fops = flow_get_drv_ops(type); 2993 return fops->validate(dev, attr, items, actions, external, 2994 hairpin, error); 2995 } 2996 2997 /** 2998 * Flow driver preparation API. This abstracts calling driver specific 2999 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3000 * calculates the size of memory required for device flow, allocates the memory, 3001 * initializes the device flow and returns the pointer. 3002 * 3003 * @note 3004 * This function initializes device flow structure such as dv or verbs in 3005 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3006 * rest. For example, adding returning device flow to flow->dev_flow list and 3007 * setting backward reference to the flow should be done out of this function. 3008 * layers field is not filled either. 3009 * 3010 * @param[in] dev 3011 * Pointer to the dev structure. 3012 * @param[in] attr 3013 * Pointer to the flow attributes. 3014 * @param[in] items 3015 * Pointer to the list of items. 3016 * @param[in] actions 3017 * Pointer to the list of actions. 3018 * @param[in] flow_idx 3019 * This memory pool index to the flow. 3020 * @param[out] error 3021 * Pointer to the error structure. 3022 * 3023 * @return 3024 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3025 */ 3026 static inline struct mlx5_flow * 3027 flow_drv_prepare(struct rte_eth_dev *dev, 3028 const struct rte_flow *flow, 3029 const struct rte_flow_attr *attr, 3030 const struct rte_flow_item items[], 3031 const struct rte_flow_action actions[], 3032 uint32_t flow_idx, 3033 struct rte_flow_error *error) 3034 { 3035 const struct mlx5_flow_driver_ops *fops; 3036 enum mlx5_flow_drv_type type = flow->drv_type; 3037 struct mlx5_flow *mlx5_flow = NULL; 3038 3039 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3040 fops = flow_get_drv_ops(type); 3041 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3042 if (mlx5_flow) 3043 mlx5_flow->flow_idx = flow_idx; 3044 return mlx5_flow; 3045 } 3046 3047 /** 3048 * Flow driver translation API. This abstracts calling driver specific 3049 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3050 * translates a generic flow into a driver flow. flow_drv_prepare() must 3051 * precede. 3052 * 3053 * @note 3054 * dev_flow->layers could be filled as a result of parsing during translation 3055 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3056 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3057 * flow->actions could be overwritten even though all the expanded dev_flows 3058 * have the same actions. 3059 * 3060 * @param[in] dev 3061 * Pointer to the rte dev structure. 3062 * @param[in, out] dev_flow 3063 * Pointer to the mlx5 flow. 3064 * @param[in] attr 3065 * Pointer to the flow attributes. 3066 * @param[in] items 3067 * Pointer to the list of items. 3068 * @param[in] actions 3069 * Pointer to the list of actions. 3070 * @param[out] error 3071 * Pointer to the error structure. 3072 * 3073 * @return 3074 * 0 on success, a negative errno value otherwise and rte_errno is set. 3075 */ 3076 static inline int 3077 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3078 const struct rte_flow_attr *attr, 3079 const struct rte_flow_item items[], 3080 const struct rte_flow_action actions[], 3081 struct rte_flow_error *error) 3082 { 3083 const struct mlx5_flow_driver_ops *fops; 3084 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3085 3086 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3087 fops = flow_get_drv_ops(type); 3088 return fops->translate(dev, dev_flow, attr, items, actions, error); 3089 } 3090 3091 /** 3092 * Flow driver apply API. This abstracts calling driver specific functions. 3093 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3094 * translated driver flows on to device. flow_drv_translate() must precede. 3095 * 3096 * @param[in] dev 3097 * Pointer to Ethernet device structure. 3098 * @param[in, out] flow 3099 * Pointer to flow structure. 3100 * @param[out] error 3101 * Pointer to error structure. 3102 * 3103 * @return 3104 * 0 on success, a negative errno value otherwise and rte_errno is set. 3105 */ 3106 static inline int 3107 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3108 struct rte_flow_error *error) 3109 { 3110 const struct mlx5_flow_driver_ops *fops; 3111 enum mlx5_flow_drv_type type = flow->drv_type; 3112 3113 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3114 fops = flow_get_drv_ops(type); 3115 return fops->apply(dev, flow, error); 3116 } 3117 3118 /** 3119 * Flow driver destroy API. This abstracts calling driver specific functions. 3120 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3121 * on device and releases resources of the flow. 3122 * 3123 * @param[in] dev 3124 * Pointer to Ethernet device. 3125 * @param[in, out] flow 3126 * Pointer to flow structure. 3127 */ 3128 static inline void 3129 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3130 { 3131 const struct mlx5_flow_driver_ops *fops; 3132 enum mlx5_flow_drv_type type = flow->drv_type; 3133 3134 flow_mreg_split_qrss_release(dev, flow); 3135 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3136 fops = flow_get_drv_ops(type); 3137 fops->destroy(dev, flow); 3138 } 3139 3140 /** 3141 * Get RSS action from the action list. 3142 * 3143 * @param[in] actions 3144 * Pointer to the list of actions. 3145 * 3146 * @return 3147 * Pointer to the RSS action if exist, else return NULL. 3148 */ 3149 static const struct rte_flow_action_rss* 3150 flow_get_rss_action(const struct rte_flow_action actions[]) 3151 { 3152 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3153 switch (actions->type) { 3154 case RTE_FLOW_ACTION_TYPE_RSS: 3155 return (const struct rte_flow_action_rss *) 3156 actions->conf; 3157 default: 3158 break; 3159 } 3160 } 3161 return NULL; 3162 } 3163 3164 /** 3165 * Get ASO age action by index. 3166 * 3167 * @param[in] dev 3168 * Pointer to the Ethernet device structure. 3169 * @param[in] age_idx 3170 * Index to the ASO age action. 3171 * 3172 * @return 3173 * The specified ASO age action. 3174 */ 3175 struct mlx5_aso_age_action* 3176 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3177 { 3178 uint16_t pool_idx = age_idx & UINT16_MAX; 3179 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3180 struct mlx5_priv *priv = dev->data->dev_private; 3181 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3182 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3183 3184 return &pool->actions[offset - 1]; 3185 } 3186 3187 /* maps shared action to translated non shared in some actions array */ 3188 struct mlx5_translated_shared_action { 3189 struct rte_flow_shared_action *action; /**< Shared action */ 3190 int index; /**< Index in related array of rte_flow_action */ 3191 }; 3192 3193 /** 3194 * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related 3195 * non shared action if translation possible. 3196 * This functionality used to run same execution path for both shared & non 3197 * shared actions on flow create. All necessary preparations for shared 3198 * action handling should be preformed on *shared* actions list returned 3199 * from this call. 3200 * 3201 * @param[in] dev 3202 * Pointer to Ethernet device. 3203 * @param[in] actions 3204 * List of actions to translate. 3205 * @param[out] shared 3206 * List to store translated shared actions. 3207 * @param[in, out] shared_n 3208 * Size of *shared* array. On return should be updated with number of shared 3209 * actions retrieved from the *actions* list. 3210 * @param[out] translated_actions 3211 * List of actions where all shared actions were translated to non shared 3212 * if possible. NULL if no translation took place. 3213 * @param[out] error 3214 * Pointer to the error structure. 3215 * 3216 * @return 3217 * 0 on success, a negative errno value otherwise and rte_errno is set. 3218 */ 3219 static int 3220 flow_shared_actions_translate(struct rte_eth_dev *dev, 3221 const struct rte_flow_action actions[], 3222 struct mlx5_translated_shared_action *shared, 3223 int *shared_n, 3224 struct rte_flow_action **translated_actions, 3225 struct rte_flow_error *error) 3226 { 3227 struct mlx5_priv *priv = dev->data->dev_private; 3228 struct rte_flow_action *translated = NULL; 3229 size_t actions_size; 3230 int n; 3231 int copied_n = 0; 3232 struct mlx5_translated_shared_action *shared_end = NULL; 3233 3234 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3235 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED) 3236 continue; 3237 if (copied_n == *shared_n) { 3238 return rte_flow_error_set 3239 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3240 NULL, "too many shared actions"); 3241 } 3242 rte_memcpy(&shared[copied_n].action, &actions[n].conf, 3243 sizeof(actions[n].conf)); 3244 shared[copied_n].index = n; 3245 copied_n++; 3246 } 3247 n++; 3248 *shared_n = copied_n; 3249 if (!copied_n) 3250 return 0; 3251 actions_size = sizeof(struct rte_flow_action) * n; 3252 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3253 if (!translated) { 3254 rte_errno = ENOMEM; 3255 return -ENOMEM; 3256 } 3257 memcpy(translated, actions, actions_size); 3258 for (shared_end = shared + copied_n; shared < shared_end; shared++) { 3259 struct mlx5_shared_action_rss *shared_rss; 3260 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; 3261 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; 3262 uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) 3263 - 1); 3264 3265 switch (type) { 3266 case MLX5_SHARED_ACTION_TYPE_RSS: 3267 shared_rss = mlx5_ipool_get 3268 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3269 translated[shared->index].type = 3270 RTE_FLOW_ACTION_TYPE_RSS; 3271 translated[shared->index].conf = 3272 &shared_rss->origin; 3273 break; 3274 case MLX5_SHARED_ACTION_TYPE_AGE: 3275 if (priv->sh->flow_hit_aso_en) { 3276 translated[shared->index].type = 3277 (enum rte_flow_action_type) 3278 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3279 translated[shared->index].conf = 3280 (void *)(uintptr_t)idx; 3281 break; 3282 } 3283 /* Fall-through */ 3284 default: 3285 mlx5_free(translated); 3286 return rte_flow_error_set 3287 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3288 NULL, "invalid shared action type"); 3289 } 3290 } 3291 *translated_actions = translated; 3292 return 0; 3293 } 3294 3295 /** 3296 * Get Shared RSS action from the action list. 3297 * 3298 * @param[in] dev 3299 * Pointer to Ethernet device. 3300 * @param[in] shared 3301 * Pointer to the list of actions. 3302 * @param[in] shared_n 3303 * Actions list length. 3304 * 3305 * @return 3306 * The MLX5 RSS action ID if exists, otherwise return 0. 3307 */ 3308 static uint32_t 3309 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3310 struct mlx5_translated_shared_action *shared, 3311 int shared_n) 3312 { 3313 struct mlx5_translated_shared_action *shared_end; 3314 struct mlx5_priv *priv = dev->data->dev_private; 3315 struct mlx5_shared_action_rss *shared_rss; 3316 3317 3318 for (shared_end = shared + shared_n; shared < shared_end; shared++) { 3319 uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; 3320 uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; 3321 uint32_t idx = act_idx & 3322 ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); 3323 switch (type) { 3324 case MLX5_SHARED_ACTION_TYPE_RSS: 3325 shared_rss = mlx5_ipool_get 3326 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3327 idx); 3328 __atomic_add_fetch(&shared_rss->refcnt, 1, 3329 __ATOMIC_RELAXED); 3330 return idx; 3331 default: 3332 break; 3333 } 3334 } 3335 return 0; 3336 } 3337 3338 static unsigned int 3339 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3340 { 3341 const struct rte_flow_item *item; 3342 unsigned int has_vlan = 0; 3343 3344 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3345 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3346 has_vlan = 1; 3347 break; 3348 } 3349 } 3350 if (has_vlan) 3351 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3352 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3353 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3354 MLX5_EXPANSION_ROOT_OUTER; 3355 } 3356 3357 /** 3358 * Get layer flags from the prefix flow. 3359 * 3360 * Some flows may be split to several subflows, the prefix subflow gets the 3361 * match items and the suffix sub flow gets the actions. 3362 * Some actions need the user defined match item flags to get the detail for 3363 * the action. 3364 * This function helps the suffix flow to get the item layer flags from prefix 3365 * subflow. 3366 * 3367 * @param[in] dev_flow 3368 * Pointer the created preifx subflow. 3369 * 3370 * @return 3371 * The layers get from prefix subflow. 3372 */ 3373 static inline uint64_t 3374 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3375 { 3376 uint64_t layers = 0; 3377 3378 /* 3379 * Layers bits could be localization, but usually the compiler will 3380 * help to do the optimization work for source code. 3381 * If no decap actions, use the layers directly. 3382 */ 3383 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3384 return dev_flow->handle->layers; 3385 /* Convert L3 layers with decap action. */ 3386 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3387 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3388 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3389 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3390 /* Convert L4 layers with decap action. */ 3391 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3392 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3393 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3394 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3395 return layers; 3396 } 3397 3398 /** 3399 * Get metadata split action information. 3400 * 3401 * @param[in] actions 3402 * Pointer to the list of actions. 3403 * @param[out] qrss 3404 * Pointer to the return pointer. 3405 * @param[out] qrss_type 3406 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3407 * if no QUEUE/RSS is found. 3408 * @param[out] encap_idx 3409 * Pointer to the index of the encap action if exists, otherwise the last 3410 * action index. 3411 * 3412 * @return 3413 * Total number of actions. 3414 */ 3415 static int 3416 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3417 const struct rte_flow_action **qrss, 3418 int *encap_idx) 3419 { 3420 const struct rte_flow_action_raw_encap *raw_encap; 3421 int actions_n = 0; 3422 int raw_decap_idx = -1; 3423 3424 *encap_idx = -1; 3425 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3426 switch (actions->type) { 3427 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3428 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3429 *encap_idx = actions_n; 3430 break; 3431 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3432 raw_decap_idx = actions_n; 3433 break; 3434 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3435 raw_encap = actions->conf; 3436 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3437 *encap_idx = raw_decap_idx != -1 ? 3438 raw_decap_idx : actions_n; 3439 break; 3440 case RTE_FLOW_ACTION_TYPE_QUEUE: 3441 case RTE_FLOW_ACTION_TYPE_RSS: 3442 *qrss = actions; 3443 break; 3444 default: 3445 break; 3446 } 3447 actions_n++; 3448 } 3449 if (*encap_idx == -1) 3450 *encap_idx = actions_n; 3451 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3452 return actions_n + 1; 3453 } 3454 3455 /** 3456 * Check meter action from the action list. 3457 * 3458 * @param[in] actions 3459 * Pointer to the list of actions. 3460 * @param[out] mtr 3461 * Pointer to the meter exist flag. 3462 * 3463 * @return 3464 * Total number of actions. 3465 */ 3466 static int 3467 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 3468 { 3469 int actions_n = 0; 3470 3471 MLX5_ASSERT(mtr); 3472 *mtr = 0; 3473 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3474 switch (actions->type) { 3475 case RTE_FLOW_ACTION_TYPE_METER: 3476 *mtr = 1; 3477 break; 3478 default: 3479 break; 3480 } 3481 actions_n++; 3482 } 3483 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3484 return actions_n + 1; 3485 } 3486 3487 /** 3488 * Check if the flow should be split due to hairpin. 3489 * The reason for the split is that in current HW we can't 3490 * support encap and push-vlan on Rx, so if a flow contains 3491 * these actions we move it to Tx. 3492 * 3493 * @param dev 3494 * Pointer to Ethernet device. 3495 * @param[in] attr 3496 * Flow rule attributes. 3497 * @param[in] actions 3498 * Associated actions (list terminated by the END action). 3499 * 3500 * @return 3501 * > 0 the number of actions and the flow should be split, 3502 * 0 when no split required. 3503 */ 3504 static int 3505 flow_check_hairpin_split(struct rte_eth_dev *dev, 3506 const struct rte_flow_attr *attr, 3507 const struct rte_flow_action actions[]) 3508 { 3509 int queue_action = 0; 3510 int action_n = 0; 3511 int split = 0; 3512 const struct rte_flow_action_queue *queue; 3513 const struct rte_flow_action_rss *rss; 3514 const struct rte_flow_action_raw_encap *raw_encap; 3515 const struct rte_eth_hairpin_conf *conf; 3516 3517 if (!attr->ingress) 3518 return 0; 3519 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3520 switch (actions->type) { 3521 case RTE_FLOW_ACTION_TYPE_QUEUE: 3522 queue = actions->conf; 3523 if (queue == NULL) 3524 return 0; 3525 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3526 if (conf != NULL && !!conf->tx_explicit) 3527 return 0; 3528 queue_action = 1; 3529 action_n++; 3530 break; 3531 case RTE_FLOW_ACTION_TYPE_RSS: 3532 rss = actions->conf; 3533 if (rss == NULL || rss->queue_num == 0) 3534 return 0; 3535 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3536 if (conf != NULL && !!conf->tx_explicit) 3537 return 0; 3538 queue_action = 1; 3539 action_n++; 3540 break; 3541 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3542 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3543 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3544 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3545 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3546 split++; 3547 action_n++; 3548 break; 3549 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3550 raw_encap = actions->conf; 3551 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3552 split++; 3553 action_n++; 3554 break; 3555 default: 3556 action_n++; 3557 break; 3558 } 3559 } 3560 if (split && queue_action) 3561 return action_n; 3562 return 0; 3563 } 3564 3565 /* Declare flow create/destroy prototype in advance. */ 3566 static uint32_t 3567 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3568 const struct rte_flow_attr *attr, 3569 const struct rte_flow_item items[], 3570 const struct rte_flow_action actions[], 3571 bool external, struct rte_flow_error *error); 3572 3573 static void 3574 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3575 uint32_t flow_idx); 3576 3577 int 3578 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused, 3579 struct mlx5_hlist_entry *entry, 3580 uint64_t key, void *cb_ctx __rte_unused) 3581 { 3582 struct mlx5_flow_mreg_copy_resource *mcp_res = 3583 container_of(entry, typeof(*mcp_res), hlist_ent); 3584 3585 return mcp_res->mark_id != key; 3586 } 3587 3588 struct mlx5_hlist_entry * 3589 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 3590 void *cb_ctx) 3591 { 3592 struct rte_eth_dev *dev = list->ctx; 3593 struct mlx5_priv *priv = dev->data->dev_private; 3594 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3595 struct mlx5_flow_mreg_copy_resource *mcp_res; 3596 struct rte_flow_error *error = ctx->error; 3597 uint32_t idx = 0; 3598 int ret; 3599 uint32_t mark_id = key; 3600 struct rte_flow_attr attr = { 3601 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3602 .ingress = 1, 3603 }; 3604 struct mlx5_rte_flow_item_tag tag_spec = { 3605 .data = mark_id, 3606 }; 3607 struct rte_flow_item items[] = { 3608 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 3609 }; 3610 struct rte_flow_action_mark ftag = { 3611 .id = mark_id, 3612 }; 3613 struct mlx5_flow_action_copy_mreg cp_mreg = { 3614 .dst = REG_B, 3615 .src = REG_NON, 3616 }; 3617 struct rte_flow_action_jump jump = { 3618 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3619 }; 3620 struct rte_flow_action actions[] = { 3621 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 3622 }; 3623 3624 /* Fill the register fileds in the flow. */ 3625 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3626 if (ret < 0) 3627 return NULL; 3628 tag_spec.id = ret; 3629 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3630 if (ret < 0) 3631 return NULL; 3632 cp_mreg.src = ret; 3633 /* Provide the full width of FLAG specific value. */ 3634 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3635 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3636 /* Build a new flow. */ 3637 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3638 items[0] = (struct rte_flow_item){ 3639 .type = (enum rte_flow_item_type) 3640 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3641 .spec = &tag_spec, 3642 }; 3643 items[1] = (struct rte_flow_item){ 3644 .type = RTE_FLOW_ITEM_TYPE_END, 3645 }; 3646 actions[0] = (struct rte_flow_action){ 3647 .type = (enum rte_flow_action_type) 3648 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3649 .conf = &ftag, 3650 }; 3651 actions[1] = (struct rte_flow_action){ 3652 .type = (enum rte_flow_action_type) 3653 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3654 .conf = &cp_mreg, 3655 }; 3656 actions[2] = (struct rte_flow_action){ 3657 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3658 .conf = &jump, 3659 }; 3660 actions[3] = (struct rte_flow_action){ 3661 .type = RTE_FLOW_ACTION_TYPE_END, 3662 }; 3663 } else { 3664 /* Default rule, wildcard match. */ 3665 attr.priority = MLX5_FLOW_PRIO_RSVD; 3666 items[0] = (struct rte_flow_item){ 3667 .type = RTE_FLOW_ITEM_TYPE_END, 3668 }; 3669 actions[0] = (struct rte_flow_action){ 3670 .type = (enum rte_flow_action_type) 3671 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3672 .conf = &cp_mreg, 3673 }; 3674 actions[1] = (struct rte_flow_action){ 3675 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3676 .conf = &jump, 3677 }; 3678 actions[2] = (struct rte_flow_action){ 3679 .type = RTE_FLOW_ACTION_TYPE_END, 3680 }; 3681 } 3682 /* Build a new entry. */ 3683 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3684 if (!mcp_res) { 3685 rte_errno = ENOMEM; 3686 return NULL; 3687 } 3688 mcp_res->idx = idx; 3689 mcp_res->mark_id = mark_id; 3690 /* 3691 * The copy Flows are not included in any list. There 3692 * ones are referenced from other Flows and can not 3693 * be applied, removed, deleted in ardbitrary order 3694 * by list traversing. 3695 */ 3696 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3697 actions, false, error); 3698 if (!mcp_res->rix_flow) { 3699 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 3700 return NULL; 3701 } 3702 return &mcp_res->hlist_ent; 3703 } 3704 3705 /** 3706 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3707 * 3708 * As mark_id is unique, if there's already a registered flow for the mark_id, 3709 * return by increasing the reference counter of the resource. Otherwise, create 3710 * the resource (mcp_res) and flow. 3711 * 3712 * Flow looks like, 3713 * - If ingress port is ANY and reg_c[1] is mark_id, 3714 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3715 * 3716 * For default flow (zero mark_id), flow is like, 3717 * - If ingress port is ANY, 3718 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 3719 * 3720 * @param dev 3721 * Pointer to Ethernet device. 3722 * @param mark_id 3723 * ID of MARK action, zero means default flow for META. 3724 * @param[out] error 3725 * Perform verbose error reporting if not NULL. 3726 * 3727 * @return 3728 * Associated resource on success, NULL otherwise and rte_errno is set. 3729 */ 3730 static struct mlx5_flow_mreg_copy_resource * 3731 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 3732 struct rte_flow_error *error) 3733 { 3734 struct mlx5_priv *priv = dev->data->dev_private; 3735 struct mlx5_hlist_entry *entry; 3736 struct mlx5_flow_cb_ctx ctx = { 3737 .dev = dev, 3738 .error = error, 3739 }; 3740 3741 /* Check if already registered. */ 3742 MLX5_ASSERT(priv->mreg_cp_tbl); 3743 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 3744 if (!entry) 3745 return NULL; 3746 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 3747 hlist_ent); 3748 } 3749 3750 void 3751 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 3752 { 3753 struct mlx5_flow_mreg_copy_resource *mcp_res = 3754 container_of(entry, typeof(*mcp_res), hlist_ent); 3755 struct rte_eth_dev *dev = list->ctx; 3756 struct mlx5_priv *priv = dev->data->dev_private; 3757 3758 MLX5_ASSERT(mcp_res->rix_flow); 3759 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3760 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3761 } 3762 3763 /** 3764 * Release flow in RX_CP_TBL. 3765 * 3766 * @param dev 3767 * Pointer to Ethernet device. 3768 * @flow 3769 * Parent flow for wich copying is provided. 3770 */ 3771 static void 3772 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3773 struct rte_flow *flow) 3774 { 3775 struct mlx5_flow_mreg_copy_resource *mcp_res; 3776 struct mlx5_priv *priv = dev->data->dev_private; 3777 3778 if (!flow->rix_mreg_copy) 3779 return; 3780 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3781 flow->rix_mreg_copy); 3782 if (!mcp_res || !priv->mreg_cp_tbl) 3783 return; 3784 MLX5_ASSERT(mcp_res->rix_flow); 3785 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3786 flow->rix_mreg_copy = 0; 3787 } 3788 3789 /** 3790 * Remove the default copy action from RX_CP_TBL. 3791 * 3792 * This functions is called in the mlx5_dev_start(). No thread safe 3793 * is guaranteed. 3794 * 3795 * @param dev 3796 * Pointer to Ethernet device. 3797 */ 3798 static void 3799 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3800 { 3801 struct mlx5_hlist_entry *entry; 3802 struct mlx5_priv *priv = dev->data->dev_private; 3803 3804 /* Check if default flow is registered. */ 3805 if (!priv->mreg_cp_tbl) 3806 return; 3807 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 3808 MLX5_DEFAULT_COPY_ID, NULL); 3809 if (!entry) 3810 return; 3811 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 3812 } 3813 3814 /** 3815 * Add the default copy action in in RX_CP_TBL. 3816 * 3817 * This functions is called in the mlx5_dev_start(). No thread safe 3818 * is guaranteed. 3819 * 3820 * @param dev 3821 * Pointer to Ethernet device. 3822 * @param[out] error 3823 * Perform verbose error reporting if not NULL. 3824 * 3825 * @return 3826 * 0 for success, negative value otherwise and rte_errno is set. 3827 */ 3828 static int 3829 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3830 struct rte_flow_error *error) 3831 { 3832 struct mlx5_priv *priv = dev->data->dev_private; 3833 struct mlx5_flow_mreg_copy_resource *mcp_res; 3834 3835 /* Check whether extensive metadata feature is engaged. */ 3836 if (!priv->config.dv_flow_en || 3837 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3838 !mlx5_flow_ext_mreg_supported(dev) || 3839 !priv->sh->dv_regc0_mask) 3840 return 0; 3841 /* 3842 * Add default mreg copy flow may be called multiple time, but 3843 * only be called once in stop. Avoid register it twice. 3844 */ 3845 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 3846 return 0; 3847 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3848 if (!mcp_res) 3849 return -rte_errno; 3850 return 0; 3851 } 3852 3853 /** 3854 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3855 * 3856 * All the flow having Q/RSS action should be split by 3857 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3858 * performs the following, 3859 * - CQE->flow_tag := reg_c[1] (MARK) 3860 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3861 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3862 * but there should be a flow per each MARK ID set by MARK action. 3863 * 3864 * For the aforementioned reason, if there's a MARK action in flow's action 3865 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3866 * the MARK ID to CQE's flow_tag like, 3867 * - If reg_c[1] is mark_id, 3868 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3869 * 3870 * For SET_META action which stores value in reg_c[0], as the destination is 3871 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3872 * MARK ID means the default flow. The default flow looks like, 3873 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3874 * 3875 * @param dev 3876 * Pointer to Ethernet device. 3877 * @param flow 3878 * Pointer to flow structure. 3879 * @param[in] actions 3880 * Pointer to the list of actions. 3881 * @param[out] error 3882 * Perform verbose error reporting if not NULL. 3883 * 3884 * @return 3885 * 0 on success, negative value otherwise and rte_errno is set. 3886 */ 3887 static int 3888 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3889 struct rte_flow *flow, 3890 const struct rte_flow_action *actions, 3891 struct rte_flow_error *error) 3892 { 3893 struct mlx5_priv *priv = dev->data->dev_private; 3894 struct mlx5_dev_config *config = &priv->config; 3895 struct mlx5_flow_mreg_copy_resource *mcp_res; 3896 const struct rte_flow_action_mark *mark; 3897 3898 /* Check whether extensive metadata feature is engaged. */ 3899 if (!config->dv_flow_en || 3900 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3901 !mlx5_flow_ext_mreg_supported(dev) || 3902 !priv->sh->dv_regc0_mask) 3903 return 0; 3904 /* Find MARK action. */ 3905 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3906 switch (actions->type) { 3907 case RTE_FLOW_ACTION_TYPE_FLAG: 3908 mcp_res = flow_mreg_add_copy_action 3909 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3910 if (!mcp_res) 3911 return -rte_errno; 3912 flow->rix_mreg_copy = mcp_res->idx; 3913 return 0; 3914 case RTE_FLOW_ACTION_TYPE_MARK: 3915 mark = (const struct rte_flow_action_mark *) 3916 actions->conf; 3917 mcp_res = 3918 flow_mreg_add_copy_action(dev, mark->id, error); 3919 if (!mcp_res) 3920 return -rte_errno; 3921 flow->rix_mreg_copy = mcp_res->idx; 3922 return 0; 3923 default: 3924 break; 3925 } 3926 } 3927 return 0; 3928 } 3929 3930 #define MLX5_MAX_SPLIT_ACTIONS 24 3931 #define MLX5_MAX_SPLIT_ITEMS 24 3932 3933 /** 3934 * Split the hairpin flow. 3935 * Since HW can't support encap and push-vlan on Rx, we move these 3936 * actions to Tx. 3937 * If the count action is after the encap then we also 3938 * move the count action. in this case the count will also measure 3939 * the outer bytes. 3940 * 3941 * @param dev 3942 * Pointer to Ethernet device. 3943 * @param[in] actions 3944 * Associated actions (list terminated by the END action). 3945 * @param[out] actions_rx 3946 * Rx flow actions. 3947 * @param[out] actions_tx 3948 * Tx flow actions.. 3949 * @param[out] pattern_tx 3950 * The pattern items for the Tx flow. 3951 * @param[out] flow_id 3952 * The flow ID connected to this flow. 3953 * 3954 * @return 3955 * 0 on success. 3956 */ 3957 static int 3958 flow_hairpin_split(struct rte_eth_dev *dev, 3959 const struct rte_flow_action actions[], 3960 struct rte_flow_action actions_rx[], 3961 struct rte_flow_action actions_tx[], 3962 struct rte_flow_item pattern_tx[], 3963 uint32_t flow_id) 3964 { 3965 const struct rte_flow_action_raw_encap *raw_encap; 3966 const struct rte_flow_action_raw_decap *raw_decap; 3967 struct mlx5_rte_flow_action_set_tag *set_tag; 3968 struct rte_flow_action *tag_action; 3969 struct mlx5_rte_flow_item_tag *tag_item; 3970 struct rte_flow_item *item; 3971 char *addr; 3972 int encap = 0; 3973 3974 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3975 switch (actions->type) { 3976 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3977 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3978 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3979 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3980 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3981 rte_memcpy(actions_tx, actions, 3982 sizeof(struct rte_flow_action)); 3983 actions_tx++; 3984 break; 3985 case RTE_FLOW_ACTION_TYPE_COUNT: 3986 if (encap) { 3987 rte_memcpy(actions_tx, actions, 3988 sizeof(struct rte_flow_action)); 3989 actions_tx++; 3990 } else { 3991 rte_memcpy(actions_rx, actions, 3992 sizeof(struct rte_flow_action)); 3993 actions_rx++; 3994 } 3995 break; 3996 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3997 raw_encap = actions->conf; 3998 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 3999 memcpy(actions_tx, actions, 4000 sizeof(struct rte_flow_action)); 4001 actions_tx++; 4002 encap = 1; 4003 } else { 4004 rte_memcpy(actions_rx, actions, 4005 sizeof(struct rte_flow_action)); 4006 actions_rx++; 4007 } 4008 break; 4009 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4010 raw_decap = actions->conf; 4011 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4012 memcpy(actions_tx, actions, 4013 sizeof(struct rte_flow_action)); 4014 actions_tx++; 4015 } else { 4016 rte_memcpy(actions_rx, actions, 4017 sizeof(struct rte_flow_action)); 4018 actions_rx++; 4019 } 4020 break; 4021 default: 4022 rte_memcpy(actions_rx, actions, 4023 sizeof(struct rte_flow_action)); 4024 actions_rx++; 4025 break; 4026 } 4027 } 4028 /* Add set meta action and end action for the Rx flow. */ 4029 tag_action = actions_rx; 4030 tag_action->type = (enum rte_flow_action_type) 4031 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4032 actions_rx++; 4033 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4034 actions_rx++; 4035 set_tag = (void *)actions_rx; 4036 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 4037 MLX5_ASSERT(set_tag->id > REG_NON); 4038 set_tag->data = flow_id; 4039 tag_action->conf = set_tag; 4040 /* Create Tx item list. */ 4041 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4042 addr = (void *)&pattern_tx[2]; 4043 item = pattern_tx; 4044 item->type = (enum rte_flow_item_type) 4045 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4046 tag_item = (void *)addr; 4047 tag_item->data = flow_id; 4048 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4049 MLX5_ASSERT(set_tag->id > REG_NON); 4050 item->spec = tag_item; 4051 addr += sizeof(struct mlx5_rte_flow_item_tag); 4052 tag_item = (void *)addr; 4053 tag_item->data = UINT32_MAX; 4054 tag_item->id = UINT16_MAX; 4055 item->mask = tag_item; 4056 item->last = NULL; 4057 item++; 4058 item->type = RTE_FLOW_ITEM_TYPE_END; 4059 return 0; 4060 } 4061 4062 /** 4063 * The last stage of splitting chain, just creates the subflow 4064 * without any modification. 4065 * 4066 * @param[in] dev 4067 * Pointer to Ethernet device. 4068 * @param[in] flow 4069 * Parent flow structure pointer. 4070 * @param[in, out] sub_flow 4071 * Pointer to return the created subflow, may be NULL. 4072 * @param[in] attr 4073 * Flow rule attributes. 4074 * @param[in] items 4075 * Pattern specification (list terminated by the END pattern item). 4076 * @param[in] actions 4077 * Associated actions (list terminated by the END action). 4078 * @param[in] flow_split_info 4079 * Pointer to flow split info structure. 4080 * @param[out] error 4081 * Perform verbose error reporting if not NULL. 4082 * @return 4083 * 0 on success, negative value otherwise 4084 */ 4085 static int 4086 flow_create_split_inner(struct rte_eth_dev *dev, 4087 struct rte_flow *flow, 4088 struct mlx5_flow **sub_flow, 4089 const struct rte_flow_attr *attr, 4090 const struct rte_flow_item items[], 4091 const struct rte_flow_action actions[], 4092 struct mlx5_flow_split_info *flow_split_info, 4093 struct rte_flow_error *error) 4094 { 4095 struct mlx5_flow *dev_flow; 4096 4097 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4098 flow_split_info->flow_idx, error); 4099 if (!dev_flow) 4100 return -rte_errno; 4101 dev_flow->flow = flow; 4102 dev_flow->external = flow_split_info->external; 4103 dev_flow->skip_scale = flow_split_info->skip_scale; 4104 /* Subflow object was created, we must include one in the list. */ 4105 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4106 dev_flow->handle, next); 4107 /* 4108 * If dev_flow is as one of the suffix flow, some actions in suffix 4109 * flow may need some user defined item layer flags, and pass the 4110 * Metadate rxq mark flag to suffix flow as well. 4111 */ 4112 if (flow_split_info->prefix_layers) 4113 dev_flow->handle->layers = flow_split_info->prefix_layers; 4114 if (flow_split_info->prefix_mark) 4115 dev_flow->handle->mark = 1; 4116 if (sub_flow) 4117 *sub_flow = dev_flow; 4118 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4119 } 4120 4121 /** 4122 * Split the meter flow. 4123 * 4124 * As meter flow will split to three sub flow, other than meter 4125 * action, the other actions make sense to only meter accepts 4126 * the packet. If it need to be dropped, no other additional 4127 * actions should be take. 4128 * 4129 * One kind of special action which decapsulates the L3 tunnel 4130 * header will be in the prefix sub flow, as not to take the 4131 * L3 tunnel header into account. 4132 * 4133 * @param dev 4134 * Pointer to Ethernet device. 4135 * @param[in] items 4136 * Pattern specification (list terminated by the END pattern item). 4137 * @param[out] sfx_items 4138 * Suffix flow match items (list terminated by the END pattern item). 4139 * @param[in] actions 4140 * Associated actions (list terminated by the END action). 4141 * @param[out] actions_sfx 4142 * Suffix flow actions. 4143 * @param[out] actions_pre 4144 * Prefix flow actions. 4145 * @param[out] pattern_sfx 4146 * The pattern items for the suffix flow. 4147 * @param[out] tag_sfx 4148 * Pointer to suffix flow tag. 4149 * 4150 * @return 4151 * 0 on success. 4152 */ 4153 static int 4154 flow_meter_split_prep(struct rte_eth_dev *dev, 4155 const struct rte_flow_item items[], 4156 struct rte_flow_item sfx_items[], 4157 const struct rte_flow_action actions[], 4158 struct rte_flow_action actions_sfx[], 4159 struct rte_flow_action actions_pre[]) 4160 { 4161 struct mlx5_priv *priv = dev->data->dev_private; 4162 struct rte_flow_action *tag_action = NULL; 4163 struct rte_flow_item *tag_item; 4164 struct mlx5_rte_flow_action_set_tag *set_tag; 4165 struct rte_flow_error error; 4166 const struct rte_flow_action_raw_encap *raw_encap; 4167 const struct rte_flow_action_raw_decap *raw_decap; 4168 struct mlx5_rte_flow_item_tag *tag_spec; 4169 struct mlx5_rte_flow_item_tag *tag_mask; 4170 uint32_t tag_id = 0; 4171 bool copy_vlan = false; 4172 4173 /* Prepare the actions for prefix and suffix flow. */ 4174 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4175 struct rte_flow_action **action_cur = NULL; 4176 4177 switch (actions->type) { 4178 case RTE_FLOW_ACTION_TYPE_METER: 4179 /* Add the extra tag action first. */ 4180 tag_action = actions_pre; 4181 tag_action->type = (enum rte_flow_action_type) 4182 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4183 actions_pre++; 4184 action_cur = &actions_pre; 4185 break; 4186 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4187 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4188 action_cur = &actions_pre; 4189 break; 4190 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4191 raw_encap = actions->conf; 4192 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4193 action_cur = &actions_pre; 4194 break; 4195 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4196 raw_decap = actions->conf; 4197 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4198 action_cur = &actions_pre; 4199 break; 4200 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4201 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4202 copy_vlan = true; 4203 break; 4204 default: 4205 break; 4206 } 4207 if (!action_cur) 4208 action_cur = &actions_sfx; 4209 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 4210 (*action_cur)++; 4211 } 4212 /* Add end action to the actions. */ 4213 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4214 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4215 actions_pre++; 4216 /* Set the tag. */ 4217 set_tag = (void *)actions_pre; 4218 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4219 mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 4220 &tag_id); 4221 if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { 4222 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", 4223 dev->data->port_id); 4224 mlx5_ipool_free(priv->sh->ipool 4225 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); 4226 return 0; 4227 } else if (!tag_id) { 4228 return 0; 4229 } 4230 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 4231 assert(tag_action); 4232 tag_action->conf = set_tag; 4233 /* Prepare the suffix subflow items. */ 4234 tag_item = sfx_items++; 4235 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4236 int item_type = items->type; 4237 4238 switch (item_type) { 4239 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4240 memcpy(sfx_items, items, sizeof(*sfx_items)); 4241 sfx_items++; 4242 break; 4243 case RTE_FLOW_ITEM_TYPE_VLAN: 4244 if (copy_vlan) { 4245 memcpy(sfx_items, items, sizeof(*sfx_items)); 4246 /* 4247 * Convert to internal match item, it is used 4248 * for vlan push and set vid. 4249 */ 4250 sfx_items->type = (enum rte_flow_item_type) 4251 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4252 sfx_items++; 4253 } 4254 break; 4255 default: 4256 break; 4257 } 4258 } 4259 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4260 sfx_items++; 4261 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4262 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 4263 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4264 tag_mask = tag_spec + 1; 4265 tag_mask->data = 0xffffff00; 4266 tag_item->type = (enum rte_flow_item_type) 4267 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4268 tag_item->spec = tag_spec; 4269 tag_item->last = NULL; 4270 tag_item->mask = tag_mask; 4271 return tag_id; 4272 } 4273 4274 /** 4275 * Split action list having QUEUE/RSS for metadata register copy. 4276 * 4277 * Once Q/RSS action is detected in user's action list, the flow action 4278 * should be split in order to copy metadata registers, which will happen in 4279 * RX_CP_TBL like, 4280 * - CQE->flow_tag := reg_c[1] (MARK) 4281 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4282 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4283 * This is because the last action of each flow must be a terminal action 4284 * (QUEUE, RSS or DROP). 4285 * 4286 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4287 * stored and kept in the mlx5_flow structure per each sub_flow. 4288 * 4289 * The Q/RSS action is replaced with, 4290 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4291 * And the following JUMP action is added at the end, 4292 * - JUMP, to RX_CP_TBL. 4293 * 4294 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4295 * flow_create_split_metadata() routine. The flow will look like, 4296 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4297 * 4298 * @param dev 4299 * Pointer to Ethernet device. 4300 * @param[out] split_actions 4301 * Pointer to store split actions to jump to CP_TBL. 4302 * @param[in] actions 4303 * Pointer to the list of original flow actions. 4304 * @param[in] qrss 4305 * Pointer to the Q/RSS action. 4306 * @param[in] actions_n 4307 * Number of original actions. 4308 * @param[out] error 4309 * Perform verbose error reporting if not NULL. 4310 * 4311 * @return 4312 * non-zero unique flow_id on success, otherwise 0 and 4313 * error/rte_error are set. 4314 */ 4315 static uint32_t 4316 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4317 struct rte_flow_action *split_actions, 4318 const struct rte_flow_action *actions, 4319 const struct rte_flow_action *qrss, 4320 int actions_n, struct rte_flow_error *error) 4321 { 4322 struct mlx5_priv *priv = dev->data->dev_private; 4323 struct mlx5_rte_flow_action_set_tag *set_tag; 4324 struct rte_flow_action_jump *jump; 4325 const int qrss_idx = qrss - actions; 4326 uint32_t flow_id = 0; 4327 int ret = 0; 4328 4329 /* 4330 * Given actions will be split 4331 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4332 * - Add jump to mreg CP_TBL. 4333 * As a result, there will be one more action. 4334 */ 4335 ++actions_n; 4336 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4337 set_tag = (void *)(split_actions + actions_n); 4338 /* 4339 * If tag action is not set to void(it means we are not the meter 4340 * suffix flow), add the tag action. Since meter suffix flow already 4341 * has the tag added. 4342 */ 4343 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4344 /* 4345 * Allocate the new subflow ID. This one is unique within 4346 * device and not shared with representors. Otherwise, 4347 * we would have to resolve multi-thread access synch 4348 * issue. Each flow on the shared device is appended 4349 * with source vport identifier, so the resulting 4350 * flows will be unique in the shared (by master and 4351 * representors) domain even if they have coinciding 4352 * IDs. 4353 */ 4354 mlx5_ipool_malloc(priv->sh->ipool 4355 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 4356 if (!flow_id) 4357 return rte_flow_error_set(error, ENOMEM, 4358 RTE_FLOW_ERROR_TYPE_ACTION, 4359 NULL, "can't allocate id " 4360 "for split Q/RSS subflow"); 4361 /* Internal SET_TAG action to set flow ID. */ 4362 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4363 .data = flow_id, 4364 }; 4365 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4366 if (ret < 0) 4367 return ret; 4368 set_tag->id = ret; 4369 /* Construct new actions array. */ 4370 /* Replace QUEUE/RSS action. */ 4371 split_actions[qrss_idx] = (struct rte_flow_action){ 4372 .type = (enum rte_flow_action_type) 4373 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4374 .conf = set_tag, 4375 }; 4376 } 4377 /* JUMP action to jump to mreg copy table (CP_TBL). */ 4378 jump = (void *)(set_tag + 1); 4379 *jump = (struct rte_flow_action_jump){ 4380 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4381 }; 4382 split_actions[actions_n - 2] = (struct rte_flow_action){ 4383 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4384 .conf = jump, 4385 }; 4386 split_actions[actions_n - 1] = (struct rte_flow_action){ 4387 .type = RTE_FLOW_ACTION_TYPE_END, 4388 }; 4389 return flow_id; 4390 } 4391 4392 /** 4393 * Extend the given action list for Tx metadata copy. 4394 * 4395 * Copy the given action list to the ext_actions and add flow metadata register 4396 * copy action in order to copy reg_a set by WQE to reg_c[0]. 4397 * 4398 * @param[out] ext_actions 4399 * Pointer to the extended action list. 4400 * @param[in] actions 4401 * Pointer to the list of actions. 4402 * @param[in] actions_n 4403 * Number of actions in the list. 4404 * @param[out] error 4405 * Perform verbose error reporting if not NULL. 4406 * @param[in] encap_idx 4407 * The encap action inndex. 4408 * 4409 * @return 4410 * 0 on success, negative value otherwise 4411 */ 4412 static int 4413 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 4414 struct rte_flow_action *ext_actions, 4415 const struct rte_flow_action *actions, 4416 int actions_n, struct rte_flow_error *error, 4417 int encap_idx) 4418 { 4419 struct mlx5_flow_action_copy_mreg *cp_mreg = 4420 (struct mlx5_flow_action_copy_mreg *) 4421 (ext_actions + actions_n + 1); 4422 int ret; 4423 4424 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4425 if (ret < 0) 4426 return ret; 4427 cp_mreg->dst = ret; 4428 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 4429 if (ret < 0) 4430 return ret; 4431 cp_mreg->src = ret; 4432 if (encap_idx != 0) 4433 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 4434 if (encap_idx == actions_n - 1) { 4435 ext_actions[actions_n - 1] = (struct rte_flow_action){ 4436 .type = (enum rte_flow_action_type) 4437 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4438 .conf = cp_mreg, 4439 }; 4440 ext_actions[actions_n] = (struct rte_flow_action){ 4441 .type = RTE_FLOW_ACTION_TYPE_END, 4442 }; 4443 } else { 4444 ext_actions[encap_idx] = (struct rte_flow_action){ 4445 .type = (enum rte_flow_action_type) 4446 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4447 .conf = cp_mreg, 4448 }; 4449 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 4450 sizeof(*ext_actions) * (actions_n - encap_idx)); 4451 } 4452 return 0; 4453 } 4454 4455 /** 4456 * Check the match action from the action list. 4457 * 4458 * @param[in] actions 4459 * Pointer to the list of actions. 4460 * @param[in] attr 4461 * Flow rule attributes. 4462 * @param[in] action 4463 * The action to be check if exist. 4464 * @param[out] match_action_pos 4465 * Pointer to the position of the matched action if exists, otherwise is -1. 4466 * @param[out] qrss_action_pos 4467 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 4468 * 4469 * @return 4470 * > 0 the total number of actions. 4471 * 0 if not found match action in action list. 4472 */ 4473 static int 4474 flow_check_match_action(const struct rte_flow_action actions[], 4475 const struct rte_flow_attr *attr, 4476 enum rte_flow_action_type action, 4477 int *match_action_pos, int *qrss_action_pos) 4478 { 4479 const struct rte_flow_action_sample *sample; 4480 int actions_n = 0; 4481 int jump_flag = 0; 4482 uint32_t ratio = 0; 4483 int sub_type = 0; 4484 int flag = 0; 4485 4486 *match_action_pos = -1; 4487 *qrss_action_pos = -1; 4488 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4489 if (actions->type == action) { 4490 flag = 1; 4491 *match_action_pos = actions_n; 4492 } 4493 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE || 4494 actions->type == RTE_FLOW_ACTION_TYPE_RSS) 4495 *qrss_action_pos = actions_n; 4496 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP) 4497 jump_flag = 1; 4498 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) { 4499 sample = actions->conf; 4500 ratio = sample->ratio; 4501 sub_type = ((const struct rte_flow_action *) 4502 (sample->actions))->type; 4503 } 4504 actions_n++; 4505 } 4506 if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) { 4507 if (ratio == 1) { 4508 /* JUMP Action not support for Mirroring; 4509 * Mirroring support multi-destination; 4510 */ 4511 if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END) 4512 flag = 0; 4513 } 4514 } 4515 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4516 return flag ? actions_n + 1 : 0; 4517 } 4518 4519 #define SAMPLE_SUFFIX_ITEM 2 4520 4521 /** 4522 * Split the sample flow. 4523 * 4524 * As sample flow will split to two sub flow, sample flow with 4525 * sample action, the other actions will move to new suffix flow. 4526 * 4527 * Also add unique tag id with tag action in the sample flow, 4528 * the same tag id will be as match in the suffix flow. 4529 * 4530 * @param dev 4531 * Pointer to Ethernet device. 4532 * @param[in] fdb_tx 4533 * FDB egress flow flag. 4534 * @param[out] sfx_items 4535 * Suffix flow match items (list terminated by the END pattern item). 4536 * @param[in] actions 4537 * Associated actions (list terminated by the END action). 4538 * @param[out] actions_sfx 4539 * Suffix flow actions. 4540 * @param[out] actions_pre 4541 * Prefix flow actions. 4542 * @param[in] actions_n 4543 * The total number of actions. 4544 * @param[in] sample_action_pos 4545 * The sample action position. 4546 * @param[in] qrss_action_pos 4547 * The Queue/RSS action position. 4548 * @param[out] error 4549 * Perform verbose error reporting if not NULL. 4550 * 4551 * @return 4552 * 0 on success, or unique flow_id, a negative errno value 4553 * otherwise and rte_errno is set. 4554 */ 4555 static int 4556 flow_sample_split_prep(struct rte_eth_dev *dev, 4557 uint32_t fdb_tx, 4558 struct rte_flow_item sfx_items[], 4559 const struct rte_flow_action actions[], 4560 struct rte_flow_action actions_sfx[], 4561 struct rte_flow_action actions_pre[], 4562 int actions_n, 4563 int sample_action_pos, 4564 int qrss_action_pos, 4565 struct rte_flow_error *error) 4566 { 4567 struct mlx5_priv *priv = dev->data->dev_private; 4568 struct mlx5_rte_flow_action_set_tag *set_tag; 4569 struct mlx5_rte_flow_item_tag *tag_spec; 4570 struct mlx5_rte_flow_item_tag *tag_mask; 4571 uint32_t tag_id = 0; 4572 int index; 4573 int ret; 4574 4575 if (sample_action_pos < 0) 4576 return rte_flow_error_set(error, EINVAL, 4577 RTE_FLOW_ERROR_TYPE_ACTION, 4578 NULL, "invalid position of sample " 4579 "action in list"); 4580 if (!fdb_tx) { 4581 /* Prepare the prefix tag action. */ 4582 set_tag = (void *)(actions_pre + actions_n + 1); 4583 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 4584 if (ret < 0) 4585 return ret; 4586 set_tag->id = ret; 4587 mlx5_ipool_malloc(priv->sh->ipool 4588 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 4589 set_tag->data = tag_id; 4590 /* Prepare the suffix subflow items. */ 4591 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 4592 tag_spec->data = tag_id; 4593 tag_spec->id = set_tag->id; 4594 tag_mask = tag_spec + 1; 4595 tag_mask->data = UINT32_MAX; 4596 sfx_items[0] = (struct rte_flow_item){ 4597 .type = (enum rte_flow_item_type) 4598 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4599 .spec = tag_spec, 4600 .last = NULL, 4601 .mask = tag_mask, 4602 }; 4603 sfx_items[1] = (struct rte_flow_item){ 4604 .type = (enum rte_flow_item_type) 4605 RTE_FLOW_ITEM_TYPE_END, 4606 }; 4607 } 4608 /* Prepare the actions for prefix and suffix flow. */ 4609 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 4610 index = qrss_action_pos; 4611 /* Put the preceding the Queue/RSS action into prefix flow. */ 4612 if (index != 0) 4613 memcpy(actions_pre, actions, 4614 sizeof(struct rte_flow_action) * index); 4615 /* Put others preceding the sample action into prefix flow. */ 4616 if (sample_action_pos > index + 1) 4617 memcpy(actions_pre + index, actions + index + 1, 4618 sizeof(struct rte_flow_action) * 4619 (sample_action_pos - index - 1)); 4620 index = sample_action_pos - 1; 4621 /* Put Queue/RSS action into Suffix flow. */ 4622 memcpy(actions_sfx, actions + qrss_action_pos, 4623 sizeof(struct rte_flow_action)); 4624 actions_sfx++; 4625 } else { 4626 index = sample_action_pos; 4627 if (index != 0) 4628 memcpy(actions_pre, actions, 4629 sizeof(struct rte_flow_action) * index); 4630 } 4631 /* Add the extra tag action for NIC-RX and E-Switch ingress. */ 4632 if (!fdb_tx) { 4633 actions_pre[index++] = 4634 (struct rte_flow_action){ 4635 .type = (enum rte_flow_action_type) 4636 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4637 .conf = set_tag, 4638 }; 4639 } 4640 memcpy(actions_pre + index, actions + sample_action_pos, 4641 sizeof(struct rte_flow_action)); 4642 index += 1; 4643 actions_pre[index] = (struct rte_flow_action){ 4644 .type = (enum rte_flow_action_type) 4645 RTE_FLOW_ACTION_TYPE_END, 4646 }; 4647 /* Put the actions after sample into Suffix flow. */ 4648 memcpy(actions_sfx, actions + sample_action_pos + 1, 4649 sizeof(struct rte_flow_action) * 4650 (actions_n - sample_action_pos - 1)); 4651 return tag_id; 4652 } 4653 4654 /** 4655 * The splitting for metadata feature. 4656 * 4657 * - Q/RSS action on NIC Rx should be split in order to pass by 4658 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4659 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4660 * 4661 * - All the actions on NIC Tx should have a mreg copy action to 4662 * copy reg_a from WQE to reg_c[0]. 4663 * 4664 * @param dev 4665 * Pointer to Ethernet device. 4666 * @param[in] flow 4667 * Parent flow structure pointer. 4668 * @param[in] attr 4669 * Flow rule attributes. 4670 * @param[in] items 4671 * Pattern specification (list terminated by the END pattern item). 4672 * @param[in] actions 4673 * Associated actions (list terminated by the END action). 4674 * @param[in] flow_split_info 4675 * Pointer to flow split info structure. 4676 * @param[out] error 4677 * Perform verbose error reporting if not NULL. 4678 * @return 4679 * 0 on success, negative value otherwise 4680 */ 4681 static int 4682 flow_create_split_metadata(struct rte_eth_dev *dev, 4683 struct rte_flow *flow, 4684 const struct rte_flow_attr *attr, 4685 const struct rte_flow_item items[], 4686 const struct rte_flow_action actions[], 4687 struct mlx5_flow_split_info *flow_split_info, 4688 struct rte_flow_error *error) 4689 { 4690 struct mlx5_priv *priv = dev->data->dev_private; 4691 struct mlx5_dev_config *config = &priv->config; 4692 const struct rte_flow_action *qrss = NULL; 4693 struct rte_flow_action *ext_actions = NULL; 4694 struct mlx5_flow *dev_flow = NULL; 4695 uint32_t qrss_id = 0; 4696 int mtr_sfx = 0; 4697 size_t act_size; 4698 int actions_n; 4699 int encap_idx; 4700 int ret; 4701 4702 /* Check whether extensive metadata feature is engaged. */ 4703 if (!config->dv_flow_en || 4704 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4705 !mlx5_flow_ext_mreg_supported(dev)) 4706 return flow_create_split_inner(dev, flow, NULL, attr, items, 4707 actions, flow_split_info, error); 4708 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 4709 &encap_idx); 4710 if (qrss) { 4711 /* Exclude hairpin flows from splitting. */ 4712 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 4713 const struct rte_flow_action_queue *queue; 4714 4715 queue = qrss->conf; 4716 if (mlx5_rxq_get_type(dev, queue->index) == 4717 MLX5_RXQ_TYPE_HAIRPIN) 4718 qrss = NULL; 4719 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 4720 const struct rte_flow_action_rss *rss; 4721 4722 rss = qrss->conf; 4723 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 4724 MLX5_RXQ_TYPE_HAIRPIN) 4725 qrss = NULL; 4726 } 4727 } 4728 if (qrss) { 4729 /* Check if it is in meter suffix table. */ 4730 mtr_sfx = attr->group == (attr->transfer ? 4731 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4732 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 4733 /* 4734 * Q/RSS action on NIC Rx should be split in order to pass by 4735 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4736 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4737 */ 4738 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4739 sizeof(struct rte_flow_action_set_tag) + 4740 sizeof(struct rte_flow_action_jump); 4741 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4742 SOCKET_ID_ANY); 4743 if (!ext_actions) 4744 return rte_flow_error_set(error, ENOMEM, 4745 RTE_FLOW_ERROR_TYPE_ACTION, 4746 NULL, "no memory to split " 4747 "metadata flow"); 4748 /* 4749 * If we are the suffix flow of meter, tag already exist. 4750 * Set the tag action to void. 4751 */ 4752 if (mtr_sfx) 4753 ext_actions[qrss - actions].type = 4754 RTE_FLOW_ACTION_TYPE_VOID; 4755 else 4756 ext_actions[qrss - actions].type = 4757 (enum rte_flow_action_type) 4758 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4759 /* 4760 * Create the new actions list with removed Q/RSS action 4761 * and appended set tag and jump to register copy table 4762 * (RX_CP_TBL). We should preallocate unique tag ID here 4763 * in advance, because it is needed for set tag action. 4764 */ 4765 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 4766 qrss, actions_n, error); 4767 if (!mtr_sfx && !qrss_id) { 4768 ret = -rte_errno; 4769 goto exit; 4770 } 4771 } else if (attr->egress && !attr->transfer) { 4772 /* 4773 * All the actions on NIC Tx should have a metadata register 4774 * copy action to copy reg_a from WQE to reg_c[meta] 4775 */ 4776 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4777 sizeof(struct mlx5_flow_action_copy_mreg); 4778 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4779 SOCKET_ID_ANY); 4780 if (!ext_actions) 4781 return rte_flow_error_set(error, ENOMEM, 4782 RTE_FLOW_ERROR_TYPE_ACTION, 4783 NULL, "no memory to split " 4784 "metadata flow"); 4785 /* Create the action list appended with copy register. */ 4786 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 4787 actions_n, error, encap_idx); 4788 if (ret < 0) 4789 goto exit; 4790 } 4791 /* Add the unmodified original or prefix subflow. */ 4792 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 4793 items, ext_actions ? ext_actions : 4794 actions, flow_split_info, error); 4795 if (ret < 0) 4796 goto exit; 4797 MLX5_ASSERT(dev_flow); 4798 if (qrss) { 4799 const struct rte_flow_attr q_attr = { 4800 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4801 .ingress = 1, 4802 }; 4803 /* Internal PMD action to set register. */ 4804 struct mlx5_rte_flow_item_tag q_tag_spec = { 4805 .data = qrss_id, 4806 .id = REG_NON, 4807 }; 4808 struct rte_flow_item q_items[] = { 4809 { 4810 .type = (enum rte_flow_item_type) 4811 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4812 .spec = &q_tag_spec, 4813 .last = NULL, 4814 .mask = NULL, 4815 }, 4816 { 4817 .type = RTE_FLOW_ITEM_TYPE_END, 4818 }, 4819 }; 4820 struct rte_flow_action q_actions[] = { 4821 { 4822 .type = qrss->type, 4823 .conf = qrss->conf, 4824 }, 4825 { 4826 .type = RTE_FLOW_ACTION_TYPE_END, 4827 }, 4828 }; 4829 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4830 4831 /* 4832 * Configure the tag item only if there is no meter subflow. 4833 * Since tag is already marked in the meter suffix subflow 4834 * we can just use the meter suffix items as is. 4835 */ 4836 if (qrss_id) { 4837 /* Not meter subflow. */ 4838 MLX5_ASSERT(!mtr_sfx); 4839 /* 4840 * Put unique id in prefix flow due to it is destroyed 4841 * after suffix flow and id will be freed after there 4842 * is no actual flows with this id and identifier 4843 * reallocation becomes possible (for example, for 4844 * other flows in other threads). 4845 */ 4846 dev_flow->handle->split_flow_id = qrss_id; 4847 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4848 error); 4849 if (ret < 0) 4850 goto exit; 4851 q_tag_spec.id = ret; 4852 } 4853 dev_flow = NULL; 4854 /* Add suffix subflow to execute Q/RSS. */ 4855 flow_split_info->prefix_layers = layers; 4856 flow_split_info->prefix_mark = 0; 4857 ret = flow_create_split_inner(dev, flow, &dev_flow, 4858 &q_attr, mtr_sfx ? items : 4859 q_items, q_actions, 4860 flow_split_info, error); 4861 if (ret < 0) 4862 goto exit; 4863 /* qrss ID should be freed if failed. */ 4864 qrss_id = 0; 4865 MLX5_ASSERT(dev_flow); 4866 } 4867 4868 exit: 4869 /* 4870 * We do not destroy the partially created sub_flows in case of error. 4871 * These ones are included into parent flow list and will be destroyed 4872 * by flow_drv_destroy. 4873 */ 4874 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 4875 qrss_id); 4876 mlx5_free(ext_actions); 4877 return ret; 4878 } 4879 4880 /** 4881 * The splitting for meter feature. 4882 * 4883 * - The meter flow will be split to two flows as prefix and 4884 * suffix flow. The packets make sense only it pass the prefix 4885 * meter action. 4886 * 4887 * - Reg_C_5 is used for the packet to match betweend prefix and 4888 * suffix flow. 4889 * 4890 * @param dev 4891 * Pointer to Ethernet device. 4892 * @param[in] flow 4893 * Parent flow structure pointer. 4894 * @param[in] attr 4895 * Flow rule attributes. 4896 * @param[in] items 4897 * Pattern specification (list terminated by the END pattern item). 4898 * @param[in] actions 4899 * Associated actions (list terminated by the END action). 4900 * @param[in] flow_split_info 4901 * Pointer to flow split info structure. 4902 * @param[out] error 4903 * Perform verbose error reporting if not NULL. 4904 * @return 4905 * 0 on success, negative value otherwise 4906 */ 4907 static int 4908 flow_create_split_meter(struct rte_eth_dev *dev, 4909 struct rte_flow *flow, 4910 const struct rte_flow_attr *attr, 4911 const struct rte_flow_item items[], 4912 const struct rte_flow_action actions[], 4913 struct mlx5_flow_split_info *flow_split_info, 4914 struct rte_flow_error *error) 4915 { 4916 struct mlx5_priv *priv = dev->data->dev_private; 4917 struct rte_flow_action *sfx_actions = NULL; 4918 struct rte_flow_action *pre_actions = NULL; 4919 struct rte_flow_item *sfx_items = NULL; 4920 struct mlx5_flow *dev_flow = NULL; 4921 struct rte_flow_attr sfx_attr = *attr; 4922 uint32_t mtr = 0; 4923 uint32_t mtr_tag_id = 0; 4924 size_t act_size; 4925 size_t item_size; 4926 int actions_n = 0; 4927 int ret; 4928 4929 if (priv->mtr_en) 4930 actions_n = flow_check_meter_action(actions, &mtr); 4931 if (mtr) { 4932 /* The five prefix actions: meter, decap, encap, tag, end. */ 4933 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4934 sizeof(struct mlx5_rte_flow_action_set_tag); 4935 /* tag, vlan, port id, end. */ 4936 #define METER_SUFFIX_ITEM 4 4937 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4938 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4939 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 4940 0, SOCKET_ID_ANY); 4941 if (!sfx_actions) 4942 return rte_flow_error_set(error, ENOMEM, 4943 RTE_FLOW_ERROR_TYPE_ACTION, 4944 NULL, "no memory to split " 4945 "meter flow"); 4946 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4947 act_size); 4948 pre_actions = sfx_actions + actions_n; 4949 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4950 actions, sfx_actions, 4951 pre_actions); 4952 if (!mtr_tag_id) { 4953 ret = -rte_errno; 4954 goto exit; 4955 } 4956 /* Add the prefix subflow. */ 4957 flow_split_info->prefix_mark = 0; 4958 ret = flow_create_split_inner(dev, flow, &dev_flow, 4959 attr, items, pre_actions, 4960 flow_split_info, error); 4961 if (ret) { 4962 ret = -rte_errno; 4963 goto exit; 4964 } 4965 dev_flow->handle->split_flow_id = mtr_tag_id; 4966 /* Setting the sfx group atrr. */ 4967 sfx_attr.group = sfx_attr.transfer ? 4968 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4969 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4970 flow_split_info->prefix_layers = 4971 flow_get_prefix_layer_flags(dev_flow); 4972 flow_split_info->prefix_mark = dev_flow->handle->mark; 4973 } 4974 /* Add the prefix subflow. */ 4975 ret = flow_create_split_metadata(dev, flow, 4976 &sfx_attr, sfx_items ? 4977 sfx_items : items, 4978 sfx_actions ? sfx_actions : actions, 4979 flow_split_info, error); 4980 exit: 4981 if (sfx_actions) 4982 mlx5_free(sfx_actions); 4983 return ret; 4984 } 4985 4986 /** 4987 * The splitting for sample feature. 4988 * 4989 * Once Sample action is detected in the action list, the flow actions should 4990 * be split into prefix sub flow and suffix sub flow. 4991 * 4992 * The original items remain in the prefix sub flow, all actions preceding the 4993 * sample action and the sample action itself will be copied to the prefix 4994 * sub flow, the actions following the sample action will be copied to the 4995 * suffix sub flow, Queue action always be located in the suffix sub flow. 4996 * 4997 * In order to make the packet from prefix sub flow matches with suffix sub 4998 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 4999 * flow uses tag item with the unique flow id. 5000 * 5001 * @param dev 5002 * Pointer to Ethernet device. 5003 * @param[in] flow 5004 * Parent flow structure pointer. 5005 * @param[in] attr 5006 * Flow rule attributes. 5007 * @param[in] items 5008 * Pattern specification (list terminated by the END pattern item). 5009 * @param[in] actions 5010 * Associated actions (list terminated by the END action). 5011 * @param[in] flow_split_info 5012 * Pointer to flow split info structure. 5013 * @param[out] error 5014 * Perform verbose error reporting if not NULL. 5015 * @return 5016 * 0 on success, negative value otherwise 5017 */ 5018 static int 5019 flow_create_split_sample(struct rte_eth_dev *dev, 5020 struct rte_flow *flow, 5021 const struct rte_flow_attr *attr, 5022 const struct rte_flow_item items[], 5023 const struct rte_flow_action actions[], 5024 struct mlx5_flow_split_info *flow_split_info, 5025 struct rte_flow_error *error) 5026 { 5027 struct mlx5_priv *priv = dev->data->dev_private; 5028 struct rte_flow_action *sfx_actions = NULL; 5029 struct rte_flow_action *pre_actions = NULL; 5030 struct rte_flow_item *sfx_items = NULL; 5031 struct mlx5_flow *dev_flow = NULL; 5032 struct rte_flow_attr sfx_attr = *attr; 5033 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5034 struct mlx5_flow_dv_sample_resource *sample_res; 5035 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5036 struct mlx5_flow_tbl_resource *sfx_tbl; 5037 #endif 5038 size_t act_size; 5039 size_t item_size; 5040 uint32_t fdb_tx = 0; 5041 int32_t tag_id = 0; 5042 int actions_n = 0; 5043 int sample_action_pos; 5044 int qrss_action_pos; 5045 int ret = 0; 5046 5047 if (priv->sampler_en) 5048 actions_n = flow_check_match_action(actions, attr, 5049 RTE_FLOW_ACTION_TYPE_SAMPLE, 5050 &sample_action_pos, &qrss_action_pos); 5051 if (actions_n) { 5052 /* The prefix actions must includes sample, tag, end. */ 5053 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5054 + sizeof(struct mlx5_rte_flow_action_set_tag); 5055 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5056 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5057 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5058 item_size), 0, SOCKET_ID_ANY); 5059 if (!sfx_actions) 5060 return rte_flow_error_set(error, ENOMEM, 5061 RTE_FLOW_ERROR_TYPE_ACTION, 5062 NULL, "no memory to split " 5063 "sample flow"); 5064 /* The representor_id is -1 for uplink. */ 5065 fdb_tx = (attr->transfer && priv->representor_id != -1); 5066 if (!fdb_tx) 5067 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5068 + act_size); 5069 pre_actions = sfx_actions + actions_n; 5070 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items, 5071 actions, sfx_actions, 5072 pre_actions, actions_n, 5073 sample_action_pos, 5074 qrss_action_pos, error); 5075 if (tag_id < 0 || (!fdb_tx && !tag_id)) { 5076 ret = -rte_errno; 5077 goto exit; 5078 } 5079 /* Add the prefix subflow. */ 5080 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5081 items, pre_actions, 5082 flow_split_info, error); 5083 if (ret) { 5084 ret = -rte_errno; 5085 goto exit; 5086 } 5087 dev_flow->handle->split_flow_id = tag_id; 5088 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5089 /* Set the sfx group attr. */ 5090 sample_res = (struct mlx5_flow_dv_sample_resource *) 5091 dev_flow->dv.sample_res; 5092 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5093 sample_res->normal_path_tbl; 5094 sfx_tbl_data = container_of(sfx_tbl, 5095 struct mlx5_flow_tbl_data_entry, tbl); 5096 sfx_attr.group = sfx_attr.transfer ? 5097 (sfx_tbl_data->table_id - 1) : 5098 sfx_tbl_data->table_id; 5099 flow_split_info->prefix_layers = 5100 flow_get_prefix_layer_flags(dev_flow); 5101 flow_split_info->prefix_mark = dev_flow->handle->mark; 5102 /* Suffix group level already be scaled with factor, set 5103 * skip_scale to 1 to avoid scale again in translation. 5104 */ 5105 flow_split_info->skip_scale = 1; 5106 #endif 5107 } 5108 /* Add the suffix subflow. */ 5109 ret = flow_create_split_meter(dev, flow, &sfx_attr, 5110 sfx_items ? sfx_items : items, 5111 sfx_actions ? sfx_actions : actions, 5112 flow_split_info, error); 5113 exit: 5114 if (sfx_actions) 5115 mlx5_free(sfx_actions); 5116 return ret; 5117 } 5118 5119 /** 5120 * Split the flow to subflow set. The splitters might be linked 5121 * in the chain, like this: 5122 * flow_create_split_outer() calls: 5123 * flow_create_split_meter() calls: 5124 * flow_create_split_metadata(meter_subflow_0) calls: 5125 * flow_create_split_inner(metadata_subflow_0) 5126 * flow_create_split_inner(metadata_subflow_1) 5127 * flow_create_split_inner(metadata_subflow_2) 5128 * flow_create_split_metadata(meter_subflow_1) calls: 5129 * flow_create_split_inner(metadata_subflow_0) 5130 * flow_create_split_inner(metadata_subflow_1) 5131 * flow_create_split_inner(metadata_subflow_2) 5132 * 5133 * This provide flexible way to add new levels of flow splitting. 5134 * The all of successfully created subflows are included to the 5135 * parent flow dev_flow list. 5136 * 5137 * @param dev 5138 * Pointer to Ethernet device. 5139 * @param[in] flow 5140 * Parent flow structure pointer. 5141 * @param[in] attr 5142 * Flow rule attributes. 5143 * @param[in] items 5144 * Pattern specification (list terminated by the END pattern item). 5145 * @param[in] actions 5146 * Associated actions (list terminated by the END action). 5147 * @param[in] flow_split_info 5148 * Pointer to flow split info structure. 5149 * @param[out] error 5150 * Perform verbose error reporting if not NULL. 5151 * @return 5152 * 0 on success, negative value otherwise 5153 */ 5154 static int 5155 flow_create_split_outer(struct rte_eth_dev *dev, 5156 struct rte_flow *flow, 5157 const struct rte_flow_attr *attr, 5158 const struct rte_flow_item items[], 5159 const struct rte_flow_action actions[], 5160 struct mlx5_flow_split_info *flow_split_info, 5161 struct rte_flow_error *error) 5162 { 5163 int ret; 5164 5165 ret = flow_create_split_sample(dev, flow, attr, items, 5166 actions, flow_split_info, error); 5167 MLX5_ASSERT(ret <= 0); 5168 return ret; 5169 } 5170 5171 static struct mlx5_flow_tunnel * 5172 flow_tunnel_from_rule(struct rte_eth_dev *dev, 5173 const struct rte_flow_attr *attr, 5174 const struct rte_flow_item items[], 5175 const struct rte_flow_action actions[]) 5176 { 5177 struct mlx5_flow_tunnel *tunnel; 5178 5179 #pragma GCC diagnostic push 5180 #pragma GCC diagnostic ignored "-Wcast-qual" 5181 if (is_flow_tunnel_match_rule(dev, attr, items, actions)) 5182 tunnel = (struct mlx5_flow_tunnel *)items[0].spec; 5183 else if (is_flow_tunnel_steer_rule(dev, attr, items, actions)) 5184 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf; 5185 else 5186 tunnel = NULL; 5187 #pragma GCC diagnostic pop 5188 5189 return tunnel; 5190 } 5191 5192 /** 5193 * Adjust flow RSS workspace if needed. 5194 * 5195 * @param wks 5196 * Pointer to thread flow work space. 5197 * @param rss_desc 5198 * Pointer to RSS descriptor. 5199 * @param[in] nrssq_num 5200 * New RSS queue number. 5201 * 5202 * @return 5203 * 0 on success, -1 otherwise and rte_errno is set. 5204 */ 5205 static int 5206 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 5207 struct mlx5_flow_rss_desc *rss_desc, 5208 uint32_t nrssq_num) 5209 { 5210 if (likely(nrssq_num <= wks->rssq_num)) 5211 return 0; 5212 rss_desc->queue = realloc(rss_desc->queue, 5213 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 5214 if (!rss_desc->queue) { 5215 rte_errno = ENOMEM; 5216 return -1; 5217 } 5218 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 5219 return 0; 5220 } 5221 5222 /** 5223 * Create a flow and add it to @p list. 5224 * 5225 * @param dev 5226 * Pointer to Ethernet device. 5227 * @param list 5228 * Pointer to a TAILQ flow list. If this parameter NULL, 5229 * no list insertion occurred, flow is just created, 5230 * this is caller's responsibility to track the 5231 * created flow. 5232 * @param[in] attr 5233 * Flow rule attributes. 5234 * @param[in] items 5235 * Pattern specification (list terminated by the END pattern item). 5236 * @param[in] actions 5237 * Associated actions (list terminated by the END action). 5238 * @param[in] external 5239 * This flow rule is created by request external to PMD. 5240 * @param[out] error 5241 * Perform verbose error reporting if not NULL. 5242 * 5243 * @return 5244 * A flow index on success, 0 otherwise and rte_errno is set. 5245 */ 5246 static uint32_t 5247 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 5248 const struct rte_flow_attr *attr, 5249 const struct rte_flow_item items[], 5250 const struct rte_flow_action original_actions[], 5251 bool external, struct rte_flow_error *error) 5252 { 5253 struct mlx5_priv *priv = dev->data->dev_private; 5254 struct rte_flow *flow = NULL; 5255 struct mlx5_flow *dev_flow; 5256 const struct rte_flow_action_rss *rss; 5257 struct mlx5_translated_shared_action 5258 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5259 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5260 union { 5261 struct mlx5_flow_expand_rss buf; 5262 uint8_t buffer[2048]; 5263 } expand_buffer; 5264 union { 5265 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5266 uint8_t buffer[2048]; 5267 } actions_rx; 5268 union { 5269 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5270 uint8_t buffer[2048]; 5271 } actions_hairpin_tx; 5272 union { 5273 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 5274 uint8_t buffer[2048]; 5275 } items_tx; 5276 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 5277 struct mlx5_flow_rss_desc *rss_desc; 5278 const struct rte_flow_action *p_actions_rx; 5279 uint32_t i; 5280 uint32_t idx = 0; 5281 int hairpin_flow; 5282 struct rte_flow_attr attr_tx = { .priority = 0 }; 5283 const struct rte_flow_action *actions; 5284 struct rte_flow_action *translated_actions = NULL; 5285 struct mlx5_flow_tunnel *tunnel; 5286 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 5287 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 5288 struct mlx5_flow_split_info flow_split_info = { 5289 .external = !!external, 5290 .skip_scale = 0, 5291 .flow_idx = 0, 5292 .prefix_mark = 0, 5293 .prefix_layers = 0 5294 }; 5295 int ret; 5296 5297 MLX5_ASSERT(wks); 5298 rss_desc = &wks->rss_desc; 5299 ret = flow_shared_actions_translate(dev, original_actions, 5300 shared_actions, 5301 &shared_actions_n, 5302 &translated_actions, error); 5303 if (ret < 0) { 5304 MLX5_ASSERT(translated_actions == NULL); 5305 return 0; 5306 } 5307 actions = translated_actions ? translated_actions : original_actions; 5308 p_actions_rx = actions; 5309 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5310 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 5311 external, hairpin_flow, error); 5312 if (ret < 0) 5313 goto error_before_hairpin_split; 5314 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 5315 if (!flow) { 5316 rte_errno = ENOMEM; 5317 goto error_before_hairpin_split; 5318 } 5319 if (hairpin_flow > 0) { 5320 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 5321 rte_errno = EINVAL; 5322 goto error_before_hairpin_split; 5323 } 5324 flow_hairpin_split(dev, actions, actions_rx.actions, 5325 actions_hairpin_tx.actions, items_tx.items, 5326 idx); 5327 p_actions_rx = actions_rx.actions; 5328 } 5329 flow_split_info.flow_idx = idx; 5330 flow->drv_type = flow_get_drv_type(dev, attr); 5331 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 5332 flow->drv_type < MLX5_FLOW_TYPE_MAX); 5333 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 5334 rss = flow_get_rss_action(p_actions_rx); 5335 if (rss) { 5336 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 5337 return 0; 5338 /* 5339 * The following information is required by 5340 * mlx5_flow_hashfields_adjust() in advance. 5341 */ 5342 rss_desc->level = rss->level; 5343 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 5344 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 5345 } 5346 flow->dev_handles = 0; 5347 if (rss && rss->types) { 5348 unsigned int graph_root; 5349 5350 graph_root = find_graph_root(items, rss->level); 5351 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 5352 items, rss->types, 5353 mlx5_support_expansion, graph_root); 5354 MLX5_ASSERT(ret > 0 && 5355 (unsigned int)ret < sizeof(expand_buffer.buffer)); 5356 } else { 5357 buf->entries = 1; 5358 buf->entry[0].pattern = (void *)(uintptr_t)items; 5359 } 5360 rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions, 5361 shared_actions_n); 5362 for (i = 0; i < buf->entries; ++i) { 5363 /* Initialize flow split data. */ 5364 flow_split_info.prefix_layers = 0; 5365 flow_split_info.prefix_mark = 0; 5366 flow_split_info.skip_scale = 0; 5367 /* 5368 * The splitter may create multiple dev_flows, 5369 * depending on configuration. In the simplest 5370 * case it just creates unmodified original flow. 5371 */ 5372 ret = flow_create_split_outer(dev, flow, attr, 5373 buf->entry[i].pattern, 5374 p_actions_rx, &flow_split_info, 5375 error); 5376 if (ret < 0) 5377 goto error; 5378 if (is_flow_tunnel_steer_rule(dev, attr, 5379 buf->entry[i].pattern, 5380 p_actions_rx)) { 5381 ret = flow_tunnel_add_default_miss(dev, flow, attr, 5382 p_actions_rx, 5383 idx, 5384 &default_miss_ctx, 5385 error); 5386 if (ret < 0) { 5387 mlx5_free(default_miss_ctx.queue); 5388 goto error; 5389 } 5390 } 5391 } 5392 /* Create the tx flow. */ 5393 if (hairpin_flow) { 5394 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 5395 attr_tx.ingress = 0; 5396 attr_tx.egress = 1; 5397 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 5398 actions_hairpin_tx.actions, 5399 idx, error); 5400 if (!dev_flow) 5401 goto error; 5402 dev_flow->flow = flow; 5403 dev_flow->external = 0; 5404 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5405 dev_flow->handle, next); 5406 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 5407 items_tx.items, 5408 actions_hairpin_tx.actions, error); 5409 if (ret < 0) 5410 goto error; 5411 } 5412 /* 5413 * Update the metadata register copy table. If extensive 5414 * metadata feature is enabled and registers are supported 5415 * we might create the extra rte_flow for each unique 5416 * MARK/FLAG action ID. 5417 * 5418 * The table is updated for ingress Flows only, because 5419 * the egress Flows belong to the different device and 5420 * copy table should be updated in peer NIC Rx domain. 5421 */ 5422 if (attr->ingress && 5423 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 5424 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 5425 if (ret) 5426 goto error; 5427 } 5428 /* 5429 * If the flow is external (from application) OR device is started, 5430 * OR mreg discover, then apply immediately. 5431 */ 5432 if (external || dev->data->dev_started || 5433 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 5434 attr->priority == MLX5_FLOW_PRIO_RSVD)) { 5435 ret = flow_drv_apply(dev, flow, error); 5436 if (ret < 0) 5437 goto error; 5438 } 5439 if (list) { 5440 rte_spinlock_lock(&priv->flow_list_lock); 5441 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 5442 flow, next); 5443 rte_spinlock_unlock(&priv->flow_list_lock); 5444 } 5445 flow_rxq_flags_set(dev, flow); 5446 rte_free(translated_actions); 5447 tunnel = flow_tunnel_from_rule(dev, attr, items, actions); 5448 if (tunnel) { 5449 flow->tunnel = 1; 5450 flow->tunnel_id = tunnel->tunnel_id; 5451 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 5452 mlx5_free(default_miss_ctx.queue); 5453 } 5454 mlx5_flow_pop_thread_workspace(); 5455 return idx; 5456 error: 5457 MLX5_ASSERT(flow); 5458 ret = rte_errno; /* Save rte_errno before cleanup. */ 5459 flow_mreg_del_copy_action(dev, flow); 5460 flow_drv_destroy(dev, flow); 5461 if (rss_desc->shared_rss) 5462 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 5463 mlx5_ipool_get 5464 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 5465 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 5466 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 5467 rte_errno = ret; /* Restore rte_errno. */ 5468 ret = rte_errno; 5469 rte_errno = ret; 5470 mlx5_flow_pop_thread_workspace(); 5471 error_before_hairpin_split: 5472 rte_free(translated_actions); 5473 return 0; 5474 } 5475 5476 /** 5477 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 5478 * incoming packets to table 1. 5479 * 5480 * Other flow rules, requested for group n, will be created in 5481 * e-switch table n+1. 5482 * Jump action to e-switch group n will be created to group n+1. 5483 * 5484 * Used when working in switchdev mode, to utilise advantages of table 1 5485 * and above. 5486 * 5487 * @param dev 5488 * Pointer to Ethernet device. 5489 * 5490 * @return 5491 * Pointer to flow on success, NULL otherwise and rte_errno is set. 5492 */ 5493 struct rte_flow * 5494 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 5495 { 5496 const struct rte_flow_attr attr = { 5497 .group = 0, 5498 .priority = 0, 5499 .ingress = 1, 5500 .egress = 0, 5501 .transfer = 1, 5502 }; 5503 const struct rte_flow_item pattern = { 5504 .type = RTE_FLOW_ITEM_TYPE_END, 5505 }; 5506 struct rte_flow_action_jump jump = { 5507 .group = 1, 5508 }; 5509 const struct rte_flow_action actions[] = { 5510 { 5511 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5512 .conf = &jump, 5513 }, 5514 { 5515 .type = RTE_FLOW_ACTION_TYPE_END, 5516 }, 5517 }; 5518 struct mlx5_priv *priv = dev->data->dev_private; 5519 struct rte_flow_error error; 5520 5521 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 5522 &attr, &pattern, 5523 actions, false, &error); 5524 } 5525 5526 /** 5527 * Validate a flow supported by the NIC. 5528 * 5529 * @see rte_flow_validate() 5530 * @see rte_flow_ops 5531 */ 5532 int 5533 mlx5_flow_validate(struct rte_eth_dev *dev, 5534 const struct rte_flow_attr *attr, 5535 const struct rte_flow_item items[], 5536 const struct rte_flow_action original_actions[], 5537 struct rte_flow_error *error) 5538 { 5539 int hairpin_flow; 5540 struct mlx5_translated_shared_action 5541 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5542 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5543 const struct rte_flow_action *actions; 5544 struct rte_flow_action *translated_actions = NULL; 5545 int ret = flow_shared_actions_translate(dev, original_actions, 5546 shared_actions, 5547 &shared_actions_n, 5548 &translated_actions, error); 5549 5550 if (ret) 5551 return ret; 5552 actions = translated_actions ? translated_actions : original_actions; 5553 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5554 ret = flow_drv_validate(dev, attr, items, actions, 5555 true, hairpin_flow, error); 5556 rte_free(translated_actions); 5557 return ret; 5558 } 5559 5560 /** 5561 * Create a flow. 5562 * 5563 * @see rte_flow_create() 5564 * @see rte_flow_ops 5565 */ 5566 struct rte_flow * 5567 mlx5_flow_create(struct rte_eth_dev *dev, 5568 const struct rte_flow_attr *attr, 5569 const struct rte_flow_item items[], 5570 const struct rte_flow_action actions[], 5571 struct rte_flow_error *error) 5572 { 5573 struct mlx5_priv *priv = dev->data->dev_private; 5574 5575 /* 5576 * If the device is not started yet, it is not allowed to created a 5577 * flow from application. PMD default flows and traffic control flows 5578 * are not affected. 5579 */ 5580 if (unlikely(!dev->data->dev_started)) { 5581 DRV_LOG(DEBUG, "port %u is not started when " 5582 "inserting a flow", dev->data->port_id); 5583 rte_flow_error_set(error, ENODEV, 5584 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5585 NULL, 5586 "port not started"); 5587 return NULL; 5588 } 5589 5590 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 5591 attr, items, actions, true, error); 5592 } 5593 5594 /** 5595 * Destroy a flow in a list. 5596 * 5597 * @param dev 5598 * Pointer to Ethernet device. 5599 * @param list 5600 * Pointer to the Indexed flow list. If this parameter NULL, 5601 * there is no flow removal from the list. Be noted that as 5602 * flow is add to the indexed list, memory of the indexed 5603 * list points to maybe changed as flow destroyed. 5604 * @param[in] flow_idx 5605 * Index of flow to destroy. 5606 */ 5607 static void 5608 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 5609 uint32_t flow_idx) 5610 { 5611 struct mlx5_priv *priv = dev->data->dev_private; 5612 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5613 [MLX5_IPOOL_RTE_FLOW], flow_idx); 5614 5615 if (!flow) 5616 return; 5617 /* 5618 * Update RX queue flags only if port is started, otherwise it is 5619 * already clean. 5620 */ 5621 if (dev->data->dev_started) 5622 flow_rxq_flags_trim(dev, flow); 5623 flow_drv_destroy(dev, flow); 5624 if (list) { 5625 rte_spinlock_lock(&priv->flow_list_lock); 5626 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 5627 flow_idx, flow, next); 5628 rte_spinlock_unlock(&priv->flow_list_lock); 5629 } 5630 if (flow->tunnel) { 5631 struct mlx5_flow_tunnel *tunnel; 5632 5633 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 5634 RTE_VERIFY(tunnel); 5635 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 5636 mlx5_flow_tunnel_free(dev, tunnel); 5637 } 5638 flow_mreg_del_copy_action(dev, flow); 5639 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5640 } 5641 5642 /** 5643 * Destroy all flows. 5644 * 5645 * @param dev 5646 * Pointer to Ethernet device. 5647 * @param list 5648 * Pointer to the Indexed flow list. 5649 * @param active 5650 * If flushing is called avtively. 5651 */ 5652 void 5653 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 5654 { 5655 uint32_t num_flushed = 0; 5656 5657 while (*list) { 5658 flow_list_destroy(dev, list, *list); 5659 num_flushed++; 5660 } 5661 if (active) { 5662 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 5663 dev->data->port_id, num_flushed); 5664 } 5665 } 5666 5667 /** 5668 * Stop all default actions for flows. 5669 * 5670 * @param dev 5671 * Pointer to Ethernet device. 5672 */ 5673 void 5674 mlx5_flow_stop_default(struct rte_eth_dev *dev) 5675 { 5676 flow_mreg_del_default_copy_action(dev); 5677 flow_rxq_flags_clear(dev); 5678 } 5679 5680 /** 5681 * Start all default actions for flows. 5682 * 5683 * @param dev 5684 * Pointer to Ethernet device. 5685 * @return 5686 * 0 on success, a negative errno value otherwise and rte_errno is set. 5687 */ 5688 int 5689 mlx5_flow_start_default(struct rte_eth_dev *dev) 5690 { 5691 struct rte_flow_error error; 5692 5693 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 5694 return flow_mreg_add_default_copy_action(dev, &error); 5695 } 5696 5697 /** 5698 * Release key of thread specific flow workspace data. 5699 */ 5700 static void 5701 flow_release_workspace(void *data) 5702 { 5703 struct mlx5_flow_workspace *wks = data; 5704 struct mlx5_flow_workspace *next; 5705 5706 while (wks) { 5707 next = wks->next; 5708 free(wks->rss_desc.queue); 5709 free(wks); 5710 wks = next; 5711 } 5712 } 5713 5714 /** 5715 * Initialize key of thread specific flow workspace data. 5716 */ 5717 static void 5718 flow_alloc_workspace(void) 5719 { 5720 if (pthread_key_create(&key_workspace, flow_release_workspace)) 5721 DRV_LOG(ERR, "Can't create flow workspace data thread key."); 5722 } 5723 5724 /** 5725 * Get thread specific current flow workspace. 5726 * 5727 * @return pointer to thread specific flow workspace data, NULL on error. 5728 */ 5729 struct mlx5_flow_workspace* 5730 mlx5_flow_get_thread_workspace(void) 5731 { 5732 struct mlx5_flow_workspace *data; 5733 5734 data = pthread_getspecific(key_workspace); 5735 MLX5_ASSERT(data && data->inuse); 5736 if (!data || !data->inuse) 5737 DRV_LOG(ERR, "flow workspace not initialized."); 5738 return data; 5739 } 5740 5741 /** 5742 * Allocate and init new flow workspace. 5743 * 5744 * @return pointer to flow workspace data, NULL on error. 5745 */ 5746 static struct mlx5_flow_workspace* 5747 flow_alloc_thread_workspace(void) 5748 { 5749 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 5750 5751 if (!data) { 5752 DRV_LOG(ERR, "Failed to allocate flow workspace " 5753 "memory."); 5754 return NULL; 5755 } 5756 data->rss_desc.queue = calloc(1, 5757 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 5758 if (!data->rss_desc.queue) 5759 goto err; 5760 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 5761 return data; 5762 err: 5763 if (data->rss_desc.queue) 5764 free(data->rss_desc.queue); 5765 free(data); 5766 return NULL; 5767 } 5768 5769 /** 5770 * Get new thread specific flow workspace. 5771 * 5772 * If current workspace inuse, create new one and set as current. 5773 * 5774 * @return pointer to thread specific flow workspace data, NULL on error. 5775 */ 5776 static struct mlx5_flow_workspace* 5777 mlx5_flow_push_thread_workspace(void) 5778 { 5779 struct mlx5_flow_workspace *curr; 5780 struct mlx5_flow_workspace *data; 5781 5782 if (pthread_once(&key_workspace_init, flow_alloc_workspace)) { 5783 DRV_LOG(ERR, "Failed to init flow workspace data thread key."); 5784 return NULL; 5785 } 5786 curr = pthread_getspecific(key_workspace); 5787 if (!curr) { 5788 data = flow_alloc_thread_workspace(); 5789 if (!data) 5790 return NULL; 5791 } else if (!curr->inuse) { 5792 data = curr; 5793 } else if (curr->next) { 5794 data = curr->next; 5795 } else { 5796 data = flow_alloc_thread_workspace(); 5797 if (!data) 5798 return NULL; 5799 curr->next = data; 5800 data->prev = curr; 5801 } 5802 data->inuse = 1; 5803 data->flow_idx = 0; 5804 /* Set as current workspace */ 5805 if (pthread_setspecific(key_workspace, data)) 5806 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 5807 return data; 5808 } 5809 5810 /** 5811 * Close current thread specific flow workspace. 5812 * 5813 * If previous workspace available, set it as current. 5814 * 5815 * @return pointer to thread specific flow workspace data, NULL on error. 5816 */ 5817 static void 5818 mlx5_flow_pop_thread_workspace(void) 5819 { 5820 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 5821 5822 if (!data) 5823 return; 5824 if (!data->inuse) { 5825 DRV_LOG(ERR, "Failed to close unused flow workspace."); 5826 return; 5827 } 5828 data->inuse = 0; 5829 if (!data->prev) 5830 return; 5831 if (pthread_setspecific(key_workspace, data->prev)) 5832 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 5833 } 5834 5835 /** 5836 * Verify the flow list is empty 5837 * 5838 * @param dev 5839 * Pointer to Ethernet device. 5840 * 5841 * @return the number of flows not released. 5842 */ 5843 int 5844 mlx5_flow_verify(struct rte_eth_dev *dev) 5845 { 5846 struct mlx5_priv *priv = dev->data->dev_private; 5847 struct rte_flow *flow; 5848 uint32_t idx; 5849 int ret = 0; 5850 5851 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 5852 flow, next) { 5853 DRV_LOG(DEBUG, "port %u flow %p still referenced", 5854 dev->data->port_id, (void *)flow); 5855 ++ret; 5856 } 5857 return ret; 5858 } 5859 5860 /** 5861 * Enable default hairpin egress flow. 5862 * 5863 * @param dev 5864 * Pointer to Ethernet device. 5865 * @param queue 5866 * The queue index. 5867 * 5868 * @return 5869 * 0 on success, a negative errno value otherwise and rte_errno is set. 5870 */ 5871 int 5872 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 5873 uint32_t queue) 5874 { 5875 struct mlx5_priv *priv = dev->data->dev_private; 5876 const struct rte_flow_attr attr = { 5877 .egress = 1, 5878 .priority = 0, 5879 }; 5880 struct mlx5_rte_flow_item_tx_queue queue_spec = { 5881 .queue = queue, 5882 }; 5883 struct mlx5_rte_flow_item_tx_queue queue_mask = { 5884 .queue = UINT32_MAX, 5885 }; 5886 struct rte_flow_item items[] = { 5887 { 5888 .type = (enum rte_flow_item_type) 5889 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 5890 .spec = &queue_spec, 5891 .last = NULL, 5892 .mask = &queue_mask, 5893 }, 5894 { 5895 .type = RTE_FLOW_ITEM_TYPE_END, 5896 }, 5897 }; 5898 struct rte_flow_action_jump jump = { 5899 .group = MLX5_HAIRPIN_TX_TABLE, 5900 }; 5901 struct rte_flow_action actions[2]; 5902 uint32_t flow_idx; 5903 struct rte_flow_error error; 5904 5905 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 5906 actions[0].conf = &jump; 5907 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 5908 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5909 &attr, items, actions, false, &error); 5910 if (!flow_idx) { 5911 DRV_LOG(DEBUG, 5912 "Failed to create ctrl flow: rte_errno(%d)," 5913 " type(%d), message(%s)", 5914 rte_errno, error.type, 5915 error.message ? error.message : " (no stated reason)"); 5916 return -rte_errno; 5917 } 5918 return 0; 5919 } 5920 5921 /** 5922 * Enable a control flow configured from the control plane. 5923 * 5924 * @param dev 5925 * Pointer to Ethernet device. 5926 * @param eth_spec 5927 * An Ethernet flow spec to apply. 5928 * @param eth_mask 5929 * An Ethernet flow mask to apply. 5930 * @param vlan_spec 5931 * A VLAN flow spec to apply. 5932 * @param vlan_mask 5933 * A VLAN flow mask to apply. 5934 * 5935 * @return 5936 * 0 on success, a negative errno value otherwise and rte_errno is set. 5937 */ 5938 int 5939 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 5940 struct rte_flow_item_eth *eth_spec, 5941 struct rte_flow_item_eth *eth_mask, 5942 struct rte_flow_item_vlan *vlan_spec, 5943 struct rte_flow_item_vlan *vlan_mask) 5944 { 5945 struct mlx5_priv *priv = dev->data->dev_private; 5946 const struct rte_flow_attr attr = { 5947 .ingress = 1, 5948 .priority = MLX5_FLOW_PRIO_RSVD, 5949 }; 5950 struct rte_flow_item items[] = { 5951 { 5952 .type = RTE_FLOW_ITEM_TYPE_ETH, 5953 .spec = eth_spec, 5954 .last = NULL, 5955 .mask = eth_mask, 5956 }, 5957 { 5958 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 5959 RTE_FLOW_ITEM_TYPE_END, 5960 .spec = vlan_spec, 5961 .last = NULL, 5962 .mask = vlan_mask, 5963 }, 5964 { 5965 .type = RTE_FLOW_ITEM_TYPE_END, 5966 }, 5967 }; 5968 uint16_t queue[priv->reta_idx_n]; 5969 struct rte_flow_action_rss action_rss = { 5970 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 5971 .level = 0, 5972 .types = priv->rss_conf.rss_hf, 5973 .key_len = priv->rss_conf.rss_key_len, 5974 .queue_num = priv->reta_idx_n, 5975 .key = priv->rss_conf.rss_key, 5976 .queue = queue, 5977 }; 5978 struct rte_flow_action actions[] = { 5979 { 5980 .type = RTE_FLOW_ACTION_TYPE_RSS, 5981 .conf = &action_rss, 5982 }, 5983 { 5984 .type = RTE_FLOW_ACTION_TYPE_END, 5985 }, 5986 }; 5987 uint32_t flow_idx; 5988 struct rte_flow_error error; 5989 unsigned int i; 5990 5991 if (!priv->reta_idx_n || !priv->rxqs_n) { 5992 return 0; 5993 } 5994 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 5995 action_rss.types = 0; 5996 for (i = 0; i != priv->reta_idx_n; ++i) 5997 queue[i] = (*priv->reta_idx)[i]; 5998 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5999 &attr, items, actions, false, &error); 6000 if (!flow_idx) 6001 return -rte_errno; 6002 return 0; 6003 } 6004 6005 /** 6006 * Enable a flow control configured from the control plane. 6007 * 6008 * @param dev 6009 * Pointer to Ethernet device. 6010 * @param eth_spec 6011 * An Ethernet flow spec to apply. 6012 * @param eth_mask 6013 * An Ethernet flow mask to apply. 6014 * 6015 * @return 6016 * 0 on success, a negative errno value otherwise and rte_errno is set. 6017 */ 6018 int 6019 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6020 struct rte_flow_item_eth *eth_spec, 6021 struct rte_flow_item_eth *eth_mask) 6022 { 6023 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6024 } 6025 6026 /** 6027 * Create default miss flow rule matching lacp traffic 6028 * 6029 * @param dev 6030 * Pointer to Ethernet device. 6031 * @param eth_spec 6032 * An Ethernet flow spec to apply. 6033 * 6034 * @return 6035 * 0 on success, a negative errno value otherwise and rte_errno is set. 6036 */ 6037 int 6038 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6039 { 6040 struct mlx5_priv *priv = dev->data->dev_private; 6041 /* 6042 * The LACP matching is done by only using ether type since using 6043 * a multicast dst mac causes kernel to give low priority to this flow. 6044 */ 6045 static const struct rte_flow_item_eth lacp_spec = { 6046 .type = RTE_BE16(0x8809), 6047 }; 6048 static const struct rte_flow_item_eth lacp_mask = { 6049 .type = 0xffff, 6050 }; 6051 const struct rte_flow_attr attr = { 6052 .ingress = 1, 6053 }; 6054 struct rte_flow_item items[] = { 6055 { 6056 .type = RTE_FLOW_ITEM_TYPE_ETH, 6057 .spec = &lacp_spec, 6058 .mask = &lacp_mask, 6059 }, 6060 { 6061 .type = RTE_FLOW_ITEM_TYPE_END, 6062 }, 6063 }; 6064 struct rte_flow_action actions[] = { 6065 { 6066 .type = (enum rte_flow_action_type) 6067 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6068 }, 6069 { 6070 .type = RTE_FLOW_ACTION_TYPE_END, 6071 }, 6072 }; 6073 struct rte_flow_error error; 6074 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6075 &attr, items, actions, false, &error); 6076 6077 if (!flow_idx) 6078 return -rte_errno; 6079 return 0; 6080 } 6081 6082 /** 6083 * Destroy a flow. 6084 * 6085 * @see rte_flow_destroy() 6086 * @see rte_flow_ops 6087 */ 6088 int 6089 mlx5_flow_destroy(struct rte_eth_dev *dev, 6090 struct rte_flow *flow, 6091 struct rte_flow_error *error __rte_unused) 6092 { 6093 struct mlx5_priv *priv = dev->data->dev_private; 6094 6095 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6096 return 0; 6097 } 6098 6099 /** 6100 * Destroy all flows. 6101 * 6102 * @see rte_flow_flush() 6103 * @see rte_flow_ops 6104 */ 6105 int 6106 mlx5_flow_flush(struct rte_eth_dev *dev, 6107 struct rte_flow_error *error __rte_unused) 6108 { 6109 struct mlx5_priv *priv = dev->data->dev_private; 6110 6111 mlx5_flow_list_flush(dev, &priv->flows, false); 6112 return 0; 6113 } 6114 6115 /** 6116 * Isolated mode. 6117 * 6118 * @see rte_flow_isolate() 6119 * @see rte_flow_ops 6120 */ 6121 int 6122 mlx5_flow_isolate(struct rte_eth_dev *dev, 6123 int enable, 6124 struct rte_flow_error *error) 6125 { 6126 struct mlx5_priv *priv = dev->data->dev_private; 6127 6128 if (dev->data->dev_started) { 6129 rte_flow_error_set(error, EBUSY, 6130 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6131 NULL, 6132 "port must be stopped first"); 6133 return -rte_errno; 6134 } 6135 priv->isolated = !!enable; 6136 if (enable) 6137 dev->dev_ops = &mlx5_os_dev_ops_isolate; 6138 else 6139 dev->dev_ops = &mlx5_os_dev_ops; 6140 6141 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 6142 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 6143 6144 return 0; 6145 } 6146 6147 /** 6148 * Query a flow. 6149 * 6150 * @see rte_flow_query() 6151 * @see rte_flow_ops 6152 */ 6153 static int 6154 flow_drv_query(struct rte_eth_dev *dev, 6155 uint32_t flow_idx, 6156 const struct rte_flow_action *actions, 6157 void *data, 6158 struct rte_flow_error *error) 6159 { 6160 struct mlx5_priv *priv = dev->data->dev_private; 6161 const struct mlx5_flow_driver_ops *fops; 6162 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6163 [MLX5_IPOOL_RTE_FLOW], 6164 flow_idx); 6165 enum mlx5_flow_drv_type ftype; 6166 6167 if (!flow) { 6168 return rte_flow_error_set(error, ENOENT, 6169 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6170 NULL, 6171 "invalid flow handle"); 6172 } 6173 ftype = flow->drv_type; 6174 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 6175 fops = flow_get_drv_ops(ftype); 6176 6177 return fops->query(dev, flow, actions, data, error); 6178 } 6179 6180 /** 6181 * Query a flow. 6182 * 6183 * @see rte_flow_query() 6184 * @see rte_flow_ops 6185 */ 6186 int 6187 mlx5_flow_query(struct rte_eth_dev *dev, 6188 struct rte_flow *flow, 6189 const struct rte_flow_action *actions, 6190 void *data, 6191 struct rte_flow_error *error) 6192 { 6193 int ret; 6194 6195 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 6196 error); 6197 if (ret < 0) 6198 return ret; 6199 return 0; 6200 } 6201 6202 /** 6203 * Manage filter operations. 6204 * 6205 * @param dev 6206 * Pointer to Ethernet device structure. 6207 * @param filter_type 6208 * Filter type. 6209 * @param filter_op 6210 * Operation to perform. 6211 * @param arg 6212 * Pointer to operation-specific structure. 6213 * 6214 * @return 6215 * 0 on success, a negative errno value otherwise and rte_errno is set. 6216 */ 6217 int 6218 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 6219 enum rte_filter_type filter_type, 6220 enum rte_filter_op filter_op, 6221 void *arg) 6222 { 6223 switch (filter_type) { 6224 case RTE_ETH_FILTER_GENERIC: 6225 if (filter_op != RTE_ETH_FILTER_GET) { 6226 rte_errno = EINVAL; 6227 return -rte_errno; 6228 } 6229 *(const void **)arg = &mlx5_flow_ops; 6230 return 0; 6231 default: 6232 DRV_LOG(ERR, "port %u filter type (%d) not supported", 6233 dev->data->port_id, filter_type); 6234 rte_errno = ENOTSUP; 6235 return -rte_errno; 6236 } 6237 return 0; 6238 } 6239 6240 /** 6241 * Create the needed meter and suffix tables. 6242 * 6243 * @param[in] dev 6244 * Pointer to Ethernet device. 6245 * @param[in] fm 6246 * Pointer to the flow meter. 6247 * 6248 * @return 6249 * Pointer to table set on success, NULL otherwise. 6250 */ 6251 struct mlx5_meter_domains_infos * 6252 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 6253 const struct mlx5_flow_meter *fm) 6254 { 6255 const struct mlx5_flow_driver_ops *fops; 6256 6257 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6258 return fops->create_mtr_tbls(dev, fm); 6259 } 6260 6261 /** 6262 * Destroy the meter table set. 6263 * 6264 * @param[in] dev 6265 * Pointer to Ethernet device. 6266 * @param[in] tbl 6267 * Pointer to the meter table set. 6268 * 6269 * @return 6270 * 0 on success. 6271 */ 6272 int 6273 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 6274 struct mlx5_meter_domains_infos *tbls) 6275 { 6276 const struct mlx5_flow_driver_ops *fops; 6277 6278 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6279 return fops->destroy_mtr_tbls(dev, tbls); 6280 } 6281 6282 /** 6283 * Create policer rules. 6284 * 6285 * @param[in] dev 6286 * Pointer to Ethernet device. 6287 * @param[in] fm 6288 * Pointer to flow meter structure. 6289 * @param[in] attr 6290 * Pointer to flow attributes. 6291 * 6292 * @return 6293 * 0 on success, -1 otherwise. 6294 */ 6295 int 6296 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 6297 struct mlx5_flow_meter *fm, 6298 const struct rte_flow_attr *attr) 6299 { 6300 const struct mlx5_flow_driver_ops *fops; 6301 6302 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6303 return fops->create_policer_rules(dev, fm, attr); 6304 } 6305 6306 /** 6307 * Destroy policer rules. 6308 * 6309 * @param[in] fm 6310 * Pointer to flow meter structure. 6311 * @param[in] attr 6312 * Pointer to flow attributes. 6313 * 6314 * @return 6315 * 0 on success, -1 otherwise. 6316 */ 6317 int 6318 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 6319 struct mlx5_flow_meter *fm, 6320 const struct rte_flow_attr *attr) 6321 { 6322 const struct mlx5_flow_driver_ops *fops; 6323 6324 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6325 return fops->destroy_policer_rules(dev, fm, attr); 6326 } 6327 6328 /** 6329 * Allocate a counter. 6330 * 6331 * @param[in] dev 6332 * Pointer to Ethernet device structure. 6333 * 6334 * @return 6335 * Index to allocated counter on success, 0 otherwise. 6336 */ 6337 uint32_t 6338 mlx5_counter_alloc(struct rte_eth_dev *dev) 6339 { 6340 const struct mlx5_flow_driver_ops *fops; 6341 struct rte_flow_attr attr = { .transfer = 0 }; 6342 6343 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6344 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6345 return fops->counter_alloc(dev); 6346 } 6347 DRV_LOG(ERR, 6348 "port %u counter allocate is not supported.", 6349 dev->data->port_id); 6350 return 0; 6351 } 6352 6353 /** 6354 * Free a counter. 6355 * 6356 * @param[in] dev 6357 * Pointer to Ethernet device structure. 6358 * @param[in] cnt 6359 * Index to counter to be free. 6360 */ 6361 void 6362 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 6363 { 6364 const struct mlx5_flow_driver_ops *fops; 6365 struct rte_flow_attr attr = { .transfer = 0 }; 6366 6367 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6368 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6369 fops->counter_free(dev, cnt); 6370 return; 6371 } 6372 DRV_LOG(ERR, 6373 "port %u counter free is not supported.", 6374 dev->data->port_id); 6375 } 6376 6377 /** 6378 * Query counter statistics. 6379 * 6380 * @param[in] dev 6381 * Pointer to Ethernet device structure. 6382 * @param[in] cnt 6383 * Index to counter to query. 6384 * @param[in] clear 6385 * Set to clear counter statistics. 6386 * @param[out] pkts 6387 * The counter hits packets number to save. 6388 * @param[out] bytes 6389 * The counter hits bytes number to save. 6390 * 6391 * @return 6392 * 0 on success, a negative errno value otherwise. 6393 */ 6394 int 6395 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 6396 bool clear, uint64_t *pkts, uint64_t *bytes) 6397 { 6398 const struct mlx5_flow_driver_ops *fops; 6399 struct rte_flow_attr attr = { .transfer = 0 }; 6400 6401 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6402 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6403 return fops->counter_query(dev, cnt, clear, pkts, bytes); 6404 } 6405 DRV_LOG(ERR, 6406 "port %u counter query is not supported.", 6407 dev->data->port_id); 6408 return -ENOTSUP; 6409 } 6410 6411 /** 6412 * Allocate a new memory for the counter values wrapped by all the needed 6413 * management. 6414 * 6415 * @param[in] sh 6416 * Pointer to mlx5_dev_ctx_shared object. 6417 * 6418 * @return 6419 * 0 on success, a negative errno value otherwise. 6420 */ 6421 static int 6422 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 6423 { 6424 struct mlx5_devx_mkey_attr mkey_attr; 6425 struct mlx5_counter_stats_mem_mng *mem_mng; 6426 volatile struct flow_counter_stats *raw_data; 6427 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 6428 int size = (sizeof(struct flow_counter_stats) * 6429 MLX5_COUNTERS_PER_POOL + 6430 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 6431 sizeof(struct mlx5_counter_stats_mem_mng); 6432 size_t pgsize = rte_mem_page_size(); 6433 uint8_t *mem; 6434 int i; 6435 6436 if (pgsize == (size_t)-1) { 6437 DRV_LOG(ERR, "Failed to get mem page size"); 6438 rte_errno = ENOMEM; 6439 return -ENOMEM; 6440 } 6441 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 6442 if (!mem) { 6443 rte_errno = ENOMEM; 6444 return -ENOMEM; 6445 } 6446 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 6447 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 6448 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 6449 IBV_ACCESS_LOCAL_WRITE); 6450 if (!mem_mng->umem) { 6451 rte_errno = errno; 6452 mlx5_free(mem); 6453 return -rte_errno; 6454 } 6455 mkey_attr.addr = (uintptr_t)mem; 6456 mkey_attr.size = size; 6457 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 6458 mkey_attr.pd = sh->pdn; 6459 mkey_attr.log_entity_size = 0; 6460 mkey_attr.pg_access = 0; 6461 mkey_attr.klm_array = NULL; 6462 mkey_attr.klm_num = 0; 6463 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 6464 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 6465 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 6466 if (!mem_mng->dm) { 6467 mlx5_glue->devx_umem_dereg(mem_mng->umem); 6468 rte_errno = errno; 6469 mlx5_free(mem); 6470 return -rte_errno; 6471 } 6472 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 6473 raw_data = (volatile struct flow_counter_stats *)mem; 6474 for (i = 0; i < raws_n; ++i) { 6475 mem_mng->raws[i].mem_mng = mem_mng; 6476 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 6477 } 6478 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 6479 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 6480 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 6481 next); 6482 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 6483 sh->cmng.mem_mng = mem_mng; 6484 return 0; 6485 } 6486 6487 /** 6488 * Set the statistic memory to the new counter pool. 6489 * 6490 * @param[in] sh 6491 * Pointer to mlx5_dev_ctx_shared object. 6492 * @param[in] pool 6493 * Pointer to the pool to set the statistic memory. 6494 * 6495 * @return 6496 * 0 on success, a negative errno value otherwise. 6497 */ 6498 static int 6499 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 6500 struct mlx5_flow_counter_pool *pool) 6501 { 6502 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6503 /* Resize statistic memory once used out. */ 6504 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 6505 mlx5_flow_create_counter_stat_mem_mng(sh)) { 6506 DRV_LOG(ERR, "Cannot resize counter stat mem."); 6507 return -1; 6508 } 6509 rte_spinlock_lock(&pool->sl); 6510 pool->raw = cmng->mem_mng->raws + pool->index % 6511 MLX5_CNT_CONTAINER_RESIZE; 6512 rte_spinlock_unlock(&pool->sl); 6513 pool->raw_hw = NULL; 6514 return 0; 6515 } 6516 6517 #define MLX5_POOL_QUERY_FREQ_US 1000000 6518 6519 /** 6520 * Set the periodic procedure for triggering asynchronous batch queries for all 6521 * the counter pools. 6522 * 6523 * @param[in] sh 6524 * Pointer to mlx5_dev_ctx_shared object. 6525 */ 6526 void 6527 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 6528 { 6529 uint32_t pools_n, us; 6530 6531 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 6532 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 6533 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 6534 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 6535 sh->cmng.query_thread_on = 0; 6536 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 6537 } else { 6538 sh->cmng.query_thread_on = 1; 6539 } 6540 } 6541 6542 /** 6543 * The periodic procedure for triggering asynchronous batch queries for all the 6544 * counter pools. This function is probably called by the host thread. 6545 * 6546 * @param[in] arg 6547 * The parameter for the alarm process. 6548 */ 6549 void 6550 mlx5_flow_query_alarm(void *arg) 6551 { 6552 struct mlx5_dev_ctx_shared *sh = arg; 6553 int ret; 6554 uint16_t pool_index = sh->cmng.pool_index; 6555 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6556 struct mlx5_flow_counter_pool *pool; 6557 uint16_t n_valid; 6558 6559 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 6560 goto set_alarm; 6561 rte_spinlock_lock(&cmng->pool_update_sl); 6562 pool = cmng->pools[pool_index]; 6563 n_valid = cmng->n_valid; 6564 rte_spinlock_unlock(&cmng->pool_update_sl); 6565 /* Set the statistic memory to the new created pool. */ 6566 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 6567 goto set_alarm; 6568 if (pool->raw_hw) 6569 /* There is a pool query in progress. */ 6570 goto set_alarm; 6571 pool->raw_hw = 6572 LIST_FIRST(&sh->cmng.free_stat_raws); 6573 if (!pool->raw_hw) 6574 /* No free counter statistics raw memory. */ 6575 goto set_alarm; 6576 /* 6577 * Identify the counters released between query trigger and query 6578 * handle more efficiently. The counter released in this gap period 6579 * should wait for a new round of query as the new arrived packets 6580 * will not be taken into account. 6581 */ 6582 pool->query_gen++; 6583 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 6584 MLX5_COUNTERS_PER_POOL, 6585 NULL, NULL, 6586 pool->raw_hw->mem_mng->dm->id, 6587 (void *)(uintptr_t) 6588 pool->raw_hw->data, 6589 sh->devx_comp, 6590 (uint64_t)(uintptr_t)pool); 6591 if (ret) { 6592 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 6593 " %d", pool->min_dcs->id); 6594 pool->raw_hw = NULL; 6595 goto set_alarm; 6596 } 6597 LIST_REMOVE(pool->raw_hw, next); 6598 sh->cmng.pending_queries++; 6599 pool_index++; 6600 if (pool_index >= n_valid) 6601 pool_index = 0; 6602 set_alarm: 6603 sh->cmng.pool_index = pool_index; 6604 mlx5_set_query_alarm(sh); 6605 } 6606 6607 /** 6608 * Check and callback event for new aged flow in the counter pool 6609 * 6610 * @param[in] sh 6611 * Pointer to mlx5_dev_ctx_shared object. 6612 * @param[in] pool 6613 * Pointer to Current counter pool. 6614 */ 6615 static void 6616 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 6617 struct mlx5_flow_counter_pool *pool) 6618 { 6619 struct mlx5_priv *priv; 6620 struct mlx5_flow_counter *cnt; 6621 struct mlx5_age_info *age_info; 6622 struct mlx5_age_param *age_param; 6623 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 6624 struct mlx5_counter_stats_raw *prev = pool->raw; 6625 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 6626 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 6627 uint16_t expected = AGE_CANDIDATE; 6628 uint32_t i; 6629 6630 pool->time_of_last_age_check = curr_time; 6631 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 6632 cnt = MLX5_POOL_GET_CNT(pool, i); 6633 age_param = MLX5_CNT_TO_AGE(cnt); 6634 if (__atomic_load_n(&age_param->state, 6635 __ATOMIC_RELAXED) != AGE_CANDIDATE) 6636 continue; 6637 if (cur->data[i].hits != prev->data[i].hits) { 6638 __atomic_store_n(&age_param->sec_since_last_hit, 0, 6639 __ATOMIC_RELAXED); 6640 continue; 6641 } 6642 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 6643 time_delta, 6644 __ATOMIC_RELAXED) <= age_param->timeout) 6645 continue; 6646 /** 6647 * Hold the lock first, or if between the 6648 * state AGE_TMOUT and tailq operation the 6649 * release happened, the release procedure 6650 * may delete a non-existent tailq node. 6651 */ 6652 priv = rte_eth_devices[age_param->port_id].data->dev_private; 6653 age_info = GET_PORT_AGE_INFO(priv); 6654 rte_spinlock_lock(&age_info->aged_sl); 6655 if (__atomic_compare_exchange_n(&age_param->state, &expected, 6656 AGE_TMOUT, false, 6657 __ATOMIC_RELAXED, 6658 __ATOMIC_RELAXED)) { 6659 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 6660 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 6661 } 6662 rte_spinlock_unlock(&age_info->aged_sl); 6663 } 6664 mlx5_age_event_prepare(sh); 6665 } 6666 6667 /** 6668 * Handler for the HW respond about ready values from an asynchronous batch 6669 * query. This function is probably called by the host thread. 6670 * 6671 * @param[in] sh 6672 * The pointer to the shared device context. 6673 * @param[in] async_id 6674 * The Devx async ID. 6675 * @param[in] status 6676 * The status of the completion. 6677 */ 6678 void 6679 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 6680 uint64_t async_id, int status) 6681 { 6682 struct mlx5_flow_counter_pool *pool = 6683 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 6684 struct mlx5_counter_stats_raw *raw_to_free; 6685 uint8_t query_gen = pool->query_gen ^ 1; 6686 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6687 enum mlx5_counter_type cnt_type = 6688 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 6689 MLX5_COUNTER_TYPE_ORIGIN; 6690 6691 if (unlikely(status)) { 6692 raw_to_free = pool->raw_hw; 6693 } else { 6694 raw_to_free = pool->raw; 6695 if (pool->is_aged) 6696 mlx5_flow_aging_check(sh, pool); 6697 rte_spinlock_lock(&pool->sl); 6698 pool->raw = pool->raw_hw; 6699 rte_spinlock_unlock(&pool->sl); 6700 /* Be sure the new raw counters data is updated in memory. */ 6701 rte_io_wmb(); 6702 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 6703 rte_spinlock_lock(&cmng->csl[cnt_type]); 6704 TAILQ_CONCAT(&cmng->counters[cnt_type], 6705 &pool->counters[query_gen], next); 6706 rte_spinlock_unlock(&cmng->csl[cnt_type]); 6707 } 6708 } 6709 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 6710 pool->raw_hw = NULL; 6711 sh->cmng.pending_queries--; 6712 } 6713 6714 static int 6715 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 6716 const struct flow_grp_info *grp_info, 6717 struct rte_flow_error *error) 6718 { 6719 if (grp_info->transfer && grp_info->external && 6720 grp_info->fdb_def_rule) { 6721 if (group == UINT32_MAX) 6722 return rte_flow_error_set 6723 (error, EINVAL, 6724 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6725 NULL, 6726 "group index not supported"); 6727 *table = group + 1; 6728 } else { 6729 *table = group; 6730 } 6731 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 6732 return 0; 6733 } 6734 6735 /** 6736 * Translate the rte_flow group index to HW table value. 6737 * 6738 * If tunnel offload is disabled, all group ids converted to flow table 6739 * id using the standard method. 6740 * If tunnel offload is enabled, group id can be converted using the 6741 * standard or tunnel conversion method. Group conversion method 6742 * selection depends on flags in `grp_info` parameter: 6743 * - Internal (grp_info.external == 0) groups conversion uses the 6744 * standard method. 6745 * - Group ids in JUMP action converted with the tunnel conversion. 6746 * - Group id in rule attribute conversion depends on a rule type and 6747 * group id value: 6748 * ** non zero group attributes converted with the tunnel method 6749 * ** zero group attribute in non-tunnel rule is converted using the 6750 * standard method - there's only one root table 6751 * ** zero group attribute in steer tunnel rule is converted with the 6752 * standard method - single root table 6753 * ** zero group attribute in match tunnel rule is a special OvS 6754 * case: that value is used for portability reasons. That group 6755 * id is converted with the tunnel conversion method. 6756 * 6757 * @param[in] dev 6758 * Port device 6759 * @param[in] tunnel 6760 * PMD tunnel offload object 6761 * @param[in] group 6762 * rte_flow group index value. 6763 * @param[out] table 6764 * HW table value. 6765 * @param[in] grp_info 6766 * flags used for conversion 6767 * @param[out] error 6768 * Pointer to error structure. 6769 * 6770 * @return 6771 * 0 on success, a negative errno value otherwise and rte_errno is set. 6772 */ 6773 int 6774 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 6775 const struct mlx5_flow_tunnel *tunnel, 6776 uint32_t group, uint32_t *table, 6777 const struct flow_grp_info *grp_info, 6778 struct rte_flow_error *error) 6779 { 6780 int ret; 6781 bool standard_translation; 6782 6783 if (!grp_info->skip_scale && grp_info->external && 6784 group < MLX5_MAX_TABLES_EXTERNAL) 6785 group *= MLX5_FLOW_TABLE_FACTOR; 6786 if (is_tunnel_offload_active(dev)) { 6787 standard_translation = !grp_info->external || 6788 grp_info->std_tbl_fix; 6789 } else { 6790 standard_translation = true; 6791 } 6792 DRV_LOG(DEBUG, 6793 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 6794 dev->data->port_id, group, grp_info->transfer, 6795 grp_info->external, grp_info->fdb_def_rule, 6796 standard_translation ? "STANDARD" : "TUNNEL"); 6797 if (standard_translation) 6798 ret = flow_group_to_table(dev->data->port_id, group, table, 6799 grp_info, error); 6800 else 6801 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 6802 table, error); 6803 6804 return ret; 6805 } 6806 6807 /** 6808 * Discover availability of metadata reg_c's. 6809 * 6810 * Iteratively use test flows to check availability. 6811 * 6812 * @param[in] dev 6813 * Pointer to the Ethernet device structure. 6814 * 6815 * @return 6816 * 0 on success, a negative errno value otherwise and rte_errno is set. 6817 */ 6818 int 6819 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 6820 { 6821 struct mlx5_priv *priv = dev->data->dev_private; 6822 struct mlx5_dev_config *config = &priv->config; 6823 enum modify_reg idx; 6824 int n = 0; 6825 6826 /* reg_c[0] and reg_c[1] are reserved. */ 6827 config->flow_mreg_c[n++] = REG_C_0; 6828 config->flow_mreg_c[n++] = REG_C_1; 6829 /* Discover availability of other reg_c's. */ 6830 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 6831 struct rte_flow_attr attr = { 6832 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6833 .priority = MLX5_FLOW_PRIO_RSVD, 6834 .ingress = 1, 6835 }; 6836 struct rte_flow_item items[] = { 6837 [0] = { 6838 .type = RTE_FLOW_ITEM_TYPE_END, 6839 }, 6840 }; 6841 struct rte_flow_action actions[] = { 6842 [0] = { 6843 .type = (enum rte_flow_action_type) 6844 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6845 .conf = &(struct mlx5_flow_action_copy_mreg){ 6846 .src = REG_C_1, 6847 .dst = idx, 6848 }, 6849 }, 6850 [1] = { 6851 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6852 .conf = &(struct rte_flow_action_jump){ 6853 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6854 }, 6855 }, 6856 [2] = { 6857 .type = RTE_FLOW_ACTION_TYPE_END, 6858 }, 6859 }; 6860 uint32_t flow_idx; 6861 struct rte_flow *flow; 6862 struct rte_flow_error error; 6863 6864 if (!config->dv_flow_en) 6865 break; 6866 /* Create internal flow, validation skips copy action. */ 6867 flow_idx = flow_list_create(dev, NULL, &attr, items, 6868 actions, false, &error); 6869 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 6870 flow_idx); 6871 if (!flow) 6872 continue; 6873 config->flow_mreg_c[n++] = idx; 6874 flow_list_destroy(dev, NULL, flow_idx); 6875 } 6876 for (; n < MLX5_MREG_C_NUM; ++n) 6877 config->flow_mreg_c[n] = REG_NON; 6878 return 0; 6879 } 6880 6881 /** 6882 * Dump flow raw hw data to file 6883 * 6884 * @param[in] dev 6885 * The pointer to Ethernet device. 6886 * @param[in] file 6887 * A pointer to a file for output. 6888 * @param[out] error 6889 * Perform verbose error reporting if not NULL. PMDs initialize this 6890 * structure in case of error only. 6891 * @return 6892 * 0 on success, a nagative value otherwise. 6893 */ 6894 int 6895 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 6896 FILE *file, 6897 struct rte_flow_error *error __rte_unused) 6898 { 6899 struct mlx5_priv *priv = dev->data->dev_private; 6900 struct mlx5_dev_ctx_shared *sh = priv->sh; 6901 6902 if (!priv->config.dv_flow_en) { 6903 if (fputs("device dv flow disabled\n", file) <= 0) 6904 return -errno; 6905 return -ENOTSUP; 6906 } 6907 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 6908 sh->tx_domain, file); 6909 } 6910 6911 /** 6912 * Get aged-out flows. 6913 * 6914 * @param[in] dev 6915 * Pointer to the Ethernet device structure. 6916 * @param[in] context 6917 * The address of an array of pointers to the aged-out flows contexts. 6918 * @param[in] nb_countexts 6919 * The length of context array pointers. 6920 * @param[out] error 6921 * Perform verbose error reporting if not NULL. Initialized in case of 6922 * error only. 6923 * 6924 * @return 6925 * how many contexts get in success, otherwise negative errno value. 6926 * if nb_contexts is 0, return the amount of all aged contexts. 6927 * if nb_contexts is not 0 , return the amount of aged flows reported 6928 * in the context array. 6929 */ 6930 int 6931 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 6932 uint32_t nb_contexts, struct rte_flow_error *error) 6933 { 6934 const struct mlx5_flow_driver_ops *fops; 6935 struct rte_flow_attr attr = { .transfer = 0 }; 6936 6937 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6938 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6939 return fops->get_aged_flows(dev, contexts, nb_contexts, 6940 error); 6941 } 6942 DRV_LOG(ERR, 6943 "port %u get aged flows is not supported.", 6944 dev->data->port_id); 6945 return -ENOTSUP; 6946 } 6947 6948 /* Wrapper for driver action_validate op callback */ 6949 static int 6950 flow_drv_action_validate(struct rte_eth_dev *dev, 6951 const struct rte_flow_shared_action_conf *conf, 6952 const struct rte_flow_action *action, 6953 const struct mlx5_flow_driver_ops *fops, 6954 struct rte_flow_error *error) 6955 { 6956 static const char err_msg[] = "shared action validation unsupported"; 6957 6958 if (!fops->action_validate) { 6959 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 6960 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 6961 NULL, err_msg); 6962 return -rte_errno; 6963 } 6964 return fops->action_validate(dev, conf, action, error); 6965 } 6966 6967 /** 6968 * Destroys the shared action by handle. 6969 * 6970 * @param dev 6971 * Pointer to Ethernet device structure. 6972 * @param[in] action 6973 * Handle for the shared action to be destroyed. 6974 * @param[out] error 6975 * Perform verbose error reporting if not NULL. PMDs initialize this 6976 * structure in case of error only. 6977 * 6978 * @return 6979 * 0 on success, a negative errno value otherwise and rte_errno is set. 6980 * 6981 * @note: wrapper for driver action_create op callback. 6982 */ 6983 static int 6984 mlx5_shared_action_destroy(struct rte_eth_dev *dev, 6985 struct rte_flow_shared_action *action, 6986 struct rte_flow_error *error) 6987 { 6988 static const char err_msg[] = "shared action destruction unsupported"; 6989 struct rte_flow_attr attr = { .transfer = 0 }; 6990 const struct mlx5_flow_driver_ops *fops = 6991 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 6992 6993 if (!fops->action_destroy) { 6994 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 6995 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 6996 NULL, err_msg); 6997 return -rte_errno; 6998 } 6999 return fops->action_destroy(dev, action, error); 7000 } 7001 7002 /* Wrapper for driver action_destroy op callback */ 7003 static int 7004 flow_drv_action_update(struct rte_eth_dev *dev, 7005 struct rte_flow_shared_action *action, 7006 const void *action_conf, 7007 const struct mlx5_flow_driver_ops *fops, 7008 struct rte_flow_error *error) 7009 { 7010 static const char err_msg[] = "shared action update unsupported"; 7011 7012 if (!fops->action_update) { 7013 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7014 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7015 NULL, err_msg); 7016 return -rte_errno; 7017 } 7018 return fops->action_update(dev, action, action_conf, error); 7019 } 7020 7021 /* Wrapper for driver action_destroy op callback */ 7022 static int 7023 flow_drv_action_query(struct rte_eth_dev *dev, 7024 const struct rte_flow_shared_action *action, 7025 void *data, 7026 const struct mlx5_flow_driver_ops *fops, 7027 struct rte_flow_error *error) 7028 { 7029 static const char err_msg[] = "shared action query unsupported"; 7030 7031 if (!fops->action_query) { 7032 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7033 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7034 NULL, err_msg); 7035 return -rte_errno; 7036 } 7037 return fops->action_query(dev, action, data, error); 7038 } 7039 7040 /** 7041 * Create shared action for reuse in multiple flow rules. 7042 * 7043 * @param dev 7044 * Pointer to Ethernet device structure. 7045 * @param[in] action 7046 * Action configuration for shared action creation. 7047 * @param[out] error 7048 * Perform verbose error reporting if not NULL. PMDs initialize this 7049 * structure in case of error only. 7050 * @return 7051 * A valid handle in case of success, NULL otherwise and rte_errno is set. 7052 */ 7053 static struct rte_flow_shared_action * 7054 mlx5_shared_action_create(struct rte_eth_dev *dev, 7055 const struct rte_flow_shared_action_conf *conf, 7056 const struct rte_flow_action *action, 7057 struct rte_flow_error *error) 7058 { 7059 static const char err_msg[] = "shared action creation unsupported"; 7060 struct rte_flow_attr attr = { .transfer = 0 }; 7061 const struct mlx5_flow_driver_ops *fops = 7062 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7063 7064 if (flow_drv_action_validate(dev, conf, action, fops, error)) 7065 return NULL; 7066 if (!fops->action_create) { 7067 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7068 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7069 NULL, err_msg); 7070 return NULL; 7071 } 7072 return fops->action_create(dev, conf, action, error); 7073 } 7074 7075 /** 7076 * Updates inplace the shared action configuration pointed by *action* handle 7077 * with the configuration provided as *action* argument. 7078 * The update of the shared action configuration effects all flow rules reusing 7079 * the action via handle. 7080 * 7081 * @param dev 7082 * Pointer to Ethernet device structure. 7083 * @param[in] shared_action 7084 * Handle for the shared action to be updated. 7085 * @param[in] action 7086 * Action specification used to modify the action pointed by handle. 7087 * *action* should be of same type with the action pointed by the *action* 7088 * handle argument, otherwise considered as invalid. 7089 * @param[out] error 7090 * Perform verbose error reporting if not NULL. PMDs initialize this 7091 * structure in case of error only. 7092 * 7093 * @return 7094 * 0 on success, a negative errno value otherwise and rte_errno is set. 7095 */ 7096 static int 7097 mlx5_shared_action_update(struct rte_eth_dev *dev, 7098 struct rte_flow_shared_action *shared_action, 7099 const struct rte_flow_action *action, 7100 struct rte_flow_error *error) 7101 { 7102 struct rte_flow_attr attr = { .transfer = 0 }; 7103 const struct mlx5_flow_driver_ops *fops = 7104 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7105 int ret; 7106 7107 ret = flow_drv_action_validate(dev, NULL, action, fops, error); 7108 if (ret) 7109 return ret; 7110 return flow_drv_action_update(dev, shared_action, action->conf, fops, 7111 error); 7112 } 7113 7114 /** 7115 * Query the shared action by handle. 7116 * 7117 * This function allows retrieving action-specific data such as counters. 7118 * Data is gathered by special action which may be present/referenced in 7119 * more than one flow rule definition. 7120 * 7121 * \see RTE_FLOW_ACTION_TYPE_COUNT 7122 * 7123 * @param dev 7124 * Pointer to Ethernet device structure. 7125 * @param[in] action 7126 * Handle for the shared action to query. 7127 * @param[in, out] data 7128 * Pointer to storage for the associated query data type. 7129 * @param[out] error 7130 * Perform verbose error reporting if not NULL. PMDs initialize this 7131 * structure in case of error only. 7132 * 7133 * @return 7134 * 0 on success, a negative errno value otherwise and rte_errno is set. 7135 */ 7136 static int 7137 mlx5_shared_action_query(struct rte_eth_dev *dev, 7138 const struct rte_flow_shared_action *action, 7139 void *data, 7140 struct rte_flow_error *error) 7141 { 7142 struct rte_flow_attr attr = { .transfer = 0 }; 7143 const struct mlx5_flow_driver_ops *fops = 7144 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7145 7146 return flow_drv_action_query(dev, action, data, fops, error); 7147 } 7148 7149 /** 7150 * Destroy all shared actions. 7151 * 7152 * @param dev 7153 * Pointer to Ethernet device. 7154 * 7155 * @return 7156 * 0 on success, a negative errno value otherwise and rte_errno is set. 7157 */ 7158 int 7159 mlx5_shared_action_flush(struct rte_eth_dev *dev) 7160 { 7161 struct rte_flow_error error; 7162 struct mlx5_priv *priv = dev->data->dev_private; 7163 struct mlx5_shared_action_rss *action; 7164 int ret = 0; 7165 uint32_t idx; 7166 7167 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7168 priv->rss_shared_actions, idx, action, next) { 7169 ret |= mlx5_shared_action_destroy(dev, 7170 (struct rte_flow_shared_action *)(uintptr_t)idx, &error); 7171 } 7172 return ret; 7173 } 7174 7175 #ifndef HAVE_MLX5DV_DR 7176 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 7177 #else 7178 #define MLX5_DOMAIN_SYNC_FLOW \ 7179 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 7180 #endif 7181 7182 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 7183 { 7184 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 7185 const struct mlx5_flow_driver_ops *fops; 7186 int ret; 7187 struct rte_flow_attr attr = { .transfer = 0 }; 7188 7189 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7190 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 7191 if (ret > 0) 7192 ret = -ret; 7193 return ret; 7194 } 7195 7196 /** 7197 * tunnel offload functionalilty is defined for DV environment only 7198 */ 7199 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7200 __extension__ 7201 union tunnel_offload_mark { 7202 uint32_t val; 7203 struct { 7204 uint32_t app_reserve:8; 7205 uint32_t table_id:15; 7206 uint32_t transfer:1; 7207 uint32_t _unused_:8; 7208 }; 7209 }; 7210 7211 static bool 7212 mlx5_access_tunnel_offload_db 7213 (struct rte_eth_dev *dev, 7214 bool (*match)(struct rte_eth_dev *, 7215 struct mlx5_flow_tunnel *, const void *), 7216 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 7217 void (*miss)(struct rte_eth_dev *, void *), 7218 void *ctx, bool lock_op); 7219 7220 static int 7221 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 7222 struct rte_flow *flow, 7223 const struct rte_flow_attr *attr, 7224 const struct rte_flow_action *app_actions, 7225 uint32_t flow_idx, 7226 struct tunnel_default_miss_ctx *ctx, 7227 struct rte_flow_error *error) 7228 { 7229 struct mlx5_priv *priv = dev->data->dev_private; 7230 struct mlx5_flow *dev_flow; 7231 struct rte_flow_attr miss_attr = *attr; 7232 const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf; 7233 const struct rte_flow_item miss_items[2] = { 7234 { 7235 .type = RTE_FLOW_ITEM_TYPE_ETH, 7236 .spec = NULL, 7237 .last = NULL, 7238 .mask = NULL 7239 }, 7240 { 7241 .type = RTE_FLOW_ITEM_TYPE_END, 7242 .spec = NULL, 7243 .last = NULL, 7244 .mask = NULL 7245 } 7246 }; 7247 union tunnel_offload_mark mark_id; 7248 struct rte_flow_action_mark miss_mark; 7249 struct rte_flow_action miss_actions[3] = { 7250 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 7251 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 7252 }; 7253 const struct rte_flow_action_jump *jump_data; 7254 uint32_t i, flow_table = 0; /* prevent compilation warning */ 7255 struct flow_grp_info grp_info = { 7256 .external = 1, 7257 .transfer = attr->transfer, 7258 .fdb_def_rule = !!priv->fdb_def_rule, 7259 .std_tbl_fix = 0, 7260 }; 7261 int ret; 7262 7263 if (!attr->transfer) { 7264 uint32_t q_size; 7265 7266 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 7267 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 7268 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 7269 0, SOCKET_ID_ANY); 7270 if (!ctx->queue) 7271 return rte_flow_error_set 7272 (error, ENOMEM, 7273 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7274 NULL, "invalid default miss RSS"); 7275 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7276 ctx->action_rss.level = 0, 7277 ctx->action_rss.types = priv->rss_conf.rss_hf, 7278 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 7279 ctx->action_rss.queue_num = priv->reta_idx_n, 7280 ctx->action_rss.key = priv->rss_conf.rss_key, 7281 ctx->action_rss.queue = ctx->queue; 7282 if (!priv->reta_idx_n || !priv->rxqs_n) 7283 return rte_flow_error_set 7284 (error, EINVAL, 7285 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7286 NULL, "invalid port configuration"); 7287 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 7288 ctx->action_rss.types = 0; 7289 for (i = 0; i != priv->reta_idx_n; ++i) 7290 ctx->queue[i] = (*priv->reta_idx)[i]; 7291 } else { 7292 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 7293 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 7294 } 7295 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 7296 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 7297 jump_data = app_actions->conf; 7298 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 7299 miss_attr.group = jump_data->group; 7300 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 7301 &flow_table, &grp_info, error); 7302 if (ret) 7303 return rte_flow_error_set(error, EINVAL, 7304 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 7305 NULL, "invalid tunnel id"); 7306 mark_id.app_reserve = 0; 7307 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 7308 mark_id.transfer = !!attr->transfer; 7309 mark_id._unused_ = 0; 7310 miss_mark.id = mark_id.val; 7311 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 7312 miss_items, miss_actions, flow_idx, error); 7313 if (!dev_flow) 7314 return -rte_errno; 7315 dev_flow->flow = flow; 7316 dev_flow->external = true; 7317 dev_flow->tunnel = tunnel; 7318 /* Subflow object was created, we must include one in the list. */ 7319 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7320 dev_flow->handle, next); 7321 DRV_LOG(DEBUG, 7322 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 7323 dev->data->port_id, tunnel->app_tunnel.type, 7324 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 7325 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 7326 miss_actions, error); 7327 if (!ret) 7328 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 7329 error); 7330 7331 return ret; 7332 } 7333 7334 static const struct mlx5_flow_tbl_data_entry * 7335 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 7336 { 7337 struct mlx5_priv *priv = dev->data->dev_private; 7338 struct mlx5_dev_ctx_shared *sh = priv->sh; 7339 struct mlx5_hlist_entry *he; 7340 union tunnel_offload_mark mbits = { .val = mark }; 7341 union mlx5_flow_tbl_key table_key = { 7342 { 7343 .table_id = tunnel_id_to_flow_tbl(mbits.table_id), 7344 .dummy = 0, 7345 .domain = !!mbits.transfer, 7346 .direction = 0, 7347 } 7348 }; 7349 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 7350 return he ? 7351 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 7352 } 7353 7354 static void 7355 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 7356 struct mlx5_hlist_entry *entry) 7357 { 7358 struct mlx5_dev_ctx_shared *sh = list->ctx; 7359 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 7360 7361 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7362 tunnel_flow_tbl_to_id(tte->flow_table)); 7363 mlx5_free(tte); 7364 } 7365 7366 static int 7367 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused, 7368 struct mlx5_hlist_entry *entry, 7369 uint64_t key, void *cb_ctx __rte_unused) 7370 { 7371 union tunnel_tbl_key tbl = { 7372 .val = key, 7373 }; 7374 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 7375 7376 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 7377 } 7378 7379 static struct mlx5_hlist_entry * 7380 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key, 7381 void *ctx __rte_unused) 7382 { 7383 struct mlx5_dev_ctx_shared *sh = list->ctx; 7384 struct tunnel_tbl_entry *tte; 7385 union tunnel_tbl_key tbl = { 7386 .val = key, 7387 }; 7388 7389 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 7390 sizeof(*tte), 0, 7391 SOCKET_ID_ANY); 7392 if (!tte) 7393 goto err; 7394 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7395 &tte->flow_table); 7396 if (tte->flow_table >= MLX5_MAX_TABLES) { 7397 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 7398 tte->flow_table); 7399 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7400 tte->flow_table); 7401 goto err; 7402 } else if (!tte->flow_table) { 7403 goto err; 7404 } 7405 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 7406 tte->tunnel_id = tbl.tunnel_id; 7407 tte->group = tbl.group; 7408 return &tte->hash; 7409 err: 7410 if (tte) 7411 mlx5_free(tte); 7412 return NULL; 7413 } 7414 7415 static uint32_t 7416 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 7417 const struct mlx5_flow_tunnel *tunnel, 7418 uint32_t group, uint32_t *table, 7419 struct rte_flow_error *error) 7420 { 7421 struct mlx5_hlist_entry *he; 7422 struct tunnel_tbl_entry *tte; 7423 union tunnel_tbl_key key = { 7424 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 7425 .group = group 7426 }; 7427 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7428 struct mlx5_hlist *group_hash; 7429 7430 group_hash = tunnel ? tunnel->groups : thub->groups; 7431 he = mlx5_hlist_register(group_hash, key.val, NULL); 7432 if (!he) 7433 return rte_flow_error_set(error, EINVAL, 7434 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7435 NULL, 7436 "tunnel group index not supported"); 7437 tte = container_of(he, typeof(*tte), hash); 7438 *table = tte->flow_table; 7439 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 7440 dev->data->port_id, key.tunnel_id, group, *table); 7441 return 0; 7442 } 7443 7444 static void 7445 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 7446 struct mlx5_flow_tunnel *tunnel) 7447 { 7448 struct mlx5_priv *priv = dev->data->dev_private; 7449 struct mlx5_indexed_pool *ipool; 7450 7451 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 7452 dev->data->port_id, tunnel->tunnel_id); 7453 LIST_REMOVE(tunnel, chain); 7454 mlx5_hlist_destroy(tunnel->groups); 7455 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 7456 mlx5_ipool_free(ipool, tunnel->tunnel_id); 7457 } 7458 7459 static bool 7460 mlx5_access_tunnel_offload_db 7461 (struct rte_eth_dev *dev, 7462 bool (*match)(struct rte_eth_dev *, 7463 struct mlx5_flow_tunnel *, const void *), 7464 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 7465 void (*miss)(struct rte_eth_dev *, void *), 7466 void *ctx, bool lock_op) 7467 { 7468 bool verdict = false; 7469 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7470 struct mlx5_flow_tunnel *tunnel; 7471 7472 rte_spinlock_lock(&thub->sl); 7473 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 7474 verdict = match(dev, tunnel, (const void *)ctx); 7475 if (verdict) 7476 break; 7477 } 7478 if (!lock_op) 7479 rte_spinlock_unlock(&thub->sl); 7480 if (verdict && hit) 7481 hit(dev, tunnel, ctx); 7482 if (!verdict && miss) 7483 miss(dev, ctx); 7484 if (lock_op) 7485 rte_spinlock_unlock(&thub->sl); 7486 7487 return verdict; 7488 } 7489 7490 struct tunnel_db_find_tunnel_id_ctx { 7491 uint32_t tunnel_id; 7492 struct mlx5_flow_tunnel *tunnel; 7493 }; 7494 7495 static bool 7496 find_tunnel_id_match(struct rte_eth_dev *dev, 7497 struct mlx5_flow_tunnel *tunnel, const void *x) 7498 { 7499 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 7500 7501 RTE_SET_USED(dev); 7502 return tunnel->tunnel_id == ctx->tunnel_id; 7503 } 7504 7505 static void 7506 find_tunnel_id_hit(struct rte_eth_dev *dev, 7507 struct mlx5_flow_tunnel *tunnel, void *x) 7508 { 7509 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 7510 RTE_SET_USED(dev); 7511 ctx->tunnel = tunnel; 7512 } 7513 7514 static struct mlx5_flow_tunnel * 7515 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 7516 { 7517 struct tunnel_db_find_tunnel_id_ctx ctx = { 7518 .tunnel_id = id, 7519 }; 7520 7521 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 7522 find_tunnel_id_hit, NULL, &ctx, true); 7523 7524 return ctx.tunnel; 7525 } 7526 7527 static struct mlx5_flow_tunnel * 7528 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 7529 const struct rte_flow_tunnel *app_tunnel) 7530 { 7531 struct mlx5_priv *priv = dev->data->dev_private; 7532 struct mlx5_indexed_pool *ipool; 7533 struct mlx5_flow_tunnel *tunnel; 7534 uint32_t id; 7535 7536 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 7537 tunnel = mlx5_ipool_zmalloc(ipool, &id); 7538 if (!tunnel) 7539 return NULL; 7540 if (id >= MLX5_MAX_TUNNELS) { 7541 mlx5_ipool_free(ipool, id); 7542 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 7543 return NULL; 7544 } 7545 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 7546 mlx5_flow_tunnel_grp2tbl_create_cb, 7547 mlx5_flow_tunnel_grp2tbl_match_cb, 7548 mlx5_flow_tunnel_grp2tbl_remove_cb); 7549 if (!tunnel->groups) { 7550 mlx5_ipool_free(ipool, id); 7551 return NULL; 7552 } 7553 tunnel->groups->ctx = priv->sh; 7554 /* initiate new PMD tunnel */ 7555 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 7556 tunnel->tunnel_id = id; 7557 tunnel->action.type = (typeof(tunnel->action.type)) 7558 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 7559 tunnel->action.conf = tunnel; 7560 tunnel->item.type = (typeof(tunnel->item.type)) 7561 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 7562 tunnel->item.spec = tunnel; 7563 tunnel->item.last = NULL; 7564 tunnel->item.mask = NULL; 7565 7566 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 7567 dev->data->port_id, tunnel->tunnel_id); 7568 7569 return tunnel; 7570 } 7571 7572 struct tunnel_db_get_tunnel_ctx { 7573 const struct rte_flow_tunnel *app_tunnel; 7574 struct mlx5_flow_tunnel *tunnel; 7575 }; 7576 7577 static bool get_tunnel_match(struct rte_eth_dev *dev, 7578 struct mlx5_flow_tunnel *tunnel, const void *x) 7579 { 7580 const struct tunnel_db_get_tunnel_ctx *ctx = x; 7581 7582 RTE_SET_USED(dev); 7583 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 7584 sizeof(*ctx->app_tunnel)); 7585 } 7586 7587 static void get_tunnel_hit(struct rte_eth_dev *dev, 7588 struct mlx5_flow_tunnel *tunnel, void *x) 7589 { 7590 /* called under tunnel spinlock protection */ 7591 struct tunnel_db_get_tunnel_ctx *ctx = x; 7592 7593 RTE_SET_USED(dev); 7594 tunnel->refctn++; 7595 ctx->tunnel = tunnel; 7596 } 7597 7598 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 7599 { 7600 /* called under tunnel spinlock protection */ 7601 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7602 struct tunnel_db_get_tunnel_ctx *ctx = x; 7603 7604 rte_spinlock_unlock(&thub->sl); 7605 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 7606 ctx->tunnel->refctn = 1; 7607 rte_spinlock_lock(&thub->sl); 7608 if (ctx->tunnel) 7609 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 7610 } 7611 7612 7613 static int 7614 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 7615 const struct rte_flow_tunnel *app_tunnel, 7616 struct mlx5_flow_tunnel **tunnel) 7617 { 7618 struct tunnel_db_get_tunnel_ctx ctx = { 7619 .app_tunnel = app_tunnel, 7620 }; 7621 7622 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 7623 get_tunnel_miss, &ctx, true); 7624 *tunnel = ctx.tunnel; 7625 return ctx.tunnel ? 0 : -ENOMEM; 7626 } 7627 7628 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 7629 { 7630 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 7631 7632 if (!thub) 7633 return; 7634 if (!LIST_EMPTY(&thub->tunnels)) 7635 DRV_LOG(WARNING, "port %u tunnels present\n", port_id); 7636 mlx5_hlist_destroy(thub->groups); 7637 mlx5_free(thub); 7638 } 7639 7640 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 7641 { 7642 int err; 7643 struct mlx5_flow_tunnel_hub *thub; 7644 7645 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 7646 0, SOCKET_ID_ANY); 7647 if (!thub) 7648 return -ENOMEM; 7649 LIST_INIT(&thub->tunnels); 7650 rte_spinlock_init(&thub->sl); 7651 thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0, 7652 0, mlx5_flow_tunnel_grp2tbl_create_cb, 7653 mlx5_flow_tunnel_grp2tbl_match_cb, 7654 mlx5_flow_tunnel_grp2tbl_remove_cb); 7655 if (!thub->groups) { 7656 err = -rte_errno; 7657 goto err; 7658 } 7659 thub->groups->ctx = sh; 7660 sh->tunnel_hub = thub; 7661 7662 return 0; 7663 7664 err: 7665 if (thub->groups) 7666 mlx5_hlist_destroy(thub->groups); 7667 if (thub) 7668 mlx5_free(thub); 7669 return err; 7670 } 7671 7672 static inline bool 7673 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 7674 struct rte_flow_tunnel *tunnel, 7675 const char *err_msg) 7676 { 7677 err_msg = NULL; 7678 if (!is_tunnel_offload_active(dev)) { 7679 err_msg = "tunnel offload was not activated"; 7680 goto out; 7681 } else if (!tunnel) { 7682 err_msg = "no application tunnel"; 7683 goto out; 7684 } 7685 7686 switch (tunnel->type) { 7687 default: 7688 err_msg = "unsupported tunnel type"; 7689 goto out; 7690 case RTE_FLOW_ITEM_TYPE_VXLAN: 7691 break; 7692 } 7693 7694 out: 7695 return !err_msg; 7696 } 7697 7698 static int 7699 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 7700 struct rte_flow_tunnel *app_tunnel, 7701 struct rte_flow_action **actions, 7702 uint32_t *num_of_actions, 7703 struct rte_flow_error *error) 7704 { 7705 int ret; 7706 struct mlx5_flow_tunnel *tunnel; 7707 const char *err_msg = NULL; 7708 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 7709 7710 if (!verdict) 7711 return rte_flow_error_set(error, EINVAL, 7712 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 7713 err_msg); 7714 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 7715 if (ret < 0) { 7716 return rte_flow_error_set(error, ret, 7717 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 7718 "failed to initialize pmd tunnel"); 7719 } 7720 *actions = &tunnel->action; 7721 *num_of_actions = 1; 7722 return 0; 7723 } 7724 7725 static int 7726 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 7727 struct rte_flow_tunnel *app_tunnel, 7728 struct rte_flow_item **items, 7729 uint32_t *num_of_items, 7730 struct rte_flow_error *error) 7731 { 7732 int ret; 7733 struct mlx5_flow_tunnel *tunnel; 7734 const char *err_msg = NULL; 7735 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 7736 7737 if (!verdict) 7738 return rte_flow_error_set(error, EINVAL, 7739 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 7740 err_msg); 7741 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 7742 if (ret < 0) { 7743 return rte_flow_error_set(error, ret, 7744 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 7745 "failed to initialize pmd tunnel"); 7746 } 7747 *items = &tunnel->item; 7748 *num_of_items = 1; 7749 return 0; 7750 } 7751 7752 struct tunnel_db_element_release_ctx { 7753 struct rte_flow_item *items; 7754 struct rte_flow_action *actions; 7755 uint32_t num_elements; 7756 struct rte_flow_error *error; 7757 int ret; 7758 }; 7759 7760 static bool 7761 tunnel_element_release_match(struct rte_eth_dev *dev, 7762 struct mlx5_flow_tunnel *tunnel, const void *x) 7763 { 7764 const struct tunnel_db_element_release_ctx *ctx = x; 7765 7766 RTE_SET_USED(dev); 7767 if (ctx->num_elements != 1) 7768 return false; 7769 else if (ctx->items) 7770 return ctx->items == &tunnel->item; 7771 else if (ctx->actions) 7772 return ctx->actions == &tunnel->action; 7773 7774 return false; 7775 } 7776 7777 static void 7778 tunnel_element_release_hit(struct rte_eth_dev *dev, 7779 struct mlx5_flow_tunnel *tunnel, void *x) 7780 { 7781 struct tunnel_db_element_release_ctx *ctx = x; 7782 ctx->ret = 0; 7783 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7784 mlx5_flow_tunnel_free(dev, tunnel); 7785 } 7786 7787 static void 7788 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 7789 { 7790 struct tunnel_db_element_release_ctx *ctx = x; 7791 RTE_SET_USED(dev); 7792 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 7793 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 7794 "invalid argument"); 7795 } 7796 7797 static int 7798 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 7799 struct rte_flow_item *pmd_items, 7800 uint32_t num_items, struct rte_flow_error *err) 7801 { 7802 struct tunnel_db_element_release_ctx ctx = { 7803 .items = pmd_items, 7804 .actions = NULL, 7805 .num_elements = num_items, 7806 .error = err, 7807 }; 7808 7809 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 7810 tunnel_element_release_hit, 7811 tunnel_element_release_miss, &ctx, false); 7812 7813 return ctx.ret; 7814 } 7815 7816 static int 7817 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 7818 struct rte_flow_action *pmd_actions, 7819 uint32_t num_actions, struct rte_flow_error *err) 7820 { 7821 struct tunnel_db_element_release_ctx ctx = { 7822 .items = NULL, 7823 .actions = pmd_actions, 7824 .num_elements = num_actions, 7825 .error = err, 7826 }; 7827 7828 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 7829 tunnel_element_release_hit, 7830 tunnel_element_release_miss, &ctx, false); 7831 7832 return ctx.ret; 7833 } 7834 7835 static int 7836 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 7837 struct rte_mbuf *m, 7838 struct rte_flow_restore_info *info, 7839 struct rte_flow_error *err) 7840 { 7841 uint64_t ol_flags = m->ol_flags; 7842 const struct mlx5_flow_tbl_data_entry *tble; 7843 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 7844 7845 if (!is_tunnel_offload_active(dev)) { 7846 info->flags = 0; 7847 return 0; 7848 } 7849 7850 if ((ol_flags & mask) != mask) 7851 goto err; 7852 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 7853 if (!tble) { 7854 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 7855 dev->data->port_id, m->hash.fdir.hi); 7856 goto err; 7857 } 7858 MLX5_ASSERT(tble->tunnel); 7859 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 7860 info->group_id = tble->group_id; 7861 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 7862 RTE_FLOW_RESTORE_INFO_GROUP_ID | 7863 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 7864 7865 return 0; 7866 7867 err: 7868 return rte_flow_error_set(err, EINVAL, 7869 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7870 "failed to get restore info"); 7871 } 7872 7873 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 7874 static int 7875 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 7876 __rte_unused struct rte_flow_tunnel *app_tunnel, 7877 __rte_unused struct rte_flow_action **actions, 7878 __rte_unused uint32_t *num_of_actions, 7879 __rte_unused struct rte_flow_error *error) 7880 { 7881 return -ENOTSUP; 7882 } 7883 7884 static int 7885 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 7886 __rte_unused struct rte_flow_tunnel *app_tunnel, 7887 __rte_unused struct rte_flow_item **items, 7888 __rte_unused uint32_t *num_of_items, 7889 __rte_unused struct rte_flow_error *error) 7890 { 7891 return -ENOTSUP; 7892 } 7893 7894 static int 7895 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 7896 __rte_unused struct rte_flow_item *pmd_items, 7897 __rte_unused uint32_t num_items, 7898 __rte_unused struct rte_flow_error *err) 7899 { 7900 return -ENOTSUP; 7901 } 7902 7903 static int 7904 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 7905 __rte_unused struct rte_flow_action *pmd_action, 7906 __rte_unused uint32_t num_actions, 7907 __rte_unused struct rte_flow_error *err) 7908 { 7909 return -ENOTSUP; 7910 } 7911 7912 static int 7913 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 7914 __rte_unused struct rte_mbuf *m, 7915 __rte_unused struct rte_flow_restore_info *i, 7916 __rte_unused struct rte_flow_error *err) 7917 { 7918 return -ENOTSUP; 7919 } 7920 7921 static int 7922 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 7923 __rte_unused struct rte_flow *flow, 7924 __rte_unused const struct rte_flow_attr *attr, 7925 __rte_unused const struct rte_flow_action *actions, 7926 __rte_unused uint32_t flow_idx, 7927 __rte_unused struct tunnel_default_miss_ctx *ctx, 7928 __rte_unused struct rte_flow_error *error) 7929 { 7930 return -ENOTSUP; 7931 } 7932 7933 static struct mlx5_flow_tunnel * 7934 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 7935 __rte_unused uint32_t id) 7936 { 7937 return NULL; 7938 } 7939 7940 static void 7941 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 7942 __rte_unused struct mlx5_flow_tunnel *tunnel) 7943 { 7944 } 7945 7946 static uint32_t 7947 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 7948 __rte_unused const struct mlx5_flow_tunnel *t, 7949 __rte_unused uint32_t group, 7950 __rte_unused uint32_t *table, 7951 struct rte_flow_error *error) 7952 { 7953 return rte_flow_error_set(error, ENOTSUP, 7954 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7955 "tunnel offload requires DV support"); 7956 } 7957 7958 void 7959 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 7960 __rte_unused uint16_t port_id) 7961 { 7962 } 7963 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 7964 7965