1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see ETH_RSS_* definitions). 102 */ 103 uint8_t optional; 104 /**< optional expand field. Default 0 to expand, 1 not go deeper. */ 105 }; 106 107 /** Object returned by mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_rss { 109 uint32_t entries; 110 /**< Number of entries @p patterns and @p priorities. */ 111 struct { 112 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 113 uint32_t priority; /**< Priority offset for each expansion. */ 114 } entry[]; 115 }; 116 117 static void 118 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 119 120 static bool 121 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 122 { 123 switch (item->type) { 124 case RTE_FLOW_ITEM_TYPE_ETH: 125 case RTE_FLOW_ITEM_TYPE_VLAN: 126 case RTE_FLOW_ITEM_TYPE_IPV4: 127 case RTE_FLOW_ITEM_TYPE_IPV6: 128 case RTE_FLOW_ITEM_TYPE_UDP: 129 case RTE_FLOW_ITEM_TYPE_TCP: 130 case RTE_FLOW_ITEM_TYPE_VXLAN: 131 case RTE_FLOW_ITEM_TYPE_NVGRE: 132 case RTE_FLOW_ITEM_TYPE_GRE: 133 case RTE_FLOW_ITEM_TYPE_GENEVE: 134 case RTE_FLOW_ITEM_TYPE_MPLS: 135 return true; 136 default: 137 break; 138 } 139 return false; 140 } 141 142 static enum rte_flow_item_type 143 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 144 { 145 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 146 uint16_t ether_type = 0; 147 uint16_t ether_type_m; 148 uint8_t ip_next_proto = 0; 149 uint8_t ip_next_proto_m; 150 151 if (item == NULL || item->spec == NULL) 152 return ret; 153 switch (item->type) { 154 case RTE_FLOW_ITEM_TYPE_ETH: 155 if (item->mask) 156 ether_type_m = ((const struct rte_flow_item_eth *) 157 (item->mask))->type; 158 else 159 ether_type_m = rte_flow_item_eth_mask.type; 160 if (ether_type_m != RTE_BE16(0xFFFF)) 161 break; 162 ether_type = ((const struct rte_flow_item_eth *) 163 (item->spec))->type; 164 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 165 ret = RTE_FLOW_ITEM_TYPE_IPV4; 166 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 167 ret = RTE_FLOW_ITEM_TYPE_IPV6; 168 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 169 ret = RTE_FLOW_ITEM_TYPE_VLAN; 170 else 171 ret = RTE_FLOW_ITEM_TYPE_END; 172 break; 173 case RTE_FLOW_ITEM_TYPE_VLAN: 174 if (item->mask) 175 ether_type_m = ((const struct rte_flow_item_vlan *) 176 (item->mask))->inner_type; 177 else 178 ether_type_m = rte_flow_item_vlan_mask.inner_type; 179 if (ether_type_m != RTE_BE16(0xFFFF)) 180 break; 181 ether_type = ((const struct rte_flow_item_vlan *) 182 (item->spec))->inner_type; 183 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 184 ret = RTE_FLOW_ITEM_TYPE_IPV4; 185 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 186 ret = RTE_FLOW_ITEM_TYPE_IPV6; 187 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 188 ret = RTE_FLOW_ITEM_TYPE_VLAN; 189 else 190 ret = RTE_FLOW_ITEM_TYPE_END; 191 break; 192 case RTE_FLOW_ITEM_TYPE_IPV4: 193 if (item->mask) 194 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 195 (item->mask))->hdr.next_proto_id; 196 else 197 ip_next_proto_m = 198 rte_flow_item_ipv4_mask.hdr.next_proto_id; 199 if (ip_next_proto_m != 0xFF) 200 break; 201 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 202 (item->spec))->hdr.next_proto_id; 203 if (ip_next_proto == IPPROTO_UDP) 204 ret = RTE_FLOW_ITEM_TYPE_UDP; 205 else if (ip_next_proto == IPPROTO_TCP) 206 ret = RTE_FLOW_ITEM_TYPE_TCP; 207 else if (ip_next_proto == IPPROTO_IP) 208 ret = RTE_FLOW_ITEM_TYPE_IPV4; 209 else if (ip_next_proto == IPPROTO_IPV6) 210 ret = RTE_FLOW_ITEM_TYPE_IPV6; 211 else 212 ret = RTE_FLOW_ITEM_TYPE_END; 213 break; 214 case RTE_FLOW_ITEM_TYPE_IPV6: 215 if (item->mask) 216 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 217 (item->mask))->hdr.proto; 218 else 219 ip_next_proto_m = 220 rte_flow_item_ipv6_mask.hdr.proto; 221 if (ip_next_proto_m != 0xFF) 222 break; 223 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 224 (item->spec))->hdr.proto; 225 if (ip_next_proto == IPPROTO_UDP) 226 ret = RTE_FLOW_ITEM_TYPE_UDP; 227 else if (ip_next_proto == IPPROTO_TCP) 228 ret = RTE_FLOW_ITEM_TYPE_TCP; 229 else if (ip_next_proto == IPPROTO_IP) 230 ret = RTE_FLOW_ITEM_TYPE_IPV4; 231 else if (ip_next_proto == IPPROTO_IPV6) 232 ret = RTE_FLOW_ITEM_TYPE_IPV6; 233 else 234 ret = RTE_FLOW_ITEM_TYPE_END; 235 break; 236 default: 237 ret = RTE_FLOW_ITEM_TYPE_VOID; 238 break; 239 } 240 return ret; 241 } 242 243 #define MLX5_RSS_EXP_ELT_N 16 244 245 /** 246 * Expand RSS flows into several possible flows according to the RSS hash 247 * fields requested and the driver capabilities. 248 * 249 * @param[out] buf 250 * Buffer to store the result expansion. 251 * @param[in] size 252 * Buffer size in bytes. If 0, @p buf can be NULL. 253 * @param[in] pattern 254 * User flow pattern. 255 * @param[in] types 256 * RSS types to expand (see ETH_RSS_* definitions). 257 * @param[in] graph 258 * Input graph to expand @p pattern according to @p types. 259 * @param[in] graph_root_index 260 * Index of root node in @p graph, typically 0. 261 * 262 * @return 263 * A positive value representing the size of @p buf in bytes regardless of 264 * @p size on success, a negative errno value otherwise and rte_errno is 265 * set, the following errors are defined: 266 * 267 * -E2BIG: graph-depth @p graph is too deep. 268 * -EINVAL: @p size has not enough space for expanded pattern. 269 */ 270 static int 271 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 272 const struct rte_flow_item *pattern, uint64_t types, 273 const struct mlx5_flow_expand_node graph[], 274 int graph_root_index) 275 { 276 const struct rte_flow_item *item; 277 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 278 const int *next_node; 279 const int *stack[MLX5_RSS_EXP_ELT_N]; 280 int stack_pos = 0; 281 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 282 unsigned int i; 283 size_t lsize; 284 size_t user_pattern_size = 0; 285 void *addr = NULL; 286 const struct mlx5_flow_expand_node *next = NULL; 287 struct rte_flow_item missed_item; 288 int missed = 0; 289 int elt = 0; 290 const struct rte_flow_item *last_item = NULL; 291 292 memset(&missed_item, 0, sizeof(missed_item)); 293 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 294 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 295 if (lsize > size) 296 return -EINVAL; 297 buf->entry[0].priority = 0; 298 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 299 buf->entries = 0; 300 addr = buf->entry[0].pattern; 301 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 302 if (!mlx5_flow_is_rss_expandable_item(item)) { 303 user_pattern_size += sizeof(*item); 304 continue; 305 } 306 last_item = item; 307 for (i = 0; node->next && node->next[i]; ++i) { 308 next = &graph[node->next[i]]; 309 if (next->type == item->type) 310 break; 311 } 312 if (next) 313 node = next; 314 user_pattern_size += sizeof(*item); 315 } 316 user_pattern_size += sizeof(*item); /* Handle END item. */ 317 lsize += user_pattern_size; 318 if (lsize > size) 319 return -EINVAL; 320 /* Copy the user pattern in the first entry of the buffer. */ 321 rte_memcpy(addr, pattern, user_pattern_size); 322 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 323 buf->entries = 1; 324 /* Start expanding. */ 325 memset(flow_items, 0, sizeof(flow_items)); 326 user_pattern_size -= sizeof(*item); 327 /* 328 * Check if the last valid item has spec set, need complete pattern, 329 * and the pattern can be used for expansion. 330 */ 331 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 332 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 333 /* Item type END indicates expansion is not required. */ 334 return lsize; 335 } 336 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 337 next = NULL; 338 missed = 1; 339 for (i = 0; node->next && node->next[i]; ++i) { 340 next = &graph[node->next[i]]; 341 if (next->type == missed_item.type) { 342 flow_items[0].type = missed_item.type; 343 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 344 break; 345 } 346 next = NULL; 347 } 348 } 349 if (next && missed) { 350 elt = 2; /* missed item + item end. */ 351 node = next; 352 lsize += elt * sizeof(*item) + user_pattern_size; 353 if (lsize > size) 354 return -EINVAL; 355 if (node->rss_types & types) { 356 buf->entry[buf->entries].priority = 1; 357 buf->entry[buf->entries].pattern = addr; 358 buf->entries++; 359 rte_memcpy(addr, buf->entry[0].pattern, 360 user_pattern_size); 361 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 362 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 363 addr = (void *)(((uintptr_t)addr) + 364 elt * sizeof(*item)); 365 } 366 } 367 memset(flow_items, 0, sizeof(flow_items)); 368 next_node = node->next; 369 stack[stack_pos] = next_node; 370 node = next_node ? &graph[*next_node] : NULL; 371 while (node) { 372 flow_items[stack_pos].type = node->type; 373 if (node->rss_types & types) { 374 size_t n; 375 /* 376 * compute the number of items to copy from the 377 * expansion and copy it. 378 * When the stack_pos is 0, there are 1 element in it, 379 * plus the addition END item. 380 */ 381 elt = stack_pos + 2; 382 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 383 lsize += elt * sizeof(*item) + user_pattern_size; 384 if (lsize > size) 385 return -EINVAL; 386 n = elt * sizeof(*item); 387 buf->entry[buf->entries].priority = 388 stack_pos + 1 + missed; 389 buf->entry[buf->entries].pattern = addr; 390 buf->entries++; 391 rte_memcpy(addr, buf->entry[0].pattern, 392 user_pattern_size); 393 addr = (void *)(((uintptr_t)addr) + 394 user_pattern_size); 395 rte_memcpy(addr, &missed_item, 396 missed * sizeof(*item)); 397 addr = (void *)(((uintptr_t)addr) + 398 missed * sizeof(*item)); 399 rte_memcpy(addr, flow_items, n); 400 addr = (void *)(((uintptr_t)addr) + n); 401 } 402 /* Go deeper. */ 403 if (!node->optional && node->next) { 404 next_node = node->next; 405 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 406 rte_errno = E2BIG; 407 return -rte_errno; 408 } 409 stack[stack_pos] = next_node; 410 } else if (*(next_node + 1)) { 411 /* Follow up with the next possibility. */ 412 ++next_node; 413 } else { 414 /* Move to the next path. */ 415 if (stack_pos) 416 next_node = stack[--stack_pos]; 417 next_node++; 418 stack[stack_pos] = next_node; 419 } 420 node = *next_node ? &graph[*next_node] : NULL; 421 }; 422 return lsize; 423 } 424 425 enum mlx5_expansion { 426 MLX5_EXPANSION_ROOT, 427 MLX5_EXPANSION_ROOT_OUTER, 428 MLX5_EXPANSION_ROOT_ETH_VLAN, 429 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 430 MLX5_EXPANSION_OUTER_ETH, 431 MLX5_EXPANSION_OUTER_ETH_VLAN, 432 MLX5_EXPANSION_OUTER_VLAN, 433 MLX5_EXPANSION_OUTER_IPV4, 434 MLX5_EXPANSION_OUTER_IPV4_UDP, 435 MLX5_EXPANSION_OUTER_IPV4_TCP, 436 MLX5_EXPANSION_OUTER_IPV6, 437 MLX5_EXPANSION_OUTER_IPV6_UDP, 438 MLX5_EXPANSION_OUTER_IPV6_TCP, 439 MLX5_EXPANSION_VXLAN, 440 MLX5_EXPANSION_VXLAN_GPE, 441 MLX5_EXPANSION_GRE, 442 MLX5_EXPANSION_NVGRE, 443 MLX5_EXPANSION_GRE_KEY, 444 MLX5_EXPANSION_MPLS, 445 MLX5_EXPANSION_ETH, 446 MLX5_EXPANSION_ETH_VLAN, 447 MLX5_EXPANSION_VLAN, 448 MLX5_EXPANSION_IPV4, 449 MLX5_EXPANSION_IPV4_UDP, 450 MLX5_EXPANSION_IPV4_TCP, 451 MLX5_EXPANSION_IPV6, 452 MLX5_EXPANSION_IPV6_UDP, 453 MLX5_EXPANSION_IPV6_TCP, 454 }; 455 456 /** Supported expansion of items. */ 457 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 458 [MLX5_EXPANSION_ROOT] = { 459 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 460 MLX5_EXPANSION_IPV4, 461 MLX5_EXPANSION_IPV6), 462 .type = RTE_FLOW_ITEM_TYPE_END, 463 }, 464 [MLX5_EXPANSION_ROOT_OUTER] = { 465 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 466 MLX5_EXPANSION_OUTER_IPV4, 467 MLX5_EXPANSION_OUTER_IPV6), 468 .type = RTE_FLOW_ITEM_TYPE_END, 469 }, 470 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 471 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 472 .type = RTE_FLOW_ITEM_TYPE_END, 473 }, 474 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 475 .next = MLX5_FLOW_EXPAND_RSS_NEXT 476 (MLX5_EXPANSION_OUTER_ETH_VLAN), 477 .type = RTE_FLOW_ITEM_TYPE_END, 478 }, 479 [MLX5_EXPANSION_OUTER_ETH] = { 480 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 481 MLX5_EXPANSION_OUTER_IPV6), 482 .type = RTE_FLOW_ITEM_TYPE_ETH, 483 .rss_types = 0, 484 }, 485 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 486 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 487 .type = RTE_FLOW_ITEM_TYPE_ETH, 488 .rss_types = 0, 489 }, 490 [MLX5_EXPANSION_OUTER_VLAN] = { 491 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 492 MLX5_EXPANSION_OUTER_IPV6), 493 .type = RTE_FLOW_ITEM_TYPE_VLAN, 494 }, 495 [MLX5_EXPANSION_OUTER_IPV4] = { 496 .next = MLX5_FLOW_EXPAND_RSS_NEXT 497 (MLX5_EXPANSION_OUTER_IPV4_UDP, 498 MLX5_EXPANSION_OUTER_IPV4_TCP, 499 MLX5_EXPANSION_GRE, 500 MLX5_EXPANSION_NVGRE, 501 MLX5_EXPANSION_IPV4, 502 MLX5_EXPANSION_IPV6), 503 .type = RTE_FLOW_ITEM_TYPE_IPV4, 504 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 505 ETH_RSS_NONFRAG_IPV4_OTHER, 506 }, 507 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 508 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 509 MLX5_EXPANSION_VXLAN_GPE, 510 MLX5_EXPANSION_MPLS), 511 .type = RTE_FLOW_ITEM_TYPE_UDP, 512 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 513 }, 514 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 515 .type = RTE_FLOW_ITEM_TYPE_TCP, 516 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 517 }, 518 [MLX5_EXPANSION_OUTER_IPV6] = { 519 .next = MLX5_FLOW_EXPAND_RSS_NEXT 520 (MLX5_EXPANSION_OUTER_IPV6_UDP, 521 MLX5_EXPANSION_OUTER_IPV6_TCP, 522 MLX5_EXPANSION_IPV4, 523 MLX5_EXPANSION_IPV6, 524 MLX5_EXPANSION_GRE, 525 MLX5_EXPANSION_NVGRE), 526 .type = RTE_FLOW_ITEM_TYPE_IPV6, 527 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 528 ETH_RSS_NONFRAG_IPV6_OTHER, 529 }, 530 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 531 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 532 MLX5_EXPANSION_VXLAN_GPE, 533 MLX5_EXPANSION_MPLS), 534 .type = RTE_FLOW_ITEM_TYPE_UDP, 535 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 536 }, 537 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 538 .type = RTE_FLOW_ITEM_TYPE_TCP, 539 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 540 }, 541 [MLX5_EXPANSION_VXLAN] = { 542 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 543 MLX5_EXPANSION_IPV4, 544 MLX5_EXPANSION_IPV6), 545 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 546 }, 547 [MLX5_EXPANSION_VXLAN_GPE] = { 548 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 549 MLX5_EXPANSION_IPV4, 550 MLX5_EXPANSION_IPV6), 551 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 552 }, 553 [MLX5_EXPANSION_GRE] = { 554 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 555 MLX5_EXPANSION_IPV6, 556 MLX5_EXPANSION_GRE_KEY, 557 MLX5_EXPANSION_MPLS), 558 .type = RTE_FLOW_ITEM_TYPE_GRE, 559 }, 560 [MLX5_EXPANSION_GRE_KEY] = { 561 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 562 MLX5_EXPANSION_IPV6, 563 MLX5_EXPANSION_MPLS), 564 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 565 .optional = 1, 566 }, 567 [MLX5_EXPANSION_NVGRE] = { 568 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 569 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 570 }, 571 [MLX5_EXPANSION_MPLS] = { 572 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 573 MLX5_EXPANSION_IPV6, 574 MLX5_EXPANSION_ETH), 575 .type = RTE_FLOW_ITEM_TYPE_MPLS, 576 }, 577 [MLX5_EXPANSION_ETH] = { 578 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 579 MLX5_EXPANSION_IPV6), 580 .type = RTE_FLOW_ITEM_TYPE_ETH, 581 }, 582 [MLX5_EXPANSION_ETH_VLAN] = { 583 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 584 .type = RTE_FLOW_ITEM_TYPE_ETH, 585 }, 586 [MLX5_EXPANSION_VLAN] = { 587 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 588 MLX5_EXPANSION_IPV6), 589 .type = RTE_FLOW_ITEM_TYPE_VLAN, 590 }, 591 [MLX5_EXPANSION_IPV4] = { 592 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 593 MLX5_EXPANSION_IPV4_TCP), 594 .type = RTE_FLOW_ITEM_TYPE_IPV4, 595 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 596 ETH_RSS_NONFRAG_IPV4_OTHER, 597 }, 598 [MLX5_EXPANSION_IPV4_UDP] = { 599 .type = RTE_FLOW_ITEM_TYPE_UDP, 600 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 601 }, 602 [MLX5_EXPANSION_IPV4_TCP] = { 603 .type = RTE_FLOW_ITEM_TYPE_TCP, 604 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 605 }, 606 [MLX5_EXPANSION_IPV6] = { 607 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 608 MLX5_EXPANSION_IPV6_TCP), 609 .type = RTE_FLOW_ITEM_TYPE_IPV6, 610 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 611 ETH_RSS_NONFRAG_IPV6_OTHER, 612 }, 613 [MLX5_EXPANSION_IPV6_UDP] = { 614 .type = RTE_FLOW_ITEM_TYPE_UDP, 615 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 616 }, 617 [MLX5_EXPANSION_IPV6_TCP] = { 618 .type = RTE_FLOW_ITEM_TYPE_TCP, 619 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 620 }, 621 }; 622 623 static struct rte_flow_action_handle * 624 mlx5_action_handle_create(struct rte_eth_dev *dev, 625 const struct rte_flow_indir_action_conf *conf, 626 const struct rte_flow_action *action, 627 struct rte_flow_error *error); 628 static int mlx5_action_handle_destroy 629 (struct rte_eth_dev *dev, 630 struct rte_flow_action_handle *handle, 631 struct rte_flow_error *error); 632 static int mlx5_action_handle_update 633 (struct rte_eth_dev *dev, 634 struct rte_flow_action_handle *handle, 635 const void *update, 636 struct rte_flow_error *error); 637 static int mlx5_action_handle_query 638 (struct rte_eth_dev *dev, 639 const struct rte_flow_action_handle *handle, 640 void *data, 641 struct rte_flow_error *error); 642 static int 643 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 644 struct rte_flow_tunnel *app_tunnel, 645 struct rte_flow_action **actions, 646 uint32_t *num_of_actions, 647 struct rte_flow_error *error); 648 static int 649 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 650 struct rte_flow_tunnel *app_tunnel, 651 struct rte_flow_item **items, 652 uint32_t *num_of_items, 653 struct rte_flow_error *error); 654 static int 655 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 656 struct rte_flow_item *pmd_items, 657 uint32_t num_items, struct rte_flow_error *err); 658 static int 659 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 660 struct rte_flow_action *pmd_actions, 661 uint32_t num_actions, 662 struct rte_flow_error *err); 663 static int 664 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 665 struct rte_mbuf *m, 666 struct rte_flow_restore_info *info, 667 struct rte_flow_error *err); 668 669 static const struct rte_flow_ops mlx5_flow_ops = { 670 .validate = mlx5_flow_validate, 671 .create = mlx5_flow_create, 672 .destroy = mlx5_flow_destroy, 673 .flush = mlx5_flow_flush, 674 .isolate = mlx5_flow_isolate, 675 .query = mlx5_flow_query, 676 .dev_dump = mlx5_flow_dev_dump, 677 .get_aged_flows = mlx5_flow_get_aged_flows, 678 .action_handle_create = mlx5_action_handle_create, 679 .action_handle_destroy = mlx5_action_handle_destroy, 680 .action_handle_update = mlx5_action_handle_update, 681 .action_handle_query = mlx5_action_handle_query, 682 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 683 .tunnel_match = mlx5_flow_tunnel_match, 684 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 685 .tunnel_item_release = mlx5_flow_tunnel_item_release, 686 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 687 }; 688 689 /* Tunnel information. */ 690 struct mlx5_flow_tunnel_info { 691 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 692 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 693 }; 694 695 static struct mlx5_flow_tunnel_info tunnels_info[] = { 696 { 697 .tunnel = MLX5_FLOW_LAYER_VXLAN, 698 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 699 }, 700 { 701 .tunnel = MLX5_FLOW_LAYER_GENEVE, 702 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 703 }, 704 { 705 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 706 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 707 }, 708 { 709 .tunnel = MLX5_FLOW_LAYER_GRE, 710 .ptype = RTE_PTYPE_TUNNEL_GRE, 711 }, 712 { 713 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 714 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 715 }, 716 { 717 .tunnel = MLX5_FLOW_LAYER_MPLS, 718 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 719 }, 720 { 721 .tunnel = MLX5_FLOW_LAYER_NVGRE, 722 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 723 }, 724 { 725 .tunnel = MLX5_FLOW_LAYER_IPIP, 726 .ptype = RTE_PTYPE_TUNNEL_IP, 727 }, 728 { 729 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 730 .ptype = RTE_PTYPE_TUNNEL_IP, 731 }, 732 { 733 .tunnel = MLX5_FLOW_LAYER_GTP, 734 .ptype = RTE_PTYPE_TUNNEL_GTPU, 735 }, 736 }; 737 738 739 740 /** 741 * Translate tag ID to register. 742 * 743 * @param[in] dev 744 * Pointer to the Ethernet device structure. 745 * @param[in] feature 746 * The feature that request the register. 747 * @param[in] id 748 * The request register ID. 749 * @param[out] error 750 * Error description in case of any. 751 * 752 * @return 753 * The request register on success, a negative errno 754 * value otherwise and rte_errno is set. 755 */ 756 int 757 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 758 enum mlx5_feature_name feature, 759 uint32_t id, 760 struct rte_flow_error *error) 761 { 762 struct mlx5_priv *priv = dev->data->dev_private; 763 struct mlx5_dev_config *config = &priv->config; 764 enum modify_reg start_reg; 765 bool skip_mtr_reg = false; 766 767 switch (feature) { 768 case MLX5_HAIRPIN_RX: 769 return REG_B; 770 case MLX5_HAIRPIN_TX: 771 return REG_A; 772 case MLX5_METADATA_RX: 773 switch (config->dv_xmeta_en) { 774 case MLX5_XMETA_MODE_LEGACY: 775 return REG_B; 776 case MLX5_XMETA_MODE_META16: 777 return REG_C_0; 778 case MLX5_XMETA_MODE_META32: 779 return REG_C_1; 780 } 781 break; 782 case MLX5_METADATA_TX: 783 return REG_A; 784 case MLX5_METADATA_FDB: 785 switch (config->dv_xmeta_en) { 786 case MLX5_XMETA_MODE_LEGACY: 787 return REG_NON; 788 case MLX5_XMETA_MODE_META16: 789 return REG_C_0; 790 case MLX5_XMETA_MODE_META32: 791 return REG_C_1; 792 } 793 break; 794 case MLX5_FLOW_MARK: 795 switch (config->dv_xmeta_en) { 796 case MLX5_XMETA_MODE_LEGACY: 797 return REG_NON; 798 case MLX5_XMETA_MODE_META16: 799 return REG_C_1; 800 case MLX5_XMETA_MODE_META32: 801 return REG_C_0; 802 } 803 break; 804 case MLX5_MTR_ID: 805 /* 806 * If meter color and meter id share one register, flow match 807 * should use the meter color register for match. 808 */ 809 if (priv->mtr_reg_share) 810 return priv->mtr_color_reg; 811 else 812 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 813 REG_C_3; 814 case MLX5_MTR_COLOR: 815 case MLX5_ASO_FLOW_HIT: 816 case MLX5_ASO_CONNTRACK: 817 /* All features use the same REG_C. */ 818 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 819 return priv->mtr_color_reg; 820 case MLX5_COPY_MARK: 821 /* 822 * Metadata COPY_MARK register using is in meter suffix sub 823 * flow while with meter. It's safe to share the same register. 824 */ 825 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 826 case MLX5_APP_TAG: 827 /* 828 * If meter is enable, it will engage the register for color 829 * match and flow match. If meter color match is not using the 830 * REG_C_2, need to skip the REG_C_x be used by meter color 831 * match. 832 * If meter is disable, free to use all available registers. 833 */ 834 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 835 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 836 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 837 if (id > (uint32_t)(REG_C_7 - start_reg)) 838 return rte_flow_error_set(error, EINVAL, 839 RTE_FLOW_ERROR_TYPE_ITEM, 840 NULL, "invalid tag id"); 841 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 842 return rte_flow_error_set(error, ENOTSUP, 843 RTE_FLOW_ERROR_TYPE_ITEM, 844 NULL, "unsupported tag id"); 845 /* 846 * This case means meter is using the REG_C_x great than 2. 847 * Take care not to conflict with meter color REG_C_x. 848 * If the available index REG_C_y >= REG_C_x, skip the 849 * color register. 850 */ 851 if (skip_mtr_reg && config->flow_mreg_c 852 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 853 if (id >= (uint32_t)(REG_C_7 - start_reg)) 854 return rte_flow_error_set(error, EINVAL, 855 RTE_FLOW_ERROR_TYPE_ITEM, 856 NULL, "invalid tag id"); 857 if (config->flow_mreg_c 858 [id + 1 + start_reg - REG_C_0] != REG_NON) 859 return config->flow_mreg_c 860 [id + 1 + start_reg - REG_C_0]; 861 return rte_flow_error_set(error, ENOTSUP, 862 RTE_FLOW_ERROR_TYPE_ITEM, 863 NULL, "unsupported tag id"); 864 } 865 return config->flow_mreg_c[id + start_reg - REG_C_0]; 866 } 867 MLX5_ASSERT(false); 868 return rte_flow_error_set(error, EINVAL, 869 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 870 NULL, "invalid feature name"); 871 } 872 873 /** 874 * Check extensive flow metadata register support. 875 * 876 * @param dev 877 * Pointer to rte_eth_dev structure. 878 * 879 * @return 880 * True if device supports extensive flow metadata register, otherwise false. 881 */ 882 bool 883 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 884 { 885 struct mlx5_priv *priv = dev->data->dev_private; 886 struct mlx5_dev_config *config = &priv->config; 887 888 /* 889 * Having available reg_c can be regarded inclusively as supporting 890 * extensive flow metadata register, which could mean, 891 * - metadata register copy action by modify header. 892 * - 16 modify header actions is supported. 893 * - reg_c's are preserved across different domain (FDB and NIC) on 894 * packet loopback by flow lookup miss. 895 */ 896 return config->flow_mreg_c[2] != REG_NON; 897 } 898 899 /** 900 * Get the lowest priority. 901 * 902 * @param[in] dev 903 * Pointer to the Ethernet device structure. 904 * @param[in] attributes 905 * Pointer to device flow rule attributes. 906 * 907 * @return 908 * The value of lowest priority of flow. 909 */ 910 uint32_t 911 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 912 const struct rte_flow_attr *attr) 913 { 914 struct mlx5_priv *priv = dev->data->dev_private; 915 916 if (!attr->group && !attr->transfer) 917 return priv->config.flow_prio - 2; 918 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 919 } 920 921 /** 922 * Calculate matcher priority of the flow. 923 * 924 * @param[in] dev 925 * Pointer to the Ethernet device structure. 926 * @param[in] attr 927 * Pointer to device flow rule attributes. 928 * @param[in] subpriority 929 * The priority based on the items. 930 * @return 931 * The matcher priority of the flow. 932 */ 933 uint16_t 934 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 935 const struct rte_flow_attr *attr, 936 uint32_t subpriority) 937 { 938 uint16_t priority = (uint16_t)attr->priority; 939 struct mlx5_priv *priv = dev->data->dev_private; 940 941 if (!attr->group && !attr->transfer) { 942 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 943 priority = priv->config.flow_prio - 1; 944 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 945 } 946 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 947 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 948 return priority * 3 + subpriority; 949 } 950 951 /** 952 * Verify the @p item specifications (spec, last, mask) are compatible with the 953 * NIC capabilities. 954 * 955 * @param[in] item 956 * Item specification. 957 * @param[in] mask 958 * @p item->mask or flow default bit-masks. 959 * @param[in] nic_mask 960 * Bit-masks covering supported fields by the NIC to compare with user mask. 961 * @param[in] size 962 * Bit-masks size in bytes. 963 * @param[in] range_accepted 964 * True if range of values is accepted for specific fields, false otherwise. 965 * @param[out] error 966 * Pointer to error structure. 967 * 968 * @return 969 * 0 on success, a negative errno value otherwise and rte_errno is set. 970 */ 971 int 972 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 973 const uint8_t *mask, 974 const uint8_t *nic_mask, 975 unsigned int size, 976 bool range_accepted, 977 struct rte_flow_error *error) 978 { 979 unsigned int i; 980 981 MLX5_ASSERT(nic_mask); 982 for (i = 0; i < size; ++i) 983 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 984 return rte_flow_error_set(error, ENOTSUP, 985 RTE_FLOW_ERROR_TYPE_ITEM, 986 item, 987 "mask enables non supported" 988 " bits"); 989 if (!item->spec && (item->mask || item->last)) 990 return rte_flow_error_set(error, EINVAL, 991 RTE_FLOW_ERROR_TYPE_ITEM, item, 992 "mask/last without a spec is not" 993 " supported"); 994 if (item->spec && item->last && !range_accepted) { 995 uint8_t spec[size]; 996 uint8_t last[size]; 997 unsigned int i; 998 int ret; 999 1000 for (i = 0; i < size; ++i) { 1001 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1002 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1003 } 1004 ret = memcmp(spec, last, size); 1005 if (ret != 0) 1006 return rte_flow_error_set(error, EINVAL, 1007 RTE_FLOW_ERROR_TYPE_ITEM, 1008 item, 1009 "range is not valid"); 1010 } 1011 return 0; 1012 } 1013 1014 /** 1015 * Adjust the hash fields according to the @p flow information. 1016 * 1017 * @param[in] dev_flow. 1018 * Pointer to the mlx5_flow. 1019 * @param[in] tunnel 1020 * 1 when the hash field is for a tunnel item. 1021 * @param[in] layer_types 1022 * ETH_RSS_* types. 1023 * @param[in] hash_fields 1024 * Item hash fields. 1025 * 1026 * @return 1027 * The hash fields that should be used. 1028 */ 1029 uint64_t 1030 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1031 int tunnel __rte_unused, uint64_t layer_types, 1032 uint64_t hash_fields) 1033 { 1034 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1035 int rss_request_inner = rss_desc->level >= 2; 1036 1037 /* Check RSS hash level for tunnel. */ 1038 if (tunnel && rss_request_inner) 1039 hash_fields |= IBV_RX_HASH_INNER; 1040 else if (tunnel || rss_request_inner) 1041 return 0; 1042 #endif 1043 /* Check if requested layer matches RSS hash fields. */ 1044 if (!(rss_desc->types & layer_types)) 1045 return 0; 1046 return hash_fields; 1047 } 1048 1049 /** 1050 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1051 * if several tunnel rules are used on this queue, the tunnel ptype will be 1052 * cleared. 1053 * 1054 * @param rxq_ctrl 1055 * Rx queue to update. 1056 */ 1057 static void 1058 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1059 { 1060 unsigned int i; 1061 uint32_t tunnel_ptype = 0; 1062 1063 /* Look up for the ptype to use. */ 1064 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1065 if (!rxq_ctrl->flow_tunnels_n[i]) 1066 continue; 1067 if (!tunnel_ptype) { 1068 tunnel_ptype = tunnels_info[i].ptype; 1069 } else { 1070 tunnel_ptype = 0; 1071 break; 1072 } 1073 } 1074 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1075 } 1076 1077 /** 1078 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1079 * flow. 1080 * 1081 * @param[in] dev 1082 * Pointer to the Ethernet device structure. 1083 * @param[in] dev_handle 1084 * Pointer to device flow handle structure. 1085 */ 1086 void 1087 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1088 struct mlx5_flow_handle *dev_handle) 1089 { 1090 struct mlx5_priv *priv = dev->data->dev_private; 1091 const int mark = dev_handle->mark; 1092 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1093 struct mlx5_ind_table_obj *ind_tbl = NULL; 1094 unsigned int i; 1095 1096 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1097 struct mlx5_hrxq *hrxq; 1098 1099 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1100 dev_handle->rix_hrxq); 1101 if (hrxq) 1102 ind_tbl = hrxq->ind_table; 1103 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1104 struct mlx5_shared_action_rss *shared_rss; 1105 1106 shared_rss = mlx5_ipool_get 1107 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1108 dev_handle->rix_srss); 1109 if (shared_rss) 1110 ind_tbl = shared_rss->ind_tbl; 1111 } 1112 if (!ind_tbl) 1113 return; 1114 for (i = 0; i != ind_tbl->queues_n; ++i) { 1115 int idx = ind_tbl->queues[i]; 1116 struct mlx5_rxq_ctrl *rxq_ctrl = 1117 container_of((*priv->rxqs)[idx], 1118 struct mlx5_rxq_ctrl, rxq); 1119 1120 /* 1121 * To support metadata register copy on Tx loopback, 1122 * this must be always enabled (metadata may arive 1123 * from other port - not from local flows only. 1124 */ 1125 if (priv->config.dv_flow_en && 1126 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1127 mlx5_flow_ext_mreg_supported(dev)) { 1128 rxq_ctrl->rxq.mark = 1; 1129 rxq_ctrl->flow_mark_n = 1; 1130 } else if (mark) { 1131 rxq_ctrl->rxq.mark = 1; 1132 rxq_ctrl->flow_mark_n++; 1133 } 1134 if (tunnel) { 1135 unsigned int j; 1136 1137 /* Increase the counter matching the flow. */ 1138 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1139 if ((tunnels_info[j].tunnel & 1140 dev_handle->layers) == 1141 tunnels_info[j].tunnel) { 1142 rxq_ctrl->flow_tunnels_n[j]++; 1143 break; 1144 } 1145 } 1146 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1147 } 1148 } 1149 } 1150 1151 /** 1152 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1153 * 1154 * @param[in] dev 1155 * Pointer to the Ethernet device structure. 1156 * @param[in] flow 1157 * Pointer to flow structure. 1158 */ 1159 static void 1160 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1161 { 1162 struct mlx5_priv *priv = dev->data->dev_private; 1163 uint32_t handle_idx; 1164 struct mlx5_flow_handle *dev_handle; 1165 1166 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1167 handle_idx, dev_handle, next) 1168 flow_drv_rxq_flags_set(dev, dev_handle); 1169 } 1170 1171 /** 1172 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1173 * device flow if no other flow uses it with the same kind of request. 1174 * 1175 * @param dev 1176 * Pointer to Ethernet device. 1177 * @param[in] dev_handle 1178 * Pointer to the device flow handle structure. 1179 */ 1180 static void 1181 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1182 struct mlx5_flow_handle *dev_handle) 1183 { 1184 struct mlx5_priv *priv = dev->data->dev_private; 1185 const int mark = dev_handle->mark; 1186 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1187 struct mlx5_ind_table_obj *ind_tbl = NULL; 1188 unsigned int i; 1189 1190 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1191 struct mlx5_hrxq *hrxq; 1192 1193 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1194 dev_handle->rix_hrxq); 1195 if (hrxq) 1196 ind_tbl = hrxq->ind_table; 1197 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1198 struct mlx5_shared_action_rss *shared_rss; 1199 1200 shared_rss = mlx5_ipool_get 1201 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1202 dev_handle->rix_srss); 1203 if (shared_rss) 1204 ind_tbl = shared_rss->ind_tbl; 1205 } 1206 if (!ind_tbl) 1207 return; 1208 MLX5_ASSERT(dev->data->dev_started); 1209 for (i = 0; i != ind_tbl->queues_n; ++i) { 1210 int idx = ind_tbl->queues[i]; 1211 struct mlx5_rxq_ctrl *rxq_ctrl = 1212 container_of((*priv->rxqs)[idx], 1213 struct mlx5_rxq_ctrl, rxq); 1214 1215 if (priv->config.dv_flow_en && 1216 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1217 mlx5_flow_ext_mreg_supported(dev)) { 1218 rxq_ctrl->rxq.mark = 1; 1219 rxq_ctrl->flow_mark_n = 1; 1220 } else if (mark) { 1221 rxq_ctrl->flow_mark_n--; 1222 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1223 } 1224 if (tunnel) { 1225 unsigned int j; 1226 1227 /* Decrease the counter matching the flow. */ 1228 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1229 if ((tunnels_info[j].tunnel & 1230 dev_handle->layers) == 1231 tunnels_info[j].tunnel) { 1232 rxq_ctrl->flow_tunnels_n[j]--; 1233 break; 1234 } 1235 } 1236 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1237 } 1238 } 1239 } 1240 1241 /** 1242 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1243 * @p flow if no other flow uses it with the same kind of request. 1244 * 1245 * @param dev 1246 * Pointer to Ethernet device. 1247 * @param[in] flow 1248 * Pointer to the flow. 1249 */ 1250 static void 1251 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1252 { 1253 struct mlx5_priv *priv = dev->data->dev_private; 1254 uint32_t handle_idx; 1255 struct mlx5_flow_handle *dev_handle; 1256 1257 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1258 handle_idx, dev_handle, next) 1259 flow_drv_rxq_flags_trim(dev, dev_handle); 1260 } 1261 1262 /** 1263 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1264 * 1265 * @param dev 1266 * Pointer to Ethernet device. 1267 */ 1268 static void 1269 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1270 { 1271 struct mlx5_priv *priv = dev->data->dev_private; 1272 unsigned int i; 1273 1274 for (i = 0; i != priv->rxqs_n; ++i) { 1275 struct mlx5_rxq_ctrl *rxq_ctrl; 1276 unsigned int j; 1277 1278 if (!(*priv->rxqs)[i]) 1279 continue; 1280 rxq_ctrl = container_of((*priv->rxqs)[i], 1281 struct mlx5_rxq_ctrl, rxq); 1282 rxq_ctrl->flow_mark_n = 0; 1283 rxq_ctrl->rxq.mark = 0; 1284 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1285 rxq_ctrl->flow_tunnels_n[j] = 0; 1286 rxq_ctrl->rxq.tunnel = 0; 1287 } 1288 } 1289 1290 /** 1291 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1292 * 1293 * @param[in] dev 1294 * Pointer to the Ethernet device structure. 1295 */ 1296 void 1297 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1298 { 1299 struct mlx5_priv *priv = dev->data->dev_private; 1300 struct mlx5_rxq_data *data; 1301 unsigned int i; 1302 1303 for (i = 0; i != priv->rxqs_n; ++i) { 1304 if (!(*priv->rxqs)[i]) 1305 continue; 1306 data = (*priv->rxqs)[i]; 1307 if (!rte_flow_dynf_metadata_avail()) { 1308 data->dynf_meta = 0; 1309 data->flow_meta_mask = 0; 1310 data->flow_meta_offset = -1; 1311 data->flow_meta_port_mask = 0; 1312 } else { 1313 data->dynf_meta = 1; 1314 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1315 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1316 data->flow_meta_port_mask = (uint32_t)~0; 1317 if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16) 1318 data->flow_meta_port_mask >>= 16; 1319 } 1320 } 1321 } 1322 1323 /* 1324 * return a pointer to the desired action in the list of actions. 1325 * 1326 * @param[in] actions 1327 * The list of actions to search the action in. 1328 * @param[in] action 1329 * The action to find. 1330 * 1331 * @return 1332 * Pointer to the action in the list, if found. NULL otherwise. 1333 */ 1334 const struct rte_flow_action * 1335 mlx5_flow_find_action(const struct rte_flow_action *actions, 1336 enum rte_flow_action_type action) 1337 { 1338 if (actions == NULL) 1339 return NULL; 1340 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1341 if (actions->type == action) 1342 return actions; 1343 return NULL; 1344 } 1345 1346 /* 1347 * Validate the flag action. 1348 * 1349 * @param[in] action_flags 1350 * Bit-fields that holds the actions detected until now. 1351 * @param[in] attr 1352 * Attributes of flow that includes this action. 1353 * @param[out] error 1354 * Pointer to error structure. 1355 * 1356 * @return 1357 * 0 on success, a negative errno value otherwise and rte_errno is set. 1358 */ 1359 int 1360 mlx5_flow_validate_action_flag(uint64_t action_flags, 1361 const struct rte_flow_attr *attr, 1362 struct rte_flow_error *error) 1363 { 1364 if (action_flags & MLX5_FLOW_ACTION_MARK) 1365 return rte_flow_error_set(error, EINVAL, 1366 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1367 "can't mark and flag in same flow"); 1368 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1369 return rte_flow_error_set(error, EINVAL, 1370 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1371 "can't have 2 flag" 1372 " actions in same flow"); 1373 if (attr->egress) 1374 return rte_flow_error_set(error, ENOTSUP, 1375 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1376 "flag action not supported for " 1377 "egress"); 1378 return 0; 1379 } 1380 1381 /* 1382 * Validate the mark action. 1383 * 1384 * @param[in] action 1385 * Pointer to the queue action. 1386 * @param[in] action_flags 1387 * Bit-fields that holds the actions detected until now. 1388 * @param[in] attr 1389 * Attributes of flow that includes this action. 1390 * @param[out] error 1391 * Pointer to error structure. 1392 * 1393 * @return 1394 * 0 on success, a negative errno value otherwise and rte_errno is set. 1395 */ 1396 int 1397 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1398 uint64_t action_flags, 1399 const struct rte_flow_attr *attr, 1400 struct rte_flow_error *error) 1401 { 1402 const struct rte_flow_action_mark *mark = action->conf; 1403 1404 if (!mark) 1405 return rte_flow_error_set(error, EINVAL, 1406 RTE_FLOW_ERROR_TYPE_ACTION, 1407 action, 1408 "configuration cannot be null"); 1409 if (mark->id >= MLX5_FLOW_MARK_MAX) 1410 return rte_flow_error_set(error, EINVAL, 1411 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1412 &mark->id, 1413 "mark id must in 0 <= id < " 1414 RTE_STR(MLX5_FLOW_MARK_MAX)); 1415 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1416 return rte_flow_error_set(error, EINVAL, 1417 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1418 "can't flag and mark in same flow"); 1419 if (action_flags & MLX5_FLOW_ACTION_MARK) 1420 return rte_flow_error_set(error, EINVAL, 1421 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1422 "can't have 2 mark actions in same" 1423 " flow"); 1424 if (attr->egress) 1425 return rte_flow_error_set(error, ENOTSUP, 1426 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1427 "mark action not supported for " 1428 "egress"); 1429 return 0; 1430 } 1431 1432 /* 1433 * Validate the drop action. 1434 * 1435 * @param[in] action_flags 1436 * Bit-fields that holds the actions detected until now. 1437 * @param[in] attr 1438 * Attributes of flow that includes this action. 1439 * @param[out] error 1440 * Pointer to error structure. 1441 * 1442 * @return 1443 * 0 on success, a negative errno value otherwise and rte_errno is set. 1444 */ 1445 int 1446 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1447 const struct rte_flow_attr *attr, 1448 struct rte_flow_error *error) 1449 { 1450 if (attr->egress) 1451 return rte_flow_error_set(error, ENOTSUP, 1452 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1453 "drop action not supported for " 1454 "egress"); 1455 return 0; 1456 } 1457 1458 /* 1459 * Validate the queue action. 1460 * 1461 * @param[in] action 1462 * Pointer to the queue action. 1463 * @param[in] action_flags 1464 * Bit-fields that holds the actions detected until now. 1465 * @param[in] dev 1466 * Pointer to the Ethernet device structure. 1467 * @param[in] attr 1468 * Attributes of flow that includes this action. 1469 * @param[out] error 1470 * Pointer to error structure. 1471 * 1472 * @return 1473 * 0 on success, a negative errno value otherwise and rte_errno is set. 1474 */ 1475 int 1476 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1477 uint64_t action_flags, 1478 struct rte_eth_dev *dev, 1479 const struct rte_flow_attr *attr, 1480 struct rte_flow_error *error) 1481 { 1482 struct mlx5_priv *priv = dev->data->dev_private; 1483 const struct rte_flow_action_queue *queue = action->conf; 1484 1485 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1486 return rte_flow_error_set(error, EINVAL, 1487 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1488 "can't have 2 fate actions in" 1489 " same flow"); 1490 if (!priv->rxqs_n) 1491 return rte_flow_error_set(error, EINVAL, 1492 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1493 NULL, "No Rx queues configured"); 1494 if (queue->index >= priv->rxqs_n) 1495 return rte_flow_error_set(error, EINVAL, 1496 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1497 &queue->index, 1498 "queue index out of range"); 1499 if (!(*priv->rxqs)[queue->index]) 1500 return rte_flow_error_set(error, EINVAL, 1501 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1502 &queue->index, 1503 "queue is not configured"); 1504 if (attr->egress) 1505 return rte_flow_error_set(error, ENOTSUP, 1506 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1507 "queue action not supported for " 1508 "egress"); 1509 return 0; 1510 } 1511 1512 /* 1513 * Validate the rss action. 1514 * 1515 * @param[in] dev 1516 * Pointer to the Ethernet device structure. 1517 * @param[in] action 1518 * Pointer to the queue action. 1519 * @param[out] error 1520 * Pointer to error structure. 1521 * 1522 * @return 1523 * 0 on success, a negative errno value otherwise and rte_errno is set. 1524 */ 1525 int 1526 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1527 const struct rte_flow_action *action, 1528 struct rte_flow_error *error) 1529 { 1530 struct mlx5_priv *priv = dev->data->dev_private; 1531 const struct rte_flow_action_rss *rss = action->conf; 1532 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1533 unsigned int i; 1534 1535 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1536 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1537 return rte_flow_error_set(error, ENOTSUP, 1538 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1539 &rss->func, 1540 "RSS hash function not supported"); 1541 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1542 if (rss->level > 2) 1543 #else 1544 if (rss->level > 1) 1545 #endif 1546 return rte_flow_error_set(error, ENOTSUP, 1547 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1548 &rss->level, 1549 "tunnel RSS is not supported"); 1550 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1551 if (rss->key_len == 0 && rss->key != NULL) 1552 return rte_flow_error_set(error, ENOTSUP, 1553 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1554 &rss->key_len, 1555 "RSS hash key length 0"); 1556 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1557 return rte_flow_error_set(error, ENOTSUP, 1558 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1559 &rss->key_len, 1560 "RSS hash key too small"); 1561 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1562 return rte_flow_error_set(error, ENOTSUP, 1563 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1564 &rss->key_len, 1565 "RSS hash key too large"); 1566 if (rss->queue_num > priv->config.ind_table_max_size) 1567 return rte_flow_error_set(error, ENOTSUP, 1568 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1569 &rss->queue_num, 1570 "number of queues too large"); 1571 if (rss->types & MLX5_RSS_HF_MASK) 1572 return rte_flow_error_set(error, ENOTSUP, 1573 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1574 &rss->types, 1575 "some RSS protocols are not" 1576 " supported"); 1577 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1578 !(rss->types & ETH_RSS_IP)) 1579 return rte_flow_error_set(error, EINVAL, 1580 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1581 "L3 partial RSS requested but L3 RSS" 1582 " type not specified"); 1583 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1584 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1585 return rte_flow_error_set(error, EINVAL, 1586 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1587 "L4 partial RSS requested but L4 RSS" 1588 " type not specified"); 1589 if (!priv->rxqs_n) 1590 return rte_flow_error_set(error, EINVAL, 1591 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1592 NULL, "No Rx queues configured"); 1593 if (!rss->queue_num) 1594 return rte_flow_error_set(error, EINVAL, 1595 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1596 NULL, "No queues configured"); 1597 for (i = 0; i != rss->queue_num; ++i) { 1598 struct mlx5_rxq_ctrl *rxq_ctrl; 1599 1600 if (rss->queue[i] >= priv->rxqs_n) 1601 return rte_flow_error_set 1602 (error, EINVAL, 1603 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1604 &rss->queue[i], "queue index out of range"); 1605 if (!(*priv->rxqs)[rss->queue[i]]) 1606 return rte_flow_error_set 1607 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1608 &rss->queue[i], "queue is not configured"); 1609 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1610 struct mlx5_rxq_ctrl, rxq); 1611 if (i == 0) 1612 rxq_type = rxq_ctrl->type; 1613 if (rxq_type != rxq_ctrl->type) 1614 return rte_flow_error_set 1615 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1616 &rss->queue[i], 1617 "combining hairpin and regular RSS queues is not supported"); 1618 } 1619 return 0; 1620 } 1621 1622 /* 1623 * Validate the rss action. 1624 * 1625 * @param[in] action 1626 * Pointer to the queue action. 1627 * @param[in] action_flags 1628 * Bit-fields that holds the actions detected until now. 1629 * @param[in] dev 1630 * Pointer to the Ethernet device structure. 1631 * @param[in] attr 1632 * Attributes of flow that includes this action. 1633 * @param[in] item_flags 1634 * Items that were detected. 1635 * @param[out] error 1636 * Pointer to error structure. 1637 * 1638 * @return 1639 * 0 on success, a negative errno value otherwise and rte_errno is set. 1640 */ 1641 int 1642 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1643 uint64_t action_flags, 1644 struct rte_eth_dev *dev, 1645 const struct rte_flow_attr *attr, 1646 uint64_t item_flags, 1647 struct rte_flow_error *error) 1648 { 1649 const struct rte_flow_action_rss *rss = action->conf; 1650 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1651 int ret; 1652 1653 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1654 return rte_flow_error_set(error, EINVAL, 1655 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1656 "can't have 2 fate actions" 1657 " in same flow"); 1658 ret = mlx5_validate_action_rss(dev, action, error); 1659 if (ret) 1660 return ret; 1661 if (attr->egress) 1662 return rte_flow_error_set(error, ENOTSUP, 1663 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1664 "rss action not supported for " 1665 "egress"); 1666 if (rss->level > 1 && !tunnel) 1667 return rte_flow_error_set(error, EINVAL, 1668 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1669 "inner RSS is not supported for " 1670 "non-tunnel flows"); 1671 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1672 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1673 return rte_flow_error_set(error, EINVAL, 1674 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1675 "RSS on eCPRI is not supported now"); 1676 } 1677 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1678 !(item_flags & 1679 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1680 rss->level > 1) 1681 return rte_flow_error_set(error, EINVAL, 1682 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1683 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1684 return 0; 1685 } 1686 1687 /* 1688 * Validate the default miss action. 1689 * 1690 * @param[in] action_flags 1691 * Bit-fields that holds the actions detected until now. 1692 * @param[out] error 1693 * Pointer to error structure. 1694 * 1695 * @return 1696 * 0 on success, a negative errno value otherwise and rte_errno is set. 1697 */ 1698 int 1699 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1700 const struct rte_flow_attr *attr, 1701 struct rte_flow_error *error) 1702 { 1703 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1704 return rte_flow_error_set(error, EINVAL, 1705 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1706 "can't have 2 fate actions in" 1707 " same flow"); 1708 if (attr->egress) 1709 return rte_flow_error_set(error, ENOTSUP, 1710 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1711 "default miss action not supported " 1712 "for egress"); 1713 if (attr->group) 1714 return rte_flow_error_set(error, ENOTSUP, 1715 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1716 "only group 0 is supported"); 1717 if (attr->transfer) 1718 return rte_flow_error_set(error, ENOTSUP, 1719 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1720 NULL, "transfer is not supported"); 1721 return 0; 1722 } 1723 1724 /* 1725 * Validate the count action. 1726 * 1727 * @param[in] dev 1728 * Pointer to the Ethernet device structure. 1729 * @param[in] attr 1730 * Attributes of flow that includes this action. 1731 * @param[out] error 1732 * Pointer to error structure. 1733 * 1734 * @return 1735 * 0 on success, a negative errno value otherwise and rte_errno is set. 1736 */ 1737 int 1738 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1739 const struct rte_flow_attr *attr, 1740 struct rte_flow_error *error) 1741 { 1742 if (attr->egress) 1743 return rte_flow_error_set(error, ENOTSUP, 1744 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1745 "count action not supported for " 1746 "egress"); 1747 return 0; 1748 } 1749 1750 /* 1751 * Validate the ASO CT action. 1752 * 1753 * @param[in] dev 1754 * Pointer to the Ethernet device structure. 1755 * @param[in] conntrack 1756 * Pointer to the CT action profile. 1757 * @param[out] error 1758 * Pointer to error structure. 1759 * 1760 * @return 1761 * 0 on success, a negative errno value otherwise and rte_errno is set. 1762 */ 1763 int 1764 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1765 const struct rte_flow_action_conntrack *conntrack, 1766 struct rte_flow_error *error) 1767 { 1768 RTE_SET_USED(dev); 1769 1770 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1771 return rte_flow_error_set(error, EINVAL, 1772 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1773 "Invalid CT state"); 1774 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1775 return rte_flow_error_set(error, EINVAL, 1776 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1777 "Invalid last TCP packet flag"); 1778 return 0; 1779 } 1780 1781 /** 1782 * Verify the @p attributes will be correctly understood by the NIC and store 1783 * them in the @p flow if everything is correct. 1784 * 1785 * @param[in] dev 1786 * Pointer to the Ethernet device structure. 1787 * @param[in] attributes 1788 * Pointer to flow attributes 1789 * @param[out] error 1790 * Pointer to error structure. 1791 * 1792 * @return 1793 * 0 on success, a negative errno value otherwise and rte_errno is set. 1794 */ 1795 int 1796 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1797 const struct rte_flow_attr *attributes, 1798 struct rte_flow_error *error) 1799 { 1800 struct mlx5_priv *priv = dev->data->dev_private; 1801 uint32_t priority_max = priv->config.flow_prio - 1; 1802 1803 if (attributes->group) 1804 return rte_flow_error_set(error, ENOTSUP, 1805 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1806 NULL, "groups is not supported"); 1807 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1808 attributes->priority >= priority_max) 1809 return rte_flow_error_set(error, ENOTSUP, 1810 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1811 NULL, "priority out of range"); 1812 if (attributes->egress) 1813 return rte_flow_error_set(error, ENOTSUP, 1814 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1815 "egress is not supported"); 1816 if (attributes->transfer && !priv->config.dv_esw_en) 1817 return rte_flow_error_set(error, ENOTSUP, 1818 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1819 NULL, "transfer is not supported"); 1820 if (!attributes->ingress) 1821 return rte_flow_error_set(error, EINVAL, 1822 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1823 NULL, 1824 "ingress attribute is mandatory"); 1825 return 0; 1826 } 1827 1828 /** 1829 * Validate ICMP6 item. 1830 * 1831 * @param[in] item 1832 * Item specification. 1833 * @param[in] item_flags 1834 * Bit-fields that holds the items detected until now. 1835 * @param[in] ext_vlan_sup 1836 * Whether extended VLAN features are supported or not. 1837 * @param[out] error 1838 * Pointer to error structure. 1839 * 1840 * @return 1841 * 0 on success, a negative errno value otherwise and rte_errno is set. 1842 */ 1843 int 1844 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1845 uint64_t item_flags, 1846 uint8_t target_protocol, 1847 struct rte_flow_error *error) 1848 { 1849 const struct rte_flow_item_icmp6 *mask = item->mask; 1850 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1851 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1852 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1853 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1854 MLX5_FLOW_LAYER_OUTER_L4; 1855 int ret; 1856 1857 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1858 return rte_flow_error_set(error, EINVAL, 1859 RTE_FLOW_ERROR_TYPE_ITEM, item, 1860 "protocol filtering not compatible" 1861 " with ICMP6 layer"); 1862 if (!(item_flags & l3m)) 1863 return rte_flow_error_set(error, EINVAL, 1864 RTE_FLOW_ERROR_TYPE_ITEM, item, 1865 "IPv6 is mandatory to filter on" 1866 " ICMP6"); 1867 if (item_flags & l4m) 1868 return rte_flow_error_set(error, EINVAL, 1869 RTE_FLOW_ERROR_TYPE_ITEM, item, 1870 "multiple L4 layers not supported"); 1871 if (!mask) 1872 mask = &rte_flow_item_icmp6_mask; 1873 ret = mlx5_flow_item_acceptable 1874 (item, (const uint8_t *)mask, 1875 (const uint8_t *)&rte_flow_item_icmp6_mask, 1876 sizeof(struct rte_flow_item_icmp6), 1877 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1878 if (ret < 0) 1879 return ret; 1880 return 0; 1881 } 1882 1883 /** 1884 * Validate ICMP item. 1885 * 1886 * @param[in] item 1887 * Item specification. 1888 * @param[in] item_flags 1889 * Bit-fields that holds the items detected until now. 1890 * @param[out] error 1891 * Pointer to error structure. 1892 * 1893 * @return 1894 * 0 on success, a negative errno value otherwise and rte_errno is set. 1895 */ 1896 int 1897 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1898 uint64_t item_flags, 1899 uint8_t target_protocol, 1900 struct rte_flow_error *error) 1901 { 1902 const struct rte_flow_item_icmp *mask = item->mask; 1903 const struct rte_flow_item_icmp nic_mask = { 1904 .hdr.icmp_type = 0xff, 1905 .hdr.icmp_code = 0xff, 1906 .hdr.icmp_ident = RTE_BE16(0xffff), 1907 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1908 }; 1909 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1910 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1911 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1912 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1913 MLX5_FLOW_LAYER_OUTER_L4; 1914 int ret; 1915 1916 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1917 return rte_flow_error_set(error, EINVAL, 1918 RTE_FLOW_ERROR_TYPE_ITEM, item, 1919 "protocol filtering not compatible" 1920 " with ICMP layer"); 1921 if (!(item_flags & l3m)) 1922 return rte_flow_error_set(error, EINVAL, 1923 RTE_FLOW_ERROR_TYPE_ITEM, item, 1924 "IPv4 is mandatory to filter" 1925 " on ICMP"); 1926 if (item_flags & l4m) 1927 return rte_flow_error_set(error, EINVAL, 1928 RTE_FLOW_ERROR_TYPE_ITEM, item, 1929 "multiple L4 layers not supported"); 1930 if (!mask) 1931 mask = &nic_mask; 1932 ret = mlx5_flow_item_acceptable 1933 (item, (const uint8_t *)mask, 1934 (const uint8_t *)&nic_mask, 1935 sizeof(struct rte_flow_item_icmp), 1936 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1937 if (ret < 0) 1938 return ret; 1939 return 0; 1940 } 1941 1942 /** 1943 * Validate Ethernet item. 1944 * 1945 * @param[in] item 1946 * Item specification. 1947 * @param[in] item_flags 1948 * Bit-fields that holds the items detected until now. 1949 * @param[out] error 1950 * Pointer to error structure. 1951 * 1952 * @return 1953 * 0 on success, a negative errno value otherwise and rte_errno is set. 1954 */ 1955 int 1956 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1957 uint64_t item_flags, bool ext_vlan_sup, 1958 struct rte_flow_error *error) 1959 { 1960 const struct rte_flow_item_eth *mask = item->mask; 1961 const struct rte_flow_item_eth nic_mask = { 1962 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1963 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1964 .type = RTE_BE16(0xffff), 1965 .has_vlan = ext_vlan_sup ? 1 : 0, 1966 }; 1967 int ret; 1968 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1969 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1970 MLX5_FLOW_LAYER_OUTER_L2; 1971 1972 if (item_flags & ethm) 1973 return rte_flow_error_set(error, ENOTSUP, 1974 RTE_FLOW_ERROR_TYPE_ITEM, item, 1975 "multiple L2 layers not supported"); 1976 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1977 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1978 return rte_flow_error_set(error, EINVAL, 1979 RTE_FLOW_ERROR_TYPE_ITEM, item, 1980 "L2 layer should not follow " 1981 "L3 layers"); 1982 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1983 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1984 return rte_flow_error_set(error, EINVAL, 1985 RTE_FLOW_ERROR_TYPE_ITEM, item, 1986 "L2 layer should not follow VLAN"); 1987 if (!mask) 1988 mask = &rte_flow_item_eth_mask; 1989 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1990 (const uint8_t *)&nic_mask, 1991 sizeof(struct rte_flow_item_eth), 1992 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1993 return ret; 1994 } 1995 1996 /** 1997 * Validate VLAN item. 1998 * 1999 * @param[in] item 2000 * Item specification. 2001 * @param[in] item_flags 2002 * Bit-fields that holds the items detected until now. 2003 * @param[in] dev 2004 * Ethernet device flow is being created on. 2005 * @param[out] error 2006 * Pointer to error structure. 2007 * 2008 * @return 2009 * 0 on success, a negative errno value otherwise and rte_errno is set. 2010 */ 2011 int 2012 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2013 uint64_t item_flags, 2014 struct rte_eth_dev *dev, 2015 struct rte_flow_error *error) 2016 { 2017 const struct rte_flow_item_vlan *spec = item->spec; 2018 const struct rte_flow_item_vlan *mask = item->mask; 2019 const struct rte_flow_item_vlan nic_mask = { 2020 .tci = RTE_BE16(UINT16_MAX), 2021 .inner_type = RTE_BE16(UINT16_MAX), 2022 }; 2023 uint16_t vlan_tag = 0; 2024 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2025 int ret; 2026 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2027 MLX5_FLOW_LAYER_INNER_L4) : 2028 (MLX5_FLOW_LAYER_OUTER_L3 | 2029 MLX5_FLOW_LAYER_OUTER_L4); 2030 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2031 MLX5_FLOW_LAYER_OUTER_VLAN; 2032 2033 if (item_flags & vlanm) 2034 return rte_flow_error_set(error, EINVAL, 2035 RTE_FLOW_ERROR_TYPE_ITEM, item, 2036 "multiple VLAN layers not supported"); 2037 else if ((item_flags & l34m) != 0) 2038 return rte_flow_error_set(error, EINVAL, 2039 RTE_FLOW_ERROR_TYPE_ITEM, item, 2040 "VLAN cannot follow L3/L4 layer"); 2041 if (!mask) 2042 mask = &rte_flow_item_vlan_mask; 2043 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2044 (const uint8_t *)&nic_mask, 2045 sizeof(struct rte_flow_item_vlan), 2046 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2047 if (ret) 2048 return ret; 2049 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2050 struct mlx5_priv *priv = dev->data->dev_private; 2051 2052 if (priv->vmwa_context) { 2053 /* 2054 * Non-NULL context means we have a virtual machine 2055 * and SR-IOV enabled, we have to create VLAN interface 2056 * to make hypervisor to setup E-Switch vport 2057 * context correctly. We avoid creating the multiple 2058 * VLAN interfaces, so we cannot support VLAN tag mask. 2059 */ 2060 return rte_flow_error_set(error, EINVAL, 2061 RTE_FLOW_ERROR_TYPE_ITEM, 2062 item, 2063 "VLAN tag mask is not" 2064 " supported in virtual" 2065 " environment"); 2066 } 2067 } 2068 if (spec) { 2069 vlan_tag = spec->tci; 2070 vlan_tag &= mask->tci; 2071 } 2072 /* 2073 * From verbs perspective an empty VLAN is equivalent 2074 * to a packet without VLAN layer. 2075 */ 2076 if (!vlan_tag) 2077 return rte_flow_error_set(error, EINVAL, 2078 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2079 item->spec, 2080 "VLAN cannot be empty"); 2081 return 0; 2082 } 2083 2084 /** 2085 * Validate IPV4 item. 2086 * 2087 * @param[in] item 2088 * Item specification. 2089 * @param[in] item_flags 2090 * Bit-fields that holds the items detected until now. 2091 * @param[in] last_item 2092 * Previous validated item in the pattern items. 2093 * @param[in] ether_type 2094 * Type in the ethernet layer header (including dot1q). 2095 * @param[in] acc_mask 2096 * Acceptable mask, if NULL default internal default mask 2097 * will be used to check whether item fields are supported. 2098 * @param[in] range_accepted 2099 * True if range of values is accepted for specific fields, false otherwise. 2100 * @param[out] error 2101 * Pointer to error structure. 2102 * 2103 * @return 2104 * 0 on success, a negative errno value otherwise and rte_errno is set. 2105 */ 2106 int 2107 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2108 uint64_t item_flags, 2109 uint64_t last_item, 2110 uint16_t ether_type, 2111 const struct rte_flow_item_ipv4 *acc_mask, 2112 bool range_accepted, 2113 struct rte_flow_error *error) 2114 { 2115 const struct rte_flow_item_ipv4 *mask = item->mask; 2116 const struct rte_flow_item_ipv4 *spec = item->spec; 2117 const struct rte_flow_item_ipv4 nic_mask = { 2118 .hdr = { 2119 .src_addr = RTE_BE32(0xffffffff), 2120 .dst_addr = RTE_BE32(0xffffffff), 2121 .type_of_service = 0xff, 2122 .next_proto_id = 0xff, 2123 }, 2124 }; 2125 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2126 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2127 MLX5_FLOW_LAYER_OUTER_L3; 2128 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2129 MLX5_FLOW_LAYER_OUTER_L4; 2130 int ret; 2131 uint8_t next_proto = 0xFF; 2132 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2133 MLX5_FLOW_LAYER_OUTER_VLAN | 2134 MLX5_FLOW_LAYER_INNER_VLAN); 2135 2136 if ((last_item & l2_vlan) && ether_type && 2137 ether_type != RTE_ETHER_TYPE_IPV4) 2138 return rte_flow_error_set(error, EINVAL, 2139 RTE_FLOW_ERROR_TYPE_ITEM, item, 2140 "IPv4 cannot follow L2/VLAN layer " 2141 "which ether type is not IPv4"); 2142 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2143 if (mask && spec) 2144 next_proto = mask->hdr.next_proto_id & 2145 spec->hdr.next_proto_id; 2146 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2147 return rte_flow_error_set(error, EINVAL, 2148 RTE_FLOW_ERROR_TYPE_ITEM, 2149 item, 2150 "multiple tunnel " 2151 "not supported"); 2152 } 2153 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2154 return rte_flow_error_set(error, EINVAL, 2155 RTE_FLOW_ERROR_TYPE_ITEM, item, 2156 "wrong tunnel type - IPv6 specified " 2157 "but IPv4 item provided"); 2158 if (item_flags & l3m) 2159 return rte_flow_error_set(error, ENOTSUP, 2160 RTE_FLOW_ERROR_TYPE_ITEM, item, 2161 "multiple L3 layers not supported"); 2162 else if (item_flags & l4m) 2163 return rte_flow_error_set(error, EINVAL, 2164 RTE_FLOW_ERROR_TYPE_ITEM, item, 2165 "L3 cannot follow an L4 layer."); 2166 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2167 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2168 return rte_flow_error_set(error, EINVAL, 2169 RTE_FLOW_ERROR_TYPE_ITEM, item, 2170 "L3 cannot follow an NVGRE layer."); 2171 if (!mask) 2172 mask = &rte_flow_item_ipv4_mask; 2173 else if (mask->hdr.next_proto_id != 0 && 2174 mask->hdr.next_proto_id != 0xff) 2175 return rte_flow_error_set(error, EINVAL, 2176 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2177 "partial mask is not supported" 2178 " for protocol"); 2179 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2180 acc_mask ? (const uint8_t *)acc_mask 2181 : (const uint8_t *)&nic_mask, 2182 sizeof(struct rte_flow_item_ipv4), 2183 range_accepted, error); 2184 if (ret < 0) 2185 return ret; 2186 return 0; 2187 } 2188 2189 /** 2190 * Validate IPV6 item. 2191 * 2192 * @param[in] item 2193 * Item specification. 2194 * @param[in] item_flags 2195 * Bit-fields that holds the items detected until now. 2196 * @param[in] last_item 2197 * Previous validated item in the pattern items. 2198 * @param[in] ether_type 2199 * Type in the ethernet layer header (including dot1q). 2200 * @param[in] acc_mask 2201 * Acceptable mask, if NULL default internal default mask 2202 * will be used to check whether item fields are supported. 2203 * @param[out] error 2204 * Pointer to error structure. 2205 * 2206 * @return 2207 * 0 on success, a negative errno value otherwise and rte_errno is set. 2208 */ 2209 int 2210 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2211 uint64_t item_flags, 2212 uint64_t last_item, 2213 uint16_t ether_type, 2214 const struct rte_flow_item_ipv6 *acc_mask, 2215 struct rte_flow_error *error) 2216 { 2217 const struct rte_flow_item_ipv6 *mask = item->mask; 2218 const struct rte_flow_item_ipv6 *spec = item->spec; 2219 const struct rte_flow_item_ipv6 nic_mask = { 2220 .hdr = { 2221 .src_addr = 2222 "\xff\xff\xff\xff\xff\xff\xff\xff" 2223 "\xff\xff\xff\xff\xff\xff\xff\xff", 2224 .dst_addr = 2225 "\xff\xff\xff\xff\xff\xff\xff\xff" 2226 "\xff\xff\xff\xff\xff\xff\xff\xff", 2227 .vtc_flow = RTE_BE32(0xffffffff), 2228 .proto = 0xff, 2229 }, 2230 }; 2231 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2232 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2233 MLX5_FLOW_LAYER_OUTER_L3; 2234 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2235 MLX5_FLOW_LAYER_OUTER_L4; 2236 int ret; 2237 uint8_t next_proto = 0xFF; 2238 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2239 MLX5_FLOW_LAYER_OUTER_VLAN | 2240 MLX5_FLOW_LAYER_INNER_VLAN); 2241 2242 if ((last_item & l2_vlan) && ether_type && 2243 ether_type != RTE_ETHER_TYPE_IPV6) 2244 return rte_flow_error_set(error, EINVAL, 2245 RTE_FLOW_ERROR_TYPE_ITEM, item, 2246 "IPv6 cannot follow L2/VLAN layer " 2247 "which ether type is not IPv6"); 2248 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2249 next_proto = spec->hdr.proto; 2250 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2251 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2252 return rte_flow_error_set(error, EINVAL, 2253 RTE_FLOW_ERROR_TYPE_ITEM, 2254 item, 2255 "multiple tunnel " 2256 "not supported"); 2257 } 2258 if (next_proto == IPPROTO_HOPOPTS || 2259 next_proto == IPPROTO_ROUTING || 2260 next_proto == IPPROTO_FRAGMENT || 2261 next_proto == IPPROTO_ESP || 2262 next_proto == IPPROTO_AH || 2263 next_proto == IPPROTO_DSTOPTS) 2264 return rte_flow_error_set(error, EINVAL, 2265 RTE_FLOW_ERROR_TYPE_ITEM, item, 2266 "IPv6 proto (next header) should " 2267 "not be set as extension header"); 2268 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2269 return rte_flow_error_set(error, EINVAL, 2270 RTE_FLOW_ERROR_TYPE_ITEM, item, 2271 "wrong tunnel type - IPv4 specified " 2272 "but IPv6 item provided"); 2273 if (item_flags & l3m) 2274 return rte_flow_error_set(error, ENOTSUP, 2275 RTE_FLOW_ERROR_TYPE_ITEM, item, 2276 "multiple L3 layers not supported"); 2277 else if (item_flags & l4m) 2278 return rte_flow_error_set(error, EINVAL, 2279 RTE_FLOW_ERROR_TYPE_ITEM, item, 2280 "L3 cannot follow an L4 layer."); 2281 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2282 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2283 return rte_flow_error_set(error, EINVAL, 2284 RTE_FLOW_ERROR_TYPE_ITEM, item, 2285 "L3 cannot follow an NVGRE layer."); 2286 if (!mask) 2287 mask = &rte_flow_item_ipv6_mask; 2288 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2289 acc_mask ? (const uint8_t *)acc_mask 2290 : (const uint8_t *)&nic_mask, 2291 sizeof(struct rte_flow_item_ipv6), 2292 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2293 if (ret < 0) 2294 return ret; 2295 return 0; 2296 } 2297 2298 /** 2299 * Validate UDP item. 2300 * 2301 * @param[in] item 2302 * Item specification. 2303 * @param[in] item_flags 2304 * Bit-fields that holds the items detected until now. 2305 * @param[in] target_protocol 2306 * The next protocol in the previous item. 2307 * @param[in] flow_mask 2308 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2309 * @param[out] error 2310 * Pointer to error structure. 2311 * 2312 * @return 2313 * 0 on success, a negative errno value otherwise and rte_errno is set. 2314 */ 2315 int 2316 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2317 uint64_t item_flags, 2318 uint8_t target_protocol, 2319 struct rte_flow_error *error) 2320 { 2321 const struct rte_flow_item_udp *mask = item->mask; 2322 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2323 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2324 MLX5_FLOW_LAYER_OUTER_L3; 2325 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2326 MLX5_FLOW_LAYER_OUTER_L4; 2327 int ret; 2328 2329 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2330 return rte_flow_error_set(error, EINVAL, 2331 RTE_FLOW_ERROR_TYPE_ITEM, item, 2332 "protocol filtering not compatible" 2333 " with UDP layer"); 2334 if (!(item_flags & l3m)) 2335 return rte_flow_error_set(error, EINVAL, 2336 RTE_FLOW_ERROR_TYPE_ITEM, item, 2337 "L3 is mandatory to filter on L4"); 2338 if (item_flags & l4m) 2339 return rte_flow_error_set(error, EINVAL, 2340 RTE_FLOW_ERROR_TYPE_ITEM, item, 2341 "multiple L4 layers not supported"); 2342 if (!mask) 2343 mask = &rte_flow_item_udp_mask; 2344 ret = mlx5_flow_item_acceptable 2345 (item, (const uint8_t *)mask, 2346 (const uint8_t *)&rte_flow_item_udp_mask, 2347 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2348 error); 2349 if (ret < 0) 2350 return ret; 2351 return 0; 2352 } 2353 2354 /** 2355 * Validate TCP item. 2356 * 2357 * @param[in] item 2358 * Item specification. 2359 * @param[in] item_flags 2360 * Bit-fields that holds the items detected until now. 2361 * @param[in] target_protocol 2362 * The next protocol in the previous item. 2363 * @param[out] error 2364 * Pointer to error structure. 2365 * 2366 * @return 2367 * 0 on success, a negative errno value otherwise and rte_errno is set. 2368 */ 2369 int 2370 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2371 uint64_t item_flags, 2372 uint8_t target_protocol, 2373 const struct rte_flow_item_tcp *flow_mask, 2374 struct rte_flow_error *error) 2375 { 2376 const struct rte_flow_item_tcp *mask = item->mask; 2377 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2378 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2379 MLX5_FLOW_LAYER_OUTER_L3; 2380 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2381 MLX5_FLOW_LAYER_OUTER_L4; 2382 int ret; 2383 2384 MLX5_ASSERT(flow_mask); 2385 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2386 return rte_flow_error_set(error, EINVAL, 2387 RTE_FLOW_ERROR_TYPE_ITEM, item, 2388 "protocol filtering not compatible" 2389 " with TCP layer"); 2390 if (!(item_flags & l3m)) 2391 return rte_flow_error_set(error, EINVAL, 2392 RTE_FLOW_ERROR_TYPE_ITEM, item, 2393 "L3 is mandatory to filter on L4"); 2394 if (item_flags & l4m) 2395 return rte_flow_error_set(error, EINVAL, 2396 RTE_FLOW_ERROR_TYPE_ITEM, item, 2397 "multiple L4 layers not supported"); 2398 if (!mask) 2399 mask = &rte_flow_item_tcp_mask; 2400 ret = mlx5_flow_item_acceptable 2401 (item, (const uint8_t *)mask, 2402 (const uint8_t *)flow_mask, 2403 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2404 error); 2405 if (ret < 0) 2406 return ret; 2407 return 0; 2408 } 2409 2410 /** 2411 * Validate VXLAN item. 2412 * 2413 * @param[in] item 2414 * Item specification. 2415 * @param[in] item_flags 2416 * Bit-fields that holds the items detected until now. 2417 * @param[in] target_protocol 2418 * The next protocol in the previous item. 2419 * @param[out] error 2420 * Pointer to error structure. 2421 * 2422 * @return 2423 * 0 on success, a negative errno value otherwise and rte_errno is set. 2424 */ 2425 int 2426 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2427 uint64_t item_flags, 2428 struct rte_flow_error *error) 2429 { 2430 const struct rte_flow_item_vxlan *spec = item->spec; 2431 const struct rte_flow_item_vxlan *mask = item->mask; 2432 int ret; 2433 union vni { 2434 uint32_t vlan_id; 2435 uint8_t vni[4]; 2436 } id = { .vlan_id = 0, }; 2437 2438 2439 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2440 return rte_flow_error_set(error, ENOTSUP, 2441 RTE_FLOW_ERROR_TYPE_ITEM, item, 2442 "multiple tunnel layers not" 2443 " supported"); 2444 /* 2445 * Verify only UDPv4 is present as defined in 2446 * https://tools.ietf.org/html/rfc7348 2447 */ 2448 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2449 return rte_flow_error_set(error, EINVAL, 2450 RTE_FLOW_ERROR_TYPE_ITEM, item, 2451 "no outer UDP layer found"); 2452 if (!mask) 2453 mask = &rte_flow_item_vxlan_mask; 2454 ret = mlx5_flow_item_acceptable 2455 (item, (const uint8_t *)mask, 2456 (const uint8_t *)&rte_flow_item_vxlan_mask, 2457 sizeof(struct rte_flow_item_vxlan), 2458 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2459 if (ret < 0) 2460 return ret; 2461 if (spec) { 2462 memcpy(&id.vni[1], spec->vni, 3); 2463 memcpy(&id.vni[1], mask->vni, 3); 2464 } 2465 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2466 return rte_flow_error_set(error, ENOTSUP, 2467 RTE_FLOW_ERROR_TYPE_ITEM, item, 2468 "VXLAN tunnel must be fully defined"); 2469 return 0; 2470 } 2471 2472 /** 2473 * Validate VXLAN_GPE item. 2474 * 2475 * @param[in] item 2476 * Item specification. 2477 * @param[in] item_flags 2478 * Bit-fields that holds the items detected until now. 2479 * @param[in] priv 2480 * Pointer to the private data structure. 2481 * @param[in] target_protocol 2482 * The next protocol in the previous item. 2483 * @param[out] error 2484 * Pointer to error structure. 2485 * 2486 * @return 2487 * 0 on success, a negative errno value otherwise and rte_errno is set. 2488 */ 2489 int 2490 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2491 uint64_t item_flags, 2492 struct rte_eth_dev *dev, 2493 struct rte_flow_error *error) 2494 { 2495 struct mlx5_priv *priv = dev->data->dev_private; 2496 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2497 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2498 int ret; 2499 union vni { 2500 uint32_t vlan_id; 2501 uint8_t vni[4]; 2502 } id = { .vlan_id = 0, }; 2503 2504 if (!priv->config.l3_vxlan_en) 2505 return rte_flow_error_set(error, ENOTSUP, 2506 RTE_FLOW_ERROR_TYPE_ITEM, item, 2507 "L3 VXLAN is not enabled by device" 2508 " parameter and/or not configured in" 2509 " firmware"); 2510 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2511 return rte_flow_error_set(error, ENOTSUP, 2512 RTE_FLOW_ERROR_TYPE_ITEM, item, 2513 "multiple tunnel layers not" 2514 " supported"); 2515 /* 2516 * Verify only UDPv4 is present as defined in 2517 * https://tools.ietf.org/html/rfc7348 2518 */ 2519 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2520 return rte_flow_error_set(error, EINVAL, 2521 RTE_FLOW_ERROR_TYPE_ITEM, item, 2522 "no outer UDP layer found"); 2523 if (!mask) 2524 mask = &rte_flow_item_vxlan_gpe_mask; 2525 ret = mlx5_flow_item_acceptable 2526 (item, (const uint8_t *)mask, 2527 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2528 sizeof(struct rte_flow_item_vxlan_gpe), 2529 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2530 if (ret < 0) 2531 return ret; 2532 if (spec) { 2533 if (spec->protocol) 2534 return rte_flow_error_set(error, ENOTSUP, 2535 RTE_FLOW_ERROR_TYPE_ITEM, 2536 item, 2537 "VxLAN-GPE protocol" 2538 " not supported"); 2539 memcpy(&id.vni[1], spec->vni, 3); 2540 memcpy(&id.vni[1], mask->vni, 3); 2541 } 2542 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2543 return rte_flow_error_set(error, ENOTSUP, 2544 RTE_FLOW_ERROR_TYPE_ITEM, item, 2545 "VXLAN-GPE tunnel must be fully" 2546 " defined"); 2547 return 0; 2548 } 2549 /** 2550 * Validate GRE Key item. 2551 * 2552 * @param[in] item 2553 * Item specification. 2554 * @param[in] item_flags 2555 * Bit flags to mark detected items. 2556 * @param[in] gre_item 2557 * Pointer to gre_item 2558 * @param[out] error 2559 * Pointer to error structure. 2560 * 2561 * @return 2562 * 0 on success, a negative errno value otherwise and rte_errno is set. 2563 */ 2564 int 2565 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2566 uint64_t item_flags, 2567 const struct rte_flow_item *gre_item, 2568 struct rte_flow_error *error) 2569 { 2570 const rte_be32_t *mask = item->mask; 2571 int ret = 0; 2572 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2573 const struct rte_flow_item_gre *gre_spec; 2574 const struct rte_flow_item_gre *gre_mask; 2575 2576 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2577 return rte_flow_error_set(error, ENOTSUP, 2578 RTE_FLOW_ERROR_TYPE_ITEM, item, 2579 "Multiple GRE key not support"); 2580 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2581 return rte_flow_error_set(error, ENOTSUP, 2582 RTE_FLOW_ERROR_TYPE_ITEM, item, 2583 "No preceding GRE header"); 2584 if (item_flags & MLX5_FLOW_LAYER_INNER) 2585 return rte_flow_error_set(error, ENOTSUP, 2586 RTE_FLOW_ERROR_TYPE_ITEM, item, 2587 "GRE key following a wrong item"); 2588 gre_mask = gre_item->mask; 2589 if (!gre_mask) 2590 gre_mask = &rte_flow_item_gre_mask; 2591 gre_spec = gre_item->spec; 2592 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2593 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2594 return rte_flow_error_set(error, EINVAL, 2595 RTE_FLOW_ERROR_TYPE_ITEM, item, 2596 "Key bit must be on"); 2597 2598 if (!mask) 2599 mask = &gre_key_default_mask; 2600 ret = mlx5_flow_item_acceptable 2601 (item, (const uint8_t *)mask, 2602 (const uint8_t *)&gre_key_default_mask, 2603 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2604 return ret; 2605 } 2606 2607 /** 2608 * Validate GRE item. 2609 * 2610 * @param[in] item 2611 * Item specification. 2612 * @param[in] item_flags 2613 * Bit flags to mark detected items. 2614 * @param[in] target_protocol 2615 * The next protocol in the previous item. 2616 * @param[out] error 2617 * Pointer to error structure. 2618 * 2619 * @return 2620 * 0 on success, a negative errno value otherwise and rte_errno is set. 2621 */ 2622 int 2623 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2624 uint64_t item_flags, 2625 uint8_t target_protocol, 2626 struct rte_flow_error *error) 2627 { 2628 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2629 const struct rte_flow_item_gre *mask = item->mask; 2630 int ret; 2631 const struct rte_flow_item_gre nic_mask = { 2632 .c_rsvd0_ver = RTE_BE16(0xB000), 2633 .protocol = RTE_BE16(UINT16_MAX), 2634 }; 2635 2636 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2637 return rte_flow_error_set(error, EINVAL, 2638 RTE_FLOW_ERROR_TYPE_ITEM, item, 2639 "protocol filtering not compatible" 2640 " with this GRE layer"); 2641 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2642 return rte_flow_error_set(error, ENOTSUP, 2643 RTE_FLOW_ERROR_TYPE_ITEM, item, 2644 "multiple tunnel layers not" 2645 " supported"); 2646 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2647 return rte_flow_error_set(error, ENOTSUP, 2648 RTE_FLOW_ERROR_TYPE_ITEM, item, 2649 "L3 Layer is missing"); 2650 if (!mask) 2651 mask = &rte_flow_item_gre_mask; 2652 ret = mlx5_flow_item_acceptable 2653 (item, (const uint8_t *)mask, 2654 (const uint8_t *)&nic_mask, 2655 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2656 error); 2657 if (ret < 0) 2658 return ret; 2659 #ifndef HAVE_MLX5DV_DR 2660 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2661 if (spec && (spec->protocol & mask->protocol)) 2662 return rte_flow_error_set(error, ENOTSUP, 2663 RTE_FLOW_ERROR_TYPE_ITEM, item, 2664 "without MPLS support the" 2665 " specification cannot be used for" 2666 " filtering"); 2667 #endif 2668 #endif 2669 return 0; 2670 } 2671 2672 /** 2673 * Validate Geneve item. 2674 * 2675 * @param[in] item 2676 * Item specification. 2677 * @param[in] itemFlags 2678 * Bit-fields that holds the items detected until now. 2679 * @param[in] enPriv 2680 * Pointer to the private data structure. 2681 * @param[out] error 2682 * Pointer to error structure. 2683 * 2684 * @return 2685 * 0 on success, a negative errno value otherwise and rte_errno is set. 2686 */ 2687 2688 int 2689 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2690 uint64_t item_flags, 2691 struct rte_eth_dev *dev, 2692 struct rte_flow_error *error) 2693 { 2694 struct mlx5_priv *priv = dev->data->dev_private; 2695 const struct rte_flow_item_geneve *spec = item->spec; 2696 const struct rte_flow_item_geneve *mask = item->mask; 2697 int ret; 2698 uint16_t gbhdr; 2699 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2700 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2701 const struct rte_flow_item_geneve nic_mask = { 2702 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2703 .vni = "\xff\xff\xff", 2704 .protocol = RTE_BE16(UINT16_MAX), 2705 }; 2706 2707 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2708 return rte_flow_error_set(error, ENOTSUP, 2709 RTE_FLOW_ERROR_TYPE_ITEM, item, 2710 "L3 Geneve is not enabled by device" 2711 " parameter and/or not configured in" 2712 " firmware"); 2713 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2714 return rte_flow_error_set(error, ENOTSUP, 2715 RTE_FLOW_ERROR_TYPE_ITEM, item, 2716 "multiple tunnel layers not" 2717 " supported"); 2718 /* 2719 * Verify only UDPv4 is present as defined in 2720 * https://tools.ietf.org/html/rfc7348 2721 */ 2722 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2723 return rte_flow_error_set(error, EINVAL, 2724 RTE_FLOW_ERROR_TYPE_ITEM, item, 2725 "no outer UDP layer found"); 2726 if (!mask) 2727 mask = &rte_flow_item_geneve_mask; 2728 ret = mlx5_flow_item_acceptable 2729 (item, (const uint8_t *)mask, 2730 (const uint8_t *)&nic_mask, 2731 sizeof(struct rte_flow_item_geneve), 2732 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2733 if (ret) 2734 return ret; 2735 if (spec) { 2736 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2737 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2738 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2739 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2740 return rte_flow_error_set(error, ENOTSUP, 2741 RTE_FLOW_ERROR_TYPE_ITEM, 2742 item, 2743 "Geneve protocol unsupported" 2744 " fields are being used"); 2745 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2746 return rte_flow_error_set 2747 (error, ENOTSUP, 2748 RTE_FLOW_ERROR_TYPE_ITEM, 2749 item, 2750 "Unsupported Geneve options length"); 2751 } 2752 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2753 return rte_flow_error_set 2754 (error, ENOTSUP, 2755 RTE_FLOW_ERROR_TYPE_ITEM, item, 2756 "Geneve tunnel must be fully defined"); 2757 return 0; 2758 } 2759 2760 /** 2761 * Validate Geneve TLV option item. 2762 * 2763 * @param[in] item 2764 * Item specification. 2765 * @param[in] last_item 2766 * Previous validated item in the pattern items. 2767 * @param[in] geneve_item 2768 * Previous GENEVE item specification. 2769 * @param[in] dev 2770 * Pointer to the rte_eth_dev structure. 2771 * @param[out] error 2772 * Pointer to error structure. 2773 * 2774 * @return 2775 * 0 on success, a negative errno value otherwise and rte_errno is set. 2776 */ 2777 int 2778 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2779 uint64_t last_item, 2780 const struct rte_flow_item *geneve_item, 2781 struct rte_eth_dev *dev, 2782 struct rte_flow_error *error) 2783 { 2784 struct mlx5_priv *priv = dev->data->dev_private; 2785 struct mlx5_dev_ctx_shared *sh = priv->sh; 2786 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2787 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2788 uint8_t data_max_supported = 2789 hca_attr->max_geneve_tlv_option_data_len * 4; 2790 struct mlx5_dev_config *config = &priv->config; 2791 const struct rte_flow_item_geneve *geneve_spec; 2792 const struct rte_flow_item_geneve *geneve_mask; 2793 const struct rte_flow_item_geneve_opt *spec = item->spec; 2794 const struct rte_flow_item_geneve_opt *mask = item->mask; 2795 unsigned int i; 2796 unsigned int data_len; 2797 uint8_t tlv_option_len; 2798 uint16_t optlen_m, optlen_v; 2799 const struct rte_flow_item_geneve_opt full_mask = { 2800 .option_class = RTE_BE16(0xffff), 2801 .option_type = 0xff, 2802 .option_len = 0x1f, 2803 }; 2804 2805 if (!mask) 2806 mask = &rte_flow_item_geneve_opt_mask; 2807 if (!spec) 2808 return rte_flow_error_set 2809 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2810 "Geneve TLV opt class/type/length must be specified"); 2811 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2812 return rte_flow_error_set 2813 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2814 "Geneve TLV opt length exceeeds the limit (31)"); 2815 /* Check if class type and length masks are full. */ 2816 if (full_mask.option_class != mask->option_class || 2817 full_mask.option_type != mask->option_type || 2818 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2819 return rte_flow_error_set 2820 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2821 "Geneve TLV opt class/type/length masks must be full"); 2822 /* Check if length is supported */ 2823 if ((uint32_t)spec->option_len > 2824 config->hca_attr.max_geneve_tlv_option_data_len) 2825 return rte_flow_error_set 2826 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2827 "Geneve TLV opt length not supported"); 2828 if (config->hca_attr.max_geneve_tlv_options > 1) 2829 DRV_LOG(DEBUG, 2830 "max_geneve_tlv_options supports more than 1 option"); 2831 /* Check GENEVE item preceding. */ 2832 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2833 return rte_flow_error_set 2834 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2835 "Geneve opt item must be preceded with Geneve item"); 2836 geneve_spec = geneve_item->spec; 2837 geneve_mask = geneve_item->mask ? geneve_item->mask : 2838 &rte_flow_item_geneve_mask; 2839 /* Check if GENEVE TLV option size doesn't exceed option length */ 2840 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2841 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2842 tlv_option_len = spec->option_len & mask->option_len; 2843 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2844 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2845 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2846 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2847 if ((optlen_v & optlen_m) <= tlv_option_len) 2848 return rte_flow_error_set 2849 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2850 "GENEVE TLV option length exceeds optlen"); 2851 } 2852 /* Check if length is 0 or data is 0. */ 2853 if (spec->data == NULL || spec->option_len == 0) 2854 return rte_flow_error_set 2855 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2856 "Geneve TLV opt with zero data/length not supported"); 2857 /* Check not all data & mask are 0. */ 2858 data_len = spec->option_len * 4; 2859 if (mask->data == NULL) { 2860 for (i = 0; i < data_len; i++) 2861 if (spec->data[i]) 2862 break; 2863 if (i == data_len) 2864 return rte_flow_error_set(error, ENOTSUP, 2865 RTE_FLOW_ERROR_TYPE_ITEM, item, 2866 "Can't match on Geneve option data 0"); 2867 } else { 2868 for (i = 0; i < data_len; i++) 2869 if (spec->data[i] & mask->data[i]) 2870 break; 2871 if (i == data_len) 2872 return rte_flow_error_set(error, ENOTSUP, 2873 RTE_FLOW_ERROR_TYPE_ITEM, item, 2874 "Can't match on Geneve option data and mask 0"); 2875 /* Check data mask supported. */ 2876 for (i = data_max_supported; i < data_len ; i++) 2877 if (mask->data[i]) 2878 return rte_flow_error_set(error, ENOTSUP, 2879 RTE_FLOW_ERROR_TYPE_ITEM, item, 2880 "Data mask is of unsupported size"); 2881 } 2882 /* Check GENEVE option is supported in NIC. */ 2883 if (!config->hca_attr.geneve_tlv_opt) 2884 return rte_flow_error_set 2885 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2886 "Geneve TLV opt not supported"); 2887 /* Check if we already have geneve option with different type/class. */ 2888 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2889 geneve_opt_resource = sh->geneve_tlv_option_resource; 2890 if (geneve_opt_resource != NULL) 2891 if (geneve_opt_resource->option_class != spec->option_class || 2892 geneve_opt_resource->option_type != spec->option_type || 2893 geneve_opt_resource->length != spec->option_len) { 2894 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2895 return rte_flow_error_set(error, ENOTSUP, 2896 RTE_FLOW_ERROR_TYPE_ITEM, item, 2897 "Only one Geneve TLV option supported"); 2898 } 2899 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2900 return 0; 2901 } 2902 2903 /** 2904 * Validate MPLS item. 2905 * 2906 * @param[in] dev 2907 * Pointer to the rte_eth_dev structure. 2908 * @param[in] item 2909 * Item specification. 2910 * @param[in] item_flags 2911 * Bit-fields that holds the items detected until now. 2912 * @param[in] prev_layer 2913 * The protocol layer indicated in previous item. 2914 * @param[out] error 2915 * Pointer to error structure. 2916 * 2917 * @return 2918 * 0 on success, a negative errno value otherwise and rte_errno is set. 2919 */ 2920 int 2921 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2922 const struct rte_flow_item *item __rte_unused, 2923 uint64_t item_flags __rte_unused, 2924 uint64_t prev_layer __rte_unused, 2925 struct rte_flow_error *error) 2926 { 2927 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2928 const struct rte_flow_item_mpls *mask = item->mask; 2929 struct mlx5_priv *priv = dev->data->dev_private; 2930 int ret; 2931 2932 if (!priv->config.mpls_en) 2933 return rte_flow_error_set(error, ENOTSUP, 2934 RTE_FLOW_ERROR_TYPE_ITEM, item, 2935 "MPLS not supported or" 2936 " disabled in firmware" 2937 " configuration."); 2938 /* MPLS over UDP, GRE is allowed */ 2939 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 2940 MLX5_FLOW_LAYER_GRE | 2941 MLX5_FLOW_LAYER_GRE_KEY))) 2942 return rte_flow_error_set(error, EINVAL, 2943 RTE_FLOW_ERROR_TYPE_ITEM, item, 2944 "protocol filtering not compatible" 2945 " with MPLS layer"); 2946 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2947 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2948 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2949 return rte_flow_error_set(error, ENOTSUP, 2950 RTE_FLOW_ERROR_TYPE_ITEM, item, 2951 "multiple tunnel layers not" 2952 " supported"); 2953 if (!mask) 2954 mask = &rte_flow_item_mpls_mask; 2955 ret = mlx5_flow_item_acceptable 2956 (item, (const uint8_t *)mask, 2957 (const uint8_t *)&rte_flow_item_mpls_mask, 2958 sizeof(struct rte_flow_item_mpls), 2959 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2960 if (ret < 0) 2961 return ret; 2962 return 0; 2963 #else 2964 return rte_flow_error_set(error, ENOTSUP, 2965 RTE_FLOW_ERROR_TYPE_ITEM, item, 2966 "MPLS is not supported by Verbs, please" 2967 " update."); 2968 #endif 2969 } 2970 2971 /** 2972 * Validate NVGRE item. 2973 * 2974 * @param[in] item 2975 * Item specification. 2976 * @param[in] item_flags 2977 * Bit flags to mark detected items. 2978 * @param[in] target_protocol 2979 * The next protocol in the previous item. 2980 * @param[out] error 2981 * Pointer to error structure. 2982 * 2983 * @return 2984 * 0 on success, a negative errno value otherwise and rte_errno is set. 2985 */ 2986 int 2987 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2988 uint64_t item_flags, 2989 uint8_t target_protocol, 2990 struct rte_flow_error *error) 2991 { 2992 const struct rte_flow_item_nvgre *mask = item->mask; 2993 int ret; 2994 2995 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2996 return rte_flow_error_set(error, EINVAL, 2997 RTE_FLOW_ERROR_TYPE_ITEM, item, 2998 "protocol filtering not compatible" 2999 " with this GRE layer"); 3000 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3001 return rte_flow_error_set(error, ENOTSUP, 3002 RTE_FLOW_ERROR_TYPE_ITEM, item, 3003 "multiple tunnel layers not" 3004 " supported"); 3005 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3006 return rte_flow_error_set(error, ENOTSUP, 3007 RTE_FLOW_ERROR_TYPE_ITEM, item, 3008 "L3 Layer is missing"); 3009 if (!mask) 3010 mask = &rte_flow_item_nvgre_mask; 3011 ret = mlx5_flow_item_acceptable 3012 (item, (const uint8_t *)mask, 3013 (const uint8_t *)&rte_flow_item_nvgre_mask, 3014 sizeof(struct rte_flow_item_nvgre), 3015 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3016 if (ret < 0) 3017 return ret; 3018 return 0; 3019 } 3020 3021 /** 3022 * Validate eCPRI item. 3023 * 3024 * @param[in] item 3025 * Item specification. 3026 * @param[in] item_flags 3027 * Bit-fields that holds the items detected until now. 3028 * @param[in] last_item 3029 * Previous validated item in the pattern items. 3030 * @param[in] ether_type 3031 * Type in the ethernet layer header (including dot1q). 3032 * @param[in] acc_mask 3033 * Acceptable mask, if NULL default internal default mask 3034 * will be used to check whether item fields are supported. 3035 * @param[out] error 3036 * Pointer to error structure. 3037 * 3038 * @return 3039 * 0 on success, a negative errno value otherwise and rte_errno is set. 3040 */ 3041 int 3042 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3043 uint64_t item_flags, 3044 uint64_t last_item, 3045 uint16_t ether_type, 3046 const struct rte_flow_item_ecpri *acc_mask, 3047 struct rte_flow_error *error) 3048 { 3049 const struct rte_flow_item_ecpri *mask = item->mask; 3050 const struct rte_flow_item_ecpri nic_mask = { 3051 .hdr = { 3052 .common = { 3053 .u32 = 3054 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3055 .type = 0xFF, 3056 }).u32), 3057 }, 3058 .dummy[0] = 0xFFFFFFFF, 3059 }, 3060 }; 3061 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3062 MLX5_FLOW_LAYER_OUTER_VLAN); 3063 struct rte_flow_item_ecpri mask_lo; 3064 3065 if (!(last_item & outer_l2_vlan) && 3066 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3067 return rte_flow_error_set(error, EINVAL, 3068 RTE_FLOW_ERROR_TYPE_ITEM, item, 3069 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3070 if ((last_item & outer_l2_vlan) && ether_type && 3071 ether_type != RTE_ETHER_TYPE_ECPRI) 3072 return rte_flow_error_set(error, EINVAL, 3073 RTE_FLOW_ERROR_TYPE_ITEM, item, 3074 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3075 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3076 return rte_flow_error_set(error, EINVAL, 3077 RTE_FLOW_ERROR_TYPE_ITEM, item, 3078 "eCPRI with tunnel is not supported right now"); 3079 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3080 return rte_flow_error_set(error, ENOTSUP, 3081 RTE_FLOW_ERROR_TYPE_ITEM, item, 3082 "multiple L3 layers not supported"); 3083 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3084 return rte_flow_error_set(error, EINVAL, 3085 RTE_FLOW_ERROR_TYPE_ITEM, item, 3086 "eCPRI cannot coexist with a TCP layer"); 3087 /* In specification, eCPRI could be over UDP layer. */ 3088 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3089 return rte_flow_error_set(error, EINVAL, 3090 RTE_FLOW_ERROR_TYPE_ITEM, item, 3091 "eCPRI over UDP layer is not yet supported right now"); 3092 /* Mask for type field in common header could be zero. */ 3093 if (!mask) 3094 mask = &rte_flow_item_ecpri_mask; 3095 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3096 /* Input mask is in big-endian format. */ 3097 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3098 return rte_flow_error_set(error, EINVAL, 3099 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3100 "partial mask is not supported for protocol"); 3101 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3102 return rte_flow_error_set(error, EINVAL, 3103 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3104 "message header mask must be after a type mask"); 3105 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3106 acc_mask ? (const uint8_t *)acc_mask 3107 : (const uint8_t *)&nic_mask, 3108 sizeof(struct rte_flow_item_ecpri), 3109 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3110 } 3111 3112 /** 3113 * Release resource related QUEUE/RSS action split. 3114 * 3115 * @param dev 3116 * Pointer to Ethernet device. 3117 * @param flow 3118 * Flow to release id's from. 3119 */ 3120 static void 3121 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 3122 struct rte_flow *flow) 3123 { 3124 struct mlx5_priv *priv = dev->data->dev_private; 3125 uint32_t handle_idx; 3126 struct mlx5_flow_handle *dev_handle; 3127 3128 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 3129 handle_idx, dev_handle, next) 3130 if (dev_handle->split_flow_id && 3131 !dev_handle->is_meter_flow_id) 3132 mlx5_ipool_free(priv->sh->ipool 3133 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 3134 dev_handle->split_flow_id); 3135 } 3136 3137 static int 3138 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3139 const struct rte_flow_attr *attr __rte_unused, 3140 const struct rte_flow_item items[] __rte_unused, 3141 const struct rte_flow_action actions[] __rte_unused, 3142 bool external __rte_unused, 3143 int hairpin __rte_unused, 3144 struct rte_flow_error *error) 3145 { 3146 return rte_flow_error_set(error, ENOTSUP, 3147 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3148 } 3149 3150 static struct mlx5_flow * 3151 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3152 const struct rte_flow_attr *attr __rte_unused, 3153 const struct rte_flow_item items[] __rte_unused, 3154 const struct rte_flow_action actions[] __rte_unused, 3155 struct rte_flow_error *error) 3156 { 3157 rte_flow_error_set(error, ENOTSUP, 3158 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3159 return NULL; 3160 } 3161 3162 static int 3163 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3164 struct mlx5_flow *dev_flow __rte_unused, 3165 const struct rte_flow_attr *attr __rte_unused, 3166 const struct rte_flow_item items[] __rte_unused, 3167 const struct rte_flow_action actions[] __rte_unused, 3168 struct rte_flow_error *error) 3169 { 3170 return rte_flow_error_set(error, ENOTSUP, 3171 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3172 } 3173 3174 static int 3175 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3176 struct rte_flow *flow __rte_unused, 3177 struct rte_flow_error *error) 3178 { 3179 return rte_flow_error_set(error, ENOTSUP, 3180 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3181 } 3182 3183 static void 3184 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3185 struct rte_flow *flow __rte_unused) 3186 { 3187 } 3188 3189 static void 3190 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3191 struct rte_flow *flow __rte_unused) 3192 { 3193 } 3194 3195 static int 3196 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3197 struct rte_flow *flow __rte_unused, 3198 const struct rte_flow_action *actions __rte_unused, 3199 void *data __rte_unused, 3200 struct rte_flow_error *error) 3201 { 3202 return rte_flow_error_set(error, ENOTSUP, 3203 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3204 } 3205 3206 static int 3207 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3208 uint32_t domains __rte_unused, 3209 uint32_t flags __rte_unused) 3210 { 3211 return 0; 3212 } 3213 3214 /* Void driver to protect from null pointer reference. */ 3215 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3216 .validate = flow_null_validate, 3217 .prepare = flow_null_prepare, 3218 .translate = flow_null_translate, 3219 .apply = flow_null_apply, 3220 .remove = flow_null_remove, 3221 .destroy = flow_null_destroy, 3222 .query = flow_null_query, 3223 .sync_domain = flow_null_sync_domain, 3224 }; 3225 3226 /** 3227 * Select flow driver type according to flow attributes and device 3228 * configuration. 3229 * 3230 * @param[in] dev 3231 * Pointer to the dev structure. 3232 * @param[in] attr 3233 * Pointer to the flow attributes. 3234 * 3235 * @return 3236 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3237 */ 3238 static enum mlx5_flow_drv_type 3239 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3240 { 3241 struct mlx5_priv *priv = dev->data->dev_private; 3242 /* The OS can determine first a specific flow type (DV, VERBS) */ 3243 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3244 3245 if (type != MLX5_FLOW_TYPE_MAX) 3246 return type; 3247 /* If no OS specific type - continue with DV/VERBS selection */ 3248 if (attr->transfer && priv->config.dv_esw_en) 3249 type = MLX5_FLOW_TYPE_DV; 3250 if (!attr->transfer) 3251 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3252 MLX5_FLOW_TYPE_VERBS; 3253 return type; 3254 } 3255 3256 #define flow_get_drv_ops(type) flow_drv_ops[type] 3257 3258 /** 3259 * Flow driver validation API. This abstracts calling driver specific functions. 3260 * The type of flow driver is determined according to flow attributes. 3261 * 3262 * @param[in] dev 3263 * Pointer to the dev structure. 3264 * @param[in] attr 3265 * Pointer to the flow attributes. 3266 * @param[in] items 3267 * Pointer to the list of items. 3268 * @param[in] actions 3269 * Pointer to the list of actions. 3270 * @param[in] external 3271 * This flow rule is created by request external to PMD. 3272 * @param[in] hairpin 3273 * Number of hairpin TX actions, 0 means classic flow. 3274 * @param[out] error 3275 * Pointer to the error structure. 3276 * 3277 * @return 3278 * 0 on success, a negative errno value otherwise and rte_errno is set. 3279 */ 3280 static inline int 3281 flow_drv_validate(struct rte_eth_dev *dev, 3282 const struct rte_flow_attr *attr, 3283 const struct rte_flow_item items[], 3284 const struct rte_flow_action actions[], 3285 bool external, int hairpin, struct rte_flow_error *error) 3286 { 3287 const struct mlx5_flow_driver_ops *fops; 3288 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3289 3290 fops = flow_get_drv_ops(type); 3291 return fops->validate(dev, attr, items, actions, external, 3292 hairpin, error); 3293 } 3294 3295 /** 3296 * Flow driver preparation API. This abstracts calling driver specific 3297 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3298 * calculates the size of memory required for device flow, allocates the memory, 3299 * initializes the device flow and returns the pointer. 3300 * 3301 * @note 3302 * This function initializes device flow structure such as dv or verbs in 3303 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3304 * rest. For example, adding returning device flow to flow->dev_flow list and 3305 * setting backward reference to the flow should be done out of this function. 3306 * layers field is not filled either. 3307 * 3308 * @param[in] dev 3309 * Pointer to the dev structure. 3310 * @param[in] attr 3311 * Pointer to the flow attributes. 3312 * @param[in] items 3313 * Pointer to the list of items. 3314 * @param[in] actions 3315 * Pointer to the list of actions. 3316 * @param[in] flow_idx 3317 * This memory pool index to the flow. 3318 * @param[out] error 3319 * Pointer to the error structure. 3320 * 3321 * @return 3322 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3323 */ 3324 static inline struct mlx5_flow * 3325 flow_drv_prepare(struct rte_eth_dev *dev, 3326 const struct rte_flow *flow, 3327 const struct rte_flow_attr *attr, 3328 const struct rte_flow_item items[], 3329 const struct rte_flow_action actions[], 3330 uint32_t flow_idx, 3331 struct rte_flow_error *error) 3332 { 3333 const struct mlx5_flow_driver_ops *fops; 3334 enum mlx5_flow_drv_type type = flow->drv_type; 3335 struct mlx5_flow *mlx5_flow = NULL; 3336 3337 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3338 fops = flow_get_drv_ops(type); 3339 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3340 if (mlx5_flow) 3341 mlx5_flow->flow_idx = flow_idx; 3342 return mlx5_flow; 3343 } 3344 3345 /** 3346 * Flow driver translation API. This abstracts calling driver specific 3347 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3348 * translates a generic flow into a driver flow. flow_drv_prepare() must 3349 * precede. 3350 * 3351 * @note 3352 * dev_flow->layers could be filled as a result of parsing during translation 3353 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3354 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3355 * flow->actions could be overwritten even though all the expanded dev_flows 3356 * have the same actions. 3357 * 3358 * @param[in] dev 3359 * Pointer to the rte dev structure. 3360 * @param[in, out] dev_flow 3361 * Pointer to the mlx5 flow. 3362 * @param[in] attr 3363 * Pointer to the flow attributes. 3364 * @param[in] items 3365 * Pointer to the list of items. 3366 * @param[in] actions 3367 * Pointer to the list of actions. 3368 * @param[out] error 3369 * Pointer to the error structure. 3370 * 3371 * @return 3372 * 0 on success, a negative errno value otherwise and rte_errno is set. 3373 */ 3374 static inline int 3375 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3376 const struct rte_flow_attr *attr, 3377 const struct rte_flow_item items[], 3378 const struct rte_flow_action actions[], 3379 struct rte_flow_error *error) 3380 { 3381 const struct mlx5_flow_driver_ops *fops; 3382 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3383 3384 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3385 fops = flow_get_drv_ops(type); 3386 return fops->translate(dev, dev_flow, attr, items, actions, error); 3387 } 3388 3389 /** 3390 * Flow driver apply API. This abstracts calling driver specific functions. 3391 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3392 * translated driver flows on to device. flow_drv_translate() must precede. 3393 * 3394 * @param[in] dev 3395 * Pointer to Ethernet device structure. 3396 * @param[in, out] flow 3397 * Pointer to flow structure. 3398 * @param[out] error 3399 * Pointer to error structure. 3400 * 3401 * @return 3402 * 0 on success, a negative errno value otherwise and rte_errno is set. 3403 */ 3404 static inline int 3405 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3406 struct rte_flow_error *error) 3407 { 3408 const struct mlx5_flow_driver_ops *fops; 3409 enum mlx5_flow_drv_type type = flow->drv_type; 3410 3411 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3412 fops = flow_get_drv_ops(type); 3413 return fops->apply(dev, flow, error); 3414 } 3415 3416 /** 3417 * Flow driver destroy API. This abstracts calling driver specific functions. 3418 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3419 * on device and releases resources of the flow. 3420 * 3421 * @param[in] dev 3422 * Pointer to Ethernet device. 3423 * @param[in, out] flow 3424 * Pointer to flow structure. 3425 */ 3426 static inline void 3427 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3428 { 3429 const struct mlx5_flow_driver_ops *fops; 3430 enum mlx5_flow_drv_type type = flow->drv_type; 3431 3432 flow_mreg_split_qrss_release(dev, flow); 3433 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3434 fops = flow_get_drv_ops(type); 3435 fops->destroy(dev, flow); 3436 } 3437 3438 /** 3439 * Flow driver find RSS policy tbl API. This abstracts calling driver 3440 * specific functions. Parent flow (rte_flow) should have driver 3441 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3442 * 3443 * @param[in] dev 3444 * Pointer to Ethernet device. 3445 * @param[in, out] flow 3446 * Pointer to flow structure. 3447 * @param[in] policy 3448 * Pointer to meter policy table. 3449 * @param[in] rss_desc 3450 * Pointer to rss_desc 3451 */ 3452 static struct mlx5_flow_meter_sub_policy * 3453 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3454 struct rte_flow *flow, 3455 struct mlx5_flow_meter_policy *policy, 3456 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3457 { 3458 const struct mlx5_flow_driver_ops *fops; 3459 enum mlx5_flow_drv_type type = flow->drv_type; 3460 3461 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3462 fops = flow_get_drv_ops(type); 3463 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3464 } 3465 3466 /** 3467 * Get RSS action from the action list. 3468 * 3469 * @param[in] dev 3470 * Pointer to Ethernet device. 3471 * @param[in] actions 3472 * Pointer to the list of actions. 3473 * @param[in] flow 3474 * Parent flow structure pointer. 3475 * 3476 * @return 3477 * Pointer to the RSS action if exist, else return NULL. 3478 */ 3479 static const struct rte_flow_action_rss* 3480 flow_get_rss_action(struct rte_eth_dev *dev, 3481 const struct rte_flow_action actions[]) 3482 { 3483 struct mlx5_priv *priv = dev->data->dev_private; 3484 const struct rte_flow_action_rss *rss = NULL; 3485 3486 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3487 switch (actions->type) { 3488 case RTE_FLOW_ACTION_TYPE_RSS: 3489 rss = actions->conf; 3490 break; 3491 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3492 { 3493 const struct rte_flow_action_sample *sample = 3494 actions->conf; 3495 const struct rte_flow_action *act = sample->actions; 3496 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3497 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3498 rss = act->conf; 3499 break; 3500 } 3501 case RTE_FLOW_ACTION_TYPE_METER: 3502 { 3503 uint32_t mtr_idx; 3504 struct mlx5_flow_meter_info *fm; 3505 struct mlx5_flow_meter_policy *policy; 3506 const struct rte_flow_action_meter *mtr = actions->conf; 3507 3508 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3509 if (fm && !fm->def_policy) { 3510 policy = mlx5_flow_meter_policy_find(dev, 3511 fm->policy_id, NULL); 3512 MLX5_ASSERT(policy); 3513 if (policy->is_hierarchy) { 3514 policy = 3515 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3516 policy); 3517 if (!policy) 3518 return NULL; 3519 } 3520 if (policy->is_rss) 3521 rss = 3522 policy->act_cnt[RTE_COLOR_GREEN].rss->conf; 3523 } 3524 break; 3525 } 3526 default: 3527 break; 3528 } 3529 } 3530 return rss; 3531 } 3532 3533 /** 3534 * Get ASO age action by index. 3535 * 3536 * @param[in] dev 3537 * Pointer to the Ethernet device structure. 3538 * @param[in] age_idx 3539 * Index to the ASO age action. 3540 * 3541 * @return 3542 * The specified ASO age action. 3543 */ 3544 struct mlx5_aso_age_action* 3545 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3546 { 3547 uint16_t pool_idx = age_idx & UINT16_MAX; 3548 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3549 struct mlx5_priv *priv = dev->data->dev_private; 3550 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3551 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3552 3553 return &pool->actions[offset - 1]; 3554 } 3555 3556 /* maps indirect action to translated direct in some actions array */ 3557 struct mlx5_translated_action_handle { 3558 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3559 int index; /**< Index in related array of rte_flow_action. */ 3560 }; 3561 3562 /** 3563 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3564 * direct action if translation possible. 3565 * This functionality used to run same execution path for both direct and 3566 * indirect actions on flow create. All necessary preparations for indirect 3567 * action handling should be performed on *handle* actions list returned 3568 * from this call. 3569 * 3570 * @param[in] dev 3571 * Pointer to Ethernet device. 3572 * @param[in] actions 3573 * List of actions to translate. 3574 * @param[out] handle 3575 * List to store translated indirect action object handles. 3576 * @param[in, out] indir_n 3577 * Size of *handle* array. On return should be updated with number of 3578 * indirect actions retrieved from the *actions* list. 3579 * @param[out] translated_actions 3580 * List of actions where all indirect actions were translated to direct 3581 * if possible. NULL if no translation took place. 3582 * @param[out] error 3583 * Pointer to the error structure. 3584 * 3585 * @return 3586 * 0 on success, a negative errno value otherwise and rte_errno is set. 3587 */ 3588 static int 3589 flow_action_handles_translate(struct rte_eth_dev *dev, 3590 const struct rte_flow_action actions[], 3591 struct mlx5_translated_action_handle *handle, 3592 int *indir_n, 3593 struct rte_flow_action **translated_actions, 3594 struct rte_flow_error *error) 3595 { 3596 struct mlx5_priv *priv = dev->data->dev_private; 3597 struct rte_flow_action *translated = NULL; 3598 size_t actions_size; 3599 int n; 3600 int copied_n = 0; 3601 struct mlx5_translated_action_handle *handle_end = NULL; 3602 3603 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3604 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3605 continue; 3606 if (copied_n == *indir_n) { 3607 return rte_flow_error_set 3608 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3609 NULL, "too many shared actions"); 3610 } 3611 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3612 sizeof(actions[n].conf)); 3613 handle[copied_n].index = n; 3614 copied_n++; 3615 } 3616 n++; 3617 *indir_n = copied_n; 3618 if (!copied_n) 3619 return 0; 3620 actions_size = sizeof(struct rte_flow_action) * n; 3621 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3622 if (!translated) { 3623 rte_errno = ENOMEM; 3624 return -ENOMEM; 3625 } 3626 memcpy(translated, actions, actions_size); 3627 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3628 struct mlx5_shared_action_rss *shared_rss; 3629 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3630 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3631 uint32_t idx = act_idx & 3632 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3633 3634 switch (type) { 3635 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3636 shared_rss = mlx5_ipool_get 3637 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3638 translated[handle->index].type = 3639 RTE_FLOW_ACTION_TYPE_RSS; 3640 translated[handle->index].conf = 3641 &shared_rss->origin; 3642 break; 3643 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3644 translated[handle->index].type = 3645 (enum rte_flow_action_type) 3646 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3647 translated[handle->index].conf = (void *)(uintptr_t)idx; 3648 break; 3649 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3650 if (priv->sh->flow_hit_aso_en) { 3651 translated[handle->index].type = 3652 (enum rte_flow_action_type) 3653 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3654 translated[handle->index].conf = 3655 (void *)(uintptr_t)idx; 3656 break; 3657 } 3658 /* Fall-through */ 3659 case MLX5_INDIRECT_ACTION_TYPE_CT: 3660 if (priv->sh->ct_aso_en) { 3661 translated[handle->index].type = 3662 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3663 translated[handle->index].conf = 3664 (void *)(uintptr_t)idx; 3665 break; 3666 } 3667 /* Fall-through */ 3668 default: 3669 mlx5_free(translated); 3670 return rte_flow_error_set 3671 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3672 NULL, "invalid indirect action type"); 3673 } 3674 } 3675 *translated_actions = translated; 3676 return 0; 3677 } 3678 3679 /** 3680 * Get Shared RSS action from the action list. 3681 * 3682 * @param[in] dev 3683 * Pointer to Ethernet device. 3684 * @param[in] shared 3685 * Pointer to the list of actions. 3686 * @param[in] shared_n 3687 * Actions list length. 3688 * 3689 * @return 3690 * The MLX5 RSS action ID if exists, otherwise return 0. 3691 */ 3692 static uint32_t 3693 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3694 struct mlx5_translated_action_handle *handle, 3695 int shared_n) 3696 { 3697 struct mlx5_translated_action_handle *handle_end; 3698 struct mlx5_priv *priv = dev->data->dev_private; 3699 struct mlx5_shared_action_rss *shared_rss; 3700 3701 3702 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3703 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3704 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3705 uint32_t idx = act_idx & 3706 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3707 switch (type) { 3708 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3709 shared_rss = mlx5_ipool_get 3710 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3711 idx); 3712 __atomic_add_fetch(&shared_rss->refcnt, 1, 3713 __ATOMIC_RELAXED); 3714 return idx; 3715 default: 3716 break; 3717 } 3718 } 3719 return 0; 3720 } 3721 3722 static unsigned int 3723 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3724 { 3725 const struct rte_flow_item *item; 3726 unsigned int has_vlan = 0; 3727 3728 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3729 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3730 has_vlan = 1; 3731 break; 3732 } 3733 } 3734 if (has_vlan) 3735 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3736 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3737 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3738 MLX5_EXPANSION_ROOT_OUTER; 3739 } 3740 3741 /** 3742 * Get layer flags from the prefix flow. 3743 * 3744 * Some flows may be split to several subflows, the prefix subflow gets the 3745 * match items and the suffix sub flow gets the actions. 3746 * Some actions need the user defined match item flags to get the detail for 3747 * the action. 3748 * This function helps the suffix flow to get the item layer flags from prefix 3749 * subflow. 3750 * 3751 * @param[in] dev_flow 3752 * Pointer the created preifx subflow. 3753 * 3754 * @return 3755 * The layers get from prefix subflow. 3756 */ 3757 static inline uint64_t 3758 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3759 { 3760 uint64_t layers = 0; 3761 3762 /* 3763 * Layers bits could be localization, but usually the compiler will 3764 * help to do the optimization work for source code. 3765 * If no decap actions, use the layers directly. 3766 */ 3767 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3768 return dev_flow->handle->layers; 3769 /* Convert L3 layers with decap action. */ 3770 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3771 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3772 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3773 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3774 /* Convert L4 layers with decap action. */ 3775 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3776 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3777 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3778 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3779 return layers; 3780 } 3781 3782 /** 3783 * Get metadata split action information. 3784 * 3785 * @param[in] actions 3786 * Pointer to the list of actions. 3787 * @param[out] qrss 3788 * Pointer to the return pointer. 3789 * @param[out] qrss_type 3790 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3791 * if no QUEUE/RSS is found. 3792 * @param[out] encap_idx 3793 * Pointer to the index of the encap action if exists, otherwise the last 3794 * action index. 3795 * 3796 * @return 3797 * Total number of actions. 3798 */ 3799 static int 3800 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3801 const struct rte_flow_action **qrss, 3802 int *encap_idx) 3803 { 3804 const struct rte_flow_action_raw_encap *raw_encap; 3805 int actions_n = 0; 3806 int raw_decap_idx = -1; 3807 3808 *encap_idx = -1; 3809 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3810 switch (actions->type) { 3811 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3812 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3813 *encap_idx = actions_n; 3814 break; 3815 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3816 raw_decap_idx = actions_n; 3817 break; 3818 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3819 raw_encap = actions->conf; 3820 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3821 *encap_idx = raw_decap_idx != -1 ? 3822 raw_decap_idx : actions_n; 3823 break; 3824 case RTE_FLOW_ACTION_TYPE_QUEUE: 3825 case RTE_FLOW_ACTION_TYPE_RSS: 3826 *qrss = actions; 3827 break; 3828 default: 3829 break; 3830 } 3831 actions_n++; 3832 } 3833 if (*encap_idx == -1) 3834 *encap_idx = actions_n; 3835 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3836 return actions_n + 1; 3837 } 3838 3839 /** 3840 * Check if the action will change packet. 3841 * 3842 * @param dev 3843 * Pointer to Ethernet device. 3844 * @param[in] type 3845 * action type. 3846 * 3847 * @return 3848 * true if action will change packet, false otherwise. 3849 */ 3850 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 3851 enum rte_flow_action_type type) 3852 { 3853 struct mlx5_priv *priv = dev->data->dev_private; 3854 3855 switch (type) { 3856 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3857 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3858 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3859 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3860 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3861 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3862 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3863 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3864 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3865 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3866 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3867 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3868 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3869 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3870 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 3871 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 3872 case RTE_FLOW_ACTION_TYPE_SET_META: 3873 case RTE_FLOW_ACTION_TYPE_SET_TAG: 3874 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 3875 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3876 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3877 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3878 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3879 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3880 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3881 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3882 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3883 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3884 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 3885 return true; 3886 case RTE_FLOW_ACTION_TYPE_FLAG: 3887 case RTE_FLOW_ACTION_TYPE_MARK: 3888 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 3889 return true; 3890 else 3891 return false; 3892 default: 3893 return false; 3894 } 3895 } 3896 3897 /** 3898 * Check meter action from the action list. 3899 * 3900 * @param dev 3901 * Pointer to Ethernet device. 3902 * @param[in] actions 3903 * Pointer to the list of actions. 3904 * @param[out] has_mtr 3905 * Pointer to the meter exist flag. 3906 * @param[out] has_modify 3907 * Pointer to the flag showing there's packet change action. 3908 * @param[out] meter_id 3909 * Pointer to the meter id. 3910 * 3911 * @return 3912 * Total number of actions. 3913 */ 3914 static int 3915 flow_check_meter_action(struct rte_eth_dev *dev, 3916 const struct rte_flow_action actions[], 3917 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 3918 { 3919 const struct rte_flow_action_meter *mtr = NULL; 3920 int actions_n = 0; 3921 3922 MLX5_ASSERT(has_mtr); 3923 *has_mtr = false; 3924 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3925 switch (actions->type) { 3926 case RTE_FLOW_ACTION_TYPE_METER: 3927 mtr = actions->conf; 3928 *meter_id = mtr->mtr_id; 3929 *has_mtr = true; 3930 break; 3931 default: 3932 break; 3933 } 3934 if (!*has_mtr) 3935 *has_modify |= flow_check_modify_action_type(dev, 3936 actions->type); 3937 actions_n++; 3938 } 3939 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3940 return actions_n + 1; 3941 } 3942 3943 /** 3944 * Check if the flow should be split due to hairpin. 3945 * The reason for the split is that in current HW we can't 3946 * support encap and push-vlan on Rx, so if a flow contains 3947 * these actions we move it to Tx. 3948 * 3949 * @param dev 3950 * Pointer to Ethernet device. 3951 * @param[in] attr 3952 * Flow rule attributes. 3953 * @param[in] actions 3954 * Associated actions (list terminated by the END action). 3955 * 3956 * @return 3957 * > 0 the number of actions and the flow should be split, 3958 * 0 when no split required. 3959 */ 3960 static int 3961 flow_check_hairpin_split(struct rte_eth_dev *dev, 3962 const struct rte_flow_attr *attr, 3963 const struct rte_flow_action actions[]) 3964 { 3965 int queue_action = 0; 3966 int action_n = 0; 3967 int split = 0; 3968 const struct rte_flow_action_queue *queue; 3969 const struct rte_flow_action_rss *rss; 3970 const struct rte_flow_action_raw_encap *raw_encap; 3971 const struct rte_eth_hairpin_conf *conf; 3972 3973 if (!attr->ingress) 3974 return 0; 3975 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3976 switch (actions->type) { 3977 case RTE_FLOW_ACTION_TYPE_QUEUE: 3978 queue = actions->conf; 3979 if (queue == NULL) 3980 return 0; 3981 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3982 if (conf == NULL || conf->tx_explicit != 0) 3983 return 0; 3984 queue_action = 1; 3985 action_n++; 3986 break; 3987 case RTE_FLOW_ACTION_TYPE_RSS: 3988 rss = actions->conf; 3989 if (rss == NULL || rss->queue_num == 0) 3990 return 0; 3991 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3992 if (conf == NULL || conf->tx_explicit != 0) 3993 return 0; 3994 queue_action = 1; 3995 action_n++; 3996 break; 3997 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3998 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3999 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4000 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4001 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4002 split++; 4003 action_n++; 4004 break; 4005 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4006 raw_encap = actions->conf; 4007 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4008 split++; 4009 action_n++; 4010 break; 4011 default: 4012 action_n++; 4013 break; 4014 } 4015 } 4016 if (split && queue_action) 4017 return action_n; 4018 return 0; 4019 } 4020 4021 /* Declare flow create/destroy prototype in advance. */ 4022 static uint32_t 4023 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 4024 const struct rte_flow_attr *attr, 4025 const struct rte_flow_item items[], 4026 const struct rte_flow_action actions[], 4027 bool external, struct rte_flow_error *error); 4028 4029 static void 4030 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 4031 uint32_t flow_idx); 4032 4033 int 4034 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused, 4035 struct mlx5_hlist_entry *entry, 4036 uint64_t key, void *cb_ctx __rte_unused) 4037 { 4038 struct mlx5_flow_mreg_copy_resource *mcp_res = 4039 container_of(entry, typeof(*mcp_res), hlist_ent); 4040 4041 return mcp_res->mark_id != key; 4042 } 4043 4044 struct mlx5_hlist_entry * 4045 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 4046 void *cb_ctx) 4047 { 4048 struct rte_eth_dev *dev = list->ctx; 4049 struct mlx5_priv *priv = dev->data->dev_private; 4050 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4051 struct mlx5_flow_mreg_copy_resource *mcp_res; 4052 struct rte_flow_error *error = ctx->error; 4053 uint32_t idx = 0; 4054 int ret; 4055 uint32_t mark_id = key; 4056 struct rte_flow_attr attr = { 4057 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4058 .ingress = 1, 4059 }; 4060 struct mlx5_rte_flow_item_tag tag_spec = { 4061 .data = mark_id, 4062 }; 4063 struct rte_flow_item items[] = { 4064 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4065 }; 4066 struct rte_flow_action_mark ftag = { 4067 .id = mark_id, 4068 }; 4069 struct mlx5_flow_action_copy_mreg cp_mreg = { 4070 .dst = REG_B, 4071 .src = REG_NON, 4072 }; 4073 struct rte_flow_action_jump jump = { 4074 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4075 }; 4076 struct rte_flow_action actions[] = { 4077 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4078 }; 4079 4080 /* Fill the register fileds in the flow. */ 4081 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4082 if (ret < 0) 4083 return NULL; 4084 tag_spec.id = ret; 4085 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4086 if (ret < 0) 4087 return NULL; 4088 cp_mreg.src = ret; 4089 /* Provide the full width of FLAG specific value. */ 4090 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4091 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4092 /* Build a new flow. */ 4093 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4094 items[0] = (struct rte_flow_item){ 4095 .type = (enum rte_flow_item_type) 4096 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4097 .spec = &tag_spec, 4098 }; 4099 items[1] = (struct rte_flow_item){ 4100 .type = RTE_FLOW_ITEM_TYPE_END, 4101 }; 4102 actions[0] = (struct rte_flow_action){ 4103 .type = (enum rte_flow_action_type) 4104 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4105 .conf = &ftag, 4106 }; 4107 actions[1] = (struct rte_flow_action){ 4108 .type = (enum rte_flow_action_type) 4109 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4110 .conf = &cp_mreg, 4111 }; 4112 actions[2] = (struct rte_flow_action){ 4113 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4114 .conf = &jump, 4115 }; 4116 actions[3] = (struct rte_flow_action){ 4117 .type = RTE_FLOW_ACTION_TYPE_END, 4118 }; 4119 } else { 4120 /* Default rule, wildcard match. */ 4121 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4122 items[0] = (struct rte_flow_item){ 4123 .type = RTE_FLOW_ITEM_TYPE_END, 4124 }; 4125 actions[0] = (struct rte_flow_action){ 4126 .type = (enum rte_flow_action_type) 4127 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4128 .conf = &cp_mreg, 4129 }; 4130 actions[1] = (struct rte_flow_action){ 4131 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4132 .conf = &jump, 4133 }; 4134 actions[2] = (struct rte_flow_action){ 4135 .type = RTE_FLOW_ACTION_TYPE_END, 4136 }; 4137 } 4138 /* Build a new entry. */ 4139 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4140 if (!mcp_res) { 4141 rte_errno = ENOMEM; 4142 return NULL; 4143 } 4144 mcp_res->idx = idx; 4145 mcp_res->mark_id = mark_id; 4146 /* 4147 * The copy Flows are not included in any list. There 4148 * ones are referenced from other Flows and can not 4149 * be applied, removed, deleted in ardbitrary order 4150 * by list traversing. 4151 */ 4152 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 4153 actions, false, error); 4154 if (!mcp_res->rix_flow) { 4155 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4156 return NULL; 4157 } 4158 return &mcp_res->hlist_ent; 4159 } 4160 4161 /** 4162 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4163 * 4164 * As mark_id is unique, if there's already a registered flow for the mark_id, 4165 * return by increasing the reference counter of the resource. Otherwise, create 4166 * the resource (mcp_res) and flow. 4167 * 4168 * Flow looks like, 4169 * - If ingress port is ANY and reg_c[1] is mark_id, 4170 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4171 * 4172 * For default flow (zero mark_id), flow is like, 4173 * - If ingress port is ANY, 4174 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4175 * 4176 * @param dev 4177 * Pointer to Ethernet device. 4178 * @param mark_id 4179 * ID of MARK action, zero means default flow for META. 4180 * @param[out] error 4181 * Perform verbose error reporting if not NULL. 4182 * 4183 * @return 4184 * Associated resource on success, NULL otherwise and rte_errno is set. 4185 */ 4186 static struct mlx5_flow_mreg_copy_resource * 4187 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4188 struct rte_flow_error *error) 4189 { 4190 struct mlx5_priv *priv = dev->data->dev_private; 4191 struct mlx5_hlist_entry *entry; 4192 struct mlx5_flow_cb_ctx ctx = { 4193 .dev = dev, 4194 .error = error, 4195 }; 4196 4197 /* Check if already registered. */ 4198 MLX5_ASSERT(priv->mreg_cp_tbl); 4199 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4200 if (!entry) 4201 return NULL; 4202 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4203 hlist_ent); 4204 } 4205 4206 void 4207 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 4208 { 4209 struct mlx5_flow_mreg_copy_resource *mcp_res = 4210 container_of(entry, typeof(*mcp_res), hlist_ent); 4211 struct rte_eth_dev *dev = list->ctx; 4212 struct mlx5_priv *priv = dev->data->dev_private; 4213 4214 MLX5_ASSERT(mcp_res->rix_flow); 4215 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 4216 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4217 } 4218 4219 /** 4220 * Release flow in RX_CP_TBL. 4221 * 4222 * @param dev 4223 * Pointer to Ethernet device. 4224 * @flow 4225 * Parent flow for wich copying is provided. 4226 */ 4227 static void 4228 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4229 struct rte_flow *flow) 4230 { 4231 struct mlx5_flow_mreg_copy_resource *mcp_res; 4232 struct mlx5_priv *priv = dev->data->dev_private; 4233 4234 if (!flow->rix_mreg_copy) 4235 return; 4236 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4237 flow->rix_mreg_copy); 4238 if (!mcp_res || !priv->mreg_cp_tbl) 4239 return; 4240 MLX5_ASSERT(mcp_res->rix_flow); 4241 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4242 flow->rix_mreg_copy = 0; 4243 } 4244 4245 /** 4246 * Remove the default copy action from RX_CP_TBL. 4247 * 4248 * This functions is called in the mlx5_dev_start(). No thread safe 4249 * is guaranteed. 4250 * 4251 * @param dev 4252 * Pointer to Ethernet device. 4253 */ 4254 static void 4255 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4256 { 4257 struct mlx5_hlist_entry *entry; 4258 struct mlx5_priv *priv = dev->data->dev_private; 4259 4260 /* Check if default flow is registered. */ 4261 if (!priv->mreg_cp_tbl) 4262 return; 4263 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 4264 MLX5_DEFAULT_COPY_ID, NULL); 4265 if (!entry) 4266 return; 4267 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4268 } 4269 4270 /** 4271 * Add the default copy action in in RX_CP_TBL. 4272 * 4273 * This functions is called in the mlx5_dev_start(). No thread safe 4274 * is guaranteed. 4275 * 4276 * @param dev 4277 * Pointer to Ethernet device. 4278 * @param[out] error 4279 * Perform verbose error reporting if not NULL. 4280 * 4281 * @return 4282 * 0 for success, negative value otherwise and rte_errno is set. 4283 */ 4284 static int 4285 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4286 struct rte_flow_error *error) 4287 { 4288 struct mlx5_priv *priv = dev->data->dev_private; 4289 struct mlx5_flow_mreg_copy_resource *mcp_res; 4290 4291 /* Check whether extensive metadata feature is engaged. */ 4292 if (!priv->config.dv_flow_en || 4293 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4294 !mlx5_flow_ext_mreg_supported(dev) || 4295 !priv->sh->dv_regc0_mask) 4296 return 0; 4297 /* 4298 * Add default mreg copy flow may be called multiple time, but 4299 * only be called once in stop. Avoid register it twice. 4300 */ 4301 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 4302 return 0; 4303 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 4304 if (!mcp_res) 4305 return -rte_errno; 4306 return 0; 4307 } 4308 4309 /** 4310 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4311 * 4312 * All the flow having Q/RSS action should be split by 4313 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4314 * performs the following, 4315 * - CQE->flow_tag := reg_c[1] (MARK) 4316 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4317 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4318 * but there should be a flow per each MARK ID set by MARK action. 4319 * 4320 * For the aforementioned reason, if there's a MARK action in flow's action 4321 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4322 * the MARK ID to CQE's flow_tag like, 4323 * - If reg_c[1] is mark_id, 4324 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4325 * 4326 * For SET_META action which stores value in reg_c[0], as the destination is 4327 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4328 * MARK ID means the default flow. The default flow looks like, 4329 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4330 * 4331 * @param dev 4332 * Pointer to Ethernet device. 4333 * @param flow 4334 * Pointer to flow structure. 4335 * @param[in] actions 4336 * Pointer to the list of actions. 4337 * @param[out] error 4338 * Perform verbose error reporting if not NULL. 4339 * 4340 * @return 4341 * 0 on success, negative value otherwise and rte_errno is set. 4342 */ 4343 static int 4344 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4345 struct rte_flow *flow, 4346 const struct rte_flow_action *actions, 4347 struct rte_flow_error *error) 4348 { 4349 struct mlx5_priv *priv = dev->data->dev_private; 4350 struct mlx5_dev_config *config = &priv->config; 4351 struct mlx5_flow_mreg_copy_resource *mcp_res; 4352 const struct rte_flow_action_mark *mark; 4353 4354 /* Check whether extensive metadata feature is engaged. */ 4355 if (!config->dv_flow_en || 4356 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4357 !mlx5_flow_ext_mreg_supported(dev) || 4358 !priv->sh->dv_regc0_mask) 4359 return 0; 4360 /* Find MARK action. */ 4361 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4362 switch (actions->type) { 4363 case RTE_FLOW_ACTION_TYPE_FLAG: 4364 mcp_res = flow_mreg_add_copy_action 4365 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4366 if (!mcp_res) 4367 return -rte_errno; 4368 flow->rix_mreg_copy = mcp_res->idx; 4369 return 0; 4370 case RTE_FLOW_ACTION_TYPE_MARK: 4371 mark = (const struct rte_flow_action_mark *) 4372 actions->conf; 4373 mcp_res = 4374 flow_mreg_add_copy_action(dev, mark->id, error); 4375 if (!mcp_res) 4376 return -rte_errno; 4377 flow->rix_mreg_copy = mcp_res->idx; 4378 return 0; 4379 default: 4380 break; 4381 } 4382 } 4383 return 0; 4384 } 4385 4386 #define MLX5_MAX_SPLIT_ACTIONS 24 4387 #define MLX5_MAX_SPLIT_ITEMS 24 4388 4389 /** 4390 * Split the hairpin flow. 4391 * Since HW can't support encap and push-vlan on Rx, we move these 4392 * actions to Tx. 4393 * If the count action is after the encap then we also 4394 * move the count action. in this case the count will also measure 4395 * the outer bytes. 4396 * 4397 * @param dev 4398 * Pointer to Ethernet device. 4399 * @param[in] actions 4400 * Associated actions (list terminated by the END action). 4401 * @param[out] actions_rx 4402 * Rx flow actions. 4403 * @param[out] actions_tx 4404 * Tx flow actions.. 4405 * @param[out] pattern_tx 4406 * The pattern items for the Tx flow. 4407 * @param[out] flow_id 4408 * The flow ID connected to this flow. 4409 * 4410 * @return 4411 * 0 on success. 4412 */ 4413 static int 4414 flow_hairpin_split(struct rte_eth_dev *dev, 4415 const struct rte_flow_action actions[], 4416 struct rte_flow_action actions_rx[], 4417 struct rte_flow_action actions_tx[], 4418 struct rte_flow_item pattern_tx[], 4419 uint32_t flow_id) 4420 { 4421 const struct rte_flow_action_raw_encap *raw_encap; 4422 const struct rte_flow_action_raw_decap *raw_decap; 4423 struct mlx5_rte_flow_action_set_tag *set_tag; 4424 struct rte_flow_action *tag_action; 4425 struct mlx5_rte_flow_item_tag *tag_item; 4426 struct rte_flow_item *item; 4427 char *addr; 4428 int encap = 0; 4429 4430 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4431 switch (actions->type) { 4432 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4433 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4434 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4435 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4436 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4437 rte_memcpy(actions_tx, actions, 4438 sizeof(struct rte_flow_action)); 4439 actions_tx++; 4440 break; 4441 case RTE_FLOW_ACTION_TYPE_COUNT: 4442 if (encap) { 4443 rte_memcpy(actions_tx, actions, 4444 sizeof(struct rte_flow_action)); 4445 actions_tx++; 4446 } else { 4447 rte_memcpy(actions_rx, actions, 4448 sizeof(struct rte_flow_action)); 4449 actions_rx++; 4450 } 4451 break; 4452 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4453 raw_encap = actions->conf; 4454 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4455 memcpy(actions_tx, actions, 4456 sizeof(struct rte_flow_action)); 4457 actions_tx++; 4458 encap = 1; 4459 } else { 4460 rte_memcpy(actions_rx, actions, 4461 sizeof(struct rte_flow_action)); 4462 actions_rx++; 4463 } 4464 break; 4465 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4466 raw_decap = actions->conf; 4467 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4468 memcpy(actions_tx, actions, 4469 sizeof(struct rte_flow_action)); 4470 actions_tx++; 4471 } else { 4472 rte_memcpy(actions_rx, actions, 4473 sizeof(struct rte_flow_action)); 4474 actions_rx++; 4475 } 4476 break; 4477 default: 4478 rte_memcpy(actions_rx, actions, 4479 sizeof(struct rte_flow_action)); 4480 actions_rx++; 4481 break; 4482 } 4483 } 4484 /* Add set meta action and end action for the Rx flow. */ 4485 tag_action = actions_rx; 4486 tag_action->type = (enum rte_flow_action_type) 4487 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4488 actions_rx++; 4489 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4490 actions_rx++; 4491 set_tag = (void *)actions_rx; 4492 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4493 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4494 .data = flow_id, 4495 }; 4496 MLX5_ASSERT(set_tag->id > REG_NON); 4497 tag_action->conf = set_tag; 4498 /* Create Tx item list. */ 4499 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4500 addr = (void *)&pattern_tx[2]; 4501 item = pattern_tx; 4502 item->type = (enum rte_flow_item_type) 4503 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4504 tag_item = (void *)addr; 4505 tag_item->data = flow_id; 4506 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4507 MLX5_ASSERT(set_tag->id > REG_NON); 4508 item->spec = tag_item; 4509 addr += sizeof(struct mlx5_rte_flow_item_tag); 4510 tag_item = (void *)addr; 4511 tag_item->data = UINT32_MAX; 4512 tag_item->id = UINT16_MAX; 4513 item->mask = tag_item; 4514 item->last = NULL; 4515 item++; 4516 item->type = RTE_FLOW_ITEM_TYPE_END; 4517 return 0; 4518 } 4519 4520 /** 4521 * The last stage of splitting chain, just creates the subflow 4522 * without any modification. 4523 * 4524 * @param[in] dev 4525 * Pointer to Ethernet device. 4526 * @param[in] flow 4527 * Parent flow structure pointer. 4528 * @param[in, out] sub_flow 4529 * Pointer to return the created subflow, may be NULL. 4530 * @param[in] attr 4531 * Flow rule attributes. 4532 * @param[in] items 4533 * Pattern specification (list terminated by the END pattern item). 4534 * @param[in] actions 4535 * Associated actions (list terminated by the END action). 4536 * @param[in] flow_split_info 4537 * Pointer to flow split info structure. 4538 * @param[out] error 4539 * Perform verbose error reporting if not NULL. 4540 * @return 4541 * 0 on success, negative value otherwise 4542 */ 4543 static int 4544 flow_create_split_inner(struct rte_eth_dev *dev, 4545 struct rte_flow *flow, 4546 struct mlx5_flow **sub_flow, 4547 const struct rte_flow_attr *attr, 4548 const struct rte_flow_item items[], 4549 const struct rte_flow_action actions[], 4550 struct mlx5_flow_split_info *flow_split_info, 4551 struct rte_flow_error *error) 4552 { 4553 struct mlx5_flow *dev_flow; 4554 4555 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4556 flow_split_info->flow_idx, error); 4557 if (!dev_flow) 4558 return -rte_errno; 4559 dev_flow->flow = flow; 4560 dev_flow->external = flow_split_info->external; 4561 dev_flow->skip_scale = flow_split_info->skip_scale; 4562 /* Subflow object was created, we must include one in the list. */ 4563 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4564 dev_flow->handle, next); 4565 /* 4566 * If dev_flow is as one of the suffix flow, some actions in suffix 4567 * flow may need some user defined item layer flags, and pass the 4568 * Metadate rxq mark flag to suffix flow as well. 4569 */ 4570 if (flow_split_info->prefix_layers) 4571 dev_flow->handle->layers = flow_split_info->prefix_layers; 4572 if (flow_split_info->prefix_mark) 4573 dev_flow->handle->mark = 1; 4574 if (sub_flow) 4575 *sub_flow = dev_flow; 4576 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4577 dev_flow->dv.table_id = flow_split_info->table_id; 4578 #endif 4579 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4580 } 4581 4582 /** 4583 * Get the sub policy of a meter. 4584 * 4585 * @param[in] dev 4586 * Pointer to Ethernet device. 4587 * @param[in] flow 4588 * Parent flow structure pointer. 4589 * @param wks 4590 * Pointer to thread flow work space. 4591 * @param[in] attr 4592 * Flow rule attributes. 4593 * @param[in] items 4594 * Pattern specification (list terminated by the END pattern item). 4595 * @param[out] error 4596 * Perform verbose error reporting if not NULL. 4597 * 4598 * @return 4599 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4600 */ 4601 static struct mlx5_flow_meter_sub_policy * 4602 get_meter_sub_policy(struct rte_eth_dev *dev, 4603 struct rte_flow *flow, 4604 struct mlx5_flow_workspace *wks, 4605 const struct rte_flow_attr *attr, 4606 const struct rte_flow_item items[], 4607 struct rte_flow_error *error) 4608 { 4609 struct mlx5_flow_meter_policy *policy; 4610 struct mlx5_flow_meter_policy *final_policy; 4611 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4612 4613 policy = wks->policy; 4614 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4615 if (final_policy->is_rss || final_policy->is_queue) { 4616 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4617 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4618 uint32_t i; 4619 4620 /** 4621 * This is a tmp dev_flow, 4622 * no need to register any matcher for it in translate. 4623 */ 4624 wks->skip_matcher_reg = 1; 4625 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4626 struct mlx5_flow dev_flow = {0}; 4627 struct mlx5_flow_handle dev_handle = { {0} }; 4628 4629 if (final_policy->is_rss) { 4630 const void *rss_act = 4631 final_policy->act_cnt[i].rss->conf; 4632 struct rte_flow_action rss_actions[2] = { 4633 [0] = { 4634 .type = RTE_FLOW_ACTION_TYPE_RSS, 4635 .conf = rss_act 4636 }, 4637 [1] = { 4638 .type = RTE_FLOW_ACTION_TYPE_END, 4639 .conf = NULL 4640 } 4641 }; 4642 4643 dev_flow.handle = &dev_handle; 4644 dev_flow.ingress = attr->ingress; 4645 dev_flow.flow = flow; 4646 dev_flow.external = 0; 4647 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4648 dev_flow.dv.transfer = attr->transfer; 4649 #endif 4650 /** 4651 * Translate RSS action to get rss hash fields. 4652 */ 4653 if (flow_drv_translate(dev, &dev_flow, attr, 4654 items, rss_actions, error)) 4655 goto exit; 4656 rss_desc_v[i] = wks->rss_desc; 4657 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4658 rss_desc_v[i].hash_fields = 4659 dev_flow.hash_fields; 4660 rss_desc_v[i].queue_num = 4661 rss_desc_v[i].hash_fields ? 4662 rss_desc_v[i].queue_num : 1; 4663 rss_desc_v[i].tunnel = 4664 !!(dev_flow.handle->layers & 4665 MLX5_FLOW_LAYER_TUNNEL); 4666 } else { 4667 /* This is queue action. */ 4668 rss_desc_v[i] = wks->rss_desc; 4669 rss_desc_v[i].key_len = 0; 4670 rss_desc_v[i].hash_fields = 0; 4671 rss_desc_v[i].queue = 4672 &final_policy->act_cnt[i].queue; 4673 rss_desc_v[i].queue_num = 1; 4674 } 4675 rss_desc[i] = &rss_desc_v[i]; 4676 } 4677 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4678 flow, policy, rss_desc); 4679 } else { 4680 enum mlx5_meter_domain mtr_domain = 4681 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4682 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4683 MLX5_MTR_DOMAIN_INGRESS; 4684 sub_policy = policy->sub_policys[mtr_domain][0]; 4685 } 4686 if (!sub_policy) { 4687 rte_flow_error_set(error, EINVAL, 4688 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4689 "Failed to get meter sub-policy."); 4690 goto exit; 4691 } 4692 exit: 4693 return sub_policy; 4694 } 4695 4696 /** 4697 * Split the meter flow. 4698 * 4699 * As meter flow will split to three sub flow, other than meter 4700 * action, the other actions make sense to only meter accepts 4701 * the packet. If it need to be dropped, no other additional 4702 * actions should be take. 4703 * 4704 * One kind of special action which decapsulates the L3 tunnel 4705 * header will be in the prefix sub flow, as not to take the 4706 * L3 tunnel header into account. 4707 * 4708 * @param[in] dev 4709 * Pointer to Ethernet device. 4710 * @param[in] flow 4711 * Parent flow structure pointer. 4712 * @param wks 4713 * Pointer to thread flow work space. 4714 * @param[in] attr 4715 * Flow rule attributes. 4716 * @param[in] items 4717 * Pattern specification (list terminated by the END pattern item). 4718 * @param[out] sfx_items 4719 * Suffix flow match items (list terminated by the END pattern item). 4720 * @param[in] actions 4721 * Associated actions (list terminated by the END action). 4722 * @param[out] actions_sfx 4723 * Suffix flow actions. 4724 * @param[out] actions_pre 4725 * Prefix flow actions. 4726 * @param[out] mtr_flow_id 4727 * Pointer to meter flow id. 4728 * @param[out] error 4729 * Perform verbose error reporting if not NULL. 4730 * 4731 * @return 4732 * 0 on success, a negative errno value otherwise and rte_errno is set. 4733 */ 4734 static int 4735 flow_meter_split_prep(struct rte_eth_dev *dev, 4736 struct rte_flow *flow, 4737 struct mlx5_flow_workspace *wks, 4738 const struct rte_flow_attr *attr, 4739 const struct rte_flow_item items[], 4740 struct rte_flow_item sfx_items[], 4741 const struct rte_flow_action actions[], 4742 struct rte_flow_action actions_sfx[], 4743 struct rte_flow_action actions_pre[], 4744 uint32_t *mtr_flow_id, 4745 struct rte_flow_error *error) 4746 { 4747 struct mlx5_priv *priv = dev->data->dev_private; 4748 struct mlx5_flow_meter_info *fm = wks->fm; 4749 struct rte_flow_action *tag_action = NULL; 4750 struct rte_flow_item *tag_item; 4751 struct mlx5_rte_flow_action_set_tag *set_tag; 4752 const struct rte_flow_action_raw_encap *raw_encap; 4753 const struct rte_flow_action_raw_decap *raw_decap; 4754 struct mlx5_rte_flow_item_tag *tag_item_spec; 4755 struct mlx5_rte_flow_item_tag *tag_item_mask; 4756 uint32_t tag_id = 0; 4757 struct rte_flow_item *vlan_item_dst = NULL; 4758 const struct rte_flow_item *vlan_item_src = NULL; 4759 struct rte_flow_action *hw_mtr_action; 4760 struct rte_flow_action *action_pre_head = NULL; 4761 int32_t flow_src_port = priv->representor_id; 4762 bool mtr_first; 4763 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 4764 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 4765 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 4766 uint32_t flow_id = 0; 4767 uint32_t flow_id_reversed = 0; 4768 uint8_t flow_id_bits = 0; 4769 int shift; 4770 4771 /* Prepare the suffix subflow items. */ 4772 tag_item = sfx_items++; 4773 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4774 struct mlx5_priv *port_priv; 4775 const struct rte_flow_item_port_id *pid_v; 4776 int item_type = items->type; 4777 4778 switch (item_type) { 4779 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4780 pid_v = items->spec; 4781 MLX5_ASSERT(pid_v); 4782 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 4783 if (!port_priv) 4784 return rte_flow_error_set(error, 4785 rte_errno, 4786 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 4787 pid_v, 4788 "Failed to get port info."); 4789 flow_src_port = port_priv->representor_id; 4790 memcpy(sfx_items, items, sizeof(*sfx_items)); 4791 sfx_items++; 4792 break; 4793 case RTE_FLOW_ITEM_TYPE_VLAN: 4794 /* Determine if copy vlan item below. */ 4795 vlan_item_src = items; 4796 vlan_item_dst = sfx_items++; 4797 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 4798 break; 4799 default: 4800 break; 4801 } 4802 } 4803 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4804 sfx_items++; 4805 mtr_first = priv->sh->meter_aso_en && 4806 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 4807 /* For ASO meter, meter must be before tag in TX direction. */ 4808 if (mtr_first) { 4809 action_pre_head = actions_pre++; 4810 /* Leave space for tag action. */ 4811 tag_action = actions_pre++; 4812 } 4813 /* Prepare the actions for prefix and suffix flow. */ 4814 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4815 struct rte_flow_action *action_cur = NULL; 4816 4817 switch (actions->type) { 4818 case RTE_FLOW_ACTION_TYPE_METER: 4819 if (mtr_first) { 4820 action_cur = action_pre_head; 4821 } else { 4822 /* Leave space for tag action. */ 4823 tag_action = actions_pre++; 4824 action_cur = actions_pre++; 4825 } 4826 break; 4827 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4828 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4829 action_cur = actions_pre++; 4830 break; 4831 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4832 raw_encap = actions->conf; 4833 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4834 action_cur = actions_pre++; 4835 break; 4836 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4837 raw_decap = actions->conf; 4838 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4839 action_cur = actions_pre++; 4840 break; 4841 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4842 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4843 if (vlan_item_dst && vlan_item_src) { 4844 memcpy(vlan_item_dst, vlan_item_src, 4845 sizeof(*vlan_item_dst)); 4846 /* 4847 * Convert to internal match item, it is used 4848 * for vlan push and set vid. 4849 */ 4850 vlan_item_dst->type = (enum rte_flow_item_type) 4851 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4852 } 4853 break; 4854 default: 4855 break; 4856 } 4857 if (!action_cur) 4858 action_cur = (fm->def_policy) ? 4859 actions_sfx++ : actions_pre++; 4860 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 4861 } 4862 /* Add end action to the actions. */ 4863 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4864 if (priv->sh->meter_aso_en) { 4865 /** 4866 * For ASO meter, need to add an extra jump action explicitly, 4867 * to jump from meter to policer table. 4868 */ 4869 struct mlx5_flow_meter_sub_policy *sub_policy; 4870 struct mlx5_flow_tbl_data_entry *tbl_data; 4871 4872 if (!fm->def_policy) { 4873 sub_policy = get_meter_sub_policy(dev, flow, wks, 4874 attr, items, error); 4875 if (!sub_policy) 4876 return -rte_errno; 4877 } else { 4878 enum mlx5_meter_domain mtr_domain = 4879 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4880 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4881 MLX5_MTR_DOMAIN_INGRESS; 4882 4883 sub_policy = 4884 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 4885 } 4886 tbl_data = container_of(sub_policy->tbl_rsc, 4887 struct mlx5_flow_tbl_data_entry, tbl); 4888 hw_mtr_action = actions_pre++; 4889 hw_mtr_action->type = (enum rte_flow_action_type) 4890 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 4891 hw_mtr_action->conf = tbl_data->jump.action; 4892 } 4893 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4894 actions_pre++; 4895 if (!tag_action) 4896 return rte_flow_error_set(error, ENOMEM, 4897 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4898 "No tag action space."); 4899 if (!mtr_flow_id) { 4900 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 4901 goto exit; 4902 } 4903 /* Only default-policy Meter creates mtr flow id. */ 4904 if (fm->def_policy) { 4905 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 4906 if (!tag_id) 4907 return rte_flow_error_set(error, ENOMEM, 4908 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4909 "Failed to allocate meter flow id."); 4910 flow_id = tag_id - 1; 4911 flow_id_bits = (!flow_id) ? 1 : 4912 (MLX5_REG_BITS - __builtin_clz(flow_id)); 4913 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 4914 mtr_reg_bits) { 4915 mlx5_ipool_free(fm->flow_ipool, tag_id); 4916 return rte_flow_error_set(error, EINVAL, 4917 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4918 "Meter flow id exceeds max limit."); 4919 } 4920 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 4921 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 4922 } 4923 /* Build tag actions and items for meter_id/meter flow_id. */ 4924 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 4925 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4926 tag_item_mask = tag_item_spec + 1; 4927 /* Both flow_id and meter_id share the same register. */ 4928 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4929 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 4930 0, error), 4931 .offset = mtr_id_offset, 4932 .length = mtr_reg_bits, 4933 .data = flow->meter, 4934 }; 4935 /* 4936 * The color Reg bits used by flow_id are growing from 4937 * msb to lsb, so must do bit reverse for flow_id val in RegC. 4938 */ 4939 for (shift = 0; shift < flow_id_bits; shift++) 4940 flow_id_reversed = (flow_id_reversed << 1) | 4941 ((flow_id >> shift) & 0x1); 4942 set_tag->data |= 4943 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 4944 tag_item_spec->id = set_tag->id; 4945 tag_item_spec->data = set_tag->data << mtr_id_offset; 4946 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 4947 tag_action->type = (enum rte_flow_action_type) 4948 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4949 tag_action->conf = set_tag; 4950 tag_item->type = (enum rte_flow_item_type) 4951 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4952 tag_item->spec = tag_item_spec; 4953 tag_item->last = NULL; 4954 tag_item->mask = tag_item_mask; 4955 exit: 4956 if (mtr_flow_id) 4957 *mtr_flow_id = tag_id; 4958 return 0; 4959 } 4960 4961 /** 4962 * Split action list having QUEUE/RSS for metadata register copy. 4963 * 4964 * Once Q/RSS action is detected in user's action list, the flow action 4965 * should be split in order to copy metadata registers, which will happen in 4966 * RX_CP_TBL like, 4967 * - CQE->flow_tag := reg_c[1] (MARK) 4968 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4969 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4970 * This is because the last action of each flow must be a terminal action 4971 * (QUEUE, RSS or DROP). 4972 * 4973 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4974 * stored and kept in the mlx5_flow structure per each sub_flow. 4975 * 4976 * The Q/RSS action is replaced with, 4977 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4978 * And the following JUMP action is added at the end, 4979 * - JUMP, to RX_CP_TBL. 4980 * 4981 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4982 * flow_create_split_metadata() routine. The flow will look like, 4983 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4984 * 4985 * @param dev 4986 * Pointer to Ethernet device. 4987 * @param[out] split_actions 4988 * Pointer to store split actions to jump to CP_TBL. 4989 * @param[in] actions 4990 * Pointer to the list of original flow actions. 4991 * @param[in] qrss 4992 * Pointer to the Q/RSS action. 4993 * @param[in] actions_n 4994 * Number of original actions. 4995 * @param[out] error 4996 * Perform verbose error reporting if not NULL. 4997 * 4998 * @return 4999 * non-zero unique flow_id on success, otherwise 0 and 5000 * error/rte_error are set. 5001 */ 5002 static uint32_t 5003 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5004 struct rte_flow_action *split_actions, 5005 const struct rte_flow_action *actions, 5006 const struct rte_flow_action *qrss, 5007 int actions_n, struct rte_flow_error *error) 5008 { 5009 struct mlx5_priv *priv = dev->data->dev_private; 5010 struct mlx5_rte_flow_action_set_tag *set_tag; 5011 struct rte_flow_action_jump *jump; 5012 const int qrss_idx = qrss - actions; 5013 uint32_t flow_id = 0; 5014 int ret = 0; 5015 5016 /* 5017 * Given actions will be split 5018 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5019 * - Add jump to mreg CP_TBL. 5020 * As a result, there will be one more action. 5021 */ 5022 ++actions_n; 5023 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5024 set_tag = (void *)(split_actions + actions_n); 5025 /* 5026 * If tag action is not set to void(it means we are not the meter 5027 * suffix flow), add the tag action. Since meter suffix flow already 5028 * has the tag added. 5029 */ 5030 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 5031 /* 5032 * Allocate the new subflow ID. This one is unique within 5033 * device and not shared with representors. Otherwise, 5034 * we would have to resolve multi-thread access synch 5035 * issue. Each flow on the shared device is appended 5036 * with source vport identifier, so the resulting 5037 * flows will be unique in the shared (by master and 5038 * representors) domain even if they have coinciding 5039 * IDs. 5040 */ 5041 mlx5_ipool_malloc(priv->sh->ipool 5042 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5043 if (!flow_id) 5044 return rte_flow_error_set(error, ENOMEM, 5045 RTE_FLOW_ERROR_TYPE_ACTION, 5046 NULL, "can't allocate id " 5047 "for split Q/RSS subflow"); 5048 /* Internal SET_TAG action to set flow ID. */ 5049 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5050 .data = flow_id, 5051 }; 5052 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5053 if (ret < 0) 5054 return ret; 5055 set_tag->id = ret; 5056 /* Construct new actions array. */ 5057 /* Replace QUEUE/RSS action. */ 5058 split_actions[qrss_idx] = (struct rte_flow_action){ 5059 .type = (enum rte_flow_action_type) 5060 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5061 .conf = set_tag, 5062 }; 5063 } 5064 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5065 jump = (void *)(set_tag + 1); 5066 *jump = (struct rte_flow_action_jump){ 5067 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5068 }; 5069 split_actions[actions_n - 2] = (struct rte_flow_action){ 5070 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5071 .conf = jump, 5072 }; 5073 split_actions[actions_n - 1] = (struct rte_flow_action){ 5074 .type = RTE_FLOW_ACTION_TYPE_END, 5075 }; 5076 return flow_id; 5077 } 5078 5079 /** 5080 * Extend the given action list for Tx metadata copy. 5081 * 5082 * Copy the given action list to the ext_actions and add flow metadata register 5083 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5084 * 5085 * @param[out] ext_actions 5086 * Pointer to the extended action list. 5087 * @param[in] actions 5088 * Pointer to the list of actions. 5089 * @param[in] actions_n 5090 * Number of actions in the list. 5091 * @param[out] error 5092 * Perform verbose error reporting if not NULL. 5093 * @param[in] encap_idx 5094 * The encap action inndex. 5095 * 5096 * @return 5097 * 0 on success, negative value otherwise 5098 */ 5099 static int 5100 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5101 struct rte_flow_action *ext_actions, 5102 const struct rte_flow_action *actions, 5103 int actions_n, struct rte_flow_error *error, 5104 int encap_idx) 5105 { 5106 struct mlx5_flow_action_copy_mreg *cp_mreg = 5107 (struct mlx5_flow_action_copy_mreg *) 5108 (ext_actions + actions_n + 1); 5109 int ret; 5110 5111 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5112 if (ret < 0) 5113 return ret; 5114 cp_mreg->dst = ret; 5115 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5116 if (ret < 0) 5117 return ret; 5118 cp_mreg->src = ret; 5119 if (encap_idx != 0) 5120 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5121 if (encap_idx == actions_n - 1) { 5122 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5123 .type = (enum rte_flow_action_type) 5124 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5125 .conf = cp_mreg, 5126 }; 5127 ext_actions[actions_n] = (struct rte_flow_action){ 5128 .type = RTE_FLOW_ACTION_TYPE_END, 5129 }; 5130 } else { 5131 ext_actions[encap_idx] = (struct rte_flow_action){ 5132 .type = (enum rte_flow_action_type) 5133 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5134 .conf = cp_mreg, 5135 }; 5136 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5137 sizeof(*ext_actions) * (actions_n - encap_idx)); 5138 } 5139 return 0; 5140 } 5141 5142 /** 5143 * Check the match action from the action list. 5144 * 5145 * @param[in] actions 5146 * Pointer to the list of actions. 5147 * @param[in] attr 5148 * Flow rule attributes. 5149 * @param[in] action 5150 * The action to be check if exist. 5151 * @param[out] match_action_pos 5152 * Pointer to the position of the matched action if exists, otherwise is -1. 5153 * @param[out] qrss_action_pos 5154 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5155 * @param[out] modify_after_mirror 5156 * Pointer to the flag of modify action after FDB mirroring. 5157 * 5158 * @return 5159 * > 0 the total number of actions. 5160 * 0 if not found match action in action list. 5161 */ 5162 static int 5163 flow_check_match_action(const struct rte_flow_action actions[], 5164 const struct rte_flow_attr *attr, 5165 enum rte_flow_action_type action, 5166 int *match_action_pos, int *qrss_action_pos, 5167 int *modify_after_mirror) 5168 { 5169 const struct rte_flow_action_sample *sample; 5170 int actions_n = 0; 5171 uint32_t ratio = 0; 5172 int sub_type = 0; 5173 int flag = 0; 5174 int fdb_mirror = 0; 5175 5176 *match_action_pos = -1; 5177 *qrss_action_pos = -1; 5178 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5179 if (actions->type == action) { 5180 flag = 1; 5181 *match_action_pos = actions_n; 5182 } 5183 switch (actions->type) { 5184 case RTE_FLOW_ACTION_TYPE_QUEUE: 5185 case RTE_FLOW_ACTION_TYPE_RSS: 5186 *qrss_action_pos = actions_n; 5187 break; 5188 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5189 sample = actions->conf; 5190 ratio = sample->ratio; 5191 sub_type = ((const struct rte_flow_action *) 5192 (sample->actions))->type; 5193 if (ratio == 1 && attr->transfer) 5194 fdb_mirror = 1; 5195 break; 5196 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5197 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5198 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5199 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5200 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5201 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5202 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5203 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5204 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5205 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5206 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5207 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5208 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5209 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5210 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5211 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5212 case RTE_FLOW_ACTION_TYPE_FLAG: 5213 case RTE_FLOW_ACTION_TYPE_MARK: 5214 case RTE_FLOW_ACTION_TYPE_SET_META: 5215 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5216 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5217 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5218 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5219 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5220 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5221 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5222 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5223 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5224 case RTE_FLOW_ACTION_TYPE_METER: 5225 if (fdb_mirror) 5226 *modify_after_mirror = 1; 5227 break; 5228 default: 5229 break; 5230 } 5231 actions_n++; 5232 } 5233 if (flag && fdb_mirror && !*modify_after_mirror) { 5234 /* FDB mirroring uses the destination array to implement 5235 * instead of FLOW_SAMPLER object. 5236 */ 5237 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5238 flag = 0; 5239 } 5240 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5241 return flag ? actions_n + 1 : 0; 5242 } 5243 5244 #define SAMPLE_SUFFIX_ITEM 2 5245 5246 /** 5247 * Split the sample flow. 5248 * 5249 * As sample flow will split to two sub flow, sample flow with 5250 * sample action, the other actions will move to new suffix flow. 5251 * 5252 * Also add unique tag id with tag action in the sample flow, 5253 * the same tag id will be as match in the suffix flow. 5254 * 5255 * @param dev 5256 * Pointer to Ethernet device. 5257 * @param[in] add_tag 5258 * Add extra tag action flag. 5259 * @param[out] sfx_items 5260 * Suffix flow match items (list terminated by the END pattern item). 5261 * @param[in] actions 5262 * Associated actions (list terminated by the END action). 5263 * @param[out] actions_sfx 5264 * Suffix flow actions. 5265 * @param[out] actions_pre 5266 * Prefix flow actions. 5267 * @param[in] actions_n 5268 * The total number of actions. 5269 * @param[in] sample_action_pos 5270 * The sample action position. 5271 * @param[in] qrss_action_pos 5272 * The Queue/RSS action position. 5273 * @param[in] jump_table 5274 * Add extra jump action flag. 5275 * @param[out] error 5276 * Perform verbose error reporting if not NULL. 5277 * 5278 * @return 5279 * 0 on success, or unique flow_id, a negative errno value 5280 * otherwise and rte_errno is set. 5281 */ 5282 static int 5283 flow_sample_split_prep(struct rte_eth_dev *dev, 5284 int add_tag, 5285 struct rte_flow_item sfx_items[], 5286 const struct rte_flow_action actions[], 5287 struct rte_flow_action actions_sfx[], 5288 struct rte_flow_action actions_pre[], 5289 int actions_n, 5290 int sample_action_pos, 5291 int qrss_action_pos, 5292 int jump_table, 5293 struct rte_flow_error *error) 5294 { 5295 struct mlx5_priv *priv = dev->data->dev_private; 5296 struct mlx5_rte_flow_action_set_tag *set_tag; 5297 struct mlx5_rte_flow_item_tag *tag_spec; 5298 struct mlx5_rte_flow_item_tag *tag_mask; 5299 struct rte_flow_action_jump *jump_action; 5300 uint32_t tag_id = 0; 5301 int index; 5302 int append_index = 0; 5303 int ret; 5304 5305 if (sample_action_pos < 0) 5306 return rte_flow_error_set(error, EINVAL, 5307 RTE_FLOW_ERROR_TYPE_ACTION, 5308 NULL, "invalid position of sample " 5309 "action in list"); 5310 /* Prepare the actions for prefix and suffix flow. */ 5311 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5312 index = qrss_action_pos; 5313 /* Put the preceding the Queue/RSS action into prefix flow. */ 5314 if (index != 0) 5315 memcpy(actions_pre, actions, 5316 sizeof(struct rte_flow_action) * index); 5317 /* Put others preceding the sample action into prefix flow. */ 5318 if (sample_action_pos > index + 1) 5319 memcpy(actions_pre + index, actions + index + 1, 5320 sizeof(struct rte_flow_action) * 5321 (sample_action_pos - index - 1)); 5322 index = sample_action_pos - 1; 5323 /* Put Queue/RSS action into Suffix flow. */ 5324 memcpy(actions_sfx, actions + qrss_action_pos, 5325 sizeof(struct rte_flow_action)); 5326 actions_sfx++; 5327 } else { 5328 index = sample_action_pos; 5329 if (index != 0) 5330 memcpy(actions_pre, actions, 5331 sizeof(struct rte_flow_action) * index); 5332 } 5333 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5334 * For CX6DX and above, metadata registers Cx preserve their value, 5335 * add an extra tag action for NIC-RX and E-Switch Domain. 5336 */ 5337 if (add_tag) { 5338 /* Prepare the prefix tag action. */ 5339 append_index++; 5340 set_tag = (void *)(actions_pre + actions_n + append_index); 5341 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5342 if (ret < 0) 5343 return ret; 5344 mlx5_ipool_malloc(priv->sh->ipool 5345 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5346 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5347 .id = ret, 5348 .data = tag_id, 5349 }; 5350 /* Prepare the suffix subflow items. */ 5351 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5352 tag_spec->data = tag_id; 5353 tag_spec->id = set_tag->id; 5354 tag_mask = tag_spec + 1; 5355 tag_mask->data = UINT32_MAX; 5356 sfx_items[0] = (struct rte_flow_item){ 5357 .type = (enum rte_flow_item_type) 5358 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5359 .spec = tag_spec, 5360 .last = NULL, 5361 .mask = tag_mask, 5362 }; 5363 sfx_items[1] = (struct rte_flow_item){ 5364 .type = (enum rte_flow_item_type) 5365 RTE_FLOW_ITEM_TYPE_END, 5366 }; 5367 /* Prepare the tag action in prefix subflow. */ 5368 actions_pre[index++] = 5369 (struct rte_flow_action){ 5370 .type = (enum rte_flow_action_type) 5371 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5372 .conf = set_tag, 5373 }; 5374 } 5375 memcpy(actions_pre + index, actions + sample_action_pos, 5376 sizeof(struct rte_flow_action)); 5377 index += 1; 5378 /* For the modify action after the sample action in E-Switch mirroring, 5379 * Add the extra jump action in prefix subflow and jump into the next 5380 * table, then do the modify action in the new table. 5381 */ 5382 if (jump_table) { 5383 /* Prepare the prefix jump action. */ 5384 append_index++; 5385 jump_action = (void *)(actions_pre + actions_n + append_index); 5386 jump_action->group = jump_table; 5387 actions_pre[index++] = 5388 (struct rte_flow_action){ 5389 .type = (enum rte_flow_action_type) 5390 RTE_FLOW_ACTION_TYPE_JUMP, 5391 .conf = jump_action, 5392 }; 5393 } 5394 actions_pre[index] = (struct rte_flow_action){ 5395 .type = (enum rte_flow_action_type) 5396 RTE_FLOW_ACTION_TYPE_END, 5397 }; 5398 /* Put the actions after sample into Suffix flow. */ 5399 memcpy(actions_sfx, actions + sample_action_pos + 1, 5400 sizeof(struct rte_flow_action) * 5401 (actions_n - sample_action_pos - 1)); 5402 return tag_id; 5403 } 5404 5405 /** 5406 * The splitting for metadata feature. 5407 * 5408 * - Q/RSS action on NIC Rx should be split in order to pass by 5409 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5410 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5411 * 5412 * - All the actions on NIC Tx should have a mreg copy action to 5413 * copy reg_a from WQE to reg_c[0]. 5414 * 5415 * @param dev 5416 * Pointer to Ethernet device. 5417 * @param[in] flow 5418 * Parent flow structure pointer. 5419 * @param[in] attr 5420 * Flow rule attributes. 5421 * @param[in] items 5422 * Pattern specification (list terminated by the END pattern item). 5423 * @param[in] actions 5424 * Associated actions (list terminated by the END action). 5425 * @param[in] flow_split_info 5426 * Pointer to flow split info structure. 5427 * @param[out] error 5428 * Perform verbose error reporting if not NULL. 5429 * @return 5430 * 0 on success, negative value otherwise 5431 */ 5432 static int 5433 flow_create_split_metadata(struct rte_eth_dev *dev, 5434 struct rte_flow *flow, 5435 const struct rte_flow_attr *attr, 5436 const struct rte_flow_item items[], 5437 const struct rte_flow_action actions[], 5438 struct mlx5_flow_split_info *flow_split_info, 5439 struct rte_flow_error *error) 5440 { 5441 struct mlx5_priv *priv = dev->data->dev_private; 5442 struct mlx5_dev_config *config = &priv->config; 5443 const struct rte_flow_action *qrss = NULL; 5444 struct rte_flow_action *ext_actions = NULL; 5445 struct mlx5_flow *dev_flow = NULL; 5446 uint32_t qrss_id = 0; 5447 int mtr_sfx = 0; 5448 size_t act_size; 5449 int actions_n; 5450 int encap_idx; 5451 int ret; 5452 5453 /* Check whether extensive metadata feature is engaged. */ 5454 if (!config->dv_flow_en || 5455 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5456 !mlx5_flow_ext_mreg_supported(dev)) 5457 return flow_create_split_inner(dev, flow, NULL, attr, items, 5458 actions, flow_split_info, error); 5459 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5460 &encap_idx); 5461 if (qrss) { 5462 /* Exclude hairpin flows from splitting. */ 5463 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5464 const struct rte_flow_action_queue *queue; 5465 5466 queue = qrss->conf; 5467 if (mlx5_rxq_get_type(dev, queue->index) == 5468 MLX5_RXQ_TYPE_HAIRPIN) 5469 qrss = NULL; 5470 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5471 const struct rte_flow_action_rss *rss; 5472 5473 rss = qrss->conf; 5474 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5475 MLX5_RXQ_TYPE_HAIRPIN) 5476 qrss = NULL; 5477 } 5478 } 5479 if (qrss) { 5480 /* Check if it is in meter suffix table. */ 5481 mtr_sfx = attr->group == (attr->transfer ? 5482 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5483 MLX5_FLOW_TABLE_LEVEL_METER); 5484 /* 5485 * Q/RSS action on NIC Rx should be split in order to pass by 5486 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5487 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5488 */ 5489 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5490 sizeof(struct rte_flow_action_set_tag) + 5491 sizeof(struct rte_flow_action_jump); 5492 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5493 SOCKET_ID_ANY); 5494 if (!ext_actions) 5495 return rte_flow_error_set(error, ENOMEM, 5496 RTE_FLOW_ERROR_TYPE_ACTION, 5497 NULL, "no memory to split " 5498 "metadata flow"); 5499 /* 5500 * If we are the suffix flow of meter, tag already exist. 5501 * Set the tag action to void. 5502 */ 5503 if (mtr_sfx) 5504 ext_actions[qrss - actions].type = 5505 RTE_FLOW_ACTION_TYPE_VOID; 5506 else 5507 ext_actions[qrss - actions].type = 5508 (enum rte_flow_action_type) 5509 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5510 /* 5511 * Create the new actions list with removed Q/RSS action 5512 * and appended set tag and jump to register copy table 5513 * (RX_CP_TBL). We should preallocate unique tag ID here 5514 * in advance, because it is needed for set tag action. 5515 */ 5516 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5517 qrss, actions_n, error); 5518 if (!mtr_sfx && !qrss_id) { 5519 ret = -rte_errno; 5520 goto exit; 5521 } 5522 } else if (attr->egress && !attr->transfer) { 5523 /* 5524 * All the actions on NIC Tx should have a metadata register 5525 * copy action to copy reg_a from WQE to reg_c[meta] 5526 */ 5527 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5528 sizeof(struct mlx5_flow_action_copy_mreg); 5529 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5530 SOCKET_ID_ANY); 5531 if (!ext_actions) 5532 return rte_flow_error_set(error, ENOMEM, 5533 RTE_FLOW_ERROR_TYPE_ACTION, 5534 NULL, "no memory to split " 5535 "metadata flow"); 5536 /* Create the action list appended with copy register. */ 5537 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5538 actions_n, error, encap_idx); 5539 if (ret < 0) 5540 goto exit; 5541 } 5542 /* Add the unmodified original or prefix subflow. */ 5543 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5544 items, ext_actions ? ext_actions : 5545 actions, flow_split_info, error); 5546 if (ret < 0) 5547 goto exit; 5548 MLX5_ASSERT(dev_flow); 5549 if (qrss) { 5550 const struct rte_flow_attr q_attr = { 5551 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5552 .ingress = 1, 5553 }; 5554 /* Internal PMD action to set register. */ 5555 struct mlx5_rte_flow_item_tag q_tag_spec = { 5556 .data = qrss_id, 5557 .id = REG_NON, 5558 }; 5559 struct rte_flow_item q_items[] = { 5560 { 5561 .type = (enum rte_flow_item_type) 5562 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5563 .spec = &q_tag_spec, 5564 .last = NULL, 5565 .mask = NULL, 5566 }, 5567 { 5568 .type = RTE_FLOW_ITEM_TYPE_END, 5569 }, 5570 }; 5571 struct rte_flow_action q_actions[] = { 5572 { 5573 .type = qrss->type, 5574 .conf = qrss->conf, 5575 }, 5576 { 5577 .type = RTE_FLOW_ACTION_TYPE_END, 5578 }, 5579 }; 5580 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5581 5582 /* 5583 * Configure the tag item only if there is no meter subflow. 5584 * Since tag is already marked in the meter suffix subflow 5585 * we can just use the meter suffix items as is. 5586 */ 5587 if (qrss_id) { 5588 /* Not meter subflow. */ 5589 MLX5_ASSERT(!mtr_sfx); 5590 /* 5591 * Put unique id in prefix flow due to it is destroyed 5592 * after suffix flow and id will be freed after there 5593 * is no actual flows with this id and identifier 5594 * reallocation becomes possible (for example, for 5595 * other flows in other threads). 5596 */ 5597 dev_flow->handle->split_flow_id = qrss_id; 5598 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5599 error); 5600 if (ret < 0) 5601 goto exit; 5602 q_tag_spec.id = ret; 5603 } 5604 dev_flow = NULL; 5605 /* Add suffix subflow to execute Q/RSS. */ 5606 flow_split_info->prefix_layers = layers; 5607 flow_split_info->prefix_mark = 0; 5608 ret = flow_create_split_inner(dev, flow, &dev_flow, 5609 &q_attr, mtr_sfx ? items : 5610 q_items, q_actions, 5611 flow_split_info, error); 5612 if (ret < 0) 5613 goto exit; 5614 /* qrss ID should be freed if failed. */ 5615 qrss_id = 0; 5616 MLX5_ASSERT(dev_flow); 5617 } 5618 5619 exit: 5620 /* 5621 * We do not destroy the partially created sub_flows in case of error. 5622 * These ones are included into parent flow list and will be destroyed 5623 * by flow_drv_destroy. 5624 */ 5625 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5626 qrss_id); 5627 mlx5_free(ext_actions); 5628 return ret; 5629 } 5630 5631 /** 5632 * Create meter internal drop flow with the original pattern. 5633 * 5634 * @param dev 5635 * Pointer to Ethernet device. 5636 * @param[in] flow 5637 * Parent flow structure pointer. 5638 * @param[in] attr 5639 * Flow rule attributes. 5640 * @param[in] items 5641 * Pattern specification (list terminated by the END pattern item). 5642 * @param[in] flow_split_info 5643 * Pointer to flow split info structure. 5644 * @param[in] fm 5645 * Pointer to flow meter structure. 5646 * @param[out] error 5647 * Perform verbose error reporting if not NULL. 5648 * @return 5649 * 0 on success, negative value otherwise 5650 */ 5651 static uint32_t 5652 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5653 struct rte_flow *flow, 5654 const struct rte_flow_attr *attr, 5655 const struct rte_flow_item items[], 5656 struct mlx5_flow_split_info *flow_split_info, 5657 struct mlx5_flow_meter_info *fm, 5658 struct rte_flow_error *error) 5659 { 5660 struct mlx5_flow *dev_flow = NULL; 5661 struct rte_flow_attr drop_attr = *attr; 5662 struct rte_flow_action drop_actions[3]; 5663 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5664 5665 MLX5_ASSERT(fm->drop_cnt); 5666 drop_actions[0].type = 5667 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5668 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5669 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5670 drop_actions[1].conf = NULL; 5671 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5672 drop_actions[2].conf = NULL; 5673 drop_split_info.external = false; 5674 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5675 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5676 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5677 return flow_create_split_inner(dev, flow, &dev_flow, 5678 &drop_attr, items, drop_actions, 5679 &drop_split_info, error); 5680 } 5681 5682 /** 5683 * The splitting for meter feature. 5684 * 5685 * - The meter flow will be split to two flows as prefix and 5686 * suffix flow. The packets make sense only it pass the prefix 5687 * meter action. 5688 * 5689 * - Reg_C_5 is used for the packet to match betweend prefix and 5690 * suffix flow. 5691 * 5692 * @param dev 5693 * Pointer to Ethernet device. 5694 * @param[in] flow 5695 * Parent flow structure pointer. 5696 * @param[in] attr 5697 * Flow rule attributes. 5698 * @param[in] items 5699 * Pattern specification (list terminated by the END pattern item). 5700 * @param[in] actions 5701 * Associated actions (list terminated by the END action). 5702 * @param[in] flow_split_info 5703 * Pointer to flow split info structure. 5704 * @param[out] error 5705 * Perform verbose error reporting if not NULL. 5706 * @return 5707 * 0 on success, negative value otherwise 5708 */ 5709 static int 5710 flow_create_split_meter(struct rte_eth_dev *dev, 5711 struct rte_flow *flow, 5712 const struct rte_flow_attr *attr, 5713 const struct rte_flow_item items[], 5714 const struct rte_flow_action actions[], 5715 struct mlx5_flow_split_info *flow_split_info, 5716 struct rte_flow_error *error) 5717 { 5718 struct mlx5_priv *priv = dev->data->dev_private; 5719 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5720 struct rte_flow_action *sfx_actions = NULL; 5721 struct rte_flow_action *pre_actions = NULL; 5722 struct rte_flow_item *sfx_items = NULL; 5723 struct mlx5_flow *dev_flow = NULL; 5724 struct rte_flow_attr sfx_attr = *attr; 5725 struct mlx5_flow_meter_info *fm = NULL; 5726 uint8_t skip_scale_restore; 5727 bool has_mtr = false; 5728 bool has_modify = false; 5729 bool set_mtr_reg = true; 5730 uint32_t meter_id = 0; 5731 uint32_t mtr_idx = 0; 5732 uint32_t mtr_flow_id = 0; 5733 size_t act_size; 5734 size_t item_size; 5735 int actions_n = 0; 5736 int ret = 0; 5737 5738 if (priv->mtr_en) 5739 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 5740 &has_modify, &meter_id); 5741 if (has_mtr) { 5742 if (flow->meter) { 5743 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 5744 if (!fm) 5745 return rte_flow_error_set(error, EINVAL, 5746 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5747 NULL, "Meter not found."); 5748 } else { 5749 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 5750 if (!fm) 5751 return rte_flow_error_set(error, EINVAL, 5752 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5753 NULL, "Meter not found."); 5754 ret = mlx5_flow_meter_attach(priv, fm, 5755 &sfx_attr, error); 5756 if (ret) 5757 return -rte_errno; 5758 flow->meter = mtr_idx; 5759 } 5760 MLX5_ASSERT(wks); 5761 wks->fm = fm; 5762 if (!fm->def_policy) { 5763 wks->policy = mlx5_flow_meter_policy_find(dev, 5764 fm->policy_id, 5765 NULL); 5766 MLX5_ASSERT(wks->policy); 5767 if (wks->policy->is_hierarchy) { 5768 wks->final_policy = 5769 mlx5_flow_meter_hierarchy_get_final_policy(dev, 5770 wks->policy); 5771 if (!wks->final_policy) 5772 return rte_flow_error_set(error, 5773 EINVAL, 5774 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5775 "Failed to find terminal policy of hierarchy."); 5776 } 5777 } 5778 /* 5779 * If it isn't default-policy Meter, and 5780 * 1. There's no action in flow to change 5781 * packet (modify/encap/decap etc.), OR 5782 * 2. No drop count needed for this meter. 5783 * no need to use regC to save meter id anymore. 5784 */ 5785 if (!fm->def_policy && (!has_modify || !fm->drop_cnt)) 5786 set_mtr_reg = false; 5787 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 5788 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 5789 sizeof(struct mlx5_rte_flow_action_set_tag); 5790 /* Suffix items: tag, vlan, port id, end. */ 5791 #define METER_SUFFIX_ITEM 4 5792 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5793 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5794 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5795 0, SOCKET_ID_ANY); 5796 if (!sfx_actions) 5797 return rte_flow_error_set(error, ENOMEM, 5798 RTE_FLOW_ERROR_TYPE_ACTION, 5799 NULL, "no memory to split " 5800 "meter flow"); 5801 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5802 act_size); 5803 /* There's no suffix flow for meter of non-default policy. */ 5804 if (!fm->def_policy) 5805 pre_actions = sfx_actions + 1; 5806 else 5807 pre_actions = sfx_actions + actions_n; 5808 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 5809 items, sfx_items, actions, 5810 sfx_actions, pre_actions, 5811 (set_mtr_reg ? &mtr_flow_id : NULL), 5812 error); 5813 if (ret) { 5814 ret = -rte_errno; 5815 goto exit; 5816 } 5817 /* Add the prefix subflow. */ 5818 flow_split_info->prefix_mark = 0; 5819 skip_scale_restore = flow_split_info->skip_scale; 5820 flow_split_info->skip_scale |= 5821 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5822 ret = flow_create_split_inner(dev, flow, &dev_flow, 5823 attr, items, pre_actions, 5824 flow_split_info, error); 5825 flow_split_info->skip_scale = skip_scale_restore; 5826 if (ret) { 5827 if (mtr_flow_id) 5828 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 5829 ret = -rte_errno; 5830 goto exit; 5831 } 5832 if (mtr_flow_id) { 5833 dev_flow->handle->split_flow_id = mtr_flow_id; 5834 dev_flow->handle->is_meter_flow_id = 1; 5835 } 5836 if (!fm->def_policy) { 5837 if (!set_mtr_reg && fm->drop_cnt) 5838 ret = 5839 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 5840 &sfx_attr, items, 5841 flow_split_info, 5842 fm, error); 5843 goto exit; 5844 } 5845 /* Setting the sfx group atrr. */ 5846 sfx_attr.group = sfx_attr.transfer ? 5847 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5848 MLX5_FLOW_TABLE_LEVEL_METER; 5849 flow_split_info->prefix_layers = 5850 flow_get_prefix_layer_flags(dev_flow); 5851 flow_split_info->prefix_mark = dev_flow->handle->mark; 5852 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 5853 } 5854 /* Add the prefix subflow. */ 5855 ret = flow_create_split_metadata(dev, flow, 5856 &sfx_attr, sfx_items ? 5857 sfx_items : items, 5858 sfx_actions ? sfx_actions : actions, 5859 flow_split_info, error); 5860 exit: 5861 if (sfx_actions) 5862 mlx5_free(sfx_actions); 5863 return ret; 5864 } 5865 5866 /** 5867 * The splitting for sample feature. 5868 * 5869 * Once Sample action is detected in the action list, the flow actions should 5870 * be split into prefix sub flow and suffix sub flow. 5871 * 5872 * The original items remain in the prefix sub flow, all actions preceding the 5873 * sample action and the sample action itself will be copied to the prefix 5874 * sub flow, the actions following the sample action will be copied to the 5875 * suffix sub flow, Queue action always be located in the suffix sub flow. 5876 * 5877 * In order to make the packet from prefix sub flow matches with suffix sub 5878 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5879 * flow uses tag item with the unique flow id. 5880 * 5881 * @param dev 5882 * Pointer to Ethernet device. 5883 * @param[in] flow 5884 * Parent flow structure pointer. 5885 * @param[in] attr 5886 * Flow rule attributes. 5887 * @param[in] items 5888 * Pattern specification (list terminated by the END pattern item). 5889 * @param[in] actions 5890 * Associated actions (list terminated by the END action). 5891 * @param[in] flow_split_info 5892 * Pointer to flow split info structure. 5893 * @param[out] error 5894 * Perform verbose error reporting if not NULL. 5895 * @return 5896 * 0 on success, negative value otherwise 5897 */ 5898 static int 5899 flow_create_split_sample(struct rte_eth_dev *dev, 5900 struct rte_flow *flow, 5901 const struct rte_flow_attr *attr, 5902 const struct rte_flow_item items[], 5903 const struct rte_flow_action actions[], 5904 struct mlx5_flow_split_info *flow_split_info, 5905 struct rte_flow_error *error) 5906 { 5907 struct mlx5_priv *priv = dev->data->dev_private; 5908 struct rte_flow_action *sfx_actions = NULL; 5909 struct rte_flow_action *pre_actions = NULL; 5910 struct rte_flow_item *sfx_items = NULL; 5911 struct mlx5_flow *dev_flow = NULL; 5912 struct rte_flow_attr sfx_attr = *attr; 5913 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5914 struct mlx5_flow_dv_sample_resource *sample_res; 5915 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5916 struct mlx5_flow_tbl_resource *sfx_tbl; 5917 #endif 5918 size_t act_size; 5919 size_t item_size; 5920 uint32_t fdb_tx = 0; 5921 int32_t tag_id = 0; 5922 int actions_n = 0; 5923 int sample_action_pos; 5924 int qrss_action_pos; 5925 int add_tag = 0; 5926 int modify_after_mirror = 0; 5927 uint16_t jump_table = 0; 5928 const uint32_t next_ft_step = 1; 5929 int ret = 0; 5930 5931 if (priv->sampler_en) 5932 actions_n = flow_check_match_action(actions, attr, 5933 RTE_FLOW_ACTION_TYPE_SAMPLE, 5934 &sample_action_pos, &qrss_action_pos, 5935 &modify_after_mirror); 5936 if (actions_n) { 5937 /* The prefix actions must includes sample, tag, end. */ 5938 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5939 + sizeof(struct mlx5_rte_flow_action_set_tag); 5940 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5941 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5942 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5943 item_size), 0, SOCKET_ID_ANY); 5944 if (!sfx_actions) 5945 return rte_flow_error_set(error, ENOMEM, 5946 RTE_FLOW_ERROR_TYPE_ACTION, 5947 NULL, "no memory to split " 5948 "sample flow"); 5949 /* The representor_id is UINT16_MAX for uplink. */ 5950 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 5951 /* 5952 * When reg_c_preserve is set, metadata registers Cx preserve 5953 * their value even through packet duplication. 5954 */ 5955 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 5956 if (add_tag) 5957 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5958 + act_size); 5959 if (modify_after_mirror) 5960 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 5961 next_ft_step; 5962 pre_actions = sfx_actions + actions_n; 5963 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 5964 actions, sfx_actions, 5965 pre_actions, actions_n, 5966 sample_action_pos, 5967 qrss_action_pos, jump_table, 5968 error); 5969 if (tag_id < 0 || (add_tag && !tag_id)) { 5970 ret = -rte_errno; 5971 goto exit; 5972 } 5973 if (modify_after_mirror) 5974 flow_split_info->skip_scale = 5975 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5976 /* Add the prefix subflow. */ 5977 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5978 items, pre_actions, 5979 flow_split_info, error); 5980 if (ret) { 5981 ret = -rte_errno; 5982 goto exit; 5983 } 5984 dev_flow->handle->split_flow_id = tag_id; 5985 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5986 if (!modify_after_mirror) { 5987 /* Set the sfx group attr. */ 5988 sample_res = (struct mlx5_flow_dv_sample_resource *) 5989 dev_flow->dv.sample_res; 5990 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5991 sample_res->normal_path_tbl; 5992 sfx_tbl_data = container_of(sfx_tbl, 5993 struct mlx5_flow_tbl_data_entry, 5994 tbl); 5995 sfx_attr.group = sfx_attr.transfer ? 5996 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 5997 } else { 5998 MLX5_ASSERT(attr->transfer); 5999 sfx_attr.group = jump_table; 6000 } 6001 flow_split_info->prefix_layers = 6002 flow_get_prefix_layer_flags(dev_flow); 6003 flow_split_info->prefix_mark = dev_flow->handle->mark; 6004 /* Suffix group level already be scaled with factor, set 6005 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6006 * again in translation. 6007 */ 6008 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6009 #endif 6010 } 6011 /* Add the suffix subflow. */ 6012 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6013 sfx_items ? sfx_items : items, 6014 sfx_actions ? sfx_actions : actions, 6015 flow_split_info, error); 6016 exit: 6017 if (sfx_actions) 6018 mlx5_free(sfx_actions); 6019 return ret; 6020 } 6021 6022 /** 6023 * Split the flow to subflow set. The splitters might be linked 6024 * in the chain, like this: 6025 * flow_create_split_outer() calls: 6026 * flow_create_split_meter() calls: 6027 * flow_create_split_metadata(meter_subflow_0) calls: 6028 * flow_create_split_inner(metadata_subflow_0) 6029 * flow_create_split_inner(metadata_subflow_1) 6030 * flow_create_split_inner(metadata_subflow_2) 6031 * flow_create_split_metadata(meter_subflow_1) calls: 6032 * flow_create_split_inner(metadata_subflow_0) 6033 * flow_create_split_inner(metadata_subflow_1) 6034 * flow_create_split_inner(metadata_subflow_2) 6035 * 6036 * This provide flexible way to add new levels of flow splitting. 6037 * The all of successfully created subflows are included to the 6038 * parent flow dev_flow list. 6039 * 6040 * @param dev 6041 * Pointer to Ethernet device. 6042 * @param[in] flow 6043 * Parent flow structure pointer. 6044 * @param[in] attr 6045 * Flow rule attributes. 6046 * @param[in] items 6047 * Pattern specification (list terminated by the END pattern item). 6048 * @param[in] actions 6049 * Associated actions (list terminated by the END action). 6050 * @param[in] flow_split_info 6051 * Pointer to flow split info structure. 6052 * @param[out] error 6053 * Perform verbose error reporting if not NULL. 6054 * @return 6055 * 0 on success, negative value otherwise 6056 */ 6057 static int 6058 flow_create_split_outer(struct rte_eth_dev *dev, 6059 struct rte_flow *flow, 6060 const struct rte_flow_attr *attr, 6061 const struct rte_flow_item items[], 6062 const struct rte_flow_action actions[], 6063 struct mlx5_flow_split_info *flow_split_info, 6064 struct rte_flow_error *error) 6065 { 6066 int ret; 6067 6068 ret = flow_create_split_sample(dev, flow, attr, items, 6069 actions, flow_split_info, error); 6070 MLX5_ASSERT(ret <= 0); 6071 return ret; 6072 } 6073 6074 static inline struct mlx5_flow_tunnel * 6075 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6076 { 6077 struct mlx5_flow_tunnel *tunnel; 6078 6079 #pragma GCC diagnostic push 6080 #pragma GCC diagnostic ignored "-Wcast-qual" 6081 tunnel = (typeof(tunnel))flow->tunnel; 6082 #pragma GCC diagnostic pop 6083 6084 return tunnel; 6085 } 6086 6087 /** 6088 * Adjust flow RSS workspace if needed. 6089 * 6090 * @param wks 6091 * Pointer to thread flow work space. 6092 * @param rss_desc 6093 * Pointer to RSS descriptor. 6094 * @param[in] nrssq_num 6095 * New RSS queue number. 6096 * 6097 * @return 6098 * 0 on success, -1 otherwise and rte_errno is set. 6099 */ 6100 static int 6101 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6102 struct mlx5_flow_rss_desc *rss_desc, 6103 uint32_t nrssq_num) 6104 { 6105 if (likely(nrssq_num <= wks->rssq_num)) 6106 return 0; 6107 rss_desc->queue = realloc(rss_desc->queue, 6108 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6109 if (!rss_desc->queue) { 6110 rte_errno = ENOMEM; 6111 return -1; 6112 } 6113 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6114 return 0; 6115 } 6116 6117 /** 6118 * Create a flow and add it to @p list. 6119 * 6120 * @param dev 6121 * Pointer to Ethernet device. 6122 * @param list 6123 * Pointer to a TAILQ flow list. If this parameter NULL, 6124 * no list insertion occurred, flow is just created, 6125 * this is caller's responsibility to track the 6126 * created flow. 6127 * @param[in] attr 6128 * Flow rule attributes. 6129 * @param[in] items 6130 * Pattern specification (list terminated by the END pattern item). 6131 * @param[in] actions 6132 * Associated actions (list terminated by the END action). 6133 * @param[in] external 6134 * This flow rule is created by request external to PMD. 6135 * @param[out] error 6136 * Perform verbose error reporting if not NULL. 6137 * 6138 * @return 6139 * A flow index on success, 0 otherwise and rte_errno is set. 6140 */ 6141 static uint32_t 6142 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 6143 const struct rte_flow_attr *attr, 6144 const struct rte_flow_item items[], 6145 const struct rte_flow_action original_actions[], 6146 bool external, struct rte_flow_error *error) 6147 { 6148 struct mlx5_priv *priv = dev->data->dev_private; 6149 struct rte_flow *flow = NULL; 6150 struct mlx5_flow *dev_flow; 6151 const struct rte_flow_action_rss *rss = NULL; 6152 struct mlx5_translated_action_handle 6153 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6154 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6155 union { 6156 struct mlx5_flow_expand_rss buf; 6157 uint8_t buffer[2048]; 6158 } expand_buffer; 6159 union { 6160 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6161 uint8_t buffer[2048]; 6162 } actions_rx; 6163 union { 6164 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6165 uint8_t buffer[2048]; 6166 } actions_hairpin_tx; 6167 union { 6168 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6169 uint8_t buffer[2048]; 6170 } items_tx; 6171 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6172 struct mlx5_flow_rss_desc *rss_desc; 6173 const struct rte_flow_action *p_actions_rx; 6174 uint32_t i; 6175 uint32_t idx = 0; 6176 int hairpin_flow; 6177 struct rte_flow_attr attr_tx = { .priority = 0 }; 6178 const struct rte_flow_action *actions; 6179 struct rte_flow_action *translated_actions = NULL; 6180 struct mlx5_flow_tunnel *tunnel; 6181 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6182 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6183 struct mlx5_flow_split_info flow_split_info = { 6184 .external = !!external, 6185 .skip_scale = 0, 6186 .flow_idx = 0, 6187 .prefix_mark = 0, 6188 .prefix_layers = 0, 6189 .table_id = 0 6190 }; 6191 int ret; 6192 6193 MLX5_ASSERT(wks); 6194 rss_desc = &wks->rss_desc; 6195 ret = flow_action_handles_translate(dev, original_actions, 6196 indir_actions, 6197 &indir_actions_n, 6198 &translated_actions, error); 6199 if (ret < 0) { 6200 MLX5_ASSERT(translated_actions == NULL); 6201 return 0; 6202 } 6203 actions = translated_actions ? translated_actions : original_actions; 6204 p_actions_rx = actions; 6205 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6206 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6207 external, hairpin_flow, error); 6208 if (ret < 0) 6209 goto error_before_hairpin_split; 6210 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 6211 if (!flow) { 6212 rte_errno = ENOMEM; 6213 goto error_before_hairpin_split; 6214 } 6215 if (hairpin_flow > 0) { 6216 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6217 rte_errno = EINVAL; 6218 goto error_before_hairpin_split; 6219 } 6220 flow_hairpin_split(dev, actions, actions_rx.actions, 6221 actions_hairpin_tx.actions, items_tx.items, 6222 idx); 6223 p_actions_rx = actions_rx.actions; 6224 } 6225 flow_split_info.flow_idx = idx; 6226 flow->drv_type = flow_get_drv_type(dev, attr); 6227 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6228 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6229 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6230 /* RSS Action only works on NIC RX domain */ 6231 if (attr->ingress && !attr->transfer) 6232 rss = flow_get_rss_action(dev, p_actions_rx); 6233 if (rss) { 6234 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6235 return 0; 6236 /* 6237 * The following information is required by 6238 * mlx5_flow_hashfields_adjust() in advance. 6239 */ 6240 rss_desc->level = rss->level; 6241 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 6242 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 6243 } 6244 flow->dev_handles = 0; 6245 if (rss && rss->types) { 6246 unsigned int graph_root; 6247 6248 graph_root = find_graph_root(items, rss->level); 6249 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6250 items, rss->types, 6251 mlx5_support_expansion, graph_root); 6252 MLX5_ASSERT(ret > 0 && 6253 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6254 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6255 for (i = 0; i < buf->entries; ++i) 6256 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6257 } 6258 } else { 6259 buf->entries = 1; 6260 buf->entry[0].pattern = (void *)(uintptr_t)items; 6261 } 6262 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6263 indir_actions_n); 6264 for (i = 0; i < buf->entries; ++i) { 6265 /* Initialize flow split data. */ 6266 flow_split_info.prefix_layers = 0; 6267 flow_split_info.prefix_mark = 0; 6268 flow_split_info.skip_scale = 0; 6269 /* 6270 * The splitter may create multiple dev_flows, 6271 * depending on configuration. In the simplest 6272 * case it just creates unmodified original flow. 6273 */ 6274 ret = flow_create_split_outer(dev, flow, attr, 6275 buf->entry[i].pattern, 6276 p_actions_rx, &flow_split_info, 6277 error); 6278 if (ret < 0) 6279 goto error; 6280 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6281 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6282 p_actions_rx, 6283 idx, 6284 wks->flows[0].tunnel, 6285 &default_miss_ctx, 6286 error); 6287 if (ret < 0) { 6288 mlx5_free(default_miss_ctx.queue); 6289 goto error; 6290 } 6291 } 6292 } 6293 /* Create the tx flow. */ 6294 if (hairpin_flow) { 6295 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6296 attr_tx.ingress = 0; 6297 attr_tx.egress = 1; 6298 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6299 actions_hairpin_tx.actions, 6300 idx, error); 6301 if (!dev_flow) 6302 goto error; 6303 dev_flow->flow = flow; 6304 dev_flow->external = 0; 6305 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6306 dev_flow->handle, next); 6307 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6308 items_tx.items, 6309 actions_hairpin_tx.actions, error); 6310 if (ret < 0) 6311 goto error; 6312 } 6313 /* 6314 * Update the metadata register copy table. If extensive 6315 * metadata feature is enabled and registers are supported 6316 * we might create the extra rte_flow for each unique 6317 * MARK/FLAG action ID. 6318 * 6319 * The table is updated for ingress Flows only, because 6320 * the egress Flows belong to the different device and 6321 * copy table should be updated in peer NIC Rx domain. 6322 */ 6323 if (attr->ingress && 6324 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6325 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6326 if (ret) 6327 goto error; 6328 } 6329 /* 6330 * If the flow is external (from application) OR device is started, 6331 * OR mreg discover, then apply immediately. 6332 */ 6333 if (external || dev->data->dev_started || 6334 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6335 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6336 ret = flow_drv_apply(dev, flow, error); 6337 if (ret < 0) 6338 goto error; 6339 } 6340 if (list) { 6341 rte_spinlock_lock(&priv->flow_list_lock); 6342 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 6343 flow, next); 6344 rte_spinlock_unlock(&priv->flow_list_lock); 6345 } 6346 flow_rxq_flags_set(dev, flow); 6347 rte_free(translated_actions); 6348 tunnel = flow_tunnel_from_rule(wks->flows); 6349 if (tunnel) { 6350 flow->tunnel = 1; 6351 flow->tunnel_id = tunnel->tunnel_id; 6352 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6353 mlx5_free(default_miss_ctx.queue); 6354 } 6355 mlx5_flow_pop_thread_workspace(); 6356 return idx; 6357 error: 6358 MLX5_ASSERT(flow); 6359 ret = rte_errno; /* Save rte_errno before cleanup. */ 6360 flow_mreg_del_copy_action(dev, flow); 6361 flow_drv_destroy(dev, flow); 6362 if (rss_desc->shared_rss) 6363 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6364 mlx5_ipool_get 6365 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6366 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6367 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 6368 rte_errno = ret; /* Restore rte_errno. */ 6369 ret = rte_errno; 6370 rte_errno = ret; 6371 mlx5_flow_pop_thread_workspace(); 6372 error_before_hairpin_split: 6373 rte_free(translated_actions); 6374 return 0; 6375 } 6376 6377 /** 6378 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6379 * incoming packets to table 1. 6380 * 6381 * Other flow rules, requested for group n, will be created in 6382 * e-switch table n+1. 6383 * Jump action to e-switch group n will be created to group n+1. 6384 * 6385 * Used when working in switchdev mode, to utilise advantages of table 1 6386 * and above. 6387 * 6388 * @param dev 6389 * Pointer to Ethernet device. 6390 * 6391 * @return 6392 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6393 */ 6394 struct rte_flow * 6395 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6396 { 6397 const struct rte_flow_attr attr = { 6398 .group = 0, 6399 .priority = 0, 6400 .ingress = 1, 6401 .egress = 0, 6402 .transfer = 1, 6403 }; 6404 const struct rte_flow_item pattern = { 6405 .type = RTE_FLOW_ITEM_TYPE_END, 6406 }; 6407 struct rte_flow_action_jump jump = { 6408 .group = 1, 6409 }; 6410 const struct rte_flow_action actions[] = { 6411 { 6412 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6413 .conf = &jump, 6414 }, 6415 { 6416 .type = RTE_FLOW_ACTION_TYPE_END, 6417 }, 6418 }; 6419 struct mlx5_priv *priv = dev->data->dev_private; 6420 struct rte_flow_error error; 6421 6422 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 6423 &attr, &pattern, 6424 actions, false, &error); 6425 } 6426 6427 /** 6428 * Validate a flow supported by the NIC. 6429 * 6430 * @see rte_flow_validate() 6431 * @see rte_flow_ops 6432 */ 6433 int 6434 mlx5_flow_validate(struct rte_eth_dev *dev, 6435 const struct rte_flow_attr *attr, 6436 const struct rte_flow_item items[], 6437 const struct rte_flow_action original_actions[], 6438 struct rte_flow_error *error) 6439 { 6440 int hairpin_flow; 6441 struct mlx5_translated_action_handle 6442 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6443 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6444 const struct rte_flow_action *actions; 6445 struct rte_flow_action *translated_actions = NULL; 6446 int ret = flow_action_handles_translate(dev, original_actions, 6447 indir_actions, 6448 &indir_actions_n, 6449 &translated_actions, error); 6450 6451 if (ret) 6452 return ret; 6453 actions = translated_actions ? translated_actions : original_actions; 6454 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6455 ret = flow_drv_validate(dev, attr, items, actions, 6456 true, hairpin_flow, error); 6457 rte_free(translated_actions); 6458 return ret; 6459 } 6460 6461 /** 6462 * Create a flow. 6463 * 6464 * @see rte_flow_create() 6465 * @see rte_flow_ops 6466 */ 6467 struct rte_flow * 6468 mlx5_flow_create(struct rte_eth_dev *dev, 6469 const struct rte_flow_attr *attr, 6470 const struct rte_flow_item items[], 6471 const struct rte_flow_action actions[], 6472 struct rte_flow_error *error) 6473 { 6474 struct mlx5_priv *priv = dev->data->dev_private; 6475 6476 /* 6477 * If the device is not started yet, it is not allowed to created a 6478 * flow from application. PMD default flows and traffic control flows 6479 * are not affected. 6480 */ 6481 if (unlikely(!dev->data->dev_started)) { 6482 DRV_LOG(DEBUG, "port %u is not started when " 6483 "inserting a flow", dev->data->port_id); 6484 rte_flow_error_set(error, ENODEV, 6485 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6486 NULL, 6487 "port not started"); 6488 return NULL; 6489 } 6490 6491 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 6492 attr, items, actions, true, error); 6493 } 6494 6495 /** 6496 * Destroy a flow in a list. 6497 * 6498 * @param dev 6499 * Pointer to Ethernet device. 6500 * @param list 6501 * Pointer to the Indexed flow list. If this parameter NULL, 6502 * there is no flow removal from the list. Be noted that as 6503 * flow is add to the indexed list, memory of the indexed 6504 * list points to maybe changed as flow destroyed. 6505 * @param[in] flow_idx 6506 * Index of flow to destroy. 6507 */ 6508 static void 6509 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 6510 uint32_t flow_idx) 6511 { 6512 struct mlx5_priv *priv = dev->data->dev_private; 6513 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6514 [MLX5_IPOOL_RTE_FLOW], flow_idx); 6515 6516 if (!flow) 6517 return; 6518 /* 6519 * Update RX queue flags only if port is started, otherwise it is 6520 * already clean. 6521 */ 6522 if (dev->data->dev_started) 6523 flow_rxq_flags_trim(dev, flow); 6524 flow_drv_destroy(dev, flow); 6525 if (list) { 6526 rte_spinlock_lock(&priv->flow_list_lock); 6527 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 6528 flow_idx, flow, next); 6529 rte_spinlock_unlock(&priv->flow_list_lock); 6530 } 6531 if (flow->tunnel) { 6532 struct mlx5_flow_tunnel *tunnel; 6533 6534 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6535 RTE_VERIFY(tunnel); 6536 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6537 mlx5_flow_tunnel_free(dev, tunnel); 6538 } 6539 flow_mreg_del_copy_action(dev, flow); 6540 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 6541 } 6542 6543 /** 6544 * Destroy all flows. 6545 * 6546 * @param dev 6547 * Pointer to Ethernet device. 6548 * @param list 6549 * Pointer to the Indexed flow list. 6550 * @param active 6551 * If flushing is called avtively. 6552 */ 6553 void 6554 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 6555 { 6556 uint32_t num_flushed = 0; 6557 6558 while (*list) { 6559 flow_list_destroy(dev, list, *list); 6560 num_flushed++; 6561 } 6562 if (active) { 6563 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6564 dev->data->port_id, num_flushed); 6565 } 6566 } 6567 6568 /** 6569 * Stop all default actions for flows. 6570 * 6571 * @param dev 6572 * Pointer to Ethernet device. 6573 */ 6574 void 6575 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6576 { 6577 flow_mreg_del_default_copy_action(dev); 6578 flow_rxq_flags_clear(dev); 6579 } 6580 6581 /** 6582 * Start all default actions for flows. 6583 * 6584 * @param dev 6585 * Pointer to Ethernet device. 6586 * @return 6587 * 0 on success, a negative errno value otherwise and rte_errno is set. 6588 */ 6589 int 6590 mlx5_flow_start_default(struct rte_eth_dev *dev) 6591 { 6592 struct rte_flow_error error; 6593 6594 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6595 return flow_mreg_add_default_copy_action(dev, &error); 6596 } 6597 6598 /** 6599 * Release key of thread specific flow workspace data. 6600 */ 6601 void 6602 flow_release_workspace(void *data) 6603 { 6604 struct mlx5_flow_workspace *wks = data; 6605 struct mlx5_flow_workspace *next; 6606 6607 while (wks) { 6608 next = wks->next; 6609 free(wks->rss_desc.queue); 6610 free(wks); 6611 wks = next; 6612 } 6613 } 6614 6615 /** 6616 * Get thread specific current flow workspace. 6617 * 6618 * @return pointer to thread specific flow workspace data, NULL on error. 6619 */ 6620 struct mlx5_flow_workspace* 6621 mlx5_flow_get_thread_workspace(void) 6622 { 6623 struct mlx5_flow_workspace *data; 6624 6625 data = mlx5_flow_os_get_specific_workspace(); 6626 MLX5_ASSERT(data && data->inuse); 6627 if (!data || !data->inuse) 6628 DRV_LOG(ERR, "flow workspace not initialized."); 6629 return data; 6630 } 6631 6632 /** 6633 * Allocate and init new flow workspace. 6634 * 6635 * @return pointer to flow workspace data, NULL on error. 6636 */ 6637 static struct mlx5_flow_workspace* 6638 flow_alloc_thread_workspace(void) 6639 { 6640 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6641 6642 if (!data) { 6643 DRV_LOG(ERR, "Failed to allocate flow workspace " 6644 "memory."); 6645 return NULL; 6646 } 6647 data->rss_desc.queue = calloc(1, 6648 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6649 if (!data->rss_desc.queue) 6650 goto err; 6651 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6652 return data; 6653 err: 6654 if (data->rss_desc.queue) 6655 free(data->rss_desc.queue); 6656 free(data); 6657 return NULL; 6658 } 6659 6660 /** 6661 * Get new thread specific flow workspace. 6662 * 6663 * If current workspace inuse, create new one and set as current. 6664 * 6665 * @return pointer to thread specific flow workspace data, NULL on error. 6666 */ 6667 static struct mlx5_flow_workspace* 6668 mlx5_flow_push_thread_workspace(void) 6669 { 6670 struct mlx5_flow_workspace *curr; 6671 struct mlx5_flow_workspace *data; 6672 6673 curr = mlx5_flow_os_get_specific_workspace(); 6674 if (!curr) { 6675 data = flow_alloc_thread_workspace(); 6676 if (!data) 6677 return NULL; 6678 } else if (!curr->inuse) { 6679 data = curr; 6680 } else if (curr->next) { 6681 data = curr->next; 6682 } else { 6683 data = flow_alloc_thread_workspace(); 6684 if (!data) 6685 return NULL; 6686 curr->next = data; 6687 data->prev = curr; 6688 } 6689 data->inuse = 1; 6690 data->flow_idx = 0; 6691 /* Set as current workspace */ 6692 if (mlx5_flow_os_set_specific_workspace(data)) 6693 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6694 return data; 6695 } 6696 6697 /** 6698 * Close current thread specific flow workspace. 6699 * 6700 * If previous workspace available, set it as current. 6701 * 6702 * @return pointer to thread specific flow workspace data, NULL on error. 6703 */ 6704 static void 6705 mlx5_flow_pop_thread_workspace(void) 6706 { 6707 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6708 6709 if (!data) 6710 return; 6711 if (!data->inuse) { 6712 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6713 return; 6714 } 6715 data->inuse = 0; 6716 if (!data->prev) 6717 return; 6718 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6719 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6720 } 6721 6722 /** 6723 * Verify the flow list is empty 6724 * 6725 * @param dev 6726 * Pointer to Ethernet device. 6727 * 6728 * @return the number of flows not released. 6729 */ 6730 int 6731 mlx5_flow_verify(struct rte_eth_dev *dev) 6732 { 6733 struct mlx5_priv *priv = dev->data->dev_private; 6734 struct rte_flow *flow; 6735 uint32_t idx; 6736 int ret = 0; 6737 6738 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 6739 flow, next) { 6740 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6741 dev->data->port_id, (void *)flow); 6742 ++ret; 6743 } 6744 return ret; 6745 } 6746 6747 /** 6748 * Enable default hairpin egress flow. 6749 * 6750 * @param dev 6751 * Pointer to Ethernet device. 6752 * @param queue 6753 * The queue index. 6754 * 6755 * @return 6756 * 0 on success, a negative errno value otherwise and rte_errno is set. 6757 */ 6758 int 6759 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6760 uint32_t queue) 6761 { 6762 struct mlx5_priv *priv = dev->data->dev_private; 6763 const struct rte_flow_attr attr = { 6764 .egress = 1, 6765 .priority = 0, 6766 }; 6767 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6768 .queue = queue, 6769 }; 6770 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6771 .queue = UINT32_MAX, 6772 }; 6773 struct rte_flow_item items[] = { 6774 { 6775 .type = (enum rte_flow_item_type) 6776 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6777 .spec = &queue_spec, 6778 .last = NULL, 6779 .mask = &queue_mask, 6780 }, 6781 { 6782 .type = RTE_FLOW_ITEM_TYPE_END, 6783 }, 6784 }; 6785 struct rte_flow_action_jump jump = { 6786 .group = MLX5_HAIRPIN_TX_TABLE, 6787 }; 6788 struct rte_flow_action actions[2]; 6789 uint32_t flow_idx; 6790 struct rte_flow_error error; 6791 6792 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6793 actions[0].conf = &jump; 6794 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6795 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6796 &attr, items, actions, false, &error); 6797 if (!flow_idx) { 6798 DRV_LOG(DEBUG, 6799 "Failed to create ctrl flow: rte_errno(%d)," 6800 " type(%d), message(%s)", 6801 rte_errno, error.type, 6802 error.message ? error.message : " (no stated reason)"); 6803 return -rte_errno; 6804 } 6805 return 0; 6806 } 6807 6808 /** 6809 * Enable a control flow configured from the control plane. 6810 * 6811 * @param dev 6812 * Pointer to Ethernet device. 6813 * @param eth_spec 6814 * An Ethernet flow spec to apply. 6815 * @param eth_mask 6816 * An Ethernet flow mask to apply. 6817 * @param vlan_spec 6818 * A VLAN flow spec to apply. 6819 * @param vlan_mask 6820 * A VLAN flow mask to apply. 6821 * 6822 * @return 6823 * 0 on success, a negative errno value otherwise and rte_errno is set. 6824 */ 6825 int 6826 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6827 struct rte_flow_item_eth *eth_spec, 6828 struct rte_flow_item_eth *eth_mask, 6829 struct rte_flow_item_vlan *vlan_spec, 6830 struct rte_flow_item_vlan *vlan_mask) 6831 { 6832 struct mlx5_priv *priv = dev->data->dev_private; 6833 const struct rte_flow_attr attr = { 6834 .ingress = 1, 6835 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6836 }; 6837 struct rte_flow_item items[] = { 6838 { 6839 .type = RTE_FLOW_ITEM_TYPE_ETH, 6840 .spec = eth_spec, 6841 .last = NULL, 6842 .mask = eth_mask, 6843 }, 6844 { 6845 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6846 RTE_FLOW_ITEM_TYPE_END, 6847 .spec = vlan_spec, 6848 .last = NULL, 6849 .mask = vlan_mask, 6850 }, 6851 { 6852 .type = RTE_FLOW_ITEM_TYPE_END, 6853 }, 6854 }; 6855 uint16_t queue[priv->reta_idx_n]; 6856 struct rte_flow_action_rss action_rss = { 6857 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6858 .level = 0, 6859 .types = priv->rss_conf.rss_hf, 6860 .key_len = priv->rss_conf.rss_key_len, 6861 .queue_num = priv->reta_idx_n, 6862 .key = priv->rss_conf.rss_key, 6863 .queue = queue, 6864 }; 6865 struct rte_flow_action actions[] = { 6866 { 6867 .type = RTE_FLOW_ACTION_TYPE_RSS, 6868 .conf = &action_rss, 6869 }, 6870 { 6871 .type = RTE_FLOW_ACTION_TYPE_END, 6872 }, 6873 }; 6874 uint32_t flow_idx; 6875 struct rte_flow_error error; 6876 unsigned int i; 6877 6878 if (!priv->reta_idx_n || !priv->rxqs_n) { 6879 return 0; 6880 } 6881 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6882 action_rss.types = 0; 6883 for (i = 0; i != priv->reta_idx_n; ++i) 6884 queue[i] = (*priv->reta_idx)[i]; 6885 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6886 &attr, items, actions, false, &error); 6887 if (!flow_idx) 6888 return -rte_errno; 6889 return 0; 6890 } 6891 6892 /** 6893 * Enable a flow control configured from the control plane. 6894 * 6895 * @param dev 6896 * Pointer to Ethernet device. 6897 * @param eth_spec 6898 * An Ethernet flow spec to apply. 6899 * @param eth_mask 6900 * An Ethernet flow mask to apply. 6901 * 6902 * @return 6903 * 0 on success, a negative errno value otherwise and rte_errno is set. 6904 */ 6905 int 6906 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6907 struct rte_flow_item_eth *eth_spec, 6908 struct rte_flow_item_eth *eth_mask) 6909 { 6910 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6911 } 6912 6913 /** 6914 * Create default miss flow rule matching lacp traffic 6915 * 6916 * @param dev 6917 * Pointer to Ethernet device. 6918 * @param eth_spec 6919 * An Ethernet flow spec to apply. 6920 * 6921 * @return 6922 * 0 on success, a negative errno value otherwise and rte_errno is set. 6923 */ 6924 int 6925 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6926 { 6927 struct mlx5_priv *priv = dev->data->dev_private; 6928 /* 6929 * The LACP matching is done by only using ether type since using 6930 * a multicast dst mac causes kernel to give low priority to this flow. 6931 */ 6932 static const struct rte_flow_item_eth lacp_spec = { 6933 .type = RTE_BE16(0x8809), 6934 }; 6935 static const struct rte_flow_item_eth lacp_mask = { 6936 .type = 0xffff, 6937 }; 6938 const struct rte_flow_attr attr = { 6939 .ingress = 1, 6940 }; 6941 struct rte_flow_item items[] = { 6942 { 6943 .type = RTE_FLOW_ITEM_TYPE_ETH, 6944 .spec = &lacp_spec, 6945 .mask = &lacp_mask, 6946 }, 6947 { 6948 .type = RTE_FLOW_ITEM_TYPE_END, 6949 }, 6950 }; 6951 struct rte_flow_action actions[] = { 6952 { 6953 .type = (enum rte_flow_action_type) 6954 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6955 }, 6956 { 6957 .type = RTE_FLOW_ACTION_TYPE_END, 6958 }, 6959 }; 6960 struct rte_flow_error error; 6961 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6962 &attr, items, actions, false, &error); 6963 6964 if (!flow_idx) 6965 return -rte_errno; 6966 return 0; 6967 } 6968 6969 /** 6970 * Destroy a flow. 6971 * 6972 * @see rte_flow_destroy() 6973 * @see rte_flow_ops 6974 */ 6975 int 6976 mlx5_flow_destroy(struct rte_eth_dev *dev, 6977 struct rte_flow *flow, 6978 struct rte_flow_error *error __rte_unused) 6979 { 6980 struct mlx5_priv *priv = dev->data->dev_private; 6981 6982 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6983 return 0; 6984 } 6985 6986 /** 6987 * Destroy all flows. 6988 * 6989 * @see rte_flow_flush() 6990 * @see rte_flow_ops 6991 */ 6992 int 6993 mlx5_flow_flush(struct rte_eth_dev *dev, 6994 struct rte_flow_error *error __rte_unused) 6995 { 6996 struct mlx5_priv *priv = dev->data->dev_private; 6997 6998 mlx5_flow_list_flush(dev, &priv->flows, false); 6999 return 0; 7000 } 7001 7002 /** 7003 * Isolated mode. 7004 * 7005 * @see rte_flow_isolate() 7006 * @see rte_flow_ops 7007 */ 7008 int 7009 mlx5_flow_isolate(struct rte_eth_dev *dev, 7010 int enable, 7011 struct rte_flow_error *error) 7012 { 7013 struct mlx5_priv *priv = dev->data->dev_private; 7014 7015 if (dev->data->dev_started) { 7016 rte_flow_error_set(error, EBUSY, 7017 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7018 NULL, 7019 "port must be stopped first"); 7020 return -rte_errno; 7021 } 7022 priv->isolated = !!enable; 7023 if (enable) 7024 dev->dev_ops = &mlx5_dev_ops_isolate; 7025 else 7026 dev->dev_ops = &mlx5_dev_ops; 7027 7028 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7029 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7030 7031 return 0; 7032 } 7033 7034 /** 7035 * Query a flow. 7036 * 7037 * @see rte_flow_query() 7038 * @see rte_flow_ops 7039 */ 7040 static int 7041 flow_drv_query(struct rte_eth_dev *dev, 7042 uint32_t flow_idx, 7043 const struct rte_flow_action *actions, 7044 void *data, 7045 struct rte_flow_error *error) 7046 { 7047 struct mlx5_priv *priv = dev->data->dev_private; 7048 const struct mlx5_flow_driver_ops *fops; 7049 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 7050 [MLX5_IPOOL_RTE_FLOW], 7051 flow_idx); 7052 enum mlx5_flow_drv_type ftype; 7053 7054 if (!flow) { 7055 return rte_flow_error_set(error, ENOENT, 7056 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7057 NULL, 7058 "invalid flow handle"); 7059 } 7060 ftype = flow->drv_type; 7061 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7062 fops = flow_get_drv_ops(ftype); 7063 7064 return fops->query(dev, flow, actions, data, error); 7065 } 7066 7067 /** 7068 * Query a flow. 7069 * 7070 * @see rte_flow_query() 7071 * @see rte_flow_ops 7072 */ 7073 int 7074 mlx5_flow_query(struct rte_eth_dev *dev, 7075 struct rte_flow *flow, 7076 const struct rte_flow_action *actions, 7077 void *data, 7078 struct rte_flow_error *error) 7079 { 7080 int ret; 7081 7082 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7083 error); 7084 if (ret < 0) 7085 return ret; 7086 return 0; 7087 } 7088 7089 /** 7090 * Get rte_flow callbacks. 7091 * 7092 * @param dev 7093 * Pointer to Ethernet device structure. 7094 * @param ops 7095 * Pointer to operation-specific structure. 7096 * 7097 * @return 0 7098 */ 7099 int 7100 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7101 const struct rte_flow_ops **ops) 7102 { 7103 *ops = &mlx5_flow_ops; 7104 return 0; 7105 } 7106 7107 /** 7108 * Validate meter policy actions. 7109 * Dispatcher for action type specific validation. 7110 * 7111 * @param[in] dev 7112 * Pointer to the Ethernet device structure. 7113 * @param[in] action 7114 * The meter policy action object to validate. 7115 * @param[in] attr 7116 * Attributes of flow to determine steering domain. 7117 * @param[out] is_rss 7118 * Is RSS or not. 7119 * @param[out] domain_bitmap 7120 * Domain bitmap. 7121 * @param[out] is_def_policy 7122 * Is default policy or not. 7123 * @param[out] error 7124 * Perform verbose error reporting if not NULL. Initialized in case of 7125 * error only. 7126 * 7127 * @return 7128 * 0 on success, otherwise negative errno value. 7129 */ 7130 int 7131 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7132 const struct rte_flow_action *actions[RTE_COLORS], 7133 struct rte_flow_attr *attr, 7134 bool *is_rss, 7135 uint8_t *domain_bitmap, 7136 bool *is_def_policy, 7137 struct rte_mtr_error *error) 7138 { 7139 const struct mlx5_flow_driver_ops *fops; 7140 7141 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7142 return fops->validate_mtr_acts(dev, actions, attr, 7143 is_rss, domain_bitmap, is_def_policy, error); 7144 } 7145 7146 /** 7147 * Destroy the meter table set. 7148 * 7149 * @param[in] dev 7150 * Pointer to Ethernet device. 7151 * @param[in] mtr_policy 7152 * Meter policy struct. 7153 */ 7154 void 7155 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7156 struct mlx5_flow_meter_policy *mtr_policy) 7157 { 7158 const struct mlx5_flow_driver_ops *fops; 7159 7160 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7161 fops->destroy_mtr_acts(dev, mtr_policy); 7162 } 7163 7164 /** 7165 * Create policy action, lock free, 7166 * (mutex should be acquired by caller). 7167 * Dispatcher for action type specific call. 7168 * 7169 * @param[in] dev 7170 * Pointer to the Ethernet device structure. 7171 * @param[in] mtr_policy 7172 * Meter policy struct. 7173 * @param[in] action 7174 * Action specification used to create meter actions. 7175 * @param[out] error 7176 * Perform verbose error reporting if not NULL. Initialized in case of 7177 * error only. 7178 * 7179 * @return 7180 * 0 on success, otherwise negative errno value. 7181 */ 7182 int 7183 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7184 struct mlx5_flow_meter_policy *mtr_policy, 7185 const struct rte_flow_action *actions[RTE_COLORS], 7186 struct rte_mtr_error *error) 7187 { 7188 const struct mlx5_flow_driver_ops *fops; 7189 7190 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7191 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7192 } 7193 7194 /** 7195 * Create policy rules, lock free, 7196 * (mutex should be acquired by caller). 7197 * Dispatcher for action type specific call. 7198 * 7199 * @param[in] dev 7200 * Pointer to the Ethernet device structure. 7201 * @param[in] mtr_policy 7202 * Meter policy struct. 7203 * 7204 * @return 7205 * 0 on success, -1 otherwise. 7206 */ 7207 int 7208 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7209 struct mlx5_flow_meter_policy *mtr_policy) 7210 { 7211 const struct mlx5_flow_driver_ops *fops; 7212 7213 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7214 return fops->create_policy_rules(dev, mtr_policy); 7215 } 7216 7217 /** 7218 * Destroy policy rules, lock free, 7219 * (mutex should be acquired by caller). 7220 * Dispatcher for action type specific call. 7221 * 7222 * @param[in] dev 7223 * Pointer to the Ethernet device structure. 7224 * @param[in] mtr_policy 7225 * Meter policy struct. 7226 */ 7227 void 7228 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7229 struct mlx5_flow_meter_policy *mtr_policy) 7230 { 7231 const struct mlx5_flow_driver_ops *fops; 7232 7233 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7234 fops->destroy_policy_rules(dev, mtr_policy); 7235 } 7236 7237 /** 7238 * Destroy the default policy table set. 7239 * 7240 * @param[in] dev 7241 * Pointer to Ethernet device. 7242 */ 7243 void 7244 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7245 { 7246 const struct mlx5_flow_driver_ops *fops; 7247 7248 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7249 fops->destroy_def_policy(dev); 7250 } 7251 7252 /** 7253 * Destroy the default policy table set. 7254 * 7255 * @param[in] dev 7256 * Pointer to Ethernet device. 7257 * 7258 * @return 7259 * 0 on success, -1 otherwise. 7260 */ 7261 int 7262 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7263 { 7264 const struct mlx5_flow_driver_ops *fops; 7265 7266 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7267 return fops->create_def_policy(dev); 7268 } 7269 7270 /** 7271 * Create the needed meter and suffix tables. 7272 * 7273 * @param[in] dev 7274 * Pointer to Ethernet device. 7275 * 7276 * @return 7277 * 0 on success, -1 otherwise. 7278 */ 7279 int 7280 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7281 struct mlx5_flow_meter_info *fm, 7282 uint32_t mtr_idx, 7283 uint8_t domain_bitmap) 7284 { 7285 const struct mlx5_flow_driver_ops *fops; 7286 7287 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7288 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7289 } 7290 7291 /** 7292 * Destroy the meter table set. 7293 * 7294 * @param[in] dev 7295 * Pointer to Ethernet device. 7296 * @param[in] tbl 7297 * Pointer to the meter table set. 7298 */ 7299 void 7300 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7301 struct mlx5_flow_meter_info *fm) 7302 { 7303 const struct mlx5_flow_driver_ops *fops; 7304 7305 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7306 fops->destroy_mtr_tbls(dev, fm); 7307 } 7308 7309 /** 7310 * Destroy the global meter drop table. 7311 * 7312 * @param[in] dev 7313 * Pointer to Ethernet device. 7314 */ 7315 void 7316 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7317 { 7318 const struct mlx5_flow_driver_ops *fops; 7319 7320 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7321 fops->destroy_mtr_drop_tbls(dev); 7322 } 7323 7324 /** 7325 * Destroy the sub policy table with RX queue. 7326 * 7327 * @param[in] dev 7328 * Pointer to Ethernet device. 7329 * @param[in] mtr_policy 7330 * Pointer to meter policy table. 7331 */ 7332 void 7333 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7334 struct mlx5_flow_meter_policy *mtr_policy) 7335 { 7336 const struct mlx5_flow_driver_ops *fops; 7337 7338 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7339 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7340 } 7341 7342 /** 7343 * Allocate the needed aso flow meter id. 7344 * 7345 * @param[in] dev 7346 * Pointer to Ethernet device. 7347 * 7348 * @return 7349 * Index to aso flow meter on success, NULL otherwise. 7350 */ 7351 uint32_t 7352 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7353 { 7354 const struct mlx5_flow_driver_ops *fops; 7355 7356 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7357 return fops->create_meter(dev); 7358 } 7359 7360 /** 7361 * Free the aso flow meter id. 7362 * 7363 * @param[in] dev 7364 * Pointer to Ethernet device. 7365 * @param[in] mtr_idx 7366 * Index to aso flow meter to be free. 7367 * 7368 * @return 7369 * 0 on success. 7370 */ 7371 void 7372 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7373 { 7374 const struct mlx5_flow_driver_ops *fops; 7375 7376 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7377 fops->free_meter(dev, mtr_idx); 7378 } 7379 7380 /** 7381 * Allocate a counter. 7382 * 7383 * @param[in] dev 7384 * Pointer to Ethernet device structure. 7385 * 7386 * @return 7387 * Index to allocated counter on success, 0 otherwise. 7388 */ 7389 uint32_t 7390 mlx5_counter_alloc(struct rte_eth_dev *dev) 7391 { 7392 const struct mlx5_flow_driver_ops *fops; 7393 struct rte_flow_attr attr = { .transfer = 0 }; 7394 7395 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7396 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7397 return fops->counter_alloc(dev); 7398 } 7399 DRV_LOG(ERR, 7400 "port %u counter allocate is not supported.", 7401 dev->data->port_id); 7402 return 0; 7403 } 7404 7405 /** 7406 * Free a counter. 7407 * 7408 * @param[in] dev 7409 * Pointer to Ethernet device structure. 7410 * @param[in] cnt 7411 * Index to counter to be free. 7412 */ 7413 void 7414 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7415 { 7416 const struct mlx5_flow_driver_ops *fops; 7417 struct rte_flow_attr attr = { .transfer = 0 }; 7418 7419 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7420 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7421 fops->counter_free(dev, cnt); 7422 return; 7423 } 7424 DRV_LOG(ERR, 7425 "port %u counter free is not supported.", 7426 dev->data->port_id); 7427 } 7428 7429 /** 7430 * Query counter statistics. 7431 * 7432 * @param[in] dev 7433 * Pointer to Ethernet device structure. 7434 * @param[in] cnt 7435 * Index to counter to query. 7436 * @param[in] clear 7437 * Set to clear counter statistics. 7438 * @param[out] pkts 7439 * The counter hits packets number to save. 7440 * @param[out] bytes 7441 * The counter hits bytes number to save. 7442 * 7443 * @return 7444 * 0 on success, a negative errno value otherwise. 7445 */ 7446 int 7447 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7448 bool clear, uint64_t *pkts, uint64_t *bytes) 7449 { 7450 const struct mlx5_flow_driver_ops *fops; 7451 struct rte_flow_attr attr = { .transfer = 0 }; 7452 7453 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7454 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7455 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7456 } 7457 DRV_LOG(ERR, 7458 "port %u counter query is not supported.", 7459 dev->data->port_id); 7460 return -ENOTSUP; 7461 } 7462 7463 /** 7464 * Allocate a new memory for the counter values wrapped by all the needed 7465 * management. 7466 * 7467 * @param[in] sh 7468 * Pointer to mlx5_dev_ctx_shared object. 7469 * 7470 * @return 7471 * 0 on success, a negative errno value otherwise. 7472 */ 7473 static int 7474 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7475 { 7476 struct mlx5_devx_mkey_attr mkey_attr; 7477 struct mlx5_counter_stats_mem_mng *mem_mng; 7478 volatile struct flow_counter_stats *raw_data; 7479 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7480 int size = (sizeof(struct flow_counter_stats) * 7481 MLX5_COUNTERS_PER_POOL + 7482 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7483 sizeof(struct mlx5_counter_stats_mem_mng); 7484 size_t pgsize = rte_mem_page_size(); 7485 uint8_t *mem; 7486 int i; 7487 7488 if (pgsize == (size_t)-1) { 7489 DRV_LOG(ERR, "Failed to get mem page size"); 7490 rte_errno = ENOMEM; 7491 return -ENOMEM; 7492 } 7493 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7494 if (!mem) { 7495 rte_errno = ENOMEM; 7496 return -ENOMEM; 7497 } 7498 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7499 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7500 mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size, 7501 IBV_ACCESS_LOCAL_WRITE); 7502 if (!mem_mng->umem) { 7503 rte_errno = errno; 7504 mlx5_free(mem); 7505 return -rte_errno; 7506 } 7507 memset(&mkey_attr, 0, sizeof(mkey_attr)); 7508 mkey_attr.addr = (uintptr_t)mem; 7509 mkey_attr.size = size; 7510 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7511 mkey_attr.pd = sh->pdn; 7512 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 7513 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 7514 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7515 if (!mem_mng->dm) { 7516 mlx5_os_umem_dereg(mem_mng->umem); 7517 rte_errno = errno; 7518 mlx5_free(mem); 7519 return -rte_errno; 7520 } 7521 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7522 raw_data = (volatile struct flow_counter_stats *)mem; 7523 for (i = 0; i < raws_n; ++i) { 7524 mem_mng->raws[i].mem_mng = mem_mng; 7525 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7526 } 7527 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7528 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7529 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7530 next); 7531 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7532 sh->cmng.mem_mng = mem_mng; 7533 return 0; 7534 } 7535 7536 /** 7537 * Set the statistic memory to the new counter pool. 7538 * 7539 * @param[in] sh 7540 * Pointer to mlx5_dev_ctx_shared object. 7541 * @param[in] pool 7542 * Pointer to the pool to set the statistic memory. 7543 * 7544 * @return 7545 * 0 on success, a negative errno value otherwise. 7546 */ 7547 static int 7548 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7549 struct mlx5_flow_counter_pool *pool) 7550 { 7551 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7552 /* Resize statistic memory once used out. */ 7553 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7554 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7555 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7556 return -1; 7557 } 7558 rte_spinlock_lock(&pool->sl); 7559 pool->raw = cmng->mem_mng->raws + pool->index % 7560 MLX5_CNT_CONTAINER_RESIZE; 7561 rte_spinlock_unlock(&pool->sl); 7562 pool->raw_hw = NULL; 7563 return 0; 7564 } 7565 7566 #define MLX5_POOL_QUERY_FREQ_US 1000000 7567 7568 /** 7569 * Set the periodic procedure for triggering asynchronous batch queries for all 7570 * the counter pools. 7571 * 7572 * @param[in] sh 7573 * Pointer to mlx5_dev_ctx_shared object. 7574 */ 7575 void 7576 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7577 { 7578 uint32_t pools_n, us; 7579 7580 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7581 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7582 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7583 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7584 sh->cmng.query_thread_on = 0; 7585 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7586 } else { 7587 sh->cmng.query_thread_on = 1; 7588 } 7589 } 7590 7591 /** 7592 * The periodic procedure for triggering asynchronous batch queries for all the 7593 * counter pools. This function is probably called by the host thread. 7594 * 7595 * @param[in] arg 7596 * The parameter for the alarm process. 7597 */ 7598 void 7599 mlx5_flow_query_alarm(void *arg) 7600 { 7601 struct mlx5_dev_ctx_shared *sh = arg; 7602 int ret; 7603 uint16_t pool_index = sh->cmng.pool_index; 7604 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7605 struct mlx5_flow_counter_pool *pool; 7606 uint16_t n_valid; 7607 7608 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7609 goto set_alarm; 7610 rte_spinlock_lock(&cmng->pool_update_sl); 7611 pool = cmng->pools[pool_index]; 7612 n_valid = cmng->n_valid; 7613 rte_spinlock_unlock(&cmng->pool_update_sl); 7614 /* Set the statistic memory to the new created pool. */ 7615 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7616 goto set_alarm; 7617 if (pool->raw_hw) 7618 /* There is a pool query in progress. */ 7619 goto set_alarm; 7620 pool->raw_hw = 7621 LIST_FIRST(&sh->cmng.free_stat_raws); 7622 if (!pool->raw_hw) 7623 /* No free counter statistics raw memory. */ 7624 goto set_alarm; 7625 /* 7626 * Identify the counters released between query trigger and query 7627 * handle more efficiently. The counter released in this gap period 7628 * should wait for a new round of query as the new arrived packets 7629 * will not be taken into account. 7630 */ 7631 pool->query_gen++; 7632 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7633 MLX5_COUNTERS_PER_POOL, 7634 NULL, NULL, 7635 pool->raw_hw->mem_mng->dm->id, 7636 (void *)(uintptr_t) 7637 pool->raw_hw->data, 7638 sh->devx_comp, 7639 (uint64_t)(uintptr_t)pool); 7640 if (ret) { 7641 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7642 " %d", pool->min_dcs->id); 7643 pool->raw_hw = NULL; 7644 goto set_alarm; 7645 } 7646 LIST_REMOVE(pool->raw_hw, next); 7647 sh->cmng.pending_queries++; 7648 pool_index++; 7649 if (pool_index >= n_valid) 7650 pool_index = 0; 7651 set_alarm: 7652 sh->cmng.pool_index = pool_index; 7653 mlx5_set_query_alarm(sh); 7654 } 7655 7656 /** 7657 * Check and callback event for new aged flow in the counter pool 7658 * 7659 * @param[in] sh 7660 * Pointer to mlx5_dev_ctx_shared object. 7661 * @param[in] pool 7662 * Pointer to Current counter pool. 7663 */ 7664 static void 7665 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7666 struct mlx5_flow_counter_pool *pool) 7667 { 7668 struct mlx5_priv *priv; 7669 struct mlx5_flow_counter *cnt; 7670 struct mlx5_age_info *age_info; 7671 struct mlx5_age_param *age_param; 7672 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7673 struct mlx5_counter_stats_raw *prev = pool->raw; 7674 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7675 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7676 uint16_t expected = AGE_CANDIDATE; 7677 uint32_t i; 7678 7679 pool->time_of_last_age_check = curr_time; 7680 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7681 cnt = MLX5_POOL_GET_CNT(pool, i); 7682 age_param = MLX5_CNT_TO_AGE(cnt); 7683 if (__atomic_load_n(&age_param->state, 7684 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7685 continue; 7686 if (cur->data[i].hits != prev->data[i].hits) { 7687 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7688 __ATOMIC_RELAXED); 7689 continue; 7690 } 7691 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7692 time_delta, 7693 __ATOMIC_RELAXED) <= age_param->timeout) 7694 continue; 7695 /** 7696 * Hold the lock first, or if between the 7697 * state AGE_TMOUT and tailq operation the 7698 * release happened, the release procedure 7699 * may delete a non-existent tailq node. 7700 */ 7701 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7702 age_info = GET_PORT_AGE_INFO(priv); 7703 rte_spinlock_lock(&age_info->aged_sl); 7704 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7705 AGE_TMOUT, false, 7706 __ATOMIC_RELAXED, 7707 __ATOMIC_RELAXED)) { 7708 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7709 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7710 } 7711 rte_spinlock_unlock(&age_info->aged_sl); 7712 } 7713 mlx5_age_event_prepare(sh); 7714 } 7715 7716 /** 7717 * Handler for the HW respond about ready values from an asynchronous batch 7718 * query. This function is probably called by the host thread. 7719 * 7720 * @param[in] sh 7721 * The pointer to the shared device context. 7722 * @param[in] async_id 7723 * The Devx async ID. 7724 * @param[in] status 7725 * The status of the completion. 7726 */ 7727 void 7728 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7729 uint64_t async_id, int status) 7730 { 7731 struct mlx5_flow_counter_pool *pool = 7732 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7733 struct mlx5_counter_stats_raw *raw_to_free; 7734 uint8_t query_gen = pool->query_gen ^ 1; 7735 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7736 enum mlx5_counter_type cnt_type = 7737 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7738 MLX5_COUNTER_TYPE_ORIGIN; 7739 7740 if (unlikely(status)) { 7741 raw_to_free = pool->raw_hw; 7742 } else { 7743 raw_to_free = pool->raw; 7744 if (pool->is_aged) 7745 mlx5_flow_aging_check(sh, pool); 7746 rte_spinlock_lock(&pool->sl); 7747 pool->raw = pool->raw_hw; 7748 rte_spinlock_unlock(&pool->sl); 7749 /* Be sure the new raw counters data is updated in memory. */ 7750 rte_io_wmb(); 7751 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7752 rte_spinlock_lock(&cmng->csl[cnt_type]); 7753 TAILQ_CONCAT(&cmng->counters[cnt_type], 7754 &pool->counters[query_gen], next); 7755 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7756 } 7757 } 7758 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7759 pool->raw_hw = NULL; 7760 sh->cmng.pending_queries--; 7761 } 7762 7763 static int 7764 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7765 const struct flow_grp_info *grp_info, 7766 struct rte_flow_error *error) 7767 { 7768 if (grp_info->transfer && grp_info->external && 7769 grp_info->fdb_def_rule) { 7770 if (group == UINT32_MAX) 7771 return rte_flow_error_set 7772 (error, EINVAL, 7773 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7774 NULL, 7775 "group index not supported"); 7776 *table = group + 1; 7777 } else { 7778 *table = group; 7779 } 7780 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7781 return 0; 7782 } 7783 7784 /** 7785 * Translate the rte_flow group index to HW table value. 7786 * 7787 * If tunnel offload is disabled, all group ids converted to flow table 7788 * id using the standard method. 7789 * If tunnel offload is enabled, group id can be converted using the 7790 * standard or tunnel conversion method. Group conversion method 7791 * selection depends on flags in `grp_info` parameter: 7792 * - Internal (grp_info.external == 0) groups conversion uses the 7793 * standard method. 7794 * - Group ids in JUMP action converted with the tunnel conversion. 7795 * - Group id in rule attribute conversion depends on a rule type and 7796 * group id value: 7797 * ** non zero group attributes converted with the tunnel method 7798 * ** zero group attribute in non-tunnel rule is converted using the 7799 * standard method - there's only one root table 7800 * ** zero group attribute in steer tunnel rule is converted with the 7801 * standard method - single root table 7802 * ** zero group attribute in match tunnel rule is a special OvS 7803 * case: that value is used for portability reasons. That group 7804 * id is converted with the tunnel conversion method. 7805 * 7806 * @param[in] dev 7807 * Port device 7808 * @param[in] tunnel 7809 * PMD tunnel offload object 7810 * @param[in] group 7811 * rte_flow group index value. 7812 * @param[out] table 7813 * HW table value. 7814 * @param[in] grp_info 7815 * flags used for conversion 7816 * @param[out] error 7817 * Pointer to error structure. 7818 * 7819 * @return 7820 * 0 on success, a negative errno value otherwise and rte_errno is set. 7821 */ 7822 int 7823 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7824 const struct mlx5_flow_tunnel *tunnel, 7825 uint32_t group, uint32_t *table, 7826 const struct flow_grp_info *grp_info, 7827 struct rte_flow_error *error) 7828 { 7829 int ret; 7830 bool standard_translation; 7831 7832 if (!grp_info->skip_scale && grp_info->external && 7833 group < MLX5_MAX_TABLES_EXTERNAL) 7834 group *= MLX5_FLOW_TABLE_FACTOR; 7835 if (is_tunnel_offload_active(dev)) { 7836 standard_translation = !grp_info->external || 7837 grp_info->std_tbl_fix; 7838 } else { 7839 standard_translation = true; 7840 } 7841 DRV_LOG(DEBUG, 7842 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 7843 dev->data->port_id, group, grp_info->transfer, 7844 grp_info->external, grp_info->fdb_def_rule, 7845 standard_translation ? "STANDARD" : "TUNNEL"); 7846 if (standard_translation) 7847 ret = flow_group_to_table(dev->data->port_id, group, table, 7848 grp_info, error); 7849 else 7850 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7851 table, error); 7852 7853 return ret; 7854 } 7855 7856 /** 7857 * Discover availability of metadata reg_c's. 7858 * 7859 * Iteratively use test flows to check availability. 7860 * 7861 * @param[in] dev 7862 * Pointer to the Ethernet device structure. 7863 * 7864 * @return 7865 * 0 on success, a negative errno value otherwise and rte_errno is set. 7866 */ 7867 int 7868 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7869 { 7870 struct mlx5_priv *priv = dev->data->dev_private; 7871 struct mlx5_dev_config *config = &priv->config; 7872 enum modify_reg idx; 7873 int n = 0; 7874 7875 /* reg_c[0] and reg_c[1] are reserved. */ 7876 config->flow_mreg_c[n++] = REG_C_0; 7877 config->flow_mreg_c[n++] = REG_C_1; 7878 /* Discover availability of other reg_c's. */ 7879 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7880 struct rte_flow_attr attr = { 7881 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7882 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7883 .ingress = 1, 7884 }; 7885 struct rte_flow_item items[] = { 7886 [0] = { 7887 .type = RTE_FLOW_ITEM_TYPE_END, 7888 }, 7889 }; 7890 struct rte_flow_action actions[] = { 7891 [0] = { 7892 .type = (enum rte_flow_action_type) 7893 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7894 .conf = &(struct mlx5_flow_action_copy_mreg){ 7895 .src = REG_C_1, 7896 .dst = idx, 7897 }, 7898 }, 7899 [1] = { 7900 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7901 .conf = &(struct rte_flow_action_jump){ 7902 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7903 }, 7904 }, 7905 [2] = { 7906 .type = RTE_FLOW_ACTION_TYPE_END, 7907 }, 7908 }; 7909 uint32_t flow_idx; 7910 struct rte_flow *flow; 7911 struct rte_flow_error error; 7912 7913 if (!config->dv_flow_en) 7914 break; 7915 /* Create internal flow, validation skips copy action. */ 7916 flow_idx = flow_list_create(dev, NULL, &attr, items, 7917 actions, false, &error); 7918 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 7919 flow_idx); 7920 if (!flow) 7921 continue; 7922 config->flow_mreg_c[n++] = idx; 7923 flow_list_destroy(dev, NULL, flow_idx); 7924 } 7925 for (; n < MLX5_MREG_C_NUM; ++n) 7926 config->flow_mreg_c[n] = REG_NON; 7927 return 0; 7928 } 7929 7930 int 7931 save_dump_file(const uint8_t *data, uint32_t size, 7932 uint32_t type, uint32_t id, void *arg, FILE *file) 7933 { 7934 char line[BUF_SIZE]; 7935 uint32_t out = 0; 7936 uint32_t k; 7937 uint32_t actions_num; 7938 struct rte_flow_query_count *count; 7939 7940 memset(line, 0, BUF_SIZE); 7941 switch (type) { 7942 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 7943 actions_num = *(uint32_t *)(arg); 7944 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,", 7945 type, id, actions_num); 7946 break; 7947 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 7948 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,", 7949 type, id); 7950 break; 7951 case DR_DUMP_REC_TYPE_PMD_COUNTER: 7952 count = (struct rte_flow_query_count *)arg; 7953 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type, 7954 id, count->hits, count->bytes); 7955 return 0; 7956 default: 7957 return -1; 7958 } 7959 7960 for (k = 0; k < size; k++) { 7961 /* Make sure we do not overrun the line buffer length. */ 7962 if (out >= BUF_SIZE - 4) { 7963 line[out] = '\0'; 7964 break; 7965 } 7966 out += snprintf(line + out, BUF_SIZE - out, "%02x", 7967 (data[k]) & 0xff); 7968 } 7969 fprintf(file, "%s\n", line); 7970 return 0; 7971 } 7972 7973 int 7974 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 7975 struct rte_flow_query_count *count, struct rte_flow_error *error) 7976 { 7977 struct rte_flow_action action[2]; 7978 enum mlx5_flow_drv_type ftype; 7979 const struct mlx5_flow_driver_ops *fops; 7980 7981 if (!flow) { 7982 return rte_flow_error_set(error, ENOENT, 7983 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7984 NULL, 7985 "invalid flow handle"); 7986 } 7987 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 7988 action[1].type = RTE_FLOW_ACTION_TYPE_END; 7989 if (flow->counter) { 7990 memset(count, 0, sizeof(struct rte_flow_query_count)); 7991 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 7992 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 7993 ftype < MLX5_FLOW_TYPE_MAX); 7994 fops = flow_get_drv_ops(ftype); 7995 return fops->query(dev, flow, action, count, error); 7996 } 7997 return -1; 7998 } 7999 8000 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8001 /** 8002 * Dump flow ipool data to file 8003 * 8004 * @param[in] dev 8005 * The pointer to Ethernet device. 8006 * @param[in] file 8007 * A pointer to a file for output. 8008 * @param[out] error 8009 * Perform verbose error reporting if not NULL. PMDs initialize this 8010 * structure in case of error only. 8011 * @return 8012 * 0 on success, a negative value otherwise. 8013 */ 8014 int 8015 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8016 struct rte_flow *flow, FILE *file, 8017 struct rte_flow_error *error) 8018 { 8019 struct mlx5_priv *priv = dev->data->dev_private; 8020 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8021 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8022 uint32_t handle_idx; 8023 struct mlx5_flow_handle *dh; 8024 struct rte_flow_query_count count; 8025 uint32_t actions_num; 8026 const uint8_t *data; 8027 size_t size; 8028 uint32_t id; 8029 uint32_t type; 8030 8031 if (!flow) { 8032 return rte_flow_error_set(error, ENOENT, 8033 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8034 NULL, 8035 "invalid flow handle"); 8036 } 8037 handle_idx = flow->dev_handles; 8038 while (handle_idx) { 8039 dh = mlx5_ipool_get(priv->sh->ipool 8040 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8041 if (!dh) 8042 continue; 8043 handle_idx = dh->next.next; 8044 id = (uint32_t)(uintptr_t)dh->drv_flow; 8045 8046 /* query counter */ 8047 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8048 if (!mlx5_flow_query_counter(dev, flow, &count, error)) 8049 save_dump_file(NULL, 0, type, 8050 id, (void *)&count, file); 8051 8052 /* Get modify_hdr and encap_decap buf from ipools. */ 8053 encap_decap = NULL; 8054 modify_hdr = dh->dvh.modify_hdr; 8055 8056 if (dh->dvh.rix_encap_decap) { 8057 encap_decap = mlx5_ipool_get(priv->sh->ipool 8058 [MLX5_IPOOL_DECAP_ENCAP], 8059 dh->dvh.rix_encap_decap); 8060 } 8061 if (modify_hdr) { 8062 data = (const uint8_t *)modify_hdr->actions; 8063 size = (size_t)(modify_hdr->actions_num) * 8; 8064 actions_num = modify_hdr->actions_num; 8065 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8066 save_dump_file(data, size, type, id, 8067 (void *)(&actions_num), file); 8068 } 8069 if (encap_decap) { 8070 data = encap_decap->buf; 8071 size = encap_decap->size; 8072 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8073 save_dump_file(data, size, type, 8074 id, NULL, file); 8075 } 8076 } 8077 return 0; 8078 } 8079 #endif 8080 8081 /** 8082 * Dump flow raw hw data to file 8083 * 8084 * @param[in] dev 8085 * The pointer to Ethernet device. 8086 * @param[in] file 8087 * A pointer to a file for output. 8088 * @param[out] error 8089 * Perform verbose error reporting if not NULL. PMDs initialize this 8090 * structure in case of error only. 8091 * @return 8092 * 0 on success, a nagative value otherwise. 8093 */ 8094 int 8095 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8096 FILE *file, 8097 struct rte_flow_error *error __rte_unused) 8098 { 8099 struct mlx5_priv *priv = dev->data->dev_private; 8100 struct mlx5_dev_ctx_shared *sh = priv->sh; 8101 uint32_t handle_idx; 8102 int ret; 8103 struct mlx5_flow_handle *dh; 8104 struct rte_flow *flow; 8105 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8106 uint32_t idx; 8107 #endif 8108 8109 if (!priv->config.dv_flow_en) { 8110 if (fputs("device dv flow disabled\n", file) <= 0) 8111 return -errno; 8112 return -ENOTSUP; 8113 } 8114 8115 /* dump all */ 8116 if (!flow_idx) { 8117 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8118 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 8119 priv->flows, idx, flow, next) 8120 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8121 #endif 8122 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8123 sh->rx_domain, 8124 sh->tx_domain, file); 8125 } 8126 /* dump one */ 8127 flow = mlx5_ipool_get(priv->sh->ipool 8128 [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx); 8129 if (!flow) 8130 return -ENOENT; 8131 8132 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8133 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8134 #endif 8135 handle_idx = flow->dev_handles; 8136 while (handle_idx) { 8137 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8138 handle_idx); 8139 if (!dh) 8140 return -ENOENT; 8141 if (dh->drv_flow) { 8142 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8143 file); 8144 if (ret) 8145 return -ENOENT; 8146 } 8147 handle_idx = dh->next.next; 8148 } 8149 return 0; 8150 } 8151 8152 /** 8153 * Get aged-out flows. 8154 * 8155 * @param[in] dev 8156 * Pointer to the Ethernet device structure. 8157 * @param[in] context 8158 * The address of an array of pointers to the aged-out flows contexts. 8159 * @param[in] nb_countexts 8160 * The length of context array pointers. 8161 * @param[out] error 8162 * Perform verbose error reporting if not NULL. Initialized in case of 8163 * error only. 8164 * 8165 * @return 8166 * how many contexts get in success, otherwise negative errno value. 8167 * if nb_contexts is 0, return the amount of all aged contexts. 8168 * if nb_contexts is not 0 , return the amount of aged flows reported 8169 * in the context array. 8170 */ 8171 int 8172 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8173 uint32_t nb_contexts, struct rte_flow_error *error) 8174 { 8175 const struct mlx5_flow_driver_ops *fops; 8176 struct rte_flow_attr attr = { .transfer = 0 }; 8177 8178 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8179 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8180 return fops->get_aged_flows(dev, contexts, nb_contexts, 8181 error); 8182 } 8183 DRV_LOG(ERR, 8184 "port %u get aged flows is not supported.", 8185 dev->data->port_id); 8186 return -ENOTSUP; 8187 } 8188 8189 /* Wrapper for driver action_validate op callback */ 8190 static int 8191 flow_drv_action_validate(struct rte_eth_dev *dev, 8192 const struct rte_flow_indir_action_conf *conf, 8193 const struct rte_flow_action *action, 8194 const struct mlx5_flow_driver_ops *fops, 8195 struct rte_flow_error *error) 8196 { 8197 static const char err_msg[] = "indirect action validation unsupported"; 8198 8199 if (!fops->action_validate) { 8200 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8201 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8202 NULL, err_msg); 8203 return -rte_errno; 8204 } 8205 return fops->action_validate(dev, conf, action, error); 8206 } 8207 8208 /** 8209 * Destroys the shared action by handle. 8210 * 8211 * @param dev 8212 * Pointer to Ethernet device structure. 8213 * @param[in] handle 8214 * Handle for the indirect action object to be destroyed. 8215 * @param[out] error 8216 * Perform verbose error reporting if not NULL. PMDs initialize this 8217 * structure in case of error only. 8218 * 8219 * @return 8220 * 0 on success, a negative errno value otherwise and rte_errno is set. 8221 * 8222 * @note: wrapper for driver action_create op callback. 8223 */ 8224 static int 8225 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8226 struct rte_flow_action_handle *handle, 8227 struct rte_flow_error *error) 8228 { 8229 static const char err_msg[] = "indirect action destruction unsupported"; 8230 struct rte_flow_attr attr = { .transfer = 0 }; 8231 const struct mlx5_flow_driver_ops *fops = 8232 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8233 8234 if (!fops->action_destroy) { 8235 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8236 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8237 NULL, err_msg); 8238 return -rte_errno; 8239 } 8240 return fops->action_destroy(dev, handle, error); 8241 } 8242 8243 /* Wrapper for driver action_destroy op callback */ 8244 static int 8245 flow_drv_action_update(struct rte_eth_dev *dev, 8246 struct rte_flow_action_handle *handle, 8247 const void *update, 8248 const struct mlx5_flow_driver_ops *fops, 8249 struct rte_flow_error *error) 8250 { 8251 static const char err_msg[] = "indirect action update unsupported"; 8252 8253 if (!fops->action_update) { 8254 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8255 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8256 NULL, err_msg); 8257 return -rte_errno; 8258 } 8259 return fops->action_update(dev, handle, update, error); 8260 } 8261 8262 /* Wrapper for driver action_destroy op callback */ 8263 static int 8264 flow_drv_action_query(struct rte_eth_dev *dev, 8265 const struct rte_flow_action_handle *handle, 8266 void *data, 8267 const struct mlx5_flow_driver_ops *fops, 8268 struct rte_flow_error *error) 8269 { 8270 static const char err_msg[] = "indirect action query unsupported"; 8271 8272 if (!fops->action_query) { 8273 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8274 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8275 NULL, err_msg); 8276 return -rte_errno; 8277 } 8278 return fops->action_query(dev, handle, data, error); 8279 } 8280 8281 /** 8282 * Create indirect action for reuse in multiple flow rules. 8283 * 8284 * @param dev 8285 * Pointer to Ethernet device structure. 8286 * @param conf 8287 * Pointer to indirect action object configuration. 8288 * @param[in] action 8289 * Action configuration for indirect action object creation. 8290 * @param[out] error 8291 * Perform verbose error reporting if not NULL. PMDs initialize this 8292 * structure in case of error only. 8293 * @return 8294 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8295 */ 8296 static struct rte_flow_action_handle * 8297 mlx5_action_handle_create(struct rte_eth_dev *dev, 8298 const struct rte_flow_indir_action_conf *conf, 8299 const struct rte_flow_action *action, 8300 struct rte_flow_error *error) 8301 { 8302 static const char err_msg[] = "indirect action creation unsupported"; 8303 struct rte_flow_attr attr = { .transfer = 0 }; 8304 const struct mlx5_flow_driver_ops *fops = 8305 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8306 8307 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8308 return NULL; 8309 if (!fops->action_create) { 8310 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8311 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8312 NULL, err_msg); 8313 return NULL; 8314 } 8315 return fops->action_create(dev, conf, action, error); 8316 } 8317 8318 /** 8319 * Updates inplace the indirect action configuration pointed by *handle* 8320 * with the configuration provided as *update* argument. 8321 * The update of the indirect action configuration effects all flow rules 8322 * reusing the action via handle. 8323 * 8324 * @param dev 8325 * Pointer to Ethernet device structure. 8326 * @param[in] handle 8327 * Handle for the indirect action to be updated. 8328 * @param[in] update 8329 * Action specification used to modify the action pointed by handle. 8330 * *update* could be of same type with the action pointed by the *handle* 8331 * handle argument, or some other structures like a wrapper, depending on 8332 * the indirect action type. 8333 * @param[out] error 8334 * Perform verbose error reporting if not NULL. PMDs initialize this 8335 * structure in case of error only. 8336 * 8337 * @return 8338 * 0 on success, a negative errno value otherwise and rte_errno is set. 8339 */ 8340 static int 8341 mlx5_action_handle_update(struct rte_eth_dev *dev, 8342 struct rte_flow_action_handle *handle, 8343 const void *update, 8344 struct rte_flow_error *error) 8345 { 8346 struct rte_flow_attr attr = { .transfer = 0 }; 8347 const struct mlx5_flow_driver_ops *fops = 8348 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8349 int ret; 8350 8351 ret = flow_drv_action_validate(dev, NULL, 8352 (const struct rte_flow_action *)update, fops, error); 8353 if (ret) 8354 return ret; 8355 return flow_drv_action_update(dev, handle, update, fops, 8356 error); 8357 } 8358 8359 /** 8360 * Query the indirect action by handle. 8361 * 8362 * This function allows retrieving action-specific data such as counters. 8363 * Data is gathered by special action which may be present/referenced in 8364 * more than one flow rule definition. 8365 * 8366 * see @RTE_FLOW_ACTION_TYPE_COUNT 8367 * 8368 * @param dev 8369 * Pointer to Ethernet device structure. 8370 * @param[in] handle 8371 * Handle for the indirect action to query. 8372 * @param[in, out] data 8373 * Pointer to storage for the associated query data type. 8374 * @param[out] error 8375 * Perform verbose error reporting if not NULL. PMDs initialize this 8376 * structure in case of error only. 8377 * 8378 * @return 8379 * 0 on success, a negative errno value otherwise and rte_errno is set. 8380 */ 8381 static int 8382 mlx5_action_handle_query(struct rte_eth_dev *dev, 8383 const struct rte_flow_action_handle *handle, 8384 void *data, 8385 struct rte_flow_error *error) 8386 { 8387 struct rte_flow_attr attr = { .transfer = 0 }; 8388 const struct mlx5_flow_driver_ops *fops = 8389 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8390 8391 return flow_drv_action_query(dev, handle, data, fops, error); 8392 } 8393 8394 /** 8395 * Destroy all indirect actions (shared RSS). 8396 * 8397 * @param dev 8398 * Pointer to Ethernet device. 8399 * 8400 * @return 8401 * 0 on success, a negative errno value otherwise and rte_errno is set. 8402 */ 8403 int 8404 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8405 { 8406 struct rte_flow_error error; 8407 struct mlx5_priv *priv = dev->data->dev_private; 8408 struct mlx5_shared_action_rss *shared_rss; 8409 int ret = 0; 8410 uint32_t idx; 8411 8412 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8413 priv->rss_shared_actions, idx, shared_rss, next) { 8414 ret |= mlx5_action_handle_destroy(dev, 8415 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8416 } 8417 return ret; 8418 } 8419 8420 #ifndef HAVE_MLX5DV_DR 8421 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8422 #else 8423 #define MLX5_DOMAIN_SYNC_FLOW \ 8424 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8425 #endif 8426 8427 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8428 { 8429 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8430 const struct mlx5_flow_driver_ops *fops; 8431 int ret; 8432 struct rte_flow_attr attr = { .transfer = 0 }; 8433 8434 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8435 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8436 if (ret > 0) 8437 ret = -ret; 8438 return ret; 8439 } 8440 8441 const struct mlx5_flow_tunnel * 8442 mlx5_get_tof(const struct rte_flow_item *item, 8443 const struct rte_flow_action *action, 8444 enum mlx5_tof_rule_type *rule_type) 8445 { 8446 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 8447 if (item->type == (typeof(item->type)) 8448 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 8449 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 8450 return flow_items_to_tunnel(item); 8451 } 8452 } 8453 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 8454 if (action->type == (typeof(action->type)) 8455 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 8456 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 8457 return flow_actions_to_tunnel(action); 8458 } 8459 } 8460 return NULL; 8461 } 8462 8463 /** 8464 * tunnel offload functionalilty is defined for DV environment only 8465 */ 8466 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8467 __extension__ 8468 union tunnel_offload_mark { 8469 uint32_t val; 8470 struct { 8471 uint32_t app_reserve:8; 8472 uint32_t table_id:15; 8473 uint32_t transfer:1; 8474 uint32_t _unused_:8; 8475 }; 8476 }; 8477 8478 static bool 8479 mlx5_access_tunnel_offload_db 8480 (struct rte_eth_dev *dev, 8481 bool (*match)(struct rte_eth_dev *, 8482 struct mlx5_flow_tunnel *, const void *), 8483 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8484 void (*miss)(struct rte_eth_dev *, void *), 8485 void *ctx, bool lock_op); 8486 8487 static int 8488 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 8489 struct rte_flow *flow, 8490 const struct rte_flow_attr *attr, 8491 const struct rte_flow_action *app_actions, 8492 uint32_t flow_idx, 8493 const struct mlx5_flow_tunnel *tunnel, 8494 struct tunnel_default_miss_ctx *ctx, 8495 struct rte_flow_error *error) 8496 { 8497 struct mlx5_priv *priv = dev->data->dev_private; 8498 struct mlx5_flow *dev_flow; 8499 struct rte_flow_attr miss_attr = *attr; 8500 const struct rte_flow_item miss_items[2] = { 8501 { 8502 .type = RTE_FLOW_ITEM_TYPE_ETH, 8503 .spec = NULL, 8504 .last = NULL, 8505 .mask = NULL 8506 }, 8507 { 8508 .type = RTE_FLOW_ITEM_TYPE_END, 8509 .spec = NULL, 8510 .last = NULL, 8511 .mask = NULL 8512 } 8513 }; 8514 union tunnel_offload_mark mark_id; 8515 struct rte_flow_action_mark miss_mark; 8516 struct rte_flow_action miss_actions[3] = { 8517 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 8518 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 8519 }; 8520 const struct rte_flow_action_jump *jump_data; 8521 uint32_t i, flow_table = 0; /* prevent compilation warning */ 8522 struct flow_grp_info grp_info = { 8523 .external = 1, 8524 .transfer = attr->transfer, 8525 .fdb_def_rule = !!priv->fdb_def_rule, 8526 .std_tbl_fix = 0, 8527 }; 8528 int ret; 8529 8530 if (!attr->transfer) { 8531 uint32_t q_size; 8532 8533 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 8534 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 8535 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 8536 0, SOCKET_ID_ANY); 8537 if (!ctx->queue) 8538 return rte_flow_error_set 8539 (error, ENOMEM, 8540 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8541 NULL, "invalid default miss RSS"); 8542 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8543 ctx->action_rss.level = 0, 8544 ctx->action_rss.types = priv->rss_conf.rss_hf, 8545 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 8546 ctx->action_rss.queue_num = priv->reta_idx_n, 8547 ctx->action_rss.key = priv->rss_conf.rss_key, 8548 ctx->action_rss.queue = ctx->queue; 8549 if (!priv->reta_idx_n || !priv->rxqs_n) 8550 return rte_flow_error_set 8551 (error, EINVAL, 8552 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8553 NULL, "invalid port configuration"); 8554 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 8555 ctx->action_rss.types = 0; 8556 for (i = 0; i != priv->reta_idx_n; ++i) 8557 ctx->queue[i] = (*priv->reta_idx)[i]; 8558 } else { 8559 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 8560 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 8561 } 8562 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 8563 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 8564 jump_data = app_actions->conf; 8565 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 8566 miss_attr.group = jump_data->group; 8567 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 8568 &flow_table, &grp_info, error); 8569 if (ret) 8570 return rte_flow_error_set(error, EINVAL, 8571 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8572 NULL, "invalid tunnel id"); 8573 mark_id.app_reserve = 0; 8574 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 8575 mark_id.transfer = !!attr->transfer; 8576 mark_id._unused_ = 0; 8577 miss_mark.id = mark_id.val; 8578 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 8579 miss_items, miss_actions, flow_idx, error); 8580 if (!dev_flow) 8581 return -rte_errno; 8582 dev_flow->flow = flow; 8583 dev_flow->external = true; 8584 dev_flow->tunnel = tunnel; 8585 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 8586 /* Subflow object was created, we must include one in the list. */ 8587 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 8588 dev_flow->handle, next); 8589 DRV_LOG(DEBUG, 8590 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 8591 dev->data->port_id, tunnel->app_tunnel.type, 8592 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 8593 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 8594 miss_actions, error); 8595 if (!ret) 8596 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 8597 error); 8598 8599 return ret; 8600 } 8601 8602 static const struct mlx5_flow_tbl_data_entry * 8603 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 8604 { 8605 struct mlx5_priv *priv = dev->data->dev_private; 8606 struct mlx5_dev_ctx_shared *sh = priv->sh; 8607 struct mlx5_hlist_entry *he; 8608 union tunnel_offload_mark mbits = { .val = mark }; 8609 union mlx5_flow_tbl_key table_key = { 8610 { 8611 .level = tunnel_id_to_flow_tbl(mbits.table_id), 8612 .id = 0, 8613 .reserved = 0, 8614 .dummy = 0, 8615 .is_fdb = !!mbits.transfer, 8616 .is_egress = 0, 8617 } 8618 }; 8619 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 8620 return he ? 8621 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 8622 } 8623 8624 static void 8625 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 8626 struct mlx5_hlist_entry *entry) 8627 { 8628 struct mlx5_dev_ctx_shared *sh = list->ctx; 8629 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8630 8631 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8632 tunnel_flow_tbl_to_id(tte->flow_table)); 8633 mlx5_free(tte); 8634 } 8635 8636 static int 8637 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused, 8638 struct mlx5_hlist_entry *entry, 8639 uint64_t key, void *cb_ctx __rte_unused) 8640 { 8641 union tunnel_tbl_key tbl = { 8642 .val = key, 8643 }; 8644 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8645 8646 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 8647 } 8648 8649 static struct mlx5_hlist_entry * 8650 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key, 8651 void *ctx __rte_unused) 8652 { 8653 struct mlx5_dev_ctx_shared *sh = list->ctx; 8654 struct tunnel_tbl_entry *tte; 8655 union tunnel_tbl_key tbl = { 8656 .val = key, 8657 }; 8658 8659 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 8660 sizeof(*tte), 0, 8661 SOCKET_ID_ANY); 8662 if (!tte) 8663 goto err; 8664 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8665 &tte->flow_table); 8666 if (tte->flow_table >= MLX5_MAX_TABLES) { 8667 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 8668 tte->flow_table); 8669 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8670 tte->flow_table); 8671 goto err; 8672 } else if (!tte->flow_table) { 8673 goto err; 8674 } 8675 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 8676 tte->tunnel_id = tbl.tunnel_id; 8677 tte->group = tbl.group; 8678 return &tte->hash; 8679 err: 8680 if (tte) 8681 mlx5_free(tte); 8682 return NULL; 8683 } 8684 8685 static uint32_t 8686 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 8687 const struct mlx5_flow_tunnel *tunnel, 8688 uint32_t group, uint32_t *table, 8689 struct rte_flow_error *error) 8690 { 8691 struct mlx5_hlist_entry *he; 8692 struct tunnel_tbl_entry *tte; 8693 union tunnel_tbl_key key = { 8694 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 8695 .group = group 8696 }; 8697 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8698 struct mlx5_hlist *group_hash; 8699 8700 group_hash = tunnel ? tunnel->groups : thub->groups; 8701 he = mlx5_hlist_register(group_hash, key.val, NULL); 8702 if (!he) 8703 return rte_flow_error_set(error, EINVAL, 8704 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8705 NULL, 8706 "tunnel group index not supported"); 8707 tte = container_of(he, typeof(*tte), hash); 8708 *table = tte->flow_table; 8709 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 8710 dev->data->port_id, key.tunnel_id, group, *table); 8711 return 0; 8712 } 8713 8714 static void 8715 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8716 struct mlx5_flow_tunnel *tunnel) 8717 { 8718 struct mlx5_priv *priv = dev->data->dev_private; 8719 struct mlx5_indexed_pool *ipool; 8720 8721 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8722 dev->data->port_id, tunnel->tunnel_id); 8723 LIST_REMOVE(tunnel, chain); 8724 mlx5_hlist_destroy(tunnel->groups); 8725 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8726 mlx5_ipool_free(ipool, tunnel->tunnel_id); 8727 } 8728 8729 static bool 8730 mlx5_access_tunnel_offload_db 8731 (struct rte_eth_dev *dev, 8732 bool (*match)(struct rte_eth_dev *, 8733 struct mlx5_flow_tunnel *, const void *), 8734 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8735 void (*miss)(struct rte_eth_dev *, void *), 8736 void *ctx, bool lock_op) 8737 { 8738 bool verdict = false; 8739 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8740 struct mlx5_flow_tunnel *tunnel; 8741 8742 rte_spinlock_lock(&thub->sl); 8743 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 8744 verdict = match(dev, tunnel, (const void *)ctx); 8745 if (verdict) 8746 break; 8747 } 8748 if (!lock_op) 8749 rte_spinlock_unlock(&thub->sl); 8750 if (verdict && hit) 8751 hit(dev, tunnel, ctx); 8752 if (!verdict && miss) 8753 miss(dev, ctx); 8754 if (lock_op) 8755 rte_spinlock_unlock(&thub->sl); 8756 8757 return verdict; 8758 } 8759 8760 struct tunnel_db_find_tunnel_id_ctx { 8761 uint32_t tunnel_id; 8762 struct mlx5_flow_tunnel *tunnel; 8763 }; 8764 8765 static bool 8766 find_tunnel_id_match(struct rte_eth_dev *dev, 8767 struct mlx5_flow_tunnel *tunnel, const void *x) 8768 { 8769 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8770 8771 RTE_SET_USED(dev); 8772 return tunnel->tunnel_id == ctx->tunnel_id; 8773 } 8774 8775 static void 8776 find_tunnel_id_hit(struct rte_eth_dev *dev, 8777 struct mlx5_flow_tunnel *tunnel, void *x) 8778 { 8779 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8780 RTE_SET_USED(dev); 8781 ctx->tunnel = tunnel; 8782 } 8783 8784 static struct mlx5_flow_tunnel * 8785 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 8786 { 8787 struct tunnel_db_find_tunnel_id_ctx ctx = { 8788 .tunnel_id = id, 8789 }; 8790 8791 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 8792 find_tunnel_id_hit, NULL, &ctx, true); 8793 8794 return ctx.tunnel; 8795 } 8796 8797 static struct mlx5_flow_tunnel * 8798 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 8799 const struct rte_flow_tunnel *app_tunnel) 8800 { 8801 struct mlx5_priv *priv = dev->data->dev_private; 8802 struct mlx5_indexed_pool *ipool; 8803 struct mlx5_flow_tunnel *tunnel; 8804 uint32_t id; 8805 8806 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8807 tunnel = mlx5_ipool_zmalloc(ipool, &id); 8808 if (!tunnel) 8809 return NULL; 8810 if (id >= MLX5_MAX_TUNNELS) { 8811 mlx5_ipool_free(ipool, id); 8812 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 8813 return NULL; 8814 } 8815 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 8816 mlx5_flow_tunnel_grp2tbl_create_cb, 8817 mlx5_flow_tunnel_grp2tbl_match_cb, 8818 mlx5_flow_tunnel_grp2tbl_remove_cb); 8819 if (!tunnel->groups) { 8820 mlx5_ipool_free(ipool, id); 8821 return NULL; 8822 } 8823 tunnel->groups->ctx = priv->sh; 8824 /* initiate new PMD tunnel */ 8825 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8826 tunnel->tunnel_id = id; 8827 tunnel->action.type = (typeof(tunnel->action.type)) 8828 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8829 tunnel->action.conf = tunnel; 8830 tunnel->item.type = (typeof(tunnel->item.type)) 8831 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8832 tunnel->item.spec = tunnel; 8833 tunnel->item.last = NULL; 8834 tunnel->item.mask = NULL; 8835 8836 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8837 dev->data->port_id, tunnel->tunnel_id); 8838 8839 return tunnel; 8840 } 8841 8842 struct tunnel_db_get_tunnel_ctx { 8843 const struct rte_flow_tunnel *app_tunnel; 8844 struct mlx5_flow_tunnel *tunnel; 8845 }; 8846 8847 static bool get_tunnel_match(struct rte_eth_dev *dev, 8848 struct mlx5_flow_tunnel *tunnel, const void *x) 8849 { 8850 const struct tunnel_db_get_tunnel_ctx *ctx = x; 8851 8852 RTE_SET_USED(dev); 8853 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 8854 sizeof(*ctx->app_tunnel)); 8855 } 8856 8857 static void get_tunnel_hit(struct rte_eth_dev *dev, 8858 struct mlx5_flow_tunnel *tunnel, void *x) 8859 { 8860 /* called under tunnel spinlock protection */ 8861 struct tunnel_db_get_tunnel_ctx *ctx = x; 8862 8863 RTE_SET_USED(dev); 8864 tunnel->refctn++; 8865 ctx->tunnel = tunnel; 8866 } 8867 8868 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 8869 { 8870 /* called under tunnel spinlock protection */ 8871 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8872 struct tunnel_db_get_tunnel_ctx *ctx = x; 8873 8874 rte_spinlock_unlock(&thub->sl); 8875 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 8876 rte_spinlock_lock(&thub->sl); 8877 if (ctx->tunnel) { 8878 ctx->tunnel->refctn = 1; 8879 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 8880 } 8881 } 8882 8883 8884 static int 8885 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 8886 const struct rte_flow_tunnel *app_tunnel, 8887 struct mlx5_flow_tunnel **tunnel) 8888 { 8889 struct tunnel_db_get_tunnel_ctx ctx = { 8890 .app_tunnel = app_tunnel, 8891 }; 8892 8893 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 8894 get_tunnel_miss, &ctx, true); 8895 *tunnel = ctx.tunnel; 8896 return ctx.tunnel ? 0 : -ENOMEM; 8897 } 8898 8899 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 8900 { 8901 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 8902 8903 if (!thub) 8904 return; 8905 if (!LIST_EMPTY(&thub->tunnels)) 8906 DRV_LOG(WARNING, "port %u tunnels present", port_id); 8907 mlx5_hlist_destroy(thub->groups); 8908 mlx5_free(thub); 8909 } 8910 8911 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 8912 { 8913 int err; 8914 struct mlx5_flow_tunnel_hub *thub; 8915 8916 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 8917 0, SOCKET_ID_ANY); 8918 if (!thub) 8919 return -ENOMEM; 8920 LIST_INIT(&thub->tunnels); 8921 rte_spinlock_init(&thub->sl); 8922 thub->groups = mlx5_hlist_create("flow groups", 8923 rte_align32pow2(MLX5_MAX_TABLES), 0, 8924 0, mlx5_flow_tunnel_grp2tbl_create_cb, 8925 mlx5_flow_tunnel_grp2tbl_match_cb, 8926 mlx5_flow_tunnel_grp2tbl_remove_cb); 8927 if (!thub->groups) { 8928 err = -rte_errno; 8929 goto err; 8930 } 8931 thub->groups->ctx = sh; 8932 sh->tunnel_hub = thub; 8933 8934 return 0; 8935 8936 err: 8937 if (thub->groups) 8938 mlx5_hlist_destroy(thub->groups); 8939 if (thub) 8940 mlx5_free(thub); 8941 return err; 8942 } 8943 8944 static inline bool 8945 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 8946 struct rte_flow_tunnel *tunnel, 8947 const char *err_msg) 8948 { 8949 err_msg = NULL; 8950 if (!is_tunnel_offload_active(dev)) { 8951 err_msg = "tunnel offload was not activated"; 8952 goto out; 8953 } else if (!tunnel) { 8954 err_msg = "no application tunnel"; 8955 goto out; 8956 } 8957 8958 switch (tunnel->type) { 8959 default: 8960 err_msg = "unsupported tunnel type"; 8961 goto out; 8962 case RTE_FLOW_ITEM_TYPE_VXLAN: 8963 break; 8964 } 8965 8966 out: 8967 return !err_msg; 8968 } 8969 8970 static int 8971 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 8972 struct rte_flow_tunnel *app_tunnel, 8973 struct rte_flow_action **actions, 8974 uint32_t *num_of_actions, 8975 struct rte_flow_error *error) 8976 { 8977 int ret; 8978 struct mlx5_flow_tunnel *tunnel; 8979 const char *err_msg = NULL; 8980 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8981 8982 if (!verdict) 8983 return rte_flow_error_set(error, EINVAL, 8984 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8985 err_msg); 8986 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8987 if (ret < 0) { 8988 return rte_flow_error_set(error, ret, 8989 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8990 "failed to initialize pmd tunnel"); 8991 } 8992 *actions = &tunnel->action; 8993 *num_of_actions = 1; 8994 return 0; 8995 } 8996 8997 static int 8998 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 8999 struct rte_flow_tunnel *app_tunnel, 9000 struct rte_flow_item **items, 9001 uint32_t *num_of_items, 9002 struct rte_flow_error *error) 9003 { 9004 int ret; 9005 struct mlx5_flow_tunnel *tunnel; 9006 const char *err_msg = NULL; 9007 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 9008 9009 if (!verdict) 9010 return rte_flow_error_set(error, EINVAL, 9011 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9012 err_msg); 9013 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9014 if (ret < 0) { 9015 return rte_flow_error_set(error, ret, 9016 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9017 "failed to initialize pmd tunnel"); 9018 } 9019 *items = &tunnel->item; 9020 *num_of_items = 1; 9021 return 0; 9022 } 9023 9024 struct tunnel_db_element_release_ctx { 9025 struct rte_flow_item *items; 9026 struct rte_flow_action *actions; 9027 uint32_t num_elements; 9028 struct rte_flow_error *error; 9029 int ret; 9030 }; 9031 9032 static bool 9033 tunnel_element_release_match(struct rte_eth_dev *dev, 9034 struct mlx5_flow_tunnel *tunnel, const void *x) 9035 { 9036 const struct tunnel_db_element_release_ctx *ctx = x; 9037 9038 RTE_SET_USED(dev); 9039 if (ctx->num_elements != 1) 9040 return false; 9041 else if (ctx->items) 9042 return ctx->items == &tunnel->item; 9043 else if (ctx->actions) 9044 return ctx->actions == &tunnel->action; 9045 9046 return false; 9047 } 9048 9049 static void 9050 tunnel_element_release_hit(struct rte_eth_dev *dev, 9051 struct mlx5_flow_tunnel *tunnel, void *x) 9052 { 9053 struct tunnel_db_element_release_ctx *ctx = x; 9054 ctx->ret = 0; 9055 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9056 mlx5_flow_tunnel_free(dev, tunnel); 9057 } 9058 9059 static void 9060 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9061 { 9062 struct tunnel_db_element_release_ctx *ctx = x; 9063 RTE_SET_USED(dev); 9064 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9065 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9066 "invalid argument"); 9067 } 9068 9069 static int 9070 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9071 struct rte_flow_item *pmd_items, 9072 uint32_t num_items, struct rte_flow_error *err) 9073 { 9074 struct tunnel_db_element_release_ctx ctx = { 9075 .items = pmd_items, 9076 .actions = NULL, 9077 .num_elements = num_items, 9078 .error = err, 9079 }; 9080 9081 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9082 tunnel_element_release_hit, 9083 tunnel_element_release_miss, &ctx, false); 9084 9085 return ctx.ret; 9086 } 9087 9088 static int 9089 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9090 struct rte_flow_action *pmd_actions, 9091 uint32_t num_actions, struct rte_flow_error *err) 9092 { 9093 struct tunnel_db_element_release_ctx ctx = { 9094 .items = NULL, 9095 .actions = pmd_actions, 9096 .num_elements = num_actions, 9097 .error = err, 9098 }; 9099 9100 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9101 tunnel_element_release_hit, 9102 tunnel_element_release_miss, &ctx, false); 9103 9104 return ctx.ret; 9105 } 9106 9107 static int 9108 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9109 struct rte_mbuf *m, 9110 struct rte_flow_restore_info *info, 9111 struct rte_flow_error *err) 9112 { 9113 uint64_t ol_flags = m->ol_flags; 9114 const struct mlx5_flow_tbl_data_entry *tble; 9115 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 9116 9117 if (!is_tunnel_offload_active(dev)) { 9118 info->flags = 0; 9119 return 0; 9120 } 9121 9122 if ((ol_flags & mask) != mask) 9123 goto err; 9124 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9125 if (!tble) { 9126 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9127 dev->data->port_id, m->hash.fdir.hi); 9128 goto err; 9129 } 9130 MLX5_ASSERT(tble->tunnel); 9131 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9132 info->group_id = tble->group_id; 9133 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9134 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9135 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9136 9137 return 0; 9138 9139 err: 9140 return rte_flow_error_set(err, EINVAL, 9141 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9142 "failed to get restore info"); 9143 } 9144 9145 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9146 static int 9147 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9148 __rte_unused struct rte_flow_tunnel *app_tunnel, 9149 __rte_unused struct rte_flow_action **actions, 9150 __rte_unused uint32_t *num_of_actions, 9151 __rte_unused struct rte_flow_error *error) 9152 { 9153 return -ENOTSUP; 9154 } 9155 9156 static int 9157 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9158 __rte_unused struct rte_flow_tunnel *app_tunnel, 9159 __rte_unused struct rte_flow_item **items, 9160 __rte_unused uint32_t *num_of_items, 9161 __rte_unused struct rte_flow_error *error) 9162 { 9163 return -ENOTSUP; 9164 } 9165 9166 static int 9167 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9168 __rte_unused struct rte_flow_item *pmd_items, 9169 __rte_unused uint32_t num_items, 9170 __rte_unused struct rte_flow_error *err) 9171 { 9172 return -ENOTSUP; 9173 } 9174 9175 static int 9176 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9177 __rte_unused struct rte_flow_action *pmd_action, 9178 __rte_unused uint32_t num_actions, 9179 __rte_unused struct rte_flow_error *err) 9180 { 9181 return -ENOTSUP; 9182 } 9183 9184 static int 9185 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9186 __rte_unused struct rte_mbuf *m, 9187 __rte_unused struct rte_flow_restore_info *i, 9188 __rte_unused struct rte_flow_error *err) 9189 { 9190 return -ENOTSUP; 9191 } 9192 9193 static int 9194 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9195 __rte_unused struct rte_flow *flow, 9196 __rte_unused const struct rte_flow_attr *attr, 9197 __rte_unused const struct rte_flow_action *actions, 9198 __rte_unused uint32_t flow_idx, 9199 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9200 __rte_unused struct tunnel_default_miss_ctx *ctx, 9201 __rte_unused struct rte_flow_error *error) 9202 { 9203 return -ENOTSUP; 9204 } 9205 9206 static struct mlx5_flow_tunnel * 9207 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9208 __rte_unused uint32_t id) 9209 { 9210 return NULL; 9211 } 9212 9213 static void 9214 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9215 __rte_unused struct mlx5_flow_tunnel *tunnel) 9216 { 9217 } 9218 9219 static uint32_t 9220 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9221 __rte_unused const struct mlx5_flow_tunnel *t, 9222 __rte_unused uint32_t group, 9223 __rte_unused uint32_t *table, 9224 struct rte_flow_error *error) 9225 { 9226 return rte_flow_error_set(error, ENOTSUP, 9227 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9228 "tunnel offload requires DV support"); 9229 } 9230 9231 void 9232 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9233 __rte_unused uint16_t port_id) 9234 { 9235 } 9236 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9237 9238 static void 9239 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9240 { 9241 int ret; 9242 struct rte_flow_error error; 9243 9244 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9245 char *item_name; 9246 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9247 sizeof(item_name), 9248 (void *)(uintptr_t)item->type, &error); 9249 if (ret > 0) 9250 printf("%s ", item_name); 9251 else 9252 printf("%d\n", (int)item->type); 9253 } 9254 printf("END\n"); 9255 } 9256