1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see ETH_RSS_* definitions). 102 */ 103 uint8_t optional; 104 /**< optional expand field. Default 0 to expand, 1 not go deeper. */ 105 }; 106 107 /** Object returned by mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_rss { 109 uint32_t entries; 110 /**< Number of entries @p patterns and @p priorities. */ 111 struct { 112 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 113 uint32_t priority; /**< Priority offset for each expansion. */ 114 } entry[]; 115 }; 116 117 static void 118 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 119 120 static bool 121 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 122 { 123 switch (item->type) { 124 case RTE_FLOW_ITEM_TYPE_ETH: 125 case RTE_FLOW_ITEM_TYPE_VLAN: 126 case RTE_FLOW_ITEM_TYPE_IPV4: 127 case RTE_FLOW_ITEM_TYPE_IPV6: 128 case RTE_FLOW_ITEM_TYPE_UDP: 129 case RTE_FLOW_ITEM_TYPE_TCP: 130 case RTE_FLOW_ITEM_TYPE_VXLAN: 131 case RTE_FLOW_ITEM_TYPE_NVGRE: 132 case RTE_FLOW_ITEM_TYPE_GRE: 133 case RTE_FLOW_ITEM_TYPE_GENEVE: 134 case RTE_FLOW_ITEM_TYPE_MPLS: 135 return true; 136 default: 137 break; 138 } 139 return false; 140 } 141 142 static enum rte_flow_item_type 143 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 144 { 145 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 146 uint16_t ether_type = 0; 147 uint16_t ether_type_m; 148 uint8_t ip_next_proto = 0; 149 uint8_t ip_next_proto_m; 150 151 if (item == NULL || item->spec == NULL) 152 return ret; 153 switch (item->type) { 154 case RTE_FLOW_ITEM_TYPE_ETH: 155 if (item->mask) 156 ether_type_m = ((const struct rte_flow_item_eth *) 157 (item->mask))->type; 158 else 159 ether_type_m = rte_flow_item_eth_mask.type; 160 if (ether_type_m != RTE_BE16(0xFFFF)) 161 break; 162 ether_type = ((const struct rte_flow_item_eth *) 163 (item->spec))->type; 164 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 165 ret = RTE_FLOW_ITEM_TYPE_IPV4; 166 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 167 ret = RTE_FLOW_ITEM_TYPE_IPV6; 168 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 169 ret = RTE_FLOW_ITEM_TYPE_VLAN; 170 else 171 ret = RTE_FLOW_ITEM_TYPE_END; 172 break; 173 case RTE_FLOW_ITEM_TYPE_VLAN: 174 if (item->mask) 175 ether_type_m = ((const struct rte_flow_item_vlan *) 176 (item->mask))->inner_type; 177 else 178 ether_type_m = rte_flow_item_vlan_mask.inner_type; 179 if (ether_type_m != RTE_BE16(0xFFFF)) 180 break; 181 ether_type = ((const struct rte_flow_item_vlan *) 182 (item->spec))->inner_type; 183 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 184 ret = RTE_FLOW_ITEM_TYPE_IPV4; 185 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 186 ret = RTE_FLOW_ITEM_TYPE_IPV6; 187 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 188 ret = RTE_FLOW_ITEM_TYPE_VLAN; 189 else 190 ret = RTE_FLOW_ITEM_TYPE_END; 191 break; 192 case RTE_FLOW_ITEM_TYPE_IPV4: 193 if (item->mask) 194 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 195 (item->mask))->hdr.next_proto_id; 196 else 197 ip_next_proto_m = 198 rte_flow_item_ipv4_mask.hdr.next_proto_id; 199 if (ip_next_proto_m != 0xFF) 200 break; 201 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 202 (item->spec))->hdr.next_proto_id; 203 if (ip_next_proto == IPPROTO_UDP) 204 ret = RTE_FLOW_ITEM_TYPE_UDP; 205 else if (ip_next_proto == IPPROTO_TCP) 206 ret = RTE_FLOW_ITEM_TYPE_TCP; 207 else if (ip_next_proto == IPPROTO_IP) 208 ret = RTE_FLOW_ITEM_TYPE_IPV4; 209 else if (ip_next_proto == IPPROTO_IPV6) 210 ret = RTE_FLOW_ITEM_TYPE_IPV6; 211 else 212 ret = RTE_FLOW_ITEM_TYPE_END; 213 break; 214 case RTE_FLOW_ITEM_TYPE_IPV6: 215 if (item->mask) 216 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 217 (item->mask))->hdr.proto; 218 else 219 ip_next_proto_m = 220 rte_flow_item_ipv6_mask.hdr.proto; 221 if (ip_next_proto_m != 0xFF) 222 break; 223 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 224 (item->spec))->hdr.proto; 225 if (ip_next_proto == IPPROTO_UDP) 226 ret = RTE_FLOW_ITEM_TYPE_UDP; 227 else if (ip_next_proto == IPPROTO_TCP) 228 ret = RTE_FLOW_ITEM_TYPE_TCP; 229 else if (ip_next_proto == IPPROTO_IP) 230 ret = RTE_FLOW_ITEM_TYPE_IPV4; 231 else if (ip_next_proto == IPPROTO_IPV6) 232 ret = RTE_FLOW_ITEM_TYPE_IPV6; 233 else 234 ret = RTE_FLOW_ITEM_TYPE_END; 235 break; 236 default: 237 ret = RTE_FLOW_ITEM_TYPE_VOID; 238 break; 239 } 240 return ret; 241 } 242 243 #define MLX5_RSS_EXP_ELT_N 16 244 245 /** 246 * Expand RSS flows into several possible flows according to the RSS hash 247 * fields requested and the driver capabilities. 248 * 249 * @param[out] buf 250 * Buffer to store the result expansion. 251 * @param[in] size 252 * Buffer size in bytes. If 0, @p buf can be NULL. 253 * @param[in] pattern 254 * User flow pattern. 255 * @param[in] types 256 * RSS types to expand (see ETH_RSS_* definitions). 257 * @param[in] graph 258 * Input graph to expand @p pattern according to @p types. 259 * @param[in] graph_root_index 260 * Index of root node in @p graph, typically 0. 261 * 262 * @return 263 * A positive value representing the size of @p buf in bytes regardless of 264 * @p size on success, a negative errno value otherwise and rte_errno is 265 * set, the following errors are defined: 266 * 267 * -E2BIG: graph-depth @p graph is too deep. 268 * -EINVAL: @p size has not enough space for expanded pattern. 269 */ 270 static int 271 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 272 const struct rte_flow_item *pattern, uint64_t types, 273 const struct mlx5_flow_expand_node graph[], 274 int graph_root_index) 275 { 276 const struct rte_flow_item *item; 277 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 278 const int *next_node; 279 const int *stack[MLX5_RSS_EXP_ELT_N]; 280 int stack_pos = 0; 281 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 282 unsigned int i; 283 size_t lsize; 284 size_t user_pattern_size = 0; 285 void *addr = NULL; 286 const struct mlx5_flow_expand_node *next = NULL; 287 struct rte_flow_item missed_item; 288 int missed = 0; 289 int elt = 0; 290 const struct rte_flow_item *last_item = NULL; 291 292 memset(&missed_item, 0, sizeof(missed_item)); 293 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 294 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 295 if (lsize > size) 296 return -EINVAL; 297 buf->entry[0].priority = 0; 298 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 299 buf->entries = 0; 300 addr = buf->entry[0].pattern; 301 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 302 if (!mlx5_flow_is_rss_expandable_item(item)) { 303 user_pattern_size += sizeof(*item); 304 continue; 305 } 306 last_item = item; 307 for (i = 0; node->next && node->next[i]; ++i) { 308 next = &graph[node->next[i]]; 309 if (next->type == item->type) 310 break; 311 } 312 if (next) 313 node = next; 314 user_pattern_size += sizeof(*item); 315 } 316 user_pattern_size += sizeof(*item); /* Handle END item. */ 317 lsize += user_pattern_size; 318 if (lsize > size) 319 return -EINVAL; 320 /* Copy the user pattern in the first entry of the buffer. */ 321 rte_memcpy(addr, pattern, user_pattern_size); 322 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 323 buf->entries = 1; 324 /* Start expanding. */ 325 memset(flow_items, 0, sizeof(flow_items)); 326 user_pattern_size -= sizeof(*item); 327 /* 328 * Check if the last valid item has spec set, need complete pattern, 329 * and the pattern can be used for expansion. 330 */ 331 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 332 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 333 /* Item type END indicates expansion is not required. */ 334 return lsize; 335 } 336 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 337 next = NULL; 338 missed = 1; 339 for (i = 0; node->next && node->next[i]; ++i) { 340 next = &graph[node->next[i]]; 341 if (next->type == missed_item.type) { 342 flow_items[0].type = missed_item.type; 343 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 344 break; 345 } 346 next = NULL; 347 } 348 } 349 if (next && missed) { 350 elt = 2; /* missed item + item end. */ 351 node = next; 352 lsize += elt * sizeof(*item) + user_pattern_size; 353 if (lsize > size) 354 return -EINVAL; 355 if (node->rss_types & types) { 356 buf->entry[buf->entries].priority = 1; 357 buf->entry[buf->entries].pattern = addr; 358 buf->entries++; 359 rte_memcpy(addr, buf->entry[0].pattern, 360 user_pattern_size); 361 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 362 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 363 addr = (void *)(((uintptr_t)addr) + 364 elt * sizeof(*item)); 365 } 366 } 367 memset(flow_items, 0, sizeof(flow_items)); 368 next_node = node->next; 369 stack[stack_pos] = next_node; 370 node = next_node ? &graph[*next_node] : NULL; 371 while (node) { 372 flow_items[stack_pos].type = node->type; 373 if (node->rss_types & types) { 374 size_t n; 375 /* 376 * compute the number of items to copy from the 377 * expansion and copy it. 378 * When the stack_pos is 0, there are 1 element in it, 379 * plus the addition END item. 380 */ 381 elt = stack_pos + 2; 382 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 383 lsize += elt * sizeof(*item) + user_pattern_size; 384 if (lsize > size) 385 return -EINVAL; 386 n = elt * sizeof(*item); 387 buf->entry[buf->entries].priority = 388 stack_pos + 1 + missed; 389 buf->entry[buf->entries].pattern = addr; 390 buf->entries++; 391 rte_memcpy(addr, buf->entry[0].pattern, 392 user_pattern_size); 393 addr = (void *)(((uintptr_t)addr) + 394 user_pattern_size); 395 rte_memcpy(addr, &missed_item, 396 missed * sizeof(*item)); 397 addr = (void *)(((uintptr_t)addr) + 398 missed * sizeof(*item)); 399 rte_memcpy(addr, flow_items, n); 400 addr = (void *)(((uintptr_t)addr) + n); 401 } 402 /* Go deeper. */ 403 if (!node->optional && node->next) { 404 next_node = node->next; 405 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 406 rte_errno = E2BIG; 407 return -rte_errno; 408 } 409 stack[stack_pos] = next_node; 410 } else if (*(next_node + 1)) { 411 /* Follow up with the next possibility. */ 412 ++next_node; 413 } else { 414 /* Move to the next path. */ 415 if (stack_pos) 416 next_node = stack[--stack_pos]; 417 next_node++; 418 stack[stack_pos] = next_node; 419 } 420 node = *next_node ? &graph[*next_node] : NULL; 421 }; 422 return lsize; 423 } 424 425 enum mlx5_expansion { 426 MLX5_EXPANSION_ROOT, 427 MLX5_EXPANSION_ROOT_OUTER, 428 MLX5_EXPANSION_ROOT_ETH_VLAN, 429 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 430 MLX5_EXPANSION_OUTER_ETH, 431 MLX5_EXPANSION_OUTER_ETH_VLAN, 432 MLX5_EXPANSION_OUTER_VLAN, 433 MLX5_EXPANSION_OUTER_IPV4, 434 MLX5_EXPANSION_OUTER_IPV4_UDP, 435 MLX5_EXPANSION_OUTER_IPV4_TCP, 436 MLX5_EXPANSION_OUTER_IPV6, 437 MLX5_EXPANSION_OUTER_IPV6_UDP, 438 MLX5_EXPANSION_OUTER_IPV6_TCP, 439 MLX5_EXPANSION_VXLAN, 440 MLX5_EXPANSION_VXLAN_GPE, 441 MLX5_EXPANSION_GRE, 442 MLX5_EXPANSION_NVGRE, 443 MLX5_EXPANSION_GRE_KEY, 444 MLX5_EXPANSION_MPLS, 445 MLX5_EXPANSION_ETH, 446 MLX5_EXPANSION_ETH_VLAN, 447 MLX5_EXPANSION_VLAN, 448 MLX5_EXPANSION_IPV4, 449 MLX5_EXPANSION_IPV4_UDP, 450 MLX5_EXPANSION_IPV4_TCP, 451 MLX5_EXPANSION_IPV6, 452 MLX5_EXPANSION_IPV6_UDP, 453 MLX5_EXPANSION_IPV6_TCP, 454 }; 455 456 /** Supported expansion of items. */ 457 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 458 [MLX5_EXPANSION_ROOT] = { 459 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 460 MLX5_EXPANSION_IPV4, 461 MLX5_EXPANSION_IPV6), 462 .type = RTE_FLOW_ITEM_TYPE_END, 463 }, 464 [MLX5_EXPANSION_ROOT_OUTER] = { 465 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 466 MLX5_EXPANSION_OUTER_IPV4, 467 MLX5_EXPANSION_OUTER_IPV6), 468 .type = RTE_FLOW_ITEM_TYPE_END, 469 }, 470 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 471 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 472 .type = RTE_FLOW_ITEM_TYPE_END, 473 }, 474 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 475 .next = MLX5_FLOW_EXPAND_RSS_NEXT 476 (MLX5_EXPANSION_OUTER_ETH_VLAN), 477 .type = RTE_FLOW_ITEM_TYPE_END, 478 }, 479 [MLX5_EXPANSION_OUTER_ETH] = { 480 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 481 MLX5_EXPANSION_OUTER_IPV6), 482 .type = RTE_FLOW_ITEM_TYPE_ETH, 483 .rss_types = 0, 484 }, 485 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 486 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 487 .type = RTE_FLOW_ITEM_TYPE_ETH, 488 .rss_types = 0, 489 }, 490 [MLX5_EXPANSION_OUTER_VLAN] = { 491 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 492 MLX5_EXPANSION_OUTER_IPV6), 493 .type = RTE_FLOW_ITEM_TYPE_VLAN, 494 }, 495 [MLX5_EXPANSION_OUTER_IPV4] = { 496 .next = MLX5_FLOW_EXPAND_RSS_NEXT 497 (MLX5_EXPANSION_OUTER_IPV4_UDP, 498 MLX5_EXPANSION_OUTER_IPV4_TCP, 499 MLX5_EXPANSION_GRE, 500 MLX5_EXPANSION_NVGRE, 501 MLX5_EXPANSION_IPV4, 502 MLX5_EXPANSION_IPV6), 503 .type = RTE_FLOW_ITEM_TYPE_IPV4, 504 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 505 ETH_RSS_NONFRAG_IPV4_OTHER, 506 }, 507 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 508 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 509 MLX5_EXPANSION_VXLAN_GPE, 510 MLX5_EXPANSION_MPLS), 511 .type = RTE_FLOW_ITEM_TYPE_UDP, 512 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 513 }, 514 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 515 .type = RTE_FLOW_ITEM_TYPE_TCP, 516 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 517 }, 518 [MLX5_EXPANSION_OUTER_IPV6] = { 519 .next = MLX5_FLOW_EXPAND_RSS_NEXT 520 (MLX5_EXPANSION_OUTER_IPV6_UDP, 521 MLX5_EXPANSION_OUTER_IPV6_TCP, 522 MLX5_EXPANSION_IPV4, 523 MLX5_EXPANSION_IPV6, 524 MLX5_EXPANSION_GRE, 525 MLX5_EXPANSION_NVGRE), 526 .type = RTE_FLOW_ITEM_TYPE_IPV6, 527 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 528 ETH_RSS_NONFRAG_IPV6_OTHER, 529 }, 530 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 531 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 532 MLX5_EXPANSION_VXLAN_GPE, 533 MLX5_EXPANSION_MPLS), 534 .type = RTE_FLOW_ITEM_TYPE_UDP, 535 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 536 }, 537 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 538 .type = RTE_FLOW_ITEM_TYPE_TCP, 539 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 540 }, 541 [MLX5_EXPANSION_VXLAN] = { 542 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 543 MLX5_EXPANSION_IPV4, 544 MLX5_EXPANSION_IPV6), 545 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 546 }, 547 [MLX5_EXPANSION_VXLAN_GPE] = { 548 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 549 MLX5_EXPANSION_IPV4, 550 MLX5_EXPANSION_IPV6), 551 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 552 }, 553 [MLX5_EXPANSION_GRE] = { 554 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 555 MLX5_EXPANSION_IPV6, 556 MLX5_EXPANSION_GRE_KEY, 557 MLX5_EXPANSION_MPLS), 558 .type = RTE_FLOW_ITEM_TYPE_GRE, 559 }, 560 [MLX5_EXPANSION_GRE_KEY] = { 561 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 562 MLX5_EXPANSION_IPV6, 563 MLX5_EXPANSION_MPLS), 564 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 565 .optional = 1, 566 }, 567 [MLX5_EXPANSION_NVGRE] = { 568 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 569 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 570 }, 571 [MLX5_EXPANSION_MPLS] = { 572 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 573 MLX5_EXPANSION_IPV6, 574 MLX5_EXPANSION_ETH), 575 .type = RTE_FLOW_ITEM_TYPE_MPLS, 576 }, 577 [MLX5_EXPANSION_ETH] = { 578 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 579 MLX5_EXPANSION_IPV6), 580 .type = RTE_FLOW_ITEM_TYPE_ETH, 581 }, 582 [MLX5_EXPANSION_ETH_VLAN] = { 583 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 584 .type = RTE_FLOW_ITEM_TYPE_ETH, 585 }, 586 [MLX5_EXPANSION_VLAN] = { 587 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 588 MLX5_EXPANSION_IPV6), 589 .type = RTE_FLOW_ITEM_TYPE_VLAN, 590 }, 591 [MLX5_EXPANSION_IPV4] = { 592 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 593 MLX5_EXPANSION_IPV4_TCP), 594 .type = RTE_FLOW_ITEM_TYPE_IPV4, 595 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 596 ETH_RSS_NONFRAG_IPV4_OTHER, 597 }, 598 [MLX5_EXPANSION_IPV4_UDP] = { 599 .type = RTE_FLOW_ITEM_TYPE_UDP, 600 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 601 }, 602 [MLX5_EXPANSION_IPV4_TCP] = { 603 .type = RTE_FLOW_ITEM_TYPE_TCP, 604 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 605 }, 606 [MLX5_EXPANSION_IPV6] = { 607 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 608 MLX5_EXPANSION_IPV6_TCP), 609 .type = RTE_FLOW_ITEM_TYPE_IPV6, 610 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 611 ETH_RSS_NONFRAG_IPV6_OTHER, 612 }, 613 [MLX5_EXPANSION_IPV6_UDP] = { 614 .type = RTE_FLOW_ITEM_TYPE_UDP, 615 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 616 }, 617 [MLX5_EXPANSION_IPV6_TCP] = { 618 .type = RTE_FLOW_ITEM_TYPE_TCP, 619 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 620 }, 621 }; 622 623 static struct rte_flow_action_handle * 624 mlx5_action_handle_create(struct rte_eth_dev *dev, 625 const struct rte_flow_indir_action_conf *conf, 626 const struct rte_flow_action *action, 627 struct rte_flow_error *error); 628 static int mlx5_action_handle_destroy 629 (struct rte_eth_dev *dev, 630 struct rte_flow_action_handle *handle, 631 struct rte_flow_error *error); 632 static int mlx5_action_handle_update 633 (struct rte_eth_dev *dev, 634 struct rte_flow_action_handle *handle, 635 const void *update, 636 struct rte_flow_error *error); 637 static int mlx5_action_handle_query 638 (struct rte_eth_dev *dev, 639 const struct rte_flow_action_handle *handle, 640 void *data, 641 struct rte_flow_error *error); 642 static int 643 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 644 struct rte_flow_tunnel *app_tunnel, 645 struct rte_flow_action **actions, 646 uint32_t *num_of_actions, 647 struct rte_flow_error *error); 648 static int 649 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 650 struct rte_flow_tunnel *app_tunnel, 651 struct rte_flow_item **items, 652 uint32_t *num_of_items, 653 struct rte_flow_error *error); 654 static int 655 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 656 struct rte_flow_item *pmd_items, 657 uint32_t num_items, struct rte_flow_error *err); 658 static int 659 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 660 struct rte_flow_action *pmd_actions, 661 uint32_t num_actions, 662 struct rte_flow_error *err); 663 static int 664 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 665 struct rte_mbuf *m, 666 struct rte_flow_restore_info *info, 667 struct rte_flow_error *err); 668 669 static const struct rte_flow_ops mlx5_flow_ops = { 670 .validate = mlx5_flow_validate, 671 .create = mlx5_flow_create, 672 .destroy = mlx5_flow_destroy, 673 .flush = mlx5_flow_flush, 674 .isolate = mlx5_flow_isolate, 675 .query = mlx5_flow_query, 676 .dev_dump = mlx5_flow_dev_dump, 677 .get_aged_flows = mlx5_flow_get_aged_flows, 678 .action_handle_create = mlx5_action_handle_create, 679 .action_handle_destroy = mlx5_action_handle_destroy, 680 .action_handle_update = mlx5_action_handle_update, 681 .action_handle_query = mlx5_action_handle_query, 682 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 683 .tunnel_match = mlx5_flow_tunnel_match, 684 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 685 .tunnel_item_release = mlx5_flow_tunnel_item_release, 686 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 687 }; 688 689 /* Tunnel information. */ 690 struct mlx5_flow_tunnel_info { 691 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 692 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 693 }; 694 695 static struct mlx5_flow_tunnel_info tunnels_info[] = { 696 { 697 .tunnel = MLX5_FLOW_LAYER_VXLAN, 698 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 699 }, 700 { 701 .tunnel = MLX5_FLOW_LAYER_GENEVE, 702 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 703 }, 704 { 705 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 706 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 707 }, 708 { 709 .tunnel = MLX5_FLOW_LAYER_GRE, 710 .ptype = RTE_PTYPE_TUNNEL_GRE, 711 }, 712 { 713 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 714 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 715 }, 716 { 717 .tunnel = MLX5_FLOW_LAYER_MPLS, 718 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 719 }, 720 { 721 .tunnel = MLX5_FLOW_LAYER_NVGRE, 722 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 723 }, 724 { 725 .tunnel = MLX5_FLOW_LAYER_IPIP, 726 .ptype = RTE_PTYPE_TUNNEL_IP, 727 }, 728 { 729 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 730 .ptype = RTE_PTYPE_TUNNEL_IP, 731 }, 732 { 733 .tunnel = MLX5_FLOW_LAYER_GTP, 734 .ptype = RTE_PTYPE_TUNNEL_GTPU, 735 }, 736 }; 737 738 739 740 /** 741 * Translate tag ID to register. 742 * 743 * @param[in] dev 744 * Pointer to the Ethernet device structure. 745 * @param[in] feature 746 * The feature that request the register. 747 * @param[in] id 748 * The request register ID. 749 * @param[out] error 750 * Error description in case of any. 751 * 752 * @return 753 * The request register on success, a negative errno 754 * value otherwise and rte_errno is set. 755 */ 756 int 757 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 758 enum mlx5_feature_name feature, 759 uint32_t id, 760 struct rte_flow_error *error) 761 { 762 struct mlx5_priv *priv = dev->data->dev_private; 763 struct mlx5_dev_config *config = &priv->config; 764 enum modify_reg start_reg; 765 bool skip_mtr_reg = false; 766 767 switch (feature) { 768 case MLX5_HAIRPIN_RX: 769 return REG_B; 770 case MLX5_HAIRPIN_TX: 771 return REG_A; 772 case MLX5_METADATA_RX: 773 switch (config->dv_xmeta_en) { 774 case MLX5_XMETA_MODE_LEGACY: 775 return REG_B; 776 case MLX5_XMETA_MODE_META16: 777 return REG_C_0; 778 case MLX5_XMETA_MODE_META32: 779 return REG_C_1; 780 } 781 break; 782 case MLX5_METADATA_TX: 783 return REG_A; 784 case MLX5_METADATA_FDB: 785 switch (config->dv_xmeta_en) { 786 case MLX5_XMETA_MODE_LEGACY: 787 return REG_NON; 788 case MLX5_XMETA_MODE_META16: 789 return REG_C_0; 790 case MLX5_XMETA_MODE_META32: 791 return REG_C_1; 792 } 793 break; 794 case MLX5_FLOW_MARK: 795 switch (config->dv_xmeta_en) { 796 case MLX5_XMETA_MODE_LEGACY: 797 return REG_NON; 798 case MLX5_XMETA_MODE_META16: 799 return REG_C_1; 800 case MLX5_XMETA_MODE_META32: 801 return REG_C_0; 802 } 803 break; 804 case MLX5_MTR_ID: 805 /* 806 * If meter color and meter id share one register, flow match 807 * should use the meter color register for match. 808 */ 809 if (priv->mtr_reg_share) 810 return priv->mtr_color_reg; 811 else 812 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 813 REG_C_3; 814 case MLX5_MTR_COLOR: 815 case MLX5_ASO_FLOW_HIT: 816 case MLX5_ASO_CONNTRACK: 817 /* All features use the same REG_C. */ 818 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 819 return priv->mtr_color_reg; 820 case MLX5_COPY_MARK: 821 /* 822 * Metadata COPY_MARK register using is in meter suffix sub 823 * flow while with meter. It's safe to share the same register. 824 */ 825 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 826 case MLX5_APP_TAG: 827 /* 828 * If meter is enable, it will engage the register for color 829 * match and flow match. If meter color match is not using the 830 * REG_C_2, need to skip the REG_C_x be used by meter color 831 * match. 832 * If meter is disable, free to use all available registers. 833 */ 834 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 835 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 836 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 837 if (id > (uint32_t)(REG_C_7 - start_reg)) 838 return rte_flow_error_set(error, EINVAL, 839 RTE_FLOW_ERROR_TYPE_ITEM, 840 NULL, "invalid tag id"); 841 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 842 return rte_flow_error_set(error, ENOTSUP, 843 RTE_FLOW_ERROR_TYPE_ITEM, 844 NULL, "unsupported tag id"); 845 /* 846 * This case means meter is using the REG_C_x great than 2. 847 * Take care not to conflict with meter color REG_C_x. 848 * If the available index REG_C_y >= REG_C_x, skip the 849 * color register. 850 */ 851 if (skip_mtr_reg && config->flow_mreg_c 852 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 853 if (id >= (uint32_t)(REG_C_7 - start_reg)) 854 return rte_flow_error_set(error, EINVAL, 855 RTE_FLOW_ERROR_TYPE_ITEM, 856 NULL, "invalid tag id"); 857 if (config->flow_mreg_c 858 [id + 1 + start_reg - REG_C_0] != REG_NON) 859 return config->flow_mreg_c 860 [id + 1 + start_reg - REG_C_0]; 861 return rte_flow_error_set(error, ENOTSUP, 862 RTE_FLOW_ERROR_TYPE_ITEM, 863 NULL, "unsupported tag id"); 864 } 865 return config->flow_mreg_c[id + start_reg - REG_C_0]; 866 } 867 MLX5_ASSERT(false); 868 return rte_flow_error_set(error, EINVAL, 869 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 870 NULL, "invalid feature name"); 871 } 872 873 /** 874 * Check extensive flow metadata register support. 875 * 876 * @param dev 877 * Pointer to rte_eth_dev structure. 878 * 879 * @return 880 * True if device supports extensive flow metadata register, otherwise false. 881 */ 882 bool 883 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 884 { 885 struct mlx5_priv *priv = dev->data->dev_private; 886 struct mlx5_dev_config *config = &priv->config; 887 888 /* 889 * Having available reg_c can be regarded inclusively as supporting 890 * extensive flow metadata register, which could mean, 891 * - metadata register copy action by modify header. 892 * - 16 modify header actions is supported. 893 * - reg_c's are preserved across different domain (FDB and NIC) on 894 * packet loopback by flow lookup miss. 895 */ 896 return config->flow_mreg_c[2] != REG_NON; 897 } 898 899 /** 900 * Get the lowest priority. 901 * 902 * @param[in] dev 903 * Pointer to the Ethernet device structure. 904 * @param[in] attributes 905 * Pointer to device flow rule attributes. 906 * 907 * @return 908 * The value of lowest priority of flow. 909 */ 910 uint32_t 911 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 912 const struct rte_flow_attr *attr) 913 { 914 struct mlx5_priv *priv = dev->data->dev_private; 915 916 if (!attr->group && !attr->transfer) 917 return priv->config.flow_prio - 2; 918 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 919 } 920 921 /** 922 * Calculate matcher priority of the flow. 923 * 924 * @param[in] dev 925 * Pointer to the Ethernet device structure. 926 * @param[in] attr 927 * Pointer to device flow rule attributes. 928 * @param[in] subpriority 929 * The priority based on the items. 930 * @return 931 * The matcher priority of the flow. 932 */ 933 uint16_t 934 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 935 const struct rte_flow_attr *attr, 936 uint32_t subpriority) 937 { 938 uint16_t priority = (uint16_t)attr->priority; 939 struct mlx5_priv *priv = dev->data->dev_private; 940 941 if (!attr->group && !attr->transfer) { 942 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 943 priority = priv->config.flow_prio - 1; 944 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 945 } 946 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 947 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 948 return priority * 3 + subpriority; 949 } 950 951 /** 952 * Verify the @p item specifications (spec, last, mask) are compatible with the 953 * NIC capabilities. 954 * 955 * @param[in] item 956 * Item specification. 957 * @param[in] mask 958 * @p item->mask or flow default bit-masks. 959 * @param[in] nic_mask 960 * Bit-masks covering supported fields by the NIC to compare with user mask. 961 * @param[in] size 962 * Bit-masks size in bytes. 963 * @param[in] range_accepted 964 * True if range of values is accepted for specific fields, false otherwise. 965 * @param[out] error 966 * Pointer to error structure. 967 * 968 * @return 969 * 0 on success, a negative errno value otherwise and rte_errno is set. 970 */ 971 int 972 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 973 const uint8_t *mask, 974 const uint8_t *nic_mask, 975 unsigned int size, 976 bool range_accepted, 977 struct rte_flow_error *error) 978 { 979 unsigned int i; 980 981 MLX5_ASSERT(nic_mask); 982 for (i = 0; i < size; ++i) 983 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 984 return rte_flow_error_set(error, ENOTSUP, 985 RTE_FLOW_ERROR_TYPE_ITEM, 986 item, 987 "mask enables non supported" 988 " bits"); 989 if (!item->spec && (item->mask || item->last)) 990 return rte_flow_error_set(error, EINVAL, 991 RTE_FLOW_ERROR_TYPE_ITEM, item, 992 "mask/last without a spec is not" 993 " supported"); 994 if (item->spec && item->last && !range_accepted) { 995 uint8_t spec[size]; 996 uint8_t last[size]; 997 unsigned int i; 998 int ret; 999 1000 for (i = 0; i < size; ++i) { 1001 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1002 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1003 } 1004 ret = memcmp(spec, last, size); 1005 if (ret != 0) 1006 return rte_flow_error_set(error, EINVAL, 1007 RTE_FLOW_ERROR_TYPE_ITEM, 1008 item, 1009 "range is not valid"); 1010 } 1011 return 0; 1012 } 1013 1014 /** 1015 * Adjust the hash fields according to the @p flow information. 1016 * 1017 * @param[in] dev_flow. 1018 * Pointer to the mlx5_flow. 1019 * @param[in] tunnel 1020 * 1 when the hash field is for a tunnel item. 1021 * @param[in] layer_types 1022 * ETH_RSS_* types. 1023 * @param[in] hash_fields 1024 * Item hash fields. 1025 * 1026 * @return 1027 * The hash fields that should be used. 1028 */ 1029 uint64_t 1030 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1031 int tunnel __rte_unused, uint64_t layer_types, 1032 uint64_t hash_fields) 1033 { 1034 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1035 int rss_request_inner = rss_desc->level >= 2; 1036 1037 /* Check RSS hash level for tunnel. */ 1038 if (tunnel && rss_request_inner) 1039 hash_fields |= IBV_RX_HASH_INNER; 1040 else if (tunnel || rss_request_inner) 1041 return 0; 1042 #endif 1043 /* Check if requested layer matches RSS hash fields. */ 1044 if (!(rss_desc->types & layer_types)) 1045 return 0; 1046 return hash_fields; 1047 } 1048 1049 /** 1050 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1051 * if several tunnel rules are used on this queue, the tunnel ptype will be 1052 * cleared. 1053 * 1054 * @param rxq_ctrl 1055 * Rx queue to update. 1056 */ 1057 static void 1058 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1059 { 1060 unsigned int i; 1061 uint32_t tunnel_ptype = 0; 1062 1063 /* Look up for the ptype to use. */ 1064 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1065 if (!rxq_ctrl->flow_tunnels_n[i]) 1066 continue; 1067 if (!tunnel_ptype) { 1068 tunnel_ptype = tunnels_info[i].ptype; 1069 } else { 1070 tunnel_ptype = 0; 1071 break; 1072 } 1073 } 1074 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1075 } 1076 1077 /** 1078 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1079 * flow. 1080 * 1081 * @param[in] dev 1082 * Pointer to the Ethernet device structure. 1083 * @param[in] dev_handle 1084 * Pointer to device flow handle structure. 1085 */ 1086 void 1087 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1088 struct mlx5_flow_handle *dev_handle) 1089 { 1090 struct mlx5_priv *priv = dev->data->dev_private; 1091 const int mark = dev_handle->mark; 1092 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1093 struct mlx5_ind_table_obj *ind_tbl = NULL; 1094 unsigned int i; 1095 1096 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1097 struct mlx5_hrxq *hrxq; 1098 1099 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1100 dev_handle->rix_hrxq); 1101 if (hrxq) 1102 ind_tbl = hrxq->ind_table; 1103 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1104 struct mlx5_shared_action_rss *shared_rss; 1105 1106 shared_rss = mlx5_ipool_get 1107 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1108 dev_handle->rix_srss); 1109 if (shared_rss) 1110 ind_tbl = shared_rss->ind_tbl; 1111 } 1112 if (!ind_tbl) 1113 return; 1114 for (i = 0; i != ind_tbl->queues_n; ++i) { 1115 int idx = ind_tbl->queues[i]; 1116 struct mlx5_rxq_ctrl *rxq_ctrl = 1117 container_of((*priv->rxqs)[idx], 1118 struct mlx5_rxq_ctrl, rxq); 1119 1120 /* 1121 * To support metadata register copy on Tx loopback, 1122 * this must be always enabled (metadata may arive 1123 * from other port - not from local flows only. 1124 */ 1125 if (priv->config.dv_flow_en && 1126 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1127 mlx5_flow_ext_mreg_supported(dev)) { 1128 rxq_ctrl->rxq.mark = 1; 1129 rxq_ctrl->flow_mark_n = 1; 1130 } else if (mark) { 1131 rxq_ctrl->rxq.mark = 1; 1132 rxq_ctrl->flow_mark_n++; 1133 } 1134 if (tunnel) { 1135 unsigned int j; 1136 1137 /* Increase the counter matching the flow. */ 1138 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1139 if ((tunnels_info[j].tunnel & 1140 dev_handle->layers) == 1141 tunnels_info[j].tunnel) { 1142 rxq_ctrl->flow_tunnels_n[j]++; 1143 break; 1144 } 1145 } 1146 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1147 } 1148 } 1149 } 1150 1151 /** 1152 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1153 * 1154 * @param[in] dev 1155 * Pointer to the Ethernet device structure. 1156 * @param[in] flow 1157 * Pointer to flow structure. 1158 */ 1159 static void 1160 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1161 { 1162 struct mlx5_priv *priv = dev->data->dev_private; 1163 uint32_t handle_idx; 1164 struct mlx5_flow_handle *dev_handle; 1165 1166 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1167 handle_idx, dev_handle, next) 1168 flow_drv_rxq_flags_set(dev, dev_handle); 1169 } 1170 1171 /** 1172 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1173 * device flow if no other flow uses it with the same kind of request. 1174 * 1175 * @param dev 1176 * Pointer to Ethernet device. 1177 * @param[in] dev_handle 1178 * Pointer to the device flow handle structure. 1179 */ 1180 static void 1181 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1182 struct mlx5_flow_handle *dev_handle) 1183 { 1184 struct mlx5_priv *priv = dev->data->dev_private; 1185 const int mark = dev_handle->mark; 1186 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1187 struct mlx5_ind_table_obj *ind_tbl = NULL; 1188 unsigned int i; 1189 1190 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1191 struct mlx5_hrxq *hrxq; 1192 1193 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1194 dev_handle->rix_hrxq); 1195 if (hrxq) 1196 ind_tbl = hrxq->ind_table; 1197 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1198 struct mlx5_shared_action_rss *shared_rss; 1199 1200 shared_rss = mlx5_ipool_get 1201 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1202 dev_handle->rix_srss); 1203 if (shared_rss) 1204 ind_tbl = shared_rss->ind_tbl; 1205 } 1206 if (!ind_tbl) 1207 return; 1208 MLX5_ASSERT(dev->data->dev_started); 1209 for (i = 0; i != ind_tbl->queues_n; ++i) { 1210 int idx = ind_tbl->queues[i]; 1211 struct mlx5_rxq_ctrl *rxq_ctrl = 1212 container_of((*priv->rxqs)[idx], 1213 struct mlx5_rxq_ctrl, rxq); 1214 1215 if (priv->config.dv_flow_en && 1216 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1217 mlx5_flow_ext_mreg_supported(dev)) { 1218 rxq_ctrl->rxq.mark = 1; 1219 rxq_ctrl->flow_mark_n = 1; 1220 } else if (mark) { 1221 rxq_ctrl->flow_mark_n--; 1222 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1223 } 1224 if (tunnel) { 1225 unsigned int j; 1226 1227 /* Decrease the counter matching the flow. */ 1228 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1229 if ((tunnels_info[j].tunnel & 1230 dev_handle->layers) == 1231 tunnels_info[j].tunnel) { 1232 rxq_ctrl->flow_tunnels_n[j]--; 1233 break; 1234 } 1235 } 1236 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1237 } 1238 } 1239 } 1240 1241 /** 1242 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1243 * @p flow if no other flow uses it with the same kind of request. 1244 * 1245 * @param dev 1246 * Pointer to Ethernet device. 1247 * @param[in] flow 1248 * Pointer to the flow. 1249 */ 1250 static void 1251 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1252 { 1253 struct mlx5_priv *priv = dev->data->dev_private; 1254 uint32_t handle_idx; 1255 struct mlx5_flow_handle *dev_handle; 1256 1257 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1258 handle_idx, dev_handle, next) 1259 flow_drv_rxq_flags_trim(dev, dev_handle); 1260 } 1261 1262 /** 1263 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1264 * 1265 * @param dev 1266 * Pointer to Ethernet device. 1267 */ 1268 static void 1269 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1270 { 1271 struct mlx5_priv *priv = dev->data->dev_private; 1272 unsigned int i; 1273 1274 for (i = 0; i != priv->rxqs_n; ++i) { 1275 struct mlx5_rxq_ctrl *rxq_ctrl; 1276 unsigned int j; 1277 1278 if (!(*priv->rxqs)[i]) 1279 continue; 1280 rxq_ctrl = container_of((*priv->rxqs)[i], 1281 struct mlx5_rxq_ctrl, rxq); 1282 rxq_ctrl->flow_mark_n = 0; 1283 rxq_ctrl->rxq.mark = 0; 1284 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1285 rxq_ctrl->flow_tunnels_n[j] = 0; 1286 rxq_ctrl->rxq.tunnel = 0; 1287 } 1288 } 1289 1290 /** 1291 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1292 * 1293 * @param[in] dev 1294 * Pointer to the Ethernet device structure. 1295 */ 1296 void 1297 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1298 { 1299 struct mlx5_priv *priv = dev->data->dev_private; 1300 struct mlx5_rxq_data *data; 1301 unsigned int i; 1302 1303 for (i = 0; i != priv->rxqs_n; ++i) { 1304 if (!(*priv->rxqs)[i]) 1305 continue; 1306 data = (*priv->rxqs)[i]; 1307 if (!rte_flow_dynf_metadata_avail()) { 1308 data->dynf_meta = 0; 1309 data->flow_meta_mask = 0; 1310 data->flow_meta_offset = -1; 1311 data->flow_meta_port_mask = 0; 1312 } else { 1313 data->dynf_meta = 1; 1314 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1315 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1316 data->flow_meta_port_mask = (uint32_t)~0; 1317 if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16) 1318 data->flow_meta_port_mask >>= 16; 1319 } 1320 } 1321 } 1322 1323 /* 1324 * return a pointer to the desired action in the list of actions. 1325 * 1326 * @param[in] actions 1327 * The list of actions to search the action in. 1328 * @param[in] action 1329 * The action to find. 1330 * 1331 * @return 1332 * Pointer to the action in the list, if found. NULL otherwise. 1333 */ 1334 const struct rte_flow_action * 1335 mlx5_flow_find_action(const struct rte_flow_action *actions, 1336 enum rte_flow_action_type action) 1337 { 1338 if (actions == NULL) 1339 return NULL; 1340 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1341 if (actions->type == action) 1342 return actions; 1343 return NULL; 1344 } 1345 1346 /* 1347 * Validate the flag action. 1348 * 1349 * @param[in] action_flags 1350 * Bit-fields that holds the actions detected until now. 1351 * @param[in] attr 1352 * Attributes of flow that includes this action. 1353 * @param[out] error 1354 * Pointer to error structure. 1355 * 1356 * @return 1357 * 0 on success, a negative errno value otherwise and rte_errno is set. 1358 */ 1359 int 1360 mlx5_flow_validate_action_flag(uint64_t action_flags, 1361 const struct rte_flow_attr *attr, 1362 struct rte_flow_error *error) 1363 { 1364 if (action_flags & MLX5_FLOW_ACTION_MARK) 1365 return rte_flow_error_set(error, EINVAL, 1366 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1367 "can't mark and flag in same flow"); 1368 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1369 return rte_flow_error_set(error, EINVAL, 1370 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1371 "can't have 2 flag" 1372 " actions in same flow"); 1373 if (attr->egress) 1374 return rte_flow_error_set(error, ENOTSUP, 1375 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1376 "flag action not supported for " 1377 "egress"); 1378 return 0; 1379 } 1380 1381 /* 1382 * Validate the mark action. 1383 * 1384 * @param[in] action 1385 * Pointer to the queue action. 1386 * @param[in] action_flags 1387 * Bit-fields that holds the actions detected until now. 1388 * @param[in] attr 1389 * Attributes of flow that includes this action. 1390 * @param[out] error 1391 * Pointer to error structure. 1392 * 1393 * @return 1394 * 0 on success, a negative errno value otherwise and rte_errno is set. 1395 */ 1396 int 1397 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1398 uint64_t action_flags, 1399 const struct rte_flow_attr *attr, 1400 struct rte_flow_error *error) 1401 { 1402 const struct rte_flow_action_mark *mark = action->conf; 1403 1404 if (!mark) 1405 return rte_flow_error_set(error, EINVAL, 1406 RTE_FLOW_ERROR_TYPE_ACTION, 1407 action, 1408 "configuration cannot be null"); 1409 if (mark->id >= MLX5_FLOW_MARK_MAX) 1410 return rte_flow_error_set(error, EINVAL, 1411 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1412 &mark->id, 1413 "mark id must in 0 <= id < " 1414 RTE_STR(MLX5_FLOW_MARK_MAX)); 1415 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1416 return rte_flow_error_set(error, EINVAL, 1417 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1418 "can't flag and mark in same flow"); 1419 if (action_flags & MLX5_FLOW_ACTION_MARK) 1420 return rte_flow_error_set(error, EINVAL, 1421 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1422 "can't have 2 mark actions in same" 1423 " flow"); 1424 if (attr->egress) 1425 return rte_flow_error_set(error, ENOTSUP, 1426 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1427 "mark action not supported for " 1428 "egress"); 1429 return 0; 1430 } 1431 1432 /* 1433 * Validate the drop action. 1434 * 1435 * @param[in] action_flags 1436 * Bit-fields that holds the actions detected until now. 1437 * @param[in] attr 1438 * Attributes of flow that includes this action. 1439 * @param[out] error 1440 * Pointer to error structure. 1441 * 1442 * @return 1443 * 0 on success, a negative errno value otherwise and rte_errno is set. 1444 */ 1445 int 1446 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1447 const struct rte_flow_attr *attr, 1448 struct rte_flow_error *error) 1449 { 1450 if (attr->egress) 1451 return rte_flow_error_set(error, ENOTSUP, 1452 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1453 "drop action not supported for " 1454 "egress"); 1455 return 0; 1456 } 1457 1458 /* 1459 * Validate the queue action. 1460 * 1461 * @param[in] action 1462 * Pointer to the queue action. 1463 * @param[in] action_flags 1464 * Bit-fields that holds the actions detected until now. 1465 * @param[in] dev 1466 * Pointer to the Ethernet device structure. 1467 * @param[in] attr 1468 * Attributes of flow that includes this action. 1469 * @param[out] error 1470 * Pointer to error structure. 1471 * 1472 * @return 1473 * 0 on success, a negative errno value otherwise and rte_errno is set. 1474 */ 1475 int 1476 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1477 uint64_t action_flags, 1478 struct rte_eth_dev *dev, 1479 const struct rte_flow_attr *attr, 1480 struct rte_flow_error *error) 1481 { 1482 struct mlx5_priv *priv = dev->data->dev_private; 1483 const struct rte_flow_action_queue *queue = action->conf; 1484 1485 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1486 return rte_flow_error_set(error, EINVAL, 1487 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1488 "can't have 2 fate actions in" 1489 " same flow"); 1490 if (!priv->rxqs_n) 1491 return rte_flow_error_set(error, EINVAL, 1492 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1493 NULL, "No Rx queues configured"); 1494 if (queue->index >= priv->rxqs_n) 1495 return rte_flow_error_set(error, EINVAL, 1496 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1497 &queue->index, 1498 "queue index out of range"); 1499 if (!(*priv->rxqs)[queue->index]) 1500 return rte_flow_error_set(error, EINVAL, 1501 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1502 &queue->index, 1503 "queue is not configured"); 1504 if (attr->egress) 1505 return rte_flow_error_set(error, ENOTSUP, 1506 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1507 "queue action not supported for " 1508 "egress"); 1509 return 0; 1510 } 1511 1512 /* 1513 * Validate the rss action. 1514 * 1515 * @param[in] dev 1516 * Pointer to the Ethernet device structure. 1517 * @param[in] action 1518 * Pointer to the queue action. 1519 * @param[out] error 1520 * Pointer to error structure. 1521 * 1522 * @return 1523 * 0 on success, a negative errno value otherwise and rte_errno is set. 1524 */ 1525 int 1526 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1527 const struct rte_flow_action *action, 1528 struct rte_flow_error *error) 1529 { 1530 struct mlx5_priv *priv = dev->data->dev_private; 1531 const struct rte_flow_action_rss *rss = action->conf; 1532 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1533 unsigned int i; 1534 1535 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1536 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1537 return rte_flow_error_set(error, ENOTSUP, 1538 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1539 &rss->func, 1540 "RSS hash function not supported"); 1541 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1542 if (rss->level > 2) 1543 #else 1544 if (rss->level > 1) 1545 #endif 1546 return rte_flow_error_set(error, ENOTSUP, 1547 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1548 &rss->level, 1549 "tunnel RSS is not supported"); 1550 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1551 if (rss->key_len == 0 && rss->key != NULL) 1552 return rte_flow_error_set(error, ENOTSUP, 1553 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1554 &rss->key_len, 1555 "RSS hash key length 0"); 1556 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1557 return rte_flow_error_set(error, ENOTSUP, 1558 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1559 &rss->key_len, 1560 "RSS hash key too small"); 1561 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1562 return rte_flow_error_set(error, ENOTSUP, 1563 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1564 &rss->key_len, 1565 "RSS hash key too large"); 1566 if (rss->queue_num > priv->config.ind_table_max_size) 1567 return rte_flow_error_set(error, ENOTSUP, 1568 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1569 &rss->queue_num, 1570 "number of queues too large"); 1571 if (rss->types & MLX5_RSS_HF_MASK) 1572 return rte_flow_error_set(error, ENOTSUP, 1573 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1574 &rss->types, 1575 "some RSS protocols are not" 1576 " supported"); 1577 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1578 !(rss->types & ETH_RSS_IP)) 1579 return rte_flow_error_set(error, EINVAL, 1580 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1581 "L3 partial RSS requested but L3 RSS" 1582 " type not specified"); 1583 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1584 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1585 return rte_flow_error_set(error, EINVAL, 1586 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1587 "L4 partial RSS requested but L4 RSS" 1588 " type not specified"); 1589 if (!priv->rxqs_n) 1590 return rte_flow_error_set(error, EINVAL, 1591 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1592 NULL, "No Rx queues configured"); 1593 if (!rss->queue_num) 1594 return rte_flow_error_set(error, EINVAL, 1595 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1596 NULL, "No queues configured"); 1597 for (i = 0; i != rss->queue_num; ++i) { 1598 struct mlx5_rxq_ctrl *rxq_ctrl; 1599 1600 if (rss->queue[i] >= priv->rxqs_n) 1601 return rte_flow_error_set 1602 (error, EINVAL, 1603 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1604 &rss->queue[i], "queue index out of range"); 1605 if (!(*priv->rxqs)[rss->queue[i]]) 1606 return rte_flow_error_set 1607 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1608 &rss->queue[i], "queue is not configured"); 1609 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1610 struct mlx5_rxq_ctrl, rxq); 1611 if (i == 0) 1612 rxq_type = rxq_ctrl->type; 1613 if (rxq_type != rxq_ctrl->type) 1614 return rte_flow_error_set 1615 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1616 &rss->queue[i], 1617 "combining hairpin and regular RSS queues is not supported"); 1618 } 1619 return 0; 1620 } 1621 1622 /* 1623 * Validate the rss action. 1624 * 1625 * @param[in] action 1626 * Pointer to the queue action. 1627 * @param[in] action_flags 1628 * Bit-fields that holds the actions detected until now. 1629 * @param[in] dev 1630 * Pointer to the Ethernet device structure. 1631 * @param[in] attr 1632 * Attributes of flow that includes this action. 1633 * @param[in] item_flags 1634 * Items that were detected. 1635 * @param[out] error 1636 * Pointer to error structure. 1637 * 1638 * @return 1639 * 0 on success, a negative errno value otherwise and rte_errno is set. 1640 */ 1641 int 1642 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1643 uint64_t action_flags, 1644 struct rte_eth_dev *dev, 1645 const struct rte_flow_attr *attr, 1646 uint64_t item_flags, 1647 struct rte_flow_error *error) 1648 { 1649 const struct rte_flow_action_rss *rss = action->conf; 1650 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1651 int ret; 1652 1653 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1654 return rte_flow_error_set(error, EINVAL, 1655 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1656 "can't have 2 fate actions" 1657 " in same flow"); 1658 ret = mlx5_validate_action_rss(dev, action, error); 1659 if (ret) 1660 return ret; 1661 if (attr->egress) 1662 return rte_flow_error_set(error, ENOTSUP, 1663 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1664 "rss action not supported for " 1665 "egress"); 1666 if (rss->level > 1 && !tunnel) 1667 return rte_flow_error_set(error, EINVAL, 1668 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1669 "inner RSS is not supported for " 1670 "non-tunnel flows"); 1671 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1672 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1673 return rte_flow_error_set(error, EINVAL, 1674 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1675 "RSS on eCPRI is not supported now"); 1676 } 1677 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1678 !(item_flags & 1679 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1680 rss->level > 1) 1681 return rte_flow_error_set(error, EINVAL, 1682 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1683 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1684 return 0; 1685 } 1686 1687 /* 1688 * Validate the default miss action. 1689 * 1690 * @param[in] action_flags 1691 * Bit-fields that holds the actions detected until now. 1692 * @param[out] error 1693 * Pointer to error structure. 1694 * 1695 * @return 1696 * 0 on success, a negative errno value otherwise and rte_errno is set. 1697 */ 1698 int 1699 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1700 const struct rte_flow_attr *attr, 1701 struct rte_flow_error *error) 1702 { 1703 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1704 return rte_flow_error_set(error, EINVAL, 1705 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1706 "can't have 2 fate actions in" 1707 " same flow"); 1708 if (attr->egress) 1709 return rte_flow_error_set(error, ENOTSUP, 1710 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1711 "default miss action not supported " 1712 "for egress"); 1713 if (attr->group) 1714 return rte_flow_error_set(error, ENOTSUP, 1715 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1716 "only group 0 is supported"); 1717 if (attr->transfer) 1718 return rte_flow_error_set(error, ENOTSUP, 1719 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1720 NULL, "transfer is not supported"); 1721 return 0; 1722 } 1723 1724 /* 1725 * Validate the count action. 1726 * 1727 * @param[in] dev 1728 * Pointer to the Ethernet device structure. 1729 * @param[in] attr 1730 * Attributes of flow that includes this action. 1731 * @param[out] error 1732 * Pointer to error structure. 1733 * 1734 * @return 1735 * 0 on success, a negative errno value otherwise and rte_errno is set. 1736 */ 1737 int 1738 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1739 const struct rte_flow_attr *attr, 1740 struct rte_flow_error *error) 1741 { 1742 if (attr->egress) 1743 return rte_flow_error_set(error, ENOTSUP, 1744 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1745 "count action not supported for " 1746 "egress"); 1747 return 0; 1748 } 1749 1750 /* 1751 * Validate the ASO CT action. 1752 * 1753 * @param[in] dev 1754 * Pointer to the Ethernet device structure. 1755 * @param[in] conntrack 1756 * Pointer to the CT action profile. 1757 * @param[out] error 1758 * Pointer to error structure. 1759 * 1760 * @return 1761 * 0 on success, a negative errno value otherwise and rte_errno is set. 1762 */ 1763 int 1764 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1765 const struct rte_flow_action_conntrack *conntrack, 1766 struct rte_flow_error *error) 1767 { 1768 RTE_SET_USED(dev); 1769 1770 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1771 return rte_flow_error_set(error, EINVAL, 1772 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1773 "Invalid CT state"); 1774 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1775 return rte_flow_error_set(error, EINVAL, 1776 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1777 "Invalid last TCP packet flag"); 1778 return 0; 1779 } 1780 1781 /** 1782 * Verify the @p attributes will be correctly understood by the NIC and store 1783 * them in the @p flow if everything is correct. 1784 * 1785 * @param[in] dev 1786 * Pointer to the Ethernet device structure. 1787 * @param[in] attributes 1788 * Pointer to flow attributes 1789 * @param[out] error 1790 * Pointer to error structure. 1791 * 1792 * @return 1793 * 0 on success, a negative errno value otherwise and rte_errno is set. 1794 */ 1795 int 1796 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1797 const struct rte_flow_attr *attributes, 1798 struct rte_flow_error *error) 1799 { 1800 struct mlx5_priv *priv = dev->data->dev_private; 1801 uint32_t priority_max = priv->config.flow_prio - 1; 1802 1803 if (attributes->group) 1804 return rte_flow_error_set(error, ENOTSUP, 1805 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1806 NULL, "groups is not supported"); 1807 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1808 attributes->priority >= priority_max) 1809 return rte_flow_error_set(error, ENOTSUP, 1810 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1811 NULL, "priority out of range"); 1812 if (attributes->egress) 1813 return rte_flow_error_set(error, ENOTSUP, 1814 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1815 "egress is not supported"); 1816 if (attributes->transfer && !priv->config.dv_esw_en) 1817 return rte_flow_error_set(error, ENOTSUP, 1818 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1819 NULL, "transfer is not supported"); 1820 if (!attributes->ingress) 1821 return rte_flow_error_set(error, EINVAL, 1822 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1823 NULL, 1824 "ingress attribute is mandatory"); 1825 return 0; 1826 } 1827 1828 /** 1829 * Validate ICMP6 item. 1830 * 1831 * @param[in] item 1832 * Item specification. 1833 * @param[in] item_flags 1834 * Bit-fields that holds the items detected until now. 1835 * @param[in] ext_vlan_sup 1836 * Whether extended VLAN features are supported or not. 1837 * @param[out] error 1838 * Pointer to error structure. 1839 * 1840 * @return 1841 * 0 on success, a negative errno value otherwise and rte_errno is set. 1842 */ 1843 int 1844 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1845 uint64_t item_flags, 1846 uint8_t target_protocol, 1847 struct rte_flow_error *error) 1848 { 1849 const struct rte_flow_item_icmp6 *mask = item->mask; 1850 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1851 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1852 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1853 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1854 MLX5_FLOW_LAYER_OUTER_L4; 1855 int ret; 1856 1857 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1858 return rte_flow_error_set(error, EINVAL, 1859 RTE_FLOW_ERROR_TYPE_ITEM, item, 1860 "protocol filtering not compatible" 1861 " with ICMP6 layer"); 1862 if (!(item_flags & l3m)) 1863 return rte_flow_error_set(error, EINVAL, 1864 RTE_FLOW_ERROR_TYPE_ITEM, item, 1865 "IPv6 is mandatory to filter on" 1866 " ICMP6"); 1867 if (item_flags & l4m) 1868 return rte_flow_error_set(error, EINVAL, 1869 RTE_FLOW_ERROR_TYPE_ITEM, item, 1870 "multiple L4 layers not supported"); 1871 if (!mask) 1872 mask = &rte_flow_item_icmp6_mask; 1873 ret = mlx5_flow_item_acceptable 1874 (item, (const uint8_t *)mask, 1875 (const uint8_t *)&rte_flow_item_icmp6_mask, 1876 sizeof(struct rte_flow_item_icmp6), 1877 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1878 if (ret < 0) 1879 return ret; 1880 return 0; 1881 } 1882 1883 /** 1884 * Validate ICMP item. 1885 * 1886 * @param[in] item 1887 * Item specification. 1888 * @param[in] item_flags 1889 * Bit-fields that holds the items detected until now. 1890 * @param[out] error 1891 * Pointer to error structure. 1892 * 1893 * @return 1894 * 0 on success, a negative errno value otherwise and rte_errno is set. 1895 */ 1896 int 1897 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1898 uint64_t item_flags, 1899 uint8_t target_protocol, 1900 struct rte_flow_error *error) 1901 { 1902 const struct rte_flow_item_icmp *mask = item->mask; 1903 const struct rte_flow_item_icmp nic_mask = { 1904 .hdr.icmp_type = 0xff, 1905 .hdr.icmp_code = 0xff, 1906 .hdr.icmp_ident = RTE_BE16(0xffff), 1907 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1908 }; 1909 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1910 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1911 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1912 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1913 MLX5_FLOW_LAYER_OUTER_L4; 1914 int ret; 1915 1916 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1917 return rte_flow_error_set(error, EINVAL, 1918 RTE_FLOW_ERROR_TYPE_ITEM, item, 1919 "protocol filtering not compatible" 1920 " with ICMP layer"); 1921 if (!(item_flags & l3m)) 1922 return rte_flow_error_set(error, EINVAL, 1923 RTE_FLOW_ERROR_TYPE_ITEM, item, 1924 "IPv4 is mandatory to filter" 1925 " on ICMP"); 1926 if (item_flags & l4m) 1927 return rte_flow_error_set(error, EINVAL, 1928 RTE_FLOW_ERROR_TYPE_ITEM, item, 1929 "multiple L4 layers not supported"); 1930 if (!mask) 1931 mask = &nic_mask; 1932 ret = mlx5_flow_item_acceptable 1933 (item, (const uint8_t *)mask, 1934 (const uint8_t *)&nic_mask, 1935 sizeof(struct rte_flow_item_icmp), 1936 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1937 if (ret < 0) 1938 return ret; 1939 return 0; 1940 } 1941 1942 /** 1943 * Validate Ethernet item. 1944 * 1945 * @param[in] item 1946 * Item specification. 1947 * @param[in] item_flags 1948 * Bit-fields that holds the items detected until now. 1949 * @param[out] error 1950 * Pointer to error structure. 1951 * 1952 * @return 1953 * 0 on success, a negative errno value otherwise and rte_errno is set. 1954 */ 1955 int 1956 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1957 uint64_t item_flags, bool ext_vlan_sup, 1958 struct rte_flow_error *error) 1959 { 1960 const struct rte_flow_item_eth *mask = item->mask; 1961 const struct rte_flow_item_eth nic_mask = { 1962 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1963 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1964 .type = RTE_BE16(0xffff), 1965 .has_vlan = ext_vlan_sup ? 1 : 0, 1966 }; 1967 int ret; 1968 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1969 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1970 MLX5_FLOW_LAYER_OUTER_L2; 1971 1972 if (item_flags & ethm) 1973 return rte_flow_error_set(error, ENOTSUP, 1974 RTE_FLOW_ERROR_TYPE_ITEM, item, 1975 "multiple L2 layers not supported"); 1976 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1977 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1978 return rte_flow_error_set(error, EINVAL, 1979 RTE_FLOW_ERROR_TYPE_ITEM, item, 1980 "L2 layer should not follow " 1981 "L3 layers"); 1982 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1983 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1984 return rte_flow_error_set(error, EINVAL, 1985 RTE_FLOW_ERROR_TYPE_ITEM, item, 1986 "L2 layer should not follow VLAN"); 1987 if (!mask) 1988 mask = &rte_flow_item_eth_mask; 1989 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1990 (const uint8_t *)&nic_mask, 1991 sizeof(struct rte_flow_item_eth), 1992 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1993 return ret; 1994 } 1995 1996 /** 1997 * Validate VLAN item. 1998 * 1999 * @param[in] item 2000 * Item specification. 2001 * @param[in] item_flags 2002 * Bit-fields that holds the items detected until now. 2003 * @param[in] dev 2004 * Ethernet device flow is being created on. 2005 * @param[out] error 2006 * Pointer to error structure. 2007 * 2008 * @return 2009 * 0 on success, a negative errno value otherwise and rte_errno is set. 2010 */ 2011 int 2012 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2013 uint64_t item_flags, 2014 struct rte_eth_dev *dev, 2015 struct rte_flow_error *error) 2016 { 2017 const struct rte_flow_item_vlan *spec = item->spec; 2018 const struct rte_flow_item_vlan *mask = item->mask; 2019 const struct rte_flow_item_vlan nic_mask = { 2020 .tci = RTE_BE16(UINT16_MAX), 2021 .inner_type = RTE_BE16(UINT16_MAX), 2022 }; 2023 uint16_t vlan_tag = 0; 2024 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2025 int ret; 2026 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2027 MLX5_FLOW_LAYER_INNER_L4) : 2028 (MLX5_FLOW_LAYER_OUTER_L3 | 2029 MLX5_FLOW_LAYER_OUTER_L4); 2030 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2031 MLX5_FLOW_LAYER_OUTER_VLAN; 2032 2033 if (item_flags & vlanm) 2034 return rte_flow_error_set(error, EINVAL, 2035 RTE_FLOW_ERROR_TYPE_ITEM, item, 2036 "multiple VLAN layers not supported"); 2037 else if ((item_flags & l34m) != 0) 2038 return rte_flow_error_set(error, EINVAL, 2039 RTE_FLOW_ERROR_TYPE_ITEM, item, 2040 "VLAN cannot follow L3/L4 layer"); 2041 if (!mask) 2042 mask = &rte_flow_item_vlan_mask; 2043 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2044 (const uint8_t *)&nic_mask, 2045 sizeof(struct rte_flow_item_vlan), 2046 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2047 if (ret) 2048 return ret; 2049 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2050 struct mlx5_priv *priv = dev->data->dev_private; 2051 2052 if (priv->vmwa_context) { 2053 /* 2054 * Non-NULL context means we have a virtual machine 2055 * and SR-IOV enabled, we have to create VLAN interface 2056 * to make hypervisor to setup E-Switch vport 2057 * context correctly. We avoid creating the multiple 2058 * VLAN interfaces, so we cannot support VLAN tag mask. 2059 */ 2060 return rte_flow_error_set(error, EINVAL, 2061 RTE_FLOW_ERROR_TYPE_ITEM, 2062 item, 2063 "VLAN tag mask is not" 2064 " supported in virtual" 2065 " environment"); 2066 } 2067 } 2068 if (spec) { 2069 vlan_tag = spec->tci; 2070 vlan_tag &= mask->tci; 2071 } 2072 /* 2073 * From verbs perspective an empty VLAN is equivalent 2074 * to a packet without VLAN layer. 2075 */ 2076 if (!vlan_tag) 2077 return rte_flow_error_set(error, EINVAL, 2078 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2079 item->spec, 2080 "VLAN cannot be empty"); 2081 return 0; 2082 } 2083 2084 /** 2085 * Validate IPV4 item. 2086 * 2087 * @param[in] item 2088 * Item specification. 2089 * @param[in] item_flags 2090 * Bit-fields that holds the items detected until now. 2091 * @param[in] last_item 2092 * Previous validated item in the pattern items. 2093 * @param[in] ether_type 2094 * Type in the ethernet layer header (including dot1q). 2095 * @param[in] acc_mask 2096 * Acceptable mask, if NULL default internal default mask 2097 * will be used to check whether item fields are supported. 2098 * @param[in] range_accepted 2099 * True if range of values is accepted for specific fields, false otherwise. 2100 * @param[out] error 2101 * Pointer to error structure. 2102 * 2103 * @return 2104 * 0 on success, a negative errno value otherwise and rte_errno is set. 2105 */ 2106 int 2107 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2108 uint64_t item_flags, 2109 uint64_t last_item, 2110 uint16_t ether_type, 2111 const struct rte_flow_item_ipv4 *acc_mask, 2112 bool range_accepted, 2113 struct rte_flow_error *error) 2114 { 2115 const struct rte_flow_item_ipv4 *mask = item->mask; 2116 const struct rte_flow_item_ipv4 *spec = item->spec; 2117 const struct rte_flow_item_ipv4 nic_mask = { 2118 .hdr = { 2119 .src_addr = RTE_BE32(0xffffffff), 2120 .dst_addr = RTE_BE32(0xffffffff), 2121 .type_of_service = 0xff, 2122 .next_proto_id = 0xff, 2123 }, 2124 }; 2125 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2126 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2127 MLX5_FLOW_LAYER_OUTER_L3; 2128 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2129 MLX5_FLOW_LAYER_OUTER_L4; 2130 int ret; 2131 uint8_t next_proto = 0xFF; 2132 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2133 MLX5_FLOW_LAYER_OUTER_VLAN | 2134 MLX5_FLOW_LAYER_INNER_VLAN); 2135 2136 if ((last_item & l2_vlan) && ether_type && 2137 ether_type != RTE_ETHER_TYPE_IPV4) 2138 return rte_flow_error_set(error, EINVAL, 2139 RTE_FLOW_ERROR_TYPE_ITEM, item, 2140 "IPv4 cannot follow L2/VLAN layer " 2141 "which ether type is not IPv4"); 2142 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2143 if (mask && spec) 2144 next_proto = mask->hdr.next_proto_id & 2145 spec->hdr.next_proto_id; 2146 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2147 return rte_flow_error_set(error, EINVAL, 2148 RTE_FLOW_ERROR_TYPE_ITEM, 2149 item, 2150 "multiple tunnel " 2151 "not supported"); 2152 } 2153 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2154 return rte_flow_error_set(error, EINVAL, 2155 RTE_FLOW_ERROR_TYPE_ITEM, item, 2156 "wrong tunnel type - IPv6 specified " 2157 "but IPv4 item provided"); 2158 if (item_flags & l3m) 2159 return rte_flow_error_set(error, ENOTSUP, 2160 RTE_FLOW_ERROR_TYPE_ITEM, item, 2161 "multiple L3 layers not supported"); 2162 else if (item_flags & l4m) 2163 return rte_flow_error_set(error, EINVAL, 2164 RTE_FLOW_ERROR_TYPE_ITEM, item, 2165 "L3 cannot follow an L4 layer."); 2166 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2167 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2168 return rte_flow_error_set(error, EINVAL, 2169 RTE_FLOW_ERROR_TYPE_ITEM, item, 2170 "L3 cannot follow an NVGRE layer."); 2171 if (!mask) 2172 mask = &rte_flow_item_ipv4_mask; 2173 else if (mask->hdr.next_proto_id != 0 && 2174 mask->hdr.next_proto_id != 0xff) 2175 return rte_flow_error_set(error, EINVAL, 2176 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2177 "partial mask is not supported" 2178 " for protocol"); 2179 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2180 acc_mask ? (const uint8_t *)acc_mask 2181 : (const uint8_t *)&nic_mask, 2182 sizeof(struct rte_flow_item_ipv4), 2183 range_accepted, error); 2184 if (ret < 0) 2185 return ret; 2186 return 0; 2187 } 2188 2189 /** 2190 * Validate IPV6 item. 2191 * 2192 * @param[in] item 2193 * Item specification. 2194 * @param[in] item_flags 2195 * Bit-fields that holds the items detected until now. 2196 * @param[in] last_item 2197 * Previous validated item in the pattern items. 2198 * @param[in] ether_type 2199 * Type in the ethernet layer header (including dot1q). 2200 * @param[in] acc_mask 2201 * Acceptable mask, if NULL default internal default mask 2202 * will be used to check whether item fields are supported. 2203 * @param[out] error 2204 * Pointer to error structure. 2205 * 2206 * @return 2207 * 0 on success, a negative errno value otherwise and rte_errno is set. 2208 */ 2209 int 2210 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2211 uint64_t item_flags, 2212 uint64_t last_item, 2213 uint16_t ether_type, 2214 const struct rte_flow_item_ipv6 *acc_mask, 2215 struct rte_flow_error *error) 2216 { 2217 const struct rte_flow_item_ipv6 *mask = item->mask; 2218 const struct rte_flow_item_ipv6 *spec = item->spec; 2219 const struct rte_flow_item_ipv6 nic_mask = { 2220 .hdr = { 2221 .src_addr = 2222 "\xff\xff\xff\xff\xff\xff\xff\xff" 2223 "\xff\xff\xff\xff\xff\xff\xff\xff", 2224 .dst_addr = 2225 "\xff\xff\xff\xff\xff\xff\xff\xff" 2226 "\xff\xff\xff\xff\xff\xff\xff\xff", 2227 .vtc_flow = RTE_BE32(0xffffffff), 2228 .proto = 0xff, 2229 }, 2230 }; 2231 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2232 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2233 MLX5_FLOW_LAYER_OUTER_L3; 2234 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2235 MLX5_FLOW_LAYER_OUTER_L4; 2236 int ret; 2237 uint8_t next_proto = 0xFF; 2238 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2239 MLX5_FLOW_LAYER_OUTER_VLAN | 2240 MLX5_FLOW_LAYER_INNER_VLAN); 2241 2242 if ((last_item & l2_vlan) && ether_type && 2243 ether_type != RTE_ETHER_TYPE_IPV6) 2244 return rte_flow_error_set(error, EINVAL, 2245 RTE_FLOW_ERROR_TYPE_ITEM, item, 2246 "IPv6 cannot follow L2/VLAN layer " 2247 "which ether type is not IPv6"); 2248 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2249 next_proto = spec->hdr.proto; 2250 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2251 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2252 return rte_flow_error_set(error, EINVAL, 2253 RTE_FLOW_ERROR_TYPE_ITEM, 2254 item, 2255 "multiple tunnel " 2256 "not supported"); 2257 } 2258 if (next_proto == IPPROTO_HOPOPTS || 2259 next_proto == IPPROTO_ROUTING || 2260 next_proto == IPPROTO_FRAGMENT || 2261 next_proto == IPPROTO_ESP || 2262 next_proto == IPPROTO_AH || 2263 next_proto == IPPROTO_DSTOPTS) 2264 return rte_flow_error_set(error, EINVAL, 2265 RTE_FLOW_ERROR_TYPE_ITEM, item, 2266 "IPv6 proto (next header) should " 2267 "not be set as extension header"); 2268 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2269 return rte_flow_error_set(error, EINVAL, 2270 RTE_FLOW_ERROR_TYPE_ITEM, item, 2271 "wrong tunnel type - IPv4 specified " 2272 "but IPv6 item provided"); 2273 if (item_flags & l3m) 2274 return rte_flow_error_set(error, ENOTSUP, 2275 RTE_FLOW_ERROR_TYPE_ITEM, item, 2276 "multiple L3 layers not supported"); 2277 else if (item_flags & l4m) 2278 return rte_flow_error_set(error, EINVAL, 2279 RTE_FLOW_ERROR_TYPE_ITEM, item, 2280 "L3 cannot follow an L4 layer."); 2281 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2282 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2283 return rte_flow_error_set(error, EINVAL, 2284 RTE_FLOW_ERROR_TYPE_ITEM, item, 2285 "L3 cannot follow an NVGRE layer."); 2286 if (!mask) 2287 mask = &rte_flow_item_ipv6_mask; 2288 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2289 acc_mask ? (const uint8_t *)acc_mask 2290 : (const uint8_t *)&nic_mask, 2291 sizeof(struct rte_flow_item_ipv6), 2292 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2293 if (ret < 0) 2294 return ret; 2295 return 0; 2296 } 2297 2298 /** 2299 * Validate UDP item. 2300 * 2301 * @param[in] item 2302 * Item specification. 2303 * @param[in] item_flags 2304 * Bit-fields that holds the items detected until now. 2305 * @param[in] target_protocol 2306 * The next protocol in the previous item. 2307 * @param[in] flow_mask 2308 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2309 * @param[out] error 2310 * Pointer to error structure. 2311 * 2312 * @return 2313 * 0 on success, a negative errno value otherwise and rte_errno is set. 2314 */ 2315 int 2316 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2317 uint64_t item_flags, 2318 uint8_t target_protocol, 2319 struct rte_flow_error *error) 2320 { 2321 const struct rte_flow_item_udp *mask = item->mask; 2322 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2323 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2324 MLX5_FLOW_LAYER_OUTER_L3; 2325 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2326 MLX5_FLOW_LAYER_OUTER_L4; 2327 int ret; 2328 2329 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2330 return rte_flow_error_set(error, EINVAL, 2331 RTE_FLOW_ERROR_TYPE_ITEM, item, 2332 "protocol filtering not compatible" 2333 " with UDP layer"); 2334 if (!(item_flags & l3m)) 2335 return rte_flow_error_set(error, EINVAL, 2336 RTE_FLOW_ERROR_TYPE_ITEM, item, 2337 "L3 is mandatory to filter on L4"); 2338 if (item_flags & l4m) 2339 return rte_flow_error_set(error, EINVAL, 2340 RTE_FLOW_ERROR_TYPE_ITEM, item, 2341 "multiple L4 layers not supported"); 2342 if (!mask) 2343 mask = &rte_flow_item_udp_mask; 2344 ret = mlx5_flow_item_acceptable 2345 (item, (const uint8_t *)mask, 2346 (const uint8_t *)&rte_flow_item_udp_mask, 2347 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2348 error); 2349 if (ret < 0) 2350 return ret; 2351 return 0; 2352 } 2353 2354 /** 2355 * Validate TCP item. 2356 * 2357 * @param[in] item 2358 * Item specification. 2359 * @param[in] item_flags 2360 * Bit-fields that holds the items detected until now. 2361 * @param[in] target_protocol 2362 * The next protocol in the previous item. 2363 * @param[out] error 2364 * Pointer to error structure. 2365 * 2366 * @return 2367 * 0 on success, a negative errno value otherwise and rte_errno is set. 2368 */ 2369 int 2370 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2371 uint64_t item_flags, 2372 uint8_t target_protocol, 2373 const struct rte_flow_item_tcp *flow_mask, 2374 struct rte_flow_error *error) 2375 { 2376 const struct rte_flow_item_tcp *mask = item->mask; 2377 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2378 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2379 MLX5_FLOW_LAYER_OUTER_L3; 2380 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2381 MLX5_FLOW_LAYER_OUTER_L4; 2382 int ret; 2383 2384 MLX5_ASSERT(flow_mask); 2385 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2386 return rte_flow_error_set(error, EINVAL, 2387 RTE_FLOW_ERROR_TYPE_ITEM, item, 2388 "protocol filtering not compatible" 2389 " with TCP layer"); 2390 if (!(item_flags & l3m)) 2391 return rte_flow_error_set(error, EINVAL, 2392 RTE_FLOW_ERROR_TYPE_ITEM, item, 2393 "L3 is mandatory to filter on L4"); 2394 if (item_flags & l4m) 2395 return rte_flow_error_set(error, EINVAL, 2396 RTE_FLOW_ERROR_TYPE_ITEM, item, 2397 "multiple L4 layers not supported"); 2398 if (!mask) 2399 mask = &rte_flow_item_tcp_mask; 2400 ret = mlx5_flow_item_acceptable 2401 (item, (const uint8_t *)mask, 2402 (const uint8_t *)flow_mask, 2403 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2404 error); 2405 if (ret < 0) 2406 return ret; 2407 return 0; 2408 } 2409 2410 /** 2411 * Validate VXLAN item. 2412 * 2413 * @param[in] dev 2414 * Pointer to the Ethernet device structure. 2415 * @param[in] item 2416 * Item specification. 2417 * @param[in] item_flags 2418 * Bit-fields that holds the items detected until now. 2419 * @param[in] attr 2420 * Flow rule attributes. 2421 * @param[out] error 2422 * Pointer to error structure. 2423 * 2424 * @return 2425 * 0 on success, a negative errno value otherwise and rte_errno is set. 2426 */ 2427 int 2428 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2429 const struct rte_flow_item *item, 2430 uint64_t item_flags, 2431 const struct rte_flow_attr *attr, 2432 struct rte_flow_error *error) 2433 { 2434 const struct rte_flow_item_vxlan *spec = item->spec; 2435 const struct rte_flow_item_vxlan *mask = item->mask; 2436 int ret; 2437 struct mlx5_priv *priv = dev->data->dev_private; 2438 union vni { 2439 uint32_t vlan_id; 2440 uint8_t vni[4]; 2441 } id = { .vlan_id = 0, }; 2442 const struct rte_flow_item_vxlan nic_mask = { 2443 .vni = "\xff\xff\xff", 2444 .rsvd1 = 0xff, 2445 }; 2446 const struct rte_flow_item_vxlan *valid_mask; 2447 2448 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2449 return rte_flow_error_set(error, ENOTSUP, 2450 RTE_FLOW_ERROR_TYPE_ITEM, item, 2451 "multiple tunnel layers not" 2452 " supported"); 2453 valid_mask = &rte_flow_item_vxlan_mask; 2454 /* 2455 * Verify only UDPv4 is present as defined in 2456 * https://tools.ietf.org/html/rfc7348 2457 */ 2458 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2459 return rte_flow_error_set(error, EINVAL, 2460 RTE_FLOW_ERROR_TYPE_ITEM, item, 2461 "no outer UDP layer found"); 2462 if (!mask) 2463 mask = &rte_flow_item_vxlan_mask; 2464 /* FDB domain & NIC domain non-zero group */ 2465 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2466 valid_mask = &nic_mask; 2467 /* Group zero in NIC domain */ 2468 if (!attr->group && !attr->transfer && priv->sh->tunnel_header_0_1) 2469 valid_mask = &nic_mask; 2470 ret = mlx5_flow_item_acceptable 2471 (item, (const uint8_t *)mask, 2472 (const uint8_t *)valid_mask, 2473 sizeof(struct rte_flow_item_vxlan), 2474 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2475 if (ret < 0) 2476 return ret; 2477 if (spec) { 2478 memcpy(&id.vni[1], spec->vni, 3); 2479 memcpy(&id.vni[1], mask->vni, 3); 2480 } 2481 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2482 return rte_flow_error_set(error, ENOTSUP, 2483 RTE_FLOW_ERROR_TYPE_ITEM, item, 2484 "VXLAN tunnel must be fully defined"); 2485 return 0; 2486 } 2487 2488 /** 2489 * Validate VXLAN_GPE item. 2490 * 2491 * @param[in] item 2492 * Item specification. 2493 * @param[in] item_flags 2494 * Bit-fields that holds the items detected until now. 2495 * @param[in] priv 2496 * Pointer to the private data structure. 2497 * @param[in] target_protocol 2498 * The next protocol in the previous item. 2499 * @param[out] error 2500 * Pointer to error structure. 2501 * 2502 * @return 2503 * 0 on success, a negative errno value otherwise and rte_errno is set. 2504 */ 2505 int 2506 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2507 uint64_t item_flags, 2508 struct rte_eth_dev *dev, 2509 struct rte_flow_error *error) 2510 { 2511 struct mlx5_priv *priv = dev->data->dev_private; 2512 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2513 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2514 int ret; 2515 union vni { 2516 uint32_t vlan_id; 2517 uint8_t vni[4]; 2518 } id = { .vlan_id = 0, }; 2519 2520 if (!priv->config.l3_vxlan_en) 2521 return rte_flow_error_set(error, ENOTSUP, 2522 RTE_FLOW_ERROR_TYPE_ITEM, item, 2523 "L3 VXLAN is not enabled by device" 2524 " parameter and/or not configured in" 2525 " firmware"); 2526 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2527 return rte_flow_error_set(error, ENOTSUP, 2528 RTE_FLOW_ERROR_TYPE_ITEM, item, 2529 "multiple tunnel layers not" 2530 " supported"); 2531 /* 2532 * Verify only UDPv4 is present as defined in 2533 * https://tools.ietf.org/html/rfc7348 2534 */ 2535 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2536 return rte_flow_error_set(error, EINVAL, 2537 RTE_FLOW_ERROR_TYPE_ITEM, item, 2538 "no outer UDP layer found"); 2539 if (!mask) 2540 mask = &rte_flow_item_vxlan_gpe_mask; 2541 ret = mlx5_flow_item_acceptable 2542 (item, (const uint8_t *)mask, 2543 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2544 sizeof(struct rte_flow_item_vxlan_gpe), 2545 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2546 if (ret < 0) 2547 return ret; 2548 if (spec) { 2549 if (spec->protocol) 2550 return rte_flow_error_set(error, ENOTSUP, 2551 RTE_FLOW_ERROR_TYPE_ITEM, 2552 item, 2553 "VxLAN-GPE protocol" 2554 " not supported"); 2555 memcpy(&id.vni[1], spec->vni, 3); 2556 memcpy(&id.vni[1], mask->vni, 3); 2557 } 2558 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2559 return rte_flow_error_set(error, ENOTSUP, 2560 RTE_FLOW_ERROR_TYPE_ITEM, item, 2561 "VXLAN-GPE tunnel must be fully" 2562 " defined"); 2563 return 0; 2564 } 2565 /** 2566 * Validate GRE Key item. 2567 * 2568 * @param[in] item 2569 * Item specification. 2570 * @param[in] item_flags 2571 * Bit flags to mark detected items. 2572 * @param[in] gre_item 2573 * Pointer to gre_item 2574 * @param[out] error 2575 * Pointer to error structure. 2576 * 2577 * @return 2578 * 0 on success, a negative errno value otherwise and rte_errno is set. 2579 */ 2580 int 2581 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2582 uint64_t item_flags, 2583 const struct rte_flow_item *gre_item, 2584 struct rte_flow_error *error) 2585 { 2586 const rte_be32_t *mask = item->mask; 2587 int ret = 0; 2588 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2589 const struct rte_flow_item_gre *gre_spec; 2590 const struct rte_flow_item_gre *gre_mask; 2591 2592 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2593 return rte_flow_error_set(error, ENOTSUP, 2594 RTE_FLOW_ERROR_TYPE_ITEM, item, 2595 "Multiple GRE key not support"); 2596 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2597 return rte_flow_error_set(error, ENOTSUP, 2598 RTE_FLOW_ERROR_TYPE_ITEM, item, 2599 "No preceding GRE header"); 2600 if (item_flags & MLX5_FLOW_LAYER_INNER) 2601 return rte_flow_error_set(error, ENOTSUP, 2602 RTE_FLOW_ERROR_TYPE_ITEM, item, 2603 "GRE key following a wrong item"); 2604 gre_mask = gre_item->mask; 2605 if (!gre_mask) 2606 gre_mask = &rte_flow_item_gre_mask; 2607 gre_spec = gre_item->spec; 2608 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2609 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2610 return rte_flow_error_set(error, EINVAL, 2611 RTE_FLOW_ERROR_TYPE_ITEM, item, 2612 "Key bit must be on"); 2613 2614 if (!mask) 2615 mask = &gre_key_default_mask; 2616 ret = mlx5_flow_item_acceptable 2617 (item, (const uint8_t *)mask, 2618 (const uint8_t *)&gre_key_default_mask, 2619 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2620 return ret; 2621 } 2622 2623 /** 2624 * Validate GRE item. 2625 * 2626 * @param[in] item 2627 * Item specification. 2628 * @param[in] item_flags 2629 * Bit flags to mark detected items. 2630 * @param[in] target_protocol 2631 * The next protocol in the previous item. 2632 * @param[out] error 2633 * Pointer to error structure. 2634 * 2635 * @return 2636 * 0 on success, a negative errno value otherwise and rte_errno is set. 2637 */ 2638 int 2639 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2640 uint64_t item_flags, 2641 uint8_t target_protocol, 2642 struct rte_flow_error *error) 2643 { 2644 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2645 const struct rte_flow_item_gre *mask = item->mask; 2646 int ret; 2647 const struct rte_flow_item_gre nic_mask = { 2648 .c_rsvd0_ver = RTE_BE16(0xB000), 2649 .protocol = RTE_BE16(UINT16_MAX), 2650 }; 2651 2652 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2653 return rte_flow_error_set(error, EINVAL, 2654 RTE_FLOW_ERROR_TYPE_ITEM, item, 2655 "protocol filtering not compatible" 2656 " with this GRE layer"); 2657 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2658 return rte_flow_error_set(error, ENOTSUP, 2659 RTE_FLOW_ERROR_TYPE_ITEM, item, 2660 "multiple tunnel layers not" 2661 " supported"); 2662 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2663 return rte_flow_error_set(error, ENOTSUP, 2664 RTE_FLOW_ERROR_TYPE_ITEM, item, 2665 "L3 Layer is missing"); 2666 if (!mask) 2667 mask = &rte_flow_item_gre_mask; 2668 ret = mlx5_flow_item_acceptable 2669 (item, (const uint8_t *)mask, 2670 (const uint8_t *)&nic_mask, 2671 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2672 error); 2673 if (ret < 0) 2674 return ret; 2675 #ifndef HAVE_MLX5DV_DR 2676 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2677 if (spec && (spec->protocol & mask->protocol)) 2678 return rte_flow_error_set(error, ENOTSUP, 2679 RTE_FLOW_ERROR_TYPE_ITEM, item, 2680 "without MPLS support the" 2681 " specification cannot be used for" 2682 " filtering"); 2683 #endif 2684 #endif 2685 return 0; 2686 } 2687 2688 /** 2689 * Validate Geneve item. 2690 * 2691 * @param[in] item 2692 * Item specification. 2693 * @param[in] itemFlags 2694 * Bit-fields that holds the items detected until now. 2695 * @param[in] enPriv 2696 * Pointer to the private data structure. 2697 * @param[out] error 2698 * Pointer to error structure. 2699 * 2700 * @return 2701 * 0 on success, a negative errno value otherwise and rte_errno is set. 2702 */ 2703 2704 int 2705 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2706 uint64_t item_flags, 2707 struct rte_eth_dev *dev, 2708 struct rte_flow_error *error) 2709 { 2710 struct mlx5_priv *priv = dev->data->dev_private; 2711 const struct rte_flow_item_geneve *spec = item->spec; 2712 const struct rte_flow_item_geneve *mask = item->mask; 2713 int ret; 2714 uint16_t gbhdr; 2715 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2716 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2717 const struct rte_flow_item_geneve nic_mask = { 2718 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2719 .vni = "\xff\xff\xff", 2720 .protocol = RTE_BE16(UINT16_MAX), 2721 }; 2722 2723 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2724 return rte_flow_error_set(error, ENOTSUP, 2725 RTE_FLOW_ERROR_TYPE_ITEM, item, 2726 "L3 Geneve is not enabled by device" 2727 " parameter and/or not configured in" 2728 " firmware"); 2729 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2730 return rte_flow_error_set(error, ENOTSUP, 2731 RTE_FLOW_ERROR_TYPE_ITEM, item, 2732 "multiple tunnel layers not" 2733 " supported"); 2734 /* 2735 * Verify only UDPv4 is present as defined in 2736 * https://tools.ietf.org/html/rfc7348 2737 */ 2738 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2739 return rte_flow_error_set(error, EINVAL, 2740 RTE_FLOW_ERROR_TYPE_ITEM, item, 2741 "no outer UDP layer found"); 2742 if (!mask) 2743 mask = &rte_flow_item_geneve_mask; 2744 ret = mlx5_flow_item_acceptable 2745 (item, (const uint8_t *)mask, 2746 (const uint8_t *)&nic_mask, 2747 sizeof(struct rte_flow_item_geneve), 2748 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2749 if (ret) 2750 return ret; 2751 if (spec) { 2752 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2753 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2754 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2755 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2756 return rte_flow_error_set(error, ENOTSUP, 2757 RTE_FLOW_ERROR_TYPE_ITEM, 2758 item, 2759 "Geneve protocol unsupported" 2760 " fields are being used"); 2761 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2762 return rte_flow_error_set 2763 (error, ENOTSUP, 2764 RTE_FLOW_ERROR_TYPE_ITEM, 2765 item, 2766 "Unsupported Geneve options length"); 2767 } 2768 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2769 return rte_flow_error_set 2770 (error, ENOTSUP, 2771 RTE_FLOW_ERROR_TYPE_ITEM, item, 2772 "Geneve tunnel must be fully defined"); 2773 return 0; 2774 } 2775 2776 /** 2777 * Validate Geneve TLV option item. 2778 * 2779 * @param[in] item 2780 * Item specification. 2781 * @param[in] last_item 2782 * Previous validated item in the pattern items. 2783 * @param[in] geneve_item 2784 * Previous GENEVE item specification. 2785 * @param[in] dev 2786 * Pointer to the rte_eth_dev structure. 2787 * @param[out] error 2788 * Pointer to error structure. 2789 * 2790 * @return 2791 * 0 on success, a negative errno value otherwise and rte_errno is set. 2792 */ 2793 int 2794 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2795 uint64_t last_item, 2796 const struct rte_flow_item *geneve_item, 2797 struct rte_eth_dev *dev, 2798 struct rte_flow_error *error) 2799 { 2800 struct mlx5_priv *priv = dev->data->dev_private; 2801 struct mlx5_dev_ctx_shared *sh = priv->sh; 2802 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2803 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2804 uint8_t data_max_supported = 2805 hca_attr->max_geneve_tlv_option_data_len * 4; 2806 struct mlx5_dev_config *config = &priv->config; 2807 const struct rte_flow_item_geneve *geneve_spec; 2808 const struct rte_flow_item_geneve *geneve_mask; 2809 const struct rte_flow_item_geneve_opt *spec = item->spec; 2810 const struct rte_flow_item_geneve_opt *mask = item->mask; 2811 unsigned int i; 2812 unsigned int data_len; 2813 uint8_t tlv_option_len; 2814 uint16_t optlen_m, optlen_v; 2815 const struct rte_flow_item_geneve_opt full_mask = { 2816 .option_class = RTE_BE16(0xffff), 2817 .option_type = 0xff, 2818 .option_len = 0x1f, 2819 }; 2820 2821 if (!mask) 2822 mask = &rte_flow_item_geneve_opt_mask; 2823 if (!spec) 2824 return rte_flow_error_set 2825 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2826 "Geneve TLV opt class/type/length must be specified"); 2827 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2828 return rte_flow_error_set 2829 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2830 "Geneve TLV opt length exceeeds the limit (31)"); 2831 /* Check if class type and length masks are full. */ 2832 if (full_mask.option_class != mask->option_class || 2833 full_mask.option_type != mask->option_type || 2834 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2835 return rte_flow_error_set 2836 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2837 "Geneve TLV opt class/type/length masks must be full"); 2838 /* Check if length is supported */ 2839 if ((uint32_t)spec->option_len > 2840 config->hca_attr.max_geneve_tlv_option_data_len) 2841 return rte_flow_error_set 2842 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2843 "Geneve TLV opt length not supported"); 2844 if (config->hca_attr.max_geneve_tlv_options > 1) 2845 DRV_LOG(DEBUG, 2846 "max_geneve_tlv_options supports more than 1 option"); 2847 /* Check GENEVE item preceding. */ 2848 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2849 return rte_flow_error_set 2850 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2851 "Geneve opt item must be preceded with Geneve item"); 2852 geneve_spec = geneve_item->spec; 2853 geneve_mask = geneve_item->mask ? geneve_item->mask : 2854 &rte_flow_item_geneve_mask; 2855 /* Check if GENEVE TLV option size doesn't exceed option length */ 2856 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2857 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2858 tlv_option_len = spec->option_len & mask->option_len; 2859 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2860 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2861 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2862 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2863 if ((optlen_v & optlen_m) <= tlv_option_len) 2864 return rte_flow_error_set 2865 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2866 "GENEVE TLV option length exceeds optlen"); 2867 } 2868 /* Check if length is 0 or data is 0. */ 2869 if (spec->data == NULL || spec->option_len == 0) 2870 return rte_flow_error_set 2871 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2872 "Geneve TLV opt with zero data/length not supported"); 2873 /* Check not all data & mask are 0. */ 2874 data_len = spec->option_len * 4; 2875 if (mask->data == NULL) { 2876 for (i = 0; i < data_len; i++) 2877 if (spec->data[i]) 2878 break; 2879 if (i == data_len) 2880 return rte_flow_error_set(error, ENOTSUP, 2881 RTE_FLOW_ERROR_TYPE_ITEM, item, 2882 "Can't match on Geneve option data 0"); 2883 } else { 2884 for (i = 0; i < data_len; i++) 2885 if (spec->data[i] & mask->data[i]) 2886 break; 2887 if (i == data_len) 2888 return rte_flow_error_set(error, ENOTSUP, 2889 RTE_FLOW_ERROR_TYPE_ITEM, item, 2890 "Can't match on Geneve option data and mask 0"); 2891 /* Check data mask supported. */ 2892 for (i = data_max_supported; i < data_len ; i++) 2893 if (mask->data[i]) 2894 return rte_flow_error_set(error, ENOTSUP, 2895 RTE_FLOW_ERROR_TYPE_ITEM, item, 2896 "Data mask is of unsupported size"); 2897 } 2898 /* Check GENEVE option is supported in NIC. */ 2899 if (!config->hca_attr.geneve_tlv_opt) 2900 return rte_flow_error_set 2901 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2902 "Geneve TLV opt not supported"); 2903 /* Check if we already have geneve option with different type/class. */ 2904 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2905 geneve_opt_resource = sh->geneve_tlv_option_resource; 2906 if (geneve_opt_resource != NULL) 2907 if (geneve_opt_resource->option_class != spec->option_class || 2908 geneve_opt_resource->option_type != spec->option_type || 2909 geneve_opt_resource->length != spec->option_len) { 2910 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2911 return rte_flow_error_set(error, ENOTSUP, 2912 RTE_FLOW_ERROR_TYPE_ITEM, item, 2913 "Only one Geneve TLV option supported"); 2914 } 2915 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2916 return 0; 2917 } 2918 2919 /** 2920 * Validate MPLS item. 2921 * 2922 * @param[in] dev 2923 * Pointer to the rte_eth_dev structure. 2924 * @param[in] item 2925 * Item specification. 2926 * @param[in] item_flags 2927 * Bit-fields that holds the items detected until now. 2928 * @param[in] prev_layer 2929 * The protocol layer indicated in previous item. 2930 * @param[out] error 2931 * Pointer to error structure. 2932 * 2933 * @return 2934 * 0 on success, a negative errno value otherwise and rte_errno is set. 2935 */ 2936 int 2937 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2938 const struct rte_flow_item *item __rte_unused, 2939 uint64_t item_flags __rte_unused, 2940 uint64_t prev_layer __rte_unused, 2941 struct rte_flow_error *error) 2942 { 2943 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2944 const struct rte_flow_item_mpls *mask = item->mask; 2945 struct mlx5_priv *priv = dev->data->dev_private; 2946 int ret; 2947 2948 if (!priv->config.mpls_en) 2949 return rte_flow_error_set(error, ENOTSUP, 2950 RTE_FLOW_ERROR_TYPE_ITEM, item, 2951 "MPLS not supported or" 2952 " disabled in firmware" 2953 " configuration."); 2954 /* MPLS over UDP, GRE is allowed */ 2955 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 2956 MLX5_FLOW_LAYER_GRE | 2957 MLX5_FLOW_LAYER_GRE_KEY))) 2958 return rte_flow_error_set(error, EINVAL, 2959 RTE_FLOW_ERROR_TYPE_ITEM, item, 2960 "protocol filtering not compatible" 2961 " with MPLS layer"); 2962 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2963 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2964 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2965 return rte_flow_error_set(error, ENOTSUP, 2966 RTE_FLOW_ERROR_TYPE_ITEM, item, 2967 "multiple tunnel layers not" 2968 " supported"); 2969 if (!mask) 2970 mask = &rte_flow_item_mpls_mask; 2971 ret = mlx5_flow_item_acceptable 2972 (item, (const uint8_t *)mask, 2973 (const uint8_t *)&rte_flow_item_mpls_mask, 2974 sizeof(struct rte_flow_item_mpls), 2975 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2976 if (ret < 0) 2977 return ret; 2978 return 0; 2979 #else 2980 return rte_flow_error_set(error, ENOTSUP, 2981 RTE_FLOW_ERROR_TYPE_ITEM, item, 2982 "MPLS is not supported by Verbs, please" 2983 " update."); 2984 #endif 2985 } 2986 2987 /** 2988 * Validate NVGRE item. 2989 * 2990 * @param[in] item 2991 * Item specification. 2992 * @param[in] item_flags 2993 * Bit flags to mark detected items. 2994 * @param[in] target_protocol 2995 * The next protocol in the previous item. 2996 * @param[out] error 2997 * Pointer to error structure. 2998 * 2999 * @return 3000 * 0 on success, a negative errno value otherwise and rte_errno is set. 3001 */ 3002 int 3003 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3004 uint64_t item_flags, 3005 uint8_t target_protocol, 3006 struct rte_flow_error *error) 3007 { 3008 const struct rte_flow_item_nvgre *mask = item->mask; 3009 int ret; 3010 3011 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3012 return rte_flow_error_set(error, EINVAL, 3013 RTE_FLOW_ERROR_TYPE_ITEM, item, 3014 "protocol filtering not compatible" 3015 " with this GRE layer"); 3016 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3017 return rte_flow_error_set(error, ENOTSUP, 3018 RTE_FLOW_ERROR_TYPE_ITEM, item, 3019 "multiple tunnel layers not" 3020 " supported"); 3021 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3022 return rte_flow_error_set(error, ENOTSUP, 3023 RTE_FLOW_ERROR_TYPE_ITEM, item, 3024 "L3 Layer is missing"); 3025 if (!mask) 3026 mask = &rte_flow_item_nvgre_mask; 3027 ret = mlx5_flow_item_acceptable 3028 (item, (const uint8_t *)mask, 3029 (const uint8_t *)&rte_flow_item_nvgre_mask, 3030 sizeof(struct rte_flow_item_nvgre), 3031 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3032 if (ret < 0) 3033 return ret; 3034 return 0; 3035 } 3036 3037 /** 3038 * Validate eCPRI item. 3039 * 3040 * @param[in] item 3041 * Item specification. 3042 * @param[in] item_flags 3043 * Bit-fields that holds the items detected until now. 3044 * @param[in] last_item 3045 * Previous validated item in the pattern items. 3046 * @param[in] ether_type 3047 * Type in the ethernet layer header (including dot1q). 3048 * @param[in] acc_mask 3049 * Acceptable mask, if NULL default internal default mask 3050 * will be used to check whether item fields are supported. 3051 * @param[out] error 3052 * Pointer to error structure. 3053 * 3054 * @return 3055 * 0 on success, a negative errno value otherwise and rte_errno is set. 3056 */ 3057 int 3058 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3059 uint64_t item_flags, 3060 uint64_t last_item, 3061 uint16_t ether_type, 3062 const struct rte_flow_item_ecpri *acc_mask, 3063 struct rte_flow_error *error) 3064 { 3065 const struct rte_flow_item_ecpri *mask = item->mask; 3066 const struct rte_flow_item_ecpri nic_mask = { 3067 .hdr = { 3068 .common = { 3069 .u32 = 3070 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3071 .type = 0xFF, 3072 }).u32), 3073 }, 3074 .dummy[0] = 0xFFFFFFFF, 3075 }, 3076 }; 3077 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3078 MLX5_FLOW_LAYER_OUTER_VLAN); 3079 struct rte_flow_item_ecpri mask_lo; 3080 3081 if (!(last_item & outer_l2_vlan) && 3082 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3083 return rte_flow_error_set(error, EINVAL, 3084 RTE_FLOW_ERROR_TYPE_ITEM, item, 3085 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3086 if ((last_item & outer_l2_vlan) && ether_type && 3087 ether_type != RTE_ETHER_TYPE_ECPRI) 3088 return rte_flow_error_set(error, EINVAL, 3089 RTE_FLOW_ERROR_TYPE_ITEM, item, 3090 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3091 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3092 return rte_flow_error_set(error, EINVAL, 3093 RTE_FLOW_ERROR_TYPE_ITEM, item, 3094 "eCPRI with tunnel is not supported right now"); 3095 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3096 return rte_flow_error_set(error, ENOTSUP, 3097 RTE_FLOW_ERROR_TYPE_ITEM, item, 3098 "multiple L3 layers not supported"); 3099 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3100 return rte_flow_error_set(error, EINVAL, 3101 RTE_FLOW_ERROR_TYPE_ITEM, item, 3102 "eCPRI cannot coexist with a TCP layer"); 3103 /* In specification, eCPRI could be over UDP layer. */ 3104 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3105 return rte_flow_error_set(error, EINVAL, 3106 RTE_FLOW_ERROR_TYPE_ITEM, item, 3107 "eCPRI over UDP layer is not yet supported right now"); 3108 /* Mask for type field in common header could be zero. */ 3109 if (!mask) 3110 mask = &rte_flow_item_ecpri_mask; 3111 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3112 /* Input mask is in big-endian format. */ 3113 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3114 return rte_flow_error_set(error, EINVAL, 3115 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3116 "partial mask is not supported for protocol"); 3117 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3118 return rte_flow_error_set(error, EINVAL, 3119 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3120 "message header mask must be after a type mask"); 3121 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3122 acc_mask ? (const uint8_t *)acc_mask 3123 : (const uint8_t *)&nic_mask, 3124 sizeof(struct rte_flow_item_ecpri), 3125 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3126 } 3127 3128 static int 3129 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3130 const struct rte_flow_attr *attr __rte_unused, 3131 const struct rte_flow_item items[] __rte_unused, 3132 const struct rte_flow_action actions[] __rte_unused, 3133 bool external __rte_unused, 3134 int hairpin __rte_unused, 3135 struct rte_flow_error *error) 3136 { 3137 return rte_flow_error_set(error, ENOTSUP, 3138 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3139 } 3140 3141 static struct mlx5_flow * 3142 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3143 const struct rte_flow_attr *attr __rte_unused, 3144 const struct rte_flow_item items[] __rte_unused, 3145 const struct rte_flow_action actions[] __rte_unused, 3146 struct rte_flow_error *error) 3147 { 3148 rte_flow_error_set(error, ENOTSUP, 3149 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3150 return NULL; 3151 } 3152 3153 static int 3154 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3155 struct mlx5_flow *dev_flow __rte_unused, 3156 const struct rte_flow_attr *attr __rte_unused, 3157 const struct rte_flow_item items[] __rte_unused, 3158 const struct rte_flow_action actions[] __rte_unused, 3159 struct rte_flow_error *error) 3160 { 3161 return rte_flow_error_set(error, ENOTSUP, 3162 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3163 } 3164 3165 static int 3166 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3167 struct rte_flow *flow __rte_unused, 3168 struct rte_flow_error *error) 3169 { 3170 return rte_flow_error_set(error, ENOTSUP, 3171 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3172 } 3173 3174 static void 3175 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3176 struct rte_flow *flow __rte_unused) 3177 { 3178 } 3179 3180 static void 3181 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3182 struct rte_flow *flow __rte_unused) 3183 { 3184 } 3185 3186 static int 3187 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3188 struct rte_flow *flow __rte_unused, 3189 const struct rte_flow_action *actions __rte_unused, 3190 void *data __rte_unused, 3191 struct rte_flow_error *error) 3192 { 3193 return rte_flow_error_set(error, ENOTSUP, 3194 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3195 } 3196 3197 static int 3198 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3199 uint32_t domains __rte_unused, 3200 uint32_t flags __rte_unused) 3201 { 3202 return 0; 3203 } 3204 3205 /* Void driver to protect from null pointer reference. */ 3206 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3207 .validate = flow_null_validate, 3208 .prepare = flow_null_prepare, 3209 .translate = flow_null_translate, 3210 .apply = flow_null_apply, 3211 .remove = flow_null_remove, 3212 .destroy = flow_null_destroy, 3213 .query = flow_null_query, 3214 .sync_domain = flow_null_sync_domain, 3215 }; 3216 3217 /** 3218 * Select flow driver type according to flow attributes and device 3219 * configuration. 3220 * 3221 * @param[in] dev 3222 * Pointer to the dev structure. 3223 * @param[in] attr 3224 * Pointer to the flow attributes. 3225 * 3226 * @return 3227 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3228 */ 3229 static enum mlx5_flow_drv_type 3230 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3231 { 3232 struct mlx5_priv *priv = dev->data->dev_private; 3233 /* The OS can determine first a specific flow type (DV, VERBS) */ 3234 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3235 3236 if (type != MLX5_FLOW_TYPE_MAX) 3237 return type; 3238 /* If no OS specific type - continue with DV/VERBS selection */ 3239 if (attr->transfer && priv->config.dv_esw_en) 3240 type = MLX5_FLOW_TYPE_DV; 3241 if (!attr->transfer) 3242 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3243 MLX5_FLOW_TYPE_VERBS; 3244 return type; 3245 } 3246 3247 #define flow_get_drv_ops(type) flow_drv_ops[type] 3248 3249 /** 3250 * Flow driver validation API. This abstracts calling driver specific functions. 3251 * The type of flow driver is determined according to flow attributes. 3252 * 3253 * @param[in] dev 3254 * Pointer to the dev structure. 3255 * @param[in] attr 3256 * Pointer to the flow attributes. 3257 * @param[in] items 3258 * Pointer to the list of items. 3259 * @param[in] actions 3260 * Pointer to the list of actions. 3261 * @param[in] external 3262 * This flow rule is created by request external to PMD. 3263 * @param[in] hairpin 3264 * Number of hairpin TX actions, 0 means classic flow. 3265 * @param[out] error 3266 * Pointer to the error structure. 3267 * 3268 * @return 3269 * 0 on success, a negative errno value otherwise and rte_errno is set. 3270 */ 3271 static inline int 3272 flow_drv_validate(struct rte_eth_dev *dev, 3273 const struct rte_flow_attr *attr, 3274 const struct rte_flow_item items[], 3275 const struct rte_flow_action actions[], 3276 bool external, int hairpin, struct rte_flow_error *error) 3277 { 3278 const struct mlx5_flow_driver_ops *fops; 3279 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3280 3281 fops = flow_get_drv_ops(type); 3282 return fops->validate(dev, attr, items, actions, external, 3283 hairpin, error); 3284 } 3285 3286 /** 3287 * Flow driver preparation API. This abstracts calling driver specific 3288 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3289 * calculates the size of memory required for device flow, allocates the memory, 3290 * initializes the device flow and returns the pointer. 3291 * 3292 * @note 3293 * This function initializes device flow structure such as dv or verbs in 3294 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3295 * rest. For example, adding returning device flow to flow->dev_flow list and 3296 * setting backward reference to the flow should be done out of this function. 3297 * layers field is not filled either. 3298 * 3299 * @param[in] dev 3300 * Pointer to the dev structure. 3301 * @param[in] attr 3302 * Pointer to the flow attributes. 3303 * @param[in] items 3304 * Pointer to the list of items. 3305 * @param[in] actions 3306 * Pointer to the list of actions. 3307 * @param[in] flow_idx 3308 * This memory pool index to the flow. 3309 * @param[out] error 3310 * Pointer to the error structure. 3311 * 3312 * @return 3313 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3314 */ 3315 static inline struct mlx5_flow * 3316 flow_drv_prepare(struct rte_eth_dev *dev, 3317 const struct rte_flow *flow, 3318 const struct rte_flow_attr *attr, 3319 const struct rte_flow_item items[], 3320 const struct rte_flow_action actions[], 3321 uint32_t flow_idx, 3322 struct rte_flow_error *error) 3323 { 3324 const struct mlx5_flow_driver_ops *fops; 3325 enum mlx5_flow_drv_type type = flow->drv_type; 3326 struct mlx5_flow *mlx5_flow = NULL; 3327 3328 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3329 fops = flow_get_drv_ops(type); 3330 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3331 if (mlx5_flow) 3332 mlx5_flow->flow_idx = flow_idx; 3333 return mlx5_flow; 3334 } 3335 3336 /** 3337 * Flow driver translation API. This abstracts calling driver specific 3338 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3339 * translates a generic flow into a driver flow. flow_drv_prepare() must 3340 * precede. 3341 * 3342 * @note 3343 * dev_flow->layers could be filled as a result of parsing during translation 3344 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3345 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3346 * flow->actions could be overwritten even though all the expanded dev_flows 3347 * have the same actions. 3348 * 3349 * @param[in] dev 3350 * Pointer to the rte dev structure. 3351 * @param[in, out] dev_flow 3352 * Pointer to the mlx5 flow. 3353 * @param[in] attr 3354 * Pointer to the flow attributes. 3355 * @param[in] items 3356 * Pointer to the list of items. 3357 * @param[in] actions 3358 * Pointer to the list of actions. 3359 * @param[out] error 3360 * Pointer to the error structure. 3361 * 3362 * @return 3363 * 0 on success, a negative errno value otherwise and rte_errno is set. 3364 */ 3365 static inline int 3366 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3367 const struct rte_flow_attr *attr, 3368 const struct rte_flow_item items[], 3369 const struct rte_flow_action actions[], 3370 struct rte_flow_error *error) 3371 { 3372 const struct mlx5_flow_driver_ops *fops; 3373 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3374 3375 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3376 fops = flow_get_drv_ops(type); 3377 return fops->translate(dev, dev_flow, attr, items, actions, error); 3378 } 3379 3380 /** 3381 * Flow driver apply API. This abstracts calling driver specific functions. 3382 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3383 * translated driver flows on to device. flow_drv_translate() must precede. 3384 * 3385 * @param[in] dev 3386 * Pointer to Ethernet device structure. 3387 * @param[in, out] flow 3388 * Pointer to flow structure. 3389 * @param[out] error 3390 * Pointer to error structure. 3391 * 3392 * @return 3393 * 0 on success, a negative errno value otherwise and rte_errno is set. 3394 */ 3395 static inline int 3396 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3397 struct rte_flow_error *error) 3398 { 3399 const struct mlx5_flow_driver_ops *fops; 3400 enum mlx5_flow_drv_type type = flow->drv_type; 3401 3402 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3403 fops = flow_get_drv_ops(type); 3404 return fops->apply(dev, flow, error); 3405 } 3406 3407 /** 3408 * Flow driver destroy API. This abstracts calling driver specific functions. 3409 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3410 * on device and releases resources of the flow. 3411 * 3412 * @param[in] dev 3413 * Pointer to Ethernet device. 3414 * @param[in, out] flow 3415 * Pointer to flow structure. 3416 */ 3417 static inline void 3418 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3419 { 3420 const struct mlx5_flow_driver_ops *fops; 3421 enum mlx5_flow_drv_type type = flow->drv_type; 3422 3423 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3424 fops = flow_get_drv_ops(type); 3425 fops->destroy(dev, flow); 3426 } 3427 3428 /** 3429 * Flow driver find RSS policy tbl API. This abstracts calling driver 3430 * specific functions. Parent flow (rte_flow) should have driver 3431 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3432 * 3433 * @param[in] dev 3434 * Pointer to Ethernet device. 3435 * @param[in, out] flow 3436 * Pointer to flow structure. 3437 * @param[in] policy 3438 * Pointer to meter policy table. 3439 * @param[in] rss_desc 3440 * Pointer to rss_desc 3441 */ 3442 static struct mlx5_flow_meter_sub_policy * 3443 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3444 struct rte_flow *flow, 3445 struct mlx5_flow_meter_policy *policy, 3446 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3447 { 3448 const struct mlx5_flow_driver_ops *fops; 3449 enum mlx5_flow_drv_type type = flow->drv_type; 3450 3451 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3452 fops = flow_get_drv_ops(type); 3453 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3454 } 3455 3456 /** 3457 * Flow driver color tag rule API. This abstracts calling driver 3458 * specific functions. Parent flow (rte_flow) should have driver 3459 * type (drv_type). It will create the color tag rules in hierarchy meter. 3460 * 3461 * @param[in] dev 3462 * Pointer to Ethernet device. 3463 * @param[in, out] flow 3464 * Pointer to flow structure. 3465 * @param[in] fm 3466 * Pointer to flow meter structure. 3467 * @param[in] src_port 3468 * The src port this extra rule should use. 3469 * @param[in] item 3470 * The src port id match item. 3471 * @param[out] error 3472 * Pointer to error structure. 3473 */ 3474 static int 3475 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3476 struct rte_flow *flow, 3477 struct mlx5_flow_meter_info *fm, 3478 int32_t src_port, 3479 const struct rte_flow_item *item, 3480 struct rte_flow_error *error) 3481 { 3482 const struct mlx5_flow_driver_ops *fops; 3483 enum mlx5_flow_drv_type type = flow->drv_type; 3484 3485 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3486 fops = flow_get_drv_ops(type); 3487 return fops->meter_hierarchy_rule_create(dev, fm, 3488 src_port, item, error); 3489 } 3490 3491 /** 3492 * Get RSS action from the action list. 3493 * 3494 * @param[in] dev 3495 * Pointer to Ethernet device. 3496 * @param[in] actions 3497 * Pointer to the list of actions. 3498 * @param[in] flow 3499 * Parent flow structure pointer. 3500 * 3501 * @return 3502 * Pointer to the RSS action if exist, else return NULL. 3503 */ 3504 static const struct rte_flow_action_rss* 3505 flow_get_rss_action(struct rte_eth_dev *dev, 3506 const struct rte_flow_action actions[]) 3507 { 3508 struct mlx5_priv *priv = dev->data->dev_private; 3509 const struct rte_flow_action_rss *rss = NULL; 3510 3511 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3512 switch (actions->type) { 3513 case RTE_FLOW_ACTION_TYPE_RSS: 3514 rss = actions->conf; 3515 break; 3516 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3517 { 3518 const struct rte_flow_action_sample *sample = 3519 actions->conf; 3520 const struct rte_flow_action *act = sample->actions; 3521 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3522 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3523 rss = act->conf; 3524 break; 3525 } 3526 case RTE_FLOW_ACTION_TYPE_METER: 3527 { 3528 uint32_t mtr_idx; 3529 struct mlx5_flow_meter_info *fm; 3530 struct mlx5_flow_meter_policy *policy; 3531 const struct rte_flow_action_meter *mtr = actions->conf; 3532 3533 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3534 if (fm && !fm->def_policy) { 3535 policy = mlx5_flow_meter_policy_find(dev, 3536 fm->policy_id, NULL); 3537 MLX5_ASSERT(policy); 3538 if (policy->is_hierarchy) { 3539 policy = 3540 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3541 policy); 3542 if (!policy) 3543 return NULL; 3544 } 3545 if (policy->is_rss) 3546 rss = 3547 policy->act_cnt[RTE_COLOR_GREEN].rss->conf; 3548 } 3549 break; 3550 } 3551 default: 3552 break; 3553 } 3554 } 3555 return rss; 3556 } 3557 3558 /** 3559 * Get ASO age action by index. 3560 * 3561 * @param[in] dev 3562 * Pointer to the Ethernet device structure. 3563 * @param[in] age_idx 3564 * Index to the ASO age action. 3565 * 3566 * @return 3567 * The specified ASO age action. 3568 */ 3569 struct mlx5_aso_age_action* 3570 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3571 { 3572 uint16_t pool_idx = age_idx & UINT16_MAX; 3573 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3574 struct mlx5_priv *priv = dev->data->dev_private; 3575 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3576 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3577 3578 return &pool->actions[offset - 1]; 3579 } 3580 3581 /* maps indirect action to translated direct in some actions array */ 3582 struct mlx5_translated_action_handle { 3583 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3584 int index; /**< Index in related array of rte_flow_action. */ 3585 }; 3586 3587 /** 3588 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3589 * direct action if translation possible. 3590 * This functionality used to run same execution path for both direct and 3591 * indirect actions on flow create. All necessary preparations for indirect 3592 * action handling should be performed on *handle* actions list returned 3593 * from this call. 3594 * 3595 * @param[in] dev 3596 * Pointer to Ethernet device. 3597 * @param[in] actions 3598 * List of actions to translate. 3599 * @param[out] handle 3600 * List to store translated indirect action object handles. 3601 * @param[in, out] indir_n 3602 * Size of *handle* array. On return should be updated with number of 3603 * indirect actions retrieved from the *actions* list. 3604 * @param[out] translated_actions 3605 * List of actions where all indirect actions were translated to direct 3606 * if possible. NULL if no translation took place. 3607 * @param[out] error 3608 * Pointer to the error structure. 3609 * 3610 * @return 3611 * 0 on success, a negative errno value otherwise and rte_errno is set. 3612 */ 3613 static int 3614 flow_action_handles_translate(struct rte_eth_dev *dev, 3615 const struct rte_flow_action actions[], 3616 struct mlx5_translated_action_handle *handle, 3617 int *indir_n, 3618 struct rte_flow_action **translated_actions, 3619 struct rte_flow_error *error) 3620 { 3621 struct mlx5_priv *priv = dev->data->dev_private; 3622 struct rte_flow_action *translated = NULL; 3623 size_t actions_size; 3624 int n; 3625 int copied_n = 0; 3626 struct mlx5_translated_action_handle *handle_end = NULL; 3627 3628 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3629 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3630 continue; 3631 if (copied_n == *indir_n) { 3632 return rte_flow_error_set 3633 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3634 NULL, "too many shared actions"); 3635 } 3636 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3637 sizeof(actions[n].conf)); 3638 handle[copied_n].index = n; 3639 copied_n++; 3640 } 3641 n++; 3642 *indir_n = copied_n; 3643 if (!copied_n) 3644 return 0; 3645 actions_size = sizeof(struct rte_flow_action) * n; 3646 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3647 if (!translated) { 3648 rte_errno = ENOMEM; 3649 return -ENOMEM; 3650 } 3651 memcpy(translated, actions, actions_size); 3652 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3653 struct mlx5_shared_action_rss *shared_rss; 3654 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3655 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3656 uint32_t idx = act_idx & 3657 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3658 3659 switch (type) { 3660 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3661 shared_rss = mlx5_ipool_get 3662 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3663 translated[handle->index].type = 3664 RTE_FLOW_ACTION_TYPE_RSS; 3665 translated[handle->index].conf = 3666 &shared_rss->origin; 3667 break; 3668 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3669 translated[handle->index].type = 3670 (enum rte_flow_action_type) 3671 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3672 translated[handle->index].conf = (void *)(uintptr_t)idx; 3673 break; 3674 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3675 if (priv->sh->flow_hit_aso_en) { 3676 translated[handle->index].type = 3677 (enum rte_flow_action_type) 3678 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3679 translated[handle->index].conf = 3680 (void *)(uintptr_t)idx; 3681 break; 3682 } 3683 /* Fall-through */ 3684 case MLX5_INDIRECT_ACTION_TYPE_CT: 3685 if (priv->sh->ct_aso_en) { 3686 translated[handle->index].type = 3687 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3688 translated[handle->index].conf = 3689 (void *)(uintptr_t)idx; 3690 break; 3691 } 3692 /* Fall-through */ 3693 default: 3694 mlx5_free(translated); 3695 return rte_flow_error_set 3696 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3697 NULL, "invalid indirect action type"); 3698 } 3699 } 3700 *translated_actions = translated; 3701 return 0; 3702 } 3703 3704 /** 3705 * Get Shared RSS action from the action list. 3706 * 3707 * @param[in] dev 3708 * Pointer to Ethernet device. 3709 * @param[in] shared 3710 * Pointer to the list of actions. 3711 * @param[in] shared_n 3712 * Actions list length. 3713 * 3714 * @return 3715 * The MLX5 RSS action ID if exists, otherwise return 0. 3716 */ 3717 static uint32_t 3718 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3719 struct mlx5_translated_action_handle *handle, 3720 int shared_n) 3721 { 3722 struct mlx5_translated_action_handle *handle_end; 3723 struct mlx5_priv *priv = dev->data->dev_private; 3724 struct mlx5_shared_action_rss *shared_rss; 3725 3726 3727 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3728 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3729 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3730 uint32_t idx = act_idx & 3731 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3732 switch (type) { 3733 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3734 shared_rss = mlx5_ipool_get 3735 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3736 idx); 3737 __atomic_add_fetch(&shared_rss->refcnt, 1, 3738 __ATOMIC_RELAXED); 3739 return idx; 3740 default: 3741 break; 3742 } 3743 } 3744 return 0; 3745 } 3746 3747 static unsigned int 3748 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3749 { 3750 const struct rte_flow_item *item; 3751 unsigned int has_vlan = 0; 3752 3753 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3754 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3755 has_vlan = 1; 3756 break; 3757 } 3758 } 3759 if (has_vlan) 3760 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3761 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3762 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3763 MLX5_EXPANSION_ROOT_OUTER; 3764 } 3765 3766 /** 3767 * Get layer flags from the prefix flow. 3768 * 3769 * Some flows may be split to several subflows, the prefix subflow gets the 3770 * match items and the suffix sub flow gets the actions. 3771 * Some actions need the user defined match item flags to get the detail for 3772 * the action. 3773 * This function helps the suffix flow to get the item layer flags from prefix 3774 * subflow. 3775 * 3776 * @param[in] dev_flow 3777 * Pointer the created preifx subflow. 3778 * 3779 * @return 3780 * The layers get from prefix subflow. 3781 */ 3782 static inline uint64_t 3783 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3784 { 3785 uint64_t layers = 0; 3786 3787 /* 3788 * Layers bits could be localization, but usually the compiler will 3789 * help to do the optimization work for source code. 3790 * If no decap actions, use the layers directly. 3791 */ 3792 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3793 return dev_flow->handle->layers; 3794 /* Convert L3 layers with decap action. */ 3795 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3796 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3797 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3798 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3799 /* Convert L4 layers with decap action. */ 3800 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3801 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3802 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3803 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3804 return layers; 3805 } 3806 3807 /** 3808 * Get metadata split action information. 3809 * 3810 * @param[in] actions 3811 * Pointer to the list of actions. 3812 * @param[out] qrss 3813 * Pointer to the return pointer. 3814 * @param[out] qrss_type 3815 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3816 * if no QUEUE/RSS is found. 3817 * @param[out] encap_idx 3818 * Pointer to the index of the encap action if exists, otherwise the last 3819 * action index. 3820 * 3821 * @return 3822 * Total number of actions. 3823 */ 3824 static int 3825 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3826 const struct rte_flow_action **qrss, 3827 int *encap_idx) 3828 { 3829 const struct rte_flow_action_raw_encap *raw_encap; 3830 int actions_n = 0; 3831 int raw_decap_idx = -1; 3832 3833 *encap_idx = -1; 3834 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3835 switch (actions->type) { 3836 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3837 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3838 *encap_idx = actions_n; 3839 break; 3840 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3841 raw_decap_idx = actions_n; 3842 break; 3843 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3844 raw_encap = actions->conf; 3845 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3846 *encap_idx = raw_decap_idx != -1 ? 3847 raw_decap_idx : actions_n; 3848 break; 3849 case RTE_FLOW_ACTION_TYPE_QUEUE: 3850 case RTE_FLOW_ACTION_TYPE_RSS: 3851 *qrss = actions; 3852 break; 3853 default: 3854 break; 3855 } 3856 actions_n++; 3857 } 3858 if (*encap_idx == -1) 3859 *encap_idx = actions_n; 3860 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3861 return actions_n + 1; 3862 } 3863 3864 /** 3865 * Check if the action will change packet. 3866 * 3867 * @param dev 3868 * Pointer to Ethernet device. 3869 * @param[in] type 3870 * action type. 3871 * 3872 * @return 3873 * true if action will change packet, false otherwise. 3874 */ 3875 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 3876 enum rte_flow_action_type type) 3877 { 3878 struct mlx5_priv *priv = dev->data->dev_private; 3879 3880 switch (type) { 3881 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3882 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3883 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3884 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3885 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3886 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3887 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3888 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3889 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3890 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3891 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3892 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3893 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3894 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3895 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 3896 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 3897 case RTE_FLOW_ACTION_TYPE_SET_META: 3898 case RTE_FLOW_ACTION_TYPE_SET_TAG: 3899 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 3900 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3901 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3902 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3903 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3904 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3905 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3906 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3907 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3908 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3909 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 3910 return true; 3911 case RTE_FLOW_ACTION_TYPE_FLAG: 3912 case RTE_FLOW_ACTION_TYPE_MARK: 3913 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 3914 return true; 3915 else 3916 return false; 3917 default: 3918 return false; 3919 } 3920 } 3921 3922 /** 3923 * Check meter action from the action list. 3924 * 3925 * @param dev 3926 * Pointer to Ethernet device. 3927 * @param[in] actions 3928 * Pointer to the list of actions. 3929 * @param[out] has_mtr 3930 * Pointer to the meter exist flag. 3931 * @param[out] has_modify 3932 * Pointer to the flag showing there's packet change action. 3933 * @param[out] meter_id 3934 * Pointer to the meter id. 3935 * 3936 * @return 3937 * Total number of actions. 3938 */ 3939 static int 3940 flow_check_meter_action(struct rte_eth_dev *dev, 3941 const struct rte_flow_action actions[], 3942 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 3943 { 3944 const struct rte_flow_action_meter *mtr = NULL; 3945 int actions_n = 0; 3946 3947 MLX5_ASSERT(has_mtr); 3948 *has_mtr = false; 3949 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3950 switch (actions->type) { 3951 case RTE_FLOW_ACTION_TYPE_METER: 3952 mtr = actions->conf; 3953 *meter_id = mtr->mtr_id; 3954 *has_mtr = true; 3955 break; 3956 default: 3957 break; 3958 } 3959 if (!*has_mtr) 3960 *has_modify |= flow_check_modify_action_type(dev, 3961 actions->type); 3962 actions_n++; 3963 } 3964 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3965 return actions_n + 1; 3966 } 3967 3968 /** 3969 * Check if the flow should be split due to hairpin. 3970 * The reason for the split is that in current HW we can't 3971 * support encap and push-vlan on Rx, so if a flow contains 3972 * these actions we move it to Tx. 3973 * 3974 * @param dev 3975 * Pointer to Ethernet device. 3976 * @param[in] attr 3977 * Flow rule attributes. 3978 * @param[in] actions 3979 * Associated actions (list terminated by the END action). 3980 * 3981 * @return 3982 * > 0 the number of actions and the flow should be split, 3983 * 0 when no split required. 3984 */ 3985 static int 3986 flow_check_hairpin_split(struct rte_eth_dev *dev, 3987 const struct rte_flow_attr *attr, 3988 const struct rte_flow_action actions[]) 3989 { 3990 int queue_action = 0; 3991 int action_n = 0; 3992 int split = 0; 3993 const struct rte_flow_action_queue *queue; 3994 const struct rte_flow_action_rss *rss; 3995 const struct rte_flow_action_raw_encap *raw_encap; 3996 const struct rte_eth_hairpin_conf *conf; 3997 3998 if (!attr->ingress) 3999 return 0; 4000 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4001 switch (actions->type) { 4002 case RTE_FLOW_ACTION_TYPE_QUEUE: 4003 queue = actions->conf; 4004 if (queue == NULL) 4005 return 0; 4006 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4007 if (conf == NULL || conf->tx_explicit != 0) 4008 return 0; 4009 queue_action = 1; 4010 action_n++; 4011 break; 4012 case RTE_FLOW_ACTION_TYPE_RSS: 4013 rss = actions->conf; 4014 if (rss == NULL || rss->queue_num == 0) 4015 return 0; 4016 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4017 if (conf == NULL || conf->tx_explicit != 0) 4018 return 0; 4019 queue_action = 1; 4020 action_n++; 4021 break; 4022 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4023 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4024 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4025 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4026 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4027 split++; 4028 action_n++; 4029 break; 4030 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4031 raw_encap = actions->conf; 4032 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4033 split++; 4034 action_n++; 4035 break; 4036 default: 4037 action_n++; 4038 break; 4039 } 4040 } 4041 if (split && queue_action) 4042 return action_n; 4043 return 0; 4044 } 4045 4046 /* Declare flow create/destroy prototype in advance. */ 4047 static uint32_t 4048 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4049 const struct rte_flow_attr *attr, 4050 const struct rte_flow_item items[], 4051 const struct rte_flow_action actions[], 4052 bool external, struct rte_flow_error *error); 4053 4054 static void 4055 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4056 uint32_t flow_idx); 4057 4058 int 4059 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused, 4060 struct mlx5_hlist_entry *entry, 4061 uint64_t key, void *cb_ctx __rte_unused) 4062 { 4063 struct mlx5_flow_mreg_copy_resource *mcp_res = 4064 container_of(entry, typeof(*mcp_res), hlist_ent); 4065 4066 return mcp_res->mark_id != key; 4067 } 4068 4069 struct mlx5_hlist_entry * 4070 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 4071 void *cb_ctx) 4072 { 4073 struct rte_eth_dev *dev = list->ctx; 4074 struct mlx5_priv *priv = dev->data->dev_private; 4075 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4076 struct mlx5_flow_mreg_copy_resource *mcp_res; 4077 struct rte_flow_error *error = ctx->error; 4078 uint32_t idx = 0; 4079 int ret; 4080 uint32_t mark_id = key; 4081 struct rte_flow_attr attr = { 4082 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4083 .ingress = 1, 4084 }; 4085 struct mlx5_rte_flow_item_tag tag_spec = { 4086 .data = mark_id, 4087 }; 4088 struct rte_flow_item items[] = { 4089 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4090 }; 4091 struct rte_flow_action_mark ftag = { 4092 .id = mark_id, 4093 }; 4094 struct mlx5_flow_action_copy_mreg cp_mreg = { 4095 .dst = REG_B, 4096 .src = REG_NON, 4097 }; 4098 struct rte_flow_action_jump jump = { 4099 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4100 }; 4101 struct rte_flow_action actions[] = { 4102 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4103 }; 4104 4105 /* Fill the register fileds in the flow. */ 4106 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4107 if (ret < 0) 4108 return NULL; 4109 tag_spec.id = ret; 4110 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4111 if (ret < 0) 4112 return NULL; 4113 cp_mreg.src = ret; 4114 /* Provide the full width of FLAG specific value. */ 4115 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4116 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4117 /* Build a new flow. */ 4118 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4119 items[0] = (struct rte_flow_item){ 4120 .type = (enum rte_flow_item_type) 4121 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4122 .spec = &tag_spec, 4123 }; 4124 items[1] = (struct rte_flow_item){ 4125 .type = RTE_FLOW_ITEM_TYPE_END, 4126 }; 4127 actions[0] = (struct rte_flow_action){ 4128 .type = (enum rte_flow_action_type) 4129 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4130 .conf = &ftag, 4131 }; 4132 actions[1] = (struct rte_flow_action){ 4133 .type = (enum rte_flow_action_type) 4134 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4135 .conf = &cp_mreg, 4136 }; 4137 actions[2] = (struct rte_flow_action){ 4138 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4139 .conf = &jump, 4140 }; 4141 actions[3] = (struct rte_flow_action){ 4142 .type = RTE_FLOW_ACTION_TYPE_END, 4143 }; 4144 } else { 4145 /* Default rule, wildcard match. */ 4146 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4147 items[0] = (struct rte_flow_item){ 4148 .type = RTE_FLOW_ITEM_TYPE_END, 4149 }; 4150 actions[0] = (struct rte_flow_action){ 4151 .type = (enum rte_flow_action_type) 4152 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4153 .conf = &cp_mreg, 4154 }; 4155 actions[1] = (struct rte_flow_action){ 4156 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4157 .conf = &jump, 4158 }; 4159 actions[2] = (struct rte_flow_action){ 4160 .type = RTE_FLOW_ACTION_TYPE_END, 4161 }; 4162 } 4163 /* Build a new entry. */ 4164 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4165 if (!mcp_res) { 4166 rte_errno = ENOMEM; 4167 return NULL; 4168 } 4169 mcp_res->idx = idx; 4170 mcp_res->mark_id = mark_id; 4171 /* 4172 * The copy Flows are not included in any list. There 4173 * ones are referenced from other Flows and can not 4174 * be applied, removed, deleted in ardbitrary order 4175 * by list traversing. 4176 */ 4177 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4178 &attr, items, actions, false, error); 4179 if (!mcp_res->rix_flow) { 4180 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4181 return NULL; 4182 } 4183 return &mcp_res->hlist_ent; 4184 } 4185 4186 /** 4187 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4188 * 4189 * As mark_id is unique, if there's already a registered flow for the mark_id, 4190 * return by increasing the reference counter of the resource. Otherwise, create 4191 * the resource (mcp_res) and flow. 4192 * 4193 * Flow looks like, 4194 * - If ingress port is ANY and reg_c[1] is mark_id, 4195 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4196 * 4197 * For default flow (zero mark_id), flow is like, 4198 * - If ingress port is ANY, 4199 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4200 * 4201 * @param dev 4202 * Pointer to Ethernet device. 4203 * @param mark_id 4204 * ID of MARK action, zero means default flow for META. 4205 * @param[out] error 4206 * Perform verbose error reporting if not NULL. 4207 * 4208 * @return 4209 * Associated resource on success, NULL otherwise and rte_errno is set. 4210 */ 4211 static struct mlx5_flow_mreg_copy_resource * 4212 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4213 struct rte_flow_error *error) 4214 { 4215 struct mlx5_priv *priv = dev->data->dev_private; 4216 struct mlx5_hlist_entry *entry; 4217 struct mlx5_flow_cb_ctx ctx = { 4218 .dev = dev, 4219 .error = error, 4220 }; 4221 4222 /* Check if already registered. */ 4223 MLX5_ASSERT(priv->mreg_cp_tbl); 4224 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4225 if (!entry) 4226 return NULL; 4227 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4228 hlist_ent); 4229 } 4230 4231 void 4232 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 4233 { 4234 struct mlx5_flow_mreg_copy_resource *mcp_res = 4235 container_of(entry, typeof(*mcp_res), hlist_ent); 4236 struct rte_eth_dev *dev = list->ctx; 4237 struct mlx5_priv *priv = dev->data->dev_private; 4238 4239 MLX5_ASSERT(mcp_res->rix_flow); 4240 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4241 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4242 } 4243 4244 /** 4245 * Release flow in RX_CP_TBL. 4246 * 4247 * @param dev 4248 * Pointer to Ethernet device. 4249 * @flow 4250 * Parent flow for wich copying is provided. 4251 */ 4252 static void 4253 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4254 struct rte_flow *flow) 4255 { 4256 struct mlx5_flow_mreg_copy_resource *mcp_res; 4257 struct mlx5_priv *priv = dev->data->dev_private; 4258 4259 if (!flow->rix_mreg_copy) 4260 return; 4261 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4262 flow->rix_mreg_copy); 4263 if (!mcp_res || !priv->mreg_cp_tbl) 4264 return; 4265 MLX5_ASSERT(mcp_res->rix_flow); 4266 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4267 flow->rix_mreg_copy = 0; 4268 } 4269 4270 /** 4271 * Remove the default copy action from RX_CP_TBL. 4272 * 4273 * This functions is called in the mlx5_dev_start(). No thread safe 4274 * is guaranteed. 4275 * 4276 * @param dev 4277 * Pointer to Ethernet device. 4278 */ 4279 static void 4280 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4281 { 4282 struct mlx5_hlist_entry *entry; 4283 struct mlx5_priv *priv = dev->data->dev_private; 4284 4285 /* Check if default flow is registered. */ 4286 if (!priv->mreg_cp_tbl) 4287 return; 4288 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 4289 MLX5_DEFAULT_COPY_ID, NULL); 4290 if (!entry) 4291 return; 4292 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4293 } 4294 4295 /** 4296 * Add the default copy action in in RX_CP_TBL. 4297 * 4298 * This functions is called in the mlx5_dev_start(). No thread safe 4299 * is guaranteed. 4300 * 4301 * @param dev 4302 * Pointer to Ethernet device. 4303 * @param[out] error 4304 * Perform verbose error reporting if not NULL. 4305 * 4306 * @return 4307 * 0 for success, negative value otherwise and rte_errno is set. 4308 */ 4309 static int 4310 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4311 struct rte_flow_error *error) 4312 { 4313 struct mlx5_priv *priv = dev->data->dev_private; 4314 struct mlx5_flow_mreg_copy_resource *mcp_res; 4315 4316 /* Check whether extensive metadata feature is engaged. */ 4317 if (!priv->config.dv_flow_en || 4318 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4319 !mlx5_flow_ext_mreg_supported(dev) || 4320 !priv->sh->dv_regc0_mask) 4321 return 0; 4322 /* 4323 * Add default mreg copy flow may be called multiple time, but 4324 * only be called once in stop. Avoid register it twice. 4325 */ 4326 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 4327 return 0; 4328 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 4329 if (!mcp_res) 4330 return -rte_errno; 4331 return 0; 4332 } 4333 4334 /** 4335 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4336 * 4337 * All the flow having Q/RSS action should be split by 4338 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4339 * performs the following, 4340 * - CQE->flow_tag := reg_c[1] (MARK) 4341 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4342 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4343 * but there should be a flow per each MARK ID set by MARK action. 4344 * 4345 * For the aforementioned reason, if there's a MARK action in flow's action 4346 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4347 * the MARK ID to CQE's flow_tag like, 4348 * - If reg_c[1] is mark_id, 4349 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4350 * 4351 * For SET_META action which stores value in reg_c[0], as the destination is 4352 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4353 * MARK ID means the default flow. The default flow looks like, 4354 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4355 * 4356 * @param dev 4357 * Pointer to Ethernet device. 4358 * @param flow 4359 * Pointer to flow structure. 4360 * @param[in] actions 4361 * Pointer to the list of actions. 4362 * @param[out] error 4363 * Perform verbose error reporting if not NULL. 4364 * 4365 * @return 4366 * 0 on success, negative value otherwise and rte_errno is set. 4367 */ 4368 static int 4369 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4370 struct rte_flow *flow, 4371 const struct rte_flow_action *actions, 4372 struct rte_flow_error *error) 4373 { 4374 struct mlx5_priv *priv = dev->data->dev_private; 4375 struct mlx5_dev_config *config = &priv->config; 4376 struct mlx5_flow_mreg_copy_resource *mcp_res; 4377 const struct rte_flow_action_mark *mark; 4378 4379 /* Check whether extensive metadata feature is engaged. */ 4380 if (!config->dv_flow_en || 4381 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4382 !mlx5_flow_ext_mreg_supported(dev) || 4383 !priv->sh->dv_regc0_mask) 4384 return 0; 4385 /* Find MARK action. */ 4386 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4387 switch (actions->type) { 4388 case RTE_FLOW_ACTION_TYPE_FLAG: 4389 mcp_res = flow_mreg_add_copy_action 4390 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4391 if (!mcp_res) 4392 return -rte_errno; 4393 flow->rix_mreg_copy = mcp_res->idx; 4394 return 0; 4395 case RTE_FLOW_ACTION_TYPE_MARK: 4396 mark = (const struct rte_flow_action_mark *) 4397 actions->conf; 4398 mcp_res = 4399 flow_mreg_add_copy_action(dev, mark->id, error); 4400 if (!mcp_res) 4401 return -rte_errno; 4402 flow->rix_mreg_copy = mcp_res->idx; 4403 return 0; 4404 default: 4405 break; 4406 } 4407 } 4408 return 0; 4409 } 4410 4411 #define MLX5_MAX_SPLIT_ACTIONS 24 4412 #define MLX5_MAX_SPLIT_ITEMS 24 4413 4414 /** 4415 * Split the hairpin flow. 4416 * Since HW can't support encap and push-vlan on Rx, we move these 4417 * actions to Tx. 4418 * If the count action is after the encap then we also 4419 * move the count action. in this case the count will also measure 4420 * the outer bytes. 4421 * 4422 * @param dev 4423 * Pointer to Ethernet device. 4424 * @param[in] actions 4425 * Associated actions (list terminated by the END action). 4426 * @param[out] actions_rx 4427 * Rx flow actions. 4428 * @param[out] actions_tx 4429 * Tx flow actions.. 4430 * @param[out] pattern_tx 4431 * The pattern items for the Tx flow. 4432 * @param[out] flow_id 4433 * The flow ID connected to this flow. 4434 * 4435 * @return 4436 * 0 on success. 4437 */ 4438 static int 4439 flow_hairpin_split(struct rte_eth_dev *dev, 4440 const struct rte_flow_action actions[], 4441 struct rte_flow_action actions_rx[], 4442 struct rte_flow_action actions_tx[], 4443 struct rte_flow_item pattern_tx[], 4444 uint32_t flow_id) 4445 { 4446 const struct rte_flow_action_raw_encap *raw_encap; 4447 const struct rte_flow_action_raw_decap *raw_decap; 4448 struct mlx5_rte_flow_action_set_tag *set_tag; 4449 struct rte_flow_action *tag_action; 4450 struct mlx5_rte_flow_item_tag *tag_item; 4451 struct rte_flow_item *item; 4452 char *addr; 4453 int encap = 0; 4454 4455 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4456 switch (actions->type) { 4457 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4458 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4459 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4460 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4461 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4462 rte_memcpy(actions_tx, actions, 4463 sizeof(struct rte_flow_action)); 4464 actions_tx++; 4465 break; 4466 case RTE_FLOW_ACTION_TYPE_COUNT: 4467 if (encap) { 4468 rte_memcpy(actions_tx, actions, 4469 sizeof(struct rte_flow_action)); 4470 actions_tx++; 4471 } else { 4472 rte_memcpy(actions_rx, actions, 4473 sizeof(struct rte_flow_action)); 4474 actions_rx++; 4475 } 4476 break; 4477 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4478 raw_encap = actions->conf; 4479 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4480 memcpy(actions_tx, actions, 4481 sizeof(struct rte_flow_action)); 4482 actions_tx++; 4483 encap = 1; 4484 } else { 4485 rte_memcpy(actions_rx, actions, 4486 sizeof(struct rte_flow_action)); 4487 actions_rx++; 4488 } 4489 break; 4490 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4491 raw_decap = actions->conf; 4492 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4493 memcpy(actions_tx, actions, 4494 sizeof(struct rte_flow_action)); 4495 actions_tx++; 4496 } else { 4497 rte_memcpy(actions_rx, actions, 4498 sizeof(struct rte_flow_action)); 4499 actions_rx++; 4500 } 4501 break; 4502 default: 4503 rte_memcpy(actions_rx, actions, 4504 sizeof(struct rte_flow_action)); 4505 actions_rx++; 4506 break; 4507 } 4508 } 4509 /* Add set meta action and end action for the Rx flow. */ 4510 tag_action = actions_rx; 4511 tag_action->type = (enum rte_flow_action_type) 4512 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4513 actions_rx++; 4514 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4515 actions_rx++; 4516 set_tag = (void *)actions_rx; 4517 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4518 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4519 .data = flow_id, 4520 }; 4521 MLX5_ASSERT(set_tag->id > REG_NON); 4522 tag_action->conf = set_tag; 4523 /* Create Tx item list. */ 4524 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4525 addr = (void *)&pattern_tx[2]; 4526 item = pattern_tx; 4527 item->type = (enum rte_flow_item_type) 4528 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4529 tag_item = (void *)addr; 4530 tag_item->data = flow_id; 4531 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4532 MLX5_ASSERT(set_tag->id > REG_NON); 4533 item->spec = tag_item; 4534 addr += sizeof(struct mlx5_rte_flow_item_tag); 4535 tag_item = (void *)addr; 4536 tag_item->data = UINT32_MAX; 4537 tag_item->id = UINT16_MAX; 4538 item->mask = tag_item; 4539 item->last = NULL; 4540 item++; 4541 item->type = RTE_FLOW_ITEM_TYPE_END; 4542 return 0; 4543 } 4544 4545 /** 4546 * The last stage of splitting chain, just creates the subflow 4547 * without any modification. 4548 * 4549 * @param[in] dev 4550 * Pointer to Ethernet device. 4551 * @param[in] flow 4552 * Parent flow structure pointer. 4553 * @param[in, out] sub_flow 4554 * Pointer to return the created subflow, may be NULL. 4555 * @param[in] attr 4556 * Flow rule attributes. 4557 * @param[in] items 4558 * Pattern specification (list terminated by the END pattern item). 4559 * @param[in] actions 4560 * Associated actions (list terminated by the END action). 4561 * @param[in] flow_split_info 4562 * Pointer to flow split info structure. 4563 * @param[out] error 4564 * Perform verbose error reporting if not NULL. 4565 * @return 4566 * 0 on success, negative value otherwise 4567 */ 4568 static int 4569 flow_create_split_inner(struct rte_eth_dev *dev, 4570 struct rte_flow *flow, 4571 struct mlx5_flow **sub_flow, 4572 const struct rte_flow_attr *attr, 4573 const struct rte_flow_item items[], 4574 const struct rte_flow_action actions[], 4575 struct mlx5_flow_split_info *flow_split_info, 4576 struct rte_flow_error *error) 4577 { 4578 struct mlx5_flow *dev_flow; 4579 4580 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4581 flow_split_info->flow_idx, error); 4582 if (!dev_flow) 4583 return -rte_errno; 4584 dev_flow->flow = flow; 4585 dev_flow->external = flow_split_info->external; 4586 dev_flow->skip_scale = flow_split_info->skip_scale; 4587 /* Subflow object was created, we must include one in the list. */ 4588 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4589 dev_flow->handle, next); 4590 /* 4591 * If dev_flow is as one of the suffix flow, some actions in suffix 4592 * flow may need some user defined item layer flags, and pass the 4593 * Metadate rxq mark flag to suffix flow as well. 4594 */ 4595 if (flow_split_info->prefix_layers) 4596 dev_flow->handle->layers = flow_split_info->prefix_layers; 4597 if (flow_split_info->prefix_mark) 4598 dev_flow->handle->mark = 1; 4599 if (sub_flow) 4600 *sub_flow = dev_flow; 4601 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4602 dev_flow->dv.table_id = flow_split_info->table_id; 4603 #endif 4604 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4605 } 4606 4607 /** 4608 * Get the sub policy of a meter. 4609 * 4610 * @param[in] dev 4611 * Pointer to Ethernet device. 4612 * @param[in] flow 4613 * Parent flow structure pointer. 4614 * @param wks 4615 * Pointer to thread flow work space. 4616 * @param[in] attr 4617 * Flow rule attributes. 4618 * @param[in] items 4619 * Pattern specification (list terminated by the END pattern item). 4620 * @param[out] error 4621 * Perform verbose error reporting if not NULL. 4622 * 4623 * @return 4624 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4625 */ 4626 static struct mlx5_flow_meter_sub_policy * 4627 get_meter_sub_policy(struct rte_eth_dev *dev, 4628 struct rte_flow *flow, 4629 struct mlx5_flow_workspace *wks, 4630 const struct rte_flow_attr *attr, 4631 const struct rte_flow_item items[], 4632 struct rte_flow_error *error) 4633 { 4634 struct mlx5_flow_meter_policy *policy; 4635 struct mlx5_flow_meter_policy *final_policy; 4636 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4637 4638 policy = wks->policy; 4639 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4640 if (final_policy->is_rss || final_policy->is_queue) { 4641 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4642 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4643 uint32_t i; 4644 4645 /** 4646 * This is a tmp dev_flow, 4647 * no need to register any matcher for it in translate. 4648 */ 4649 wks->skip_matcher_reg = 1; 4650 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4651 struct mlx5_flow dev_flow = {0}; 4652 struct mlx5_flow_handle dev_handle = { {0} }; 4653 4654 if (final_policy->is_rss) { 4655 const void *rss_act = 4656 final_policy->act_cnt[i].rss->conf; 4657 struct rte_flow_action rss_actions[2] = { 4658 [0] = { 4659 .type = RTE_FLOW_ACTION_TYPE_RSS, 4660 .conf = rss_act 4661 }, 4662 [1] = { 4663 .type = RTE_FLOW_ACTION_TYPE_END, 4664 .conf = NULL 4665 } 4666 }; 4667 4668 dev_flow.handle = &dev_handle; 4669 dev_flow.ingress = attr->ingress; 4670 dev_flow.flow = flow; 4671 dev_flow.external = 0; 4672 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4673 dev_flow.dv.transfer = attr->transfer; 4674 #endif 4675 /** 4676 * Translate RSS action to get rss hash fields. 4677 */ 4678 if (flow_drv_translate(dev, &dev_flow, attr, 4679 items, rss_actions, error)) 4680 goto exit; 4681 rss_desc_v[i] = wks->rss_desc; 4682 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4683 rss_desc_v[i].hash_fields = 4684 dev_flow.hash_fields; 4685 rss_desc_v[i].queue_num = 4686 rss_desc_v[i].hash_fields ? 4687 rss_desc_v[i].queue_num : 1; 4688 rss_desc_v[i].tunnel = 4689 !!(dev_flow.handle->layers & 4690 MLX5_FLOW_LAYER_TUNNEL); 4691 } else { 4692 /* This is queue action. */ 4693 rss_desc_v[i] = wks->rss_desc; 4694 rss_desc_v[i].key_len = 0; 4695 rss_desc_v[i].hash_fields = 0; 4696 rss_desc_v[i].queue = 4697 &final_policy->act_cnt[i].queue; 4698 rss_desc_v[i].queue_num = 1; 4699 } 4700 rss_desc[i] = &rss_desc_v[i]; 4701 } 4702 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4703 flow, policy, rss_desc); 4704 } else { 4705 enum mlx5_meter_domain mtr_domain = 4706 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4707 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4708 MLX5_MTR_DOMAIN_INGRESS; 4709 sub_policy = policy->sub_policys[mtr_domain][0]; 4710 } 4711 if (!sub_policy) { 4712 rte_flow_error_set(error, EINVAL, 4713 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4714 "Failed to get meter sub-policy."); 4715 goto exit; 4716 } 4717 exit: 4718 return sub_policy; 4719 } 4720 4721 /** 4722 * Split the meter flow. 4723 * 4724 * As meter flow will split to three sub flow, other than meter 4725 * action, the other actions make sense to only meter accepts 4726 * the packet. If it need to be dropped, no other additional 4727 * actions should be take. 4728 * 4729 * One kind of special action which decapsulates the L3 tunnel 4730 * header will be in the prefix sub flow, as not to take the 4731 * L3 tunnel header into account. 4732 * 4733 * @param[in] dev 4734 * Pointer to Ethernet device. 4735 * @param[in] flow 4736 * Parent flow structure pointer. 4737 * @param wks 4738 * Pointer to thread flow work space. 4739 * @param[in] attr 4740 * Flow rule attributes. 4741 * @param[in] items 4742 * Pattern specification (list terminated by the END pattern item). 4743 * @param[out] sfx_items 4744 * Suffix flow match items (list terminated by the END pattern item). 4745 * @param[in] actions 4746 * Associated actions (list terminated by the END action). 4747 * @param[out] actions_sfx 4748 * Suffix flow actions. 4749 * @param[out] actions_pre 4750 * Prefix flow actions. 4751 * @param[out] mtr_flow_id 4752 * Pointer to meter flow id. 4753 * @param[out] error 4754 * Perform verbose error reporting if not NULL. 4755 * 4756 * @return 4757 * 0 on success, a negative errno value otherwise and rte_errno is set. 4758 */ 4759 static int 4760 flow_meter_split_prep(struct rte_eth_dev *dev, 4761 struct rte_flow *flow, 4762 struct mlx5_flow_workspace *wks, 4763 const struct rte_flow_attr *attr, 4764 const struct rte_flow_item items[], 4765 struct rte_flow_item sfx_items[], 4766 const struct rte_flow_action actions[], 4767 struct rte_flow_action actions_sfx[], 4768 struct rte_flow_action actions_pre[], 4769 uint32_t *mtr_flow_id, 4770 struct rte_flow_error *error) 4771 { 4772 struct mlx5_priv *priv = dev->data->dev_private; 4773 struct mlx5_flow_meter_info *fm = wks->fm; 4774 struct rte_flow_action *tag_action = NULL; 4775 struct rte_flow_item *tag_item; 4776 struct mlx5_rte_flow_action_set_tag *set_tag; 4777 const struct rte_flow_action_raw_encap *raw_encap; 4778 const struct rte_flow_action_raw_decap *raw_decap; 4779 struct mlx5_rte_flow_item_tag *tag_item_spec; 4780 struct mlx5_rte_flow_item_tag *tag_item_mask; 4781 uint32_t tag_id = 0; 4782 struct rte_flow_item *vlan_item_dst = NULL; 4783 const struct rte_flow_item *vlan_item_src = NULL; 4784 struct rte_flow_action *hw_mtr_action; 4785 struct rte_flow_action *action_pre_head = NULL; 4786 int32_t flow_src_port = priv->representor_id; 4787 bool mtr_first; 4788 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 4789 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 4790 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 4791 uint32_t flow_id = 0; 4792 uint32_t flow_id_reversed = 0; 4793 uint8_t flow_id_bits = 0; 4794 int shift; 4795 4796 /* Prepare the suffix subflow items. */ 4797 tag_item = sfx_items++; 4798 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4799 struct mlx5_priv *port_priv; 4800 const struct rte_flow_item_port_id *pid_v; 4801 int item_type = items->type; 4802 4803 switch (item_type) { 4804 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4805 pid_v = items->spec; 4806 MLX5_ASSERT(pid_v); 4807 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 4808 if (!port_priv) 4809 return rte_flow_error_set(error, 4810 rte_errno, 4811 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 4812 pid_v, 4813 "Failed to get port info."); 4814 flow_src_port = port_priv->representor_id; 4815 if (!fm->def_policy && wks->policy->is_hierarchy && 4816 flow_src_port != priv->representor_id) { 4817 if (flow_drv_mtr_hierarchy_rule_create(dev, 4818 flow, fm, 4819 flow_src_port, 4820 items, 4821 error)) 4822 return -rte_errno; 4823 } 4824 memcpy(sfx_items, items, sizeof(*sfx_items)); 4825 sfx_items++; 4826 break; 4827 case RTE_FLOW_ITEM_TYPE_VLAN: 4828 /* Determine if copy vlan item below. */ 4829 vlan_item_src = items; 4830 vlan_item_dst = sfx_items++; 4831 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 4832 break; 4833 default: 4834 break; 4835 } 4836 } 4837 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4838 sfx_items++; 4839 mtr_first = priv->sh->meter_aso_en && 4840 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 4841 /* For ASO meter, meter must be before tag in TX direction. */ 4842 if (mtr_first) { 4843 action_pre_head = actions_pre++; 4844 /* Leave space for tag action. */ 4845 tag_action = actions_pre++; 4846 } 4847 /* Prepare the actions for prefix and suffix flow. */ 4848 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4849 struct rte_flow_action *action_cur = NULL; 4850 4851 switch (actions->type) { 4852 case RTE_FLOW_ACTION_TYPE_METER: 4853 if (mtr_first) { 4854 action_cur = action_pre_head; 4855 } else { 4856 /* Leave space for tag action. */ 4857 tag_action = actions_pre++; 4858 action_cur = actions_pre++; 4859 } 4860 break; 4861 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4862 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4863 action_cur = actions_pre++; 4864 break; 4865 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4866 raw_encap = actions->conf; 4867 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4868 action_cur = actions_pre++; 4869 break; 4870 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4871 raw_decap = actions->conf; 4872 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4873 action_cur = actions_pre++; 4874 break; 4875 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4876 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4877 if (vlan_item_dst && vlan_item_src) { 4878 memcpy(vlan_item_dst, vlan_item_src, 4879 sizeof(*vlan_item_dst)); 4880 /* 4881 * Convert to internal match item, it is used 4882 * for vlan push and set vid. 4883 */ 4884 vlan_item_dst->type = (enum rte_flow_item_type) 4885 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4886 } 4887 break; 4888 default: 4889 break; 4890 } 4891 if (!action_cur) 4892 action_cur = (fm->def_policy) ? 4893 actions_sfx++ : actions_pre++; 4894 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 4895 } 4896 /* Add end action to the actions. */ 4897 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4898 if (priv->sh->meter_aso_en) { 4899 /** 4900 * For ASO meter, need to add an extra jump action explicitly, 4901 * to jump from meter to policer table. 4902 */ 4903 struct mlx5_flow_meter_sub_policy *sub_policy; 4904 struct mlx5_flow_tbl_data_entry *tbl_data; 4905 4906 if (!fm->def_policy) { 4907 sub_policy = get_meter_sub_policy(dev, flow, wks, 4908 attr, items, error); 4909 if (!sub_policy) 4910 return -rte_errno; 4911 } else { 4912 enum mlx5_meter_domain mtr_domain = 4913 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4914 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4915 MLX5_MTR_DOMAIN_INGRESS; 4916 4917 sub_policy = 4918 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 4919 } 4920 tbl_data = container_of(sub_policy->tbl_rsc, 4921 struct mlx5_flow_tbl_data_entry, tbl); 4922 hw_mtr_action = actions_pre++; 4923 hw_mtr_action->type = (enum rte_flow_action_type) 4924 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 4925 hw_mtr_action->conf = tbl_data->jump.action; 4926 } 4927 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4928 actions_pre++; 4929 if (!tag_action) 4930 return rte_flow_error_set(error, ENOMEM, 4931 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4932 "No tag action space."); 4933 if (!mtr_flow_id) { 4934 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 4935 goto exit; 4936 } 4937 /* Only default-policy Meter creates mtr flow id. */ 4938 if (fm->def_policy) { 4939 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 4940 if (!tag_id) 4941 return rte_flow_error_set(error, ENOMEM, 4942 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4943 "Failed to allocate meter flow id."); 4944 flow_id = tag_id - 1; 4945 flow_id_bits = (!flow_id) ? 1 : 4946 (MLX5_REG_BITS - __builtin_clz(flow_id)); 4947 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 4948 mtr_reg_bits) { 4949 mlx5_ipool_free(fm->flow_ipool, tag_id); 4950 return rte_flow_error_set(error, EINVAL, 4951 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4952 "Meter flow id exceeds max limit."); 4953 } 4954 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 4955 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 4956 } 4957 /* Build tag actions and items for meter_id/meter flow_id. */ 4958 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 4959 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4960 tag_item_mask = tag_item_spec + 1; 4961 /* Both flow_id and meter_id share the same register. */ 4962 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4963 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 4964 0, error), 4965 .offset = mtr_id_offset, 4966 .length = mtr_reg_bits, 4967 .data = flow->meter, 4968 }; 4969 /* 4970 * The color Reg bits used by flow_id are growing from 4971 * msb to lsb, so must do bit reverse for flow_id val in RegC. 4972 */ 4973 for (shift = 0; shift < flow_id_bits; shift++) 4974 flow_id_reversed = (flow_id_reversed << 1) | 4975 ((flow_id >> shift) & 0x1); 4976 set_tag->data |= 4977 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 4978 tag_item_spec->id = set_tag->id; 4979 tag_item_spec->data = set_tag->data << mtr_id_offset; 4980 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 4981 tag_action->type = (enum rte_flow_action_type) 4982 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4983 tag_action->conf = set_tag; 4984 tag_item->type = (enum rte_flow_item_type) 4985 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4986 tag_item->spec = tag_item_spec; 4987 tag_item->last = NULL; 4988 tag_item->mask = tag_item_mask; 4989 exit: 4990 if (mtr_flow_id) 4991 *mtr_flow_id = tag_id; 4992 return 0; 4993 } 4994 4995 /** 4996 * Split action list having QUEUE/RSS for metadata register copy. 4997 * 4998 * Once Q/RSS action is detected in user's action list, the flow action 4999 * should be split in order to copy metadata registers, which will happen in 5000 * RX_CP_TBL like, 5001 * - CQE->flow_tag := reg_c[1] (MARK) 5002 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5003 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5004 * This is because the last action of each flow must be a terminal action 5005 * (QUEUE, RSS or DROP). 5006 * 5007 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5008 * stored and kept in the mlx5_flow structure per each sub_flow. 5009 * 5010 * The Q/RSS action is replaced with, 5011 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5012 * And the following JUMP action is added at the end, 5013 * - JUMP, to RX_CP_TBL. 5014 * 5015 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5016 * flow_create_split_metadata() routine. The flow will look like, 5017 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5018 * 5019 * @param dev 5020 * Pointer to Ethernet device. 5021 * @param[out] split_actions 5022 * Pointer to store split actions to jump to CP_TBL. 5023 * @param[in] actions 5024 * Pointer to the list of original flow actions. 5025 * @param[in] qrss 5026 * Pointer to the Q/RSS action. 5027 * @param[in] actions_n 5028 * Number of original actions. 5029 * @param[out] error 5030 * Perform verbose error reporting if not NULL. 5031 * 5032 * @return 5033 * non-zero unique flow_id on success, otherwise 0 and 5034 * error/rte_error are set. 5035 */ 5036 static uint32_t 5037 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5038 struct rte_flow_action *split_actions, 5039 const struct rte_flow_action *actions, 5040 const struct rte_flow_action *qrss, 5041 int actions_n, struct rte_flow_error *error) 5042 { 5043 struct mlx5_priv *priv = dev->data->dev_private; 5044 struct mlx5_rte_flow_action_set_tag *set_tag; 5045 struct rte_flow_action_jump *jump; 5046 const int qrss_idx = qrss - actions; 5047 uint32_t flow_id = 0; 5048 int ret = 0; 5049 5050 /* 5051 * Given actions will be split 5052 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5053 * - Add jump to mreg CP_TBL. 5054 * As a result, there will be one more action. 5055 */ 5056 ++actions_n; 5057 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5058 set_tag = (void *)(split_actions + actions_n); 5059 /* 5060 * If tag action is not set to void(it means we are not the meter 5061 * suffix flow), add the tag action. Since meter suffix flow already 5062 * has the tag added. 5063 */ 5064 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 5065 /* 5066 * Allocate the new subflow ID. This one is unique within 5067 * device and not shared with representors. Otherwise, 5068 * we would have to resolve multi-thread access synch 5069 * issue. Each flow on the shared device is appended 5070 * with source vport identifier, so the resulting 5071 * flows will be unique in the shared (by master and 5072 * representors) domain even if they have coinciding 5073 * IDs. 5074 */ 5075 mlx5_ipool_malloc(priv->sh->ipool 5076 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5077 if (!flow_id) 5078 return rte_flow_error_set(error, ENOMEM, 5079 RTE_FLOW_ERROR_TYPE_ACTION, 5080 NULL, "can't allocate id " 5081 "for split Q/RSS subflow"); 5082 /* Internal SET_TAG action to set flow ID. */ 5083 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5084 .data = flow_id, 5085 }; 5086 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5087 if (ret < 0) 5088 return ret; 5089 set_tag->id = ret; 5090 /* Construct new actions array. */ 5091 /* Replace QUEUE/RSS action. */ 5092 split_actions[qrss_idx] = (struct rte_flow_action){ 5093 .type = (enum rte_flow_action_type) 5094 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5095 .conf = set_tag, 5096 }; 5097 } 5098 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5099 jump = (void *)(set_tag + 1); 5100 *jump = (struct rte_flow_action_jump){ 5101 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5102 }; 5103 split_actions[actions_n - 2] = (struct rte_flow_action){ 5104 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5105 .conf = jump, 5106 }; 5107 split_actions[actions_n - 1] = (struct rte_flow_action){ 5108 .type = RTE_FLOW_ACTION_TYPE_END, 5109 }; 5110 return flow_id; 5111 } 5112 5113 /** 5114 * Extend the given action list for Tx metadata copy. 5115 * 5116 * Copy the given action list to the ext_actions and add flow metadata register 5117 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5118 * 5119 * @param[out] ext_actions 5120 * Pointer to the extended action list. 5121 * @param[in] actions 5122 * Pointer to the list of actions. 5123 * @param[in] actions_n 5124 * Number of actions in the list. 5125 * @param[out] error 5126 * Perform verbose error reporting if not NULL. 5127 * @param[in] encap_idx 5128 * The encap action inndex. 5129 * 5130 * @return 5131 * 0 on success, negative value otherwise 5132 */ 5133 static int 5134 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5135 struct rte_flow_action *ext_actions, 5136 const struct rte_flow_action *actions, 5137 int actions_n, struct rte_flow_error *error, 5138 int encap_idx) 5139 { 5140 struct mlx5_flow_action_copy_mreg *cp_mreg = 5141 (struct mlx5_flow_action_copy_mreg *) 5142 (ext_actions + actions_n + 1); 5143 int ret; 5144 5145 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5146 if (ret < 0) 5147 return ret; 5148 cp_mreg->dst = ret; 5149 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5150 if (ret < 0) 5151 return ret; 5152 cp_mreg->src = ret; 5153 if (encap_idx != 0) 5154 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5155 if (encap_idx == actions_n - 1) { 5156 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5157 .type = (enum rte_flow_action_type) 5158 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5159 .conf = cp_mreg, 5160 }; 5161 ext_actions[actions_n] = (struct rte_flow_action){ 5162 .type = RTE_FLOW_ACTION_TYPE_END, 5163 }; 5164 } else { 5165 ext_actions[encap_idx] = (struct rte_flow_action){ 5166 .type = (enum rte_flow_action_type) 5167 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5168 .conf = cp_mreg, 5169 }; 5170 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5171 sizeof(*ext_actions) * (actions_n - encap_idx)); 5172 } 5173 return 0; 5174 } 5175 5176 /** 5177 * Check the match action from the action list. 5178 * 5179 * @param[in] actions 5180 * Pointer to the list of actions. 5181 * @param[in] attr 5182 * Flow rule attributes. 5183 * @param[in] action 5184 * The action to be check if exist. 5185 * @param[out] match_action_pos 5186 * Pointer to the position of the matched action if exists, otherwise is -1. 5187 * @param[out] qrss_action_pos 5188 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5189 * @param[out] modify_after_mirror 5190 * Pointer to the flag of modify action after FDB mirroring. 5191 * 5192 * @return 5193 * > 0 the total number of actions. 5194 * 0 if not found match action in action list. 5195 */ 5196 static int 5197 flow_check_match_action(const struct rte_flow_action actions[], 5198 const struct rte_flow_attr *attr, 5199 enum rte_flow_action_type action, 5200 int *match_action_pos, int *qrss_action_pos, 5201 int *modify_after_mirror) 5202 { 5203 const struct rte_flow_action_sample *sample; 5204 int actions_n = 0; 5205 uint32_t ratio = 0; 5206 int sub_type = 0; 5207 int flag = 0; 5208 int fdb_mirror = 0; 5209 5210 *match_action_pos = -1; 5211 *qrss_action_pos = -1; 5212 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5213 if (actions->type == action) { 5214 flag = 1; 5215 *match_action_pos = actions_n; 5216 } 5217 switch (actions->type) { 5218 case RTE_FLOW_ACTION_TYPE_QUEUE: 5219 case RTE_FLOW_ACTION_TYPE_RSS: 5220 *qrss_action_pos = actions_n; 5221 break; 5222 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5223 sample = actions->conf; 5224 ratio = sample->ratio; 5225 sub_type = ((const struct rte_flow_action *) 5226 (sample->actions))->type; 5227 if (ratio == 1 && attr->transfer) 5228 fdb_mirror = 1; 5229 break; 5230 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5231 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5232 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5233 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5234 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5235 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5236 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5237 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5238 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5239 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5240 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5241 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5242 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5243 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5244 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5245 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5246 case RTE_FLOW_ACTION_TYPE_FLAG: 5247 case RTE_FLOW_ACTION_TYPE_MARK: 5248 case RTE_FLOW_ACTION_TYPE_SET_META: 5249 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5250 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5251 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5252 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5253 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5254 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5255 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5256 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5257 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5258 case RTE_FLOW_ACTION_TYPE_METER: 5259 if (fdb_mirror) 5260 *modify_after_mirror = 1; 5261 break; 5262 default: 5263 break; 5264 } 5265 actions_n++; 5266 } 5267 if (flag && fdb_mirror && !*modify_after_mirror) { 5268 /* FDB mirroring uses the destination array to implement 5269 * instead of FLOW_SAMPLER object. 5270 */ 5271 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5272 flag = 0; 5273 } 5274 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5275 return flag ? actions_n + 1 : 0; 5276 } 5277 5278 #define SAMPLE_SUFFIX_ITEM 2 5279 5280 /** 5281 * Split the sample flow. 5282 * 5283 * As sample flow will split to two sub flow, sample flow with 5284 * sample action, the other actions will move to new suffix flow. 5285 * 5286 * Also add unique tag id with tag action in the sample flow, 5287 * the same tag id will be as match in the suffix flow. 5288 * 5289 * @param dev 5290 * Pointer to Ethernet device. 5291 * @param[in] add_tag 5292 * Add extra tag action flag. 5293 * @param[out] sfx_items 5294 * Suffix flow match items (list terminated by the END pattern item). 5295 * @param[in] actions 5296 * Associated actions (list terminated by the END action). 5297 * @param[out] actions_sfx 5298 * Suffix flow actions. 5299 * @param[out] actions_pre 5300 * Prefix flow actions. 5301 * @param[in] actions_n 5302 * The total number of actions. 5303 * @param[in] sample_action_pos 5304 * The sample action position. 5305 * @param[in] qrss_action_pos 5306 * The Queue/RSS action position. 5307 * @param[in] jump_table 5308 * Add extra jump action flag. 5309 * @param[out] error 5310 * Perform verbose error reporting if not NULL. 5311 * 5312 * @return 5313 * 0 on success, or unique flow_id, a negative errno value 5314 * otherwise and rte_errno is set. 5315 */ 5316 static int 5317 flow_sample_split_prep(struct rte_eth_dev *dev, 5318 int add_tag, 5319 struct rte_flow_item sfx_items[], 5320 const struct rte_flow_action actions[], 5321 struct rte_flow_action actions_sfx[], 5322 struct rte_flow_action actions_pre[], 5323 int actions_n, 5324 int sample_action_pos, 5325 int qrss_action_pos, 5326 int jump_table, 5327 struct rte_flow_error *error) 5328 { 5329 struct mlx5_priv *priv = dev->data->dev_private; 5330 struct mlx5_rte_flow_action_set_tag *set_tag; 5331 struct mlx5_rte_flow_item_tag *tag_spec; 5332 struct mlx5_rte_flow_item_tag *tag_mask; 5333 struct rte_flow_action_jump *jump_action; 5334 uint32_t tag_id = 0; 5335 int index; 5336 int append_index = 0; 5337 int ret; 5338 5339 if (sample_action_pos < 0) 5340 return rte_flow_error_set(error, EINVAL, 5341 RTE_FLOW_ERROR_TYPE_ACTION, 5342 NULL, "invalid position of sample " 5343 "action in list"); 5344 /* Prepare the actions for prefix and suffix flow. */ 5345 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5346 index = qrss_action_pos; 5347 /* Put the preceding the Queue/RSS action into prefix flow. */ 5348 if (index != 0) 5349 memcpy(actions_pre, actions, 5350 sizeof(struct rte_flow_action) * index); 5351 /* Put others preceding the sample action into prefix flow. */ 5352 if (sample_action_pos > index + 1) 5353 memcpy(actions_pre + index, actions + index + 1, 5354 sizeof(struct rte_flow_action) * 5355 (sample_action_pos - index - 1)); 5356 index = sample_action_pos - 1; 5357 /* Put Queue/RSS action into Suffix flow. */ 5358 memcpy(actions_sfx, actions + qrss_action_pos, 5359 sizeof(struct rte_flow_action)); 5360 actions_sfx++; 5361 } else { 5362 index = sample_action_pos; 5363 if (index != 0) 5364 memcpy(actions_pre, actions, 5365 sizeof(struct rte_flow_action) * index); 5366 } 5367 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5368 * For CX6DX and above, metadata registers Cx preserve their value, 5369 * add an extra tag action for NIC-RX and E-Switch Domain. 5370 */ 5371 if (add_tag) { 5372 /* Prepare the prefix tag action. */ 5373 append_index++; 5374 set_tag = (void *)(actions_pre + actions_n + append_index); 5375 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5376 if (ret < 0) 5377 return ret; 5378 mlx5_ipool_malloc(priv->sh->ipool 5379 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5380 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5381 .id = ret, 5382 .data = tag_id, 5383 }; 5384 /* Prepare the suffix subflow items. */ 5385 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5386 tag_spec->data = tag_id; 5387 tag_spec->id = set_tag->id; 5388 tag_mask = tag_spec + 1; 5389 tag_mask->data = UINT32_MAX; 5390 sfx_items[0] = (struct rte_flow_item){ 5391 .type = (enum rte_flow_item_type) 5392 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5393 .spec = tag_spec, 5394 .last = NULL, 5395 .mask = tag_mask, 5396 }; 5397 sfx_items[1] = (struct rte_flow_item){ 5398 .type = (enum rte_flow_item_type) 5399 RTE_FLOW_ITEM_TYPE_END, 5400 }; 5401 /* Prepare the tag action in prefix subflow. */ 5402 actions_pre[index++] = 5403 (struct rte_flow_action){ 5404 .type = (enum rte_flow_action_type) 5405 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5406 .conf = set_tag, 5407 }; 5408 } 5409 memcpy(actions_pre + index, actions + sample_action_pos, 5410 sizeof(struct rte_flow_action)); 5411 index += 1; 5412 /* For the modify action after the sample action in E-Switch mirroring, 5413 * Add the extra jump action in prefix subflow and jump into the next 5414 * table, then do the modify action in the new table. 5415 */ 5416 if (jump_table) { 5417 /* Prepare the prefix jump action. */ 5418 append_index++; 5419 jump_action = (void *)(actions_pre + actions_n + append_index); 5420 jump_action->group = jump_table; 5421 actions_pre[index++] = 5422 (struct rte_flow_action){ 5423 .type = (enum rte_flow_action_type) 5424 RTE_FLOW_ACTION_TYPE_JUMP, 5425 .conf = jump_action, 5426 }; 5427 } 5428 actions_pre[index] = (struct rte_flow_action){ 5429 .type = (enum rte_flow_action_type) 5430 RTE_FLOW_ACTION_TYPE_END, 5431 }; 5432 /* Put the actions after sample into Suffix flow. */ 5433 memcpy(actions_sfx, actions + sample_action_pos + 1, 5434 sizeof(struct rte_flow_action) * 5435 (actions_n - sample_action_pos - 1)); 5436 return tag_id; 5437 } 5438 5439 /** 5440 * The splitting for metadata feature. 5441 * 5442 * - Q/RSS action on NIC Rx should be split in order to pass by 5443 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5444 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5445 * 5446 * - All the actions on NIC Tx should have a mreg copy action to 5447 * copy reg_a from WQE to reg_c[0]. 5448 * 5449 * @param dev 5450 * Pointer to Ethernet device. 5451 * @param[in] flow 5452 * Parent flow structure pointer. 5453 * @param[in] attr 5454 * Flow rule attributes. 5455 * @param[in] items 5456 * Pattern specification (list terminated by the END pattern item). 5457 * @param[in] actions 5458 * Associated actions (list terminated by the END action). 5459 * @param[in] flow_split_info 5460 * Pointer to flow split info structure. 5461 * @param[out] error 5462 * Perform verbose error reporting if not NULL. 5463 * @return 5464 * 0 on success, negative value otherwise 5465 */ 5466 static int 5467 flow_create_split_metadata(struct rte_eth_dev *dev, 5468 struct rte_flow *flow, 5469 const struct rte_flow_attr *attr, 5470 const struct rte_flow_item items[], 5471 const struct rte_flow_action actions[], 5472 struct mlx5_flow_split_info *flow_split_info, 5473 struct rte_flow_error *error) 5474 { 5475 struct mlx5_priv *priv = dev->data->dev_private; 5476 struct mlx5_dev_config *config = &priv->config; 5477 const struct rte_flow_action *qrss = NULL; 5478 struct rte_flow_action *ext_actions = NULL; 5479 struct mlx5_flow *dev_flow = NULL; 5480 uint32_t qrss_id = 0; 5481 int mtr_sfx = 0; 5482 size_t act_size; 5483 int actions_n; 5484 int encap_idx; 5485 int ret; 5486 5487 /* Check whether extensive metadata feature is engaged. */ 5488 if (!config->dv_flow_en || 5489 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5490 !mlx5_flow_ext_mreg_supported(dev)) 5491 return flow_create_split_inner(dev, flow, NULL, attr, items, 5492 actions, flow_split_info, error); 5493 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5494 &encap_idx); 5495 if (qrss) { 5496 /* Exclude hairpin flows from splitting. */ 5497 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5498 const struct rte_flow_action_queue *queue; 5499 5500 queue = qrss->conf; 5501 if (mlx5_rxq_get_type(dev, queue->index) == 5502 MLX5_RXQ_TYPE_HAIRPIN) 5503 qrss = NULL; 5504 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5505 const struct rte_flow_action_rss *rss; 5506 5507 rss = qrss->conf; 5508 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5509 MLX5_RXQ_TYPE_HAIRPIN) 5510 qrss = NULL; 5511 } 5512 } 5513 if (qrss) { 5514 /* Check if it is in meter suffix table. */ 5515 mtr_sfx = attr->group == (attr->transfer ? 5516 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5517 MLX5_FLOW_TABLE_LEVEL_METER); 5518 /* 5519 * Q/RSS action on NIC Rx should be split in order to pass by 5520 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5521 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5522 */ 5523 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5524 sizeof(struct rte_flow_action_set_tag) + 5525 sizeof(struct rte_flow_action_jump); 5526 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5527 SOCKET_ID_ANY); 5528 if (!ext_actions) 5529 return rte_flow_error_set(error, ENOMEM, 5530 RTE_FLOW_ERROR_TYPE_ACTION, 5531 NULL, "no memory to split " 5532 "metadata flow"); 5533 /* 5534 * If we are the suffix flow of meter, tag already exist. 5535 * Set the tag action to void. 5536 */ 5537 if (mtr_sfx) 5538 ext_actions[qrss - actions].type = 5539 RTE_FLOW_ACTION_TYPE_VOID; 5540 else 5541 ext_actions[qrss - actions].type = 5542 (enum rte_flow_action_type) 5543 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5544 /* 5545 * Create the new actions list with removed Q/RSS action 5546 * and appended set tag and jump to register copy table 5547 * (RX_CP_TBL). We should preallocate unique tag ID here 5548 * in advance, because it is needed for set tag action. 5549 */ 5550 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5551 qrss, actions_n, error); 5552 if (!mtr_sfx && !qrss_id) { 5553 ret = -rte_errno; 5554 goto exit; 5555 } 5556 } else if (attr->egress && !attr->transfer) { 5557 /* 5558 * All the actions on NIC Tx should have a metadata register 5559 * copy action to copy reg_a from WQE to reg_c[meta] 5560 */ 5561 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5562 sizeof(struct mlx5_flow_action_copy_mreg); 5563 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5564 SOCKET_ID_ANY); 5565 if (!ext_actions) 5566 return rte_flow_error_set(error, ENOMEM, 5567 RTE_FLOW_ERROR_TYPE_ACTION, 5568 NULL, "no memory to split " 5569 "metadata flow"); 5570 /* Create the action list appended with copy register. */ 5571 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5572 actions_n, error, encap_idx); 5573 if (ret < 0) 5574 goto exit; 5575 } 5576 /* Add the unmodified original or prefix subflow. */ 5577 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5578 items, ext_actions ? ext_actions : 5579 actions, flow_split_info, error); 5580 if (ret < 0) 5581 goto exit; 5582 MLX5_ASSERT(dev_flow); 5583 if (qrss) { 5584 const struct rte_flow_attr q_attr = { 5585 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5586 .ingress = 1, 5587 }; 5588 /* Internal PMD action to set register. */ 5589 struct mlx5_rte_flow_item_tag q_tag_spec = { 5590 .data = qrss_id, 5591 .id = REG_NON, 5592 }; 5593 struct rte_flow_item q_items[] = { 5594 { 5595 .type = (enum rte_flow_item_type) 5596 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5597 .spec = &q_tag_spec, 5598 .last = NULL, 5599 .mask = NULL, 5600 }, 5601 { 5602 .type = RTE_FLOW_ITEM_TYPE_END, 5603 }, 5604 }; 5605 struct rte_flow_action q_actions[] = { 5606 { 5607 .type = qrss->type, 5608 .conf = qrss->conf, 5609 }, 5610 { 5611 .type = RTE_FLOW_ACTION_TYPE_END, 5612 }, 5613 }; 5614 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5615 5616 /* 5617 * Configure the tag item only if there is no meter subflow. 5618 * Since tag is already marked in the meter suffix subflow 5619 * we can just use the meter suffix items as is. 5620 */ 5621 if (qrss_id) { 5622 /* Not meter subflow. */ 5623 MLX5_ASSERT(!mtr_sfx); 5624 /* 5625 * Put unique id in prefix flow due to it is destroyed 5626 * after suffix flow and id will be freed after there 5627 * is no actual flows with this id and identifier 5628 * reallocation becomes possible (for example, for 5629 * other flows in other threads). 5630 */ 5631 dev_flow->handle->split_flow_id = qrss_id; 5632 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5633 error); 5634 if (ret < 0) 5635 goto exit; 5636 q_tag_spec.id = ret; 5637 } 5638 dev_flow = NULL; 5639 /* Add suffix subflow to execute Q/RSS. */ 5640 flow_split_info->prefix_layers = layers; 5641 flow_split_info->prefix_mark = 0; 5642 ret = flow_create_split_inner(dev, flow, &dev_flow, 5643 &q_attr, mtr_sfx ? items : 5644 q_items, q_actions, 5645 flow_split_info, error); 5646 if (ret < 0) 5647 goto exit; 5648 /* qrss ID should be freed if failed. */ 5649 qrss_id = 0; 5650 MLX5_ASSERT(dev_flow); 5651 } 5652 5653 exit: 5654 /* 5655 * We do not destroy the partially created sub_flows in case of error. 5656 * These ones are included into parent flow list and will be destroyed 5657 * by flow_drv_destroy. 5658 */ 5659 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5660 qrss_id); 5661 mlx5_free(ext_actions); 5662 return ret; 5663 } 5664 5665 /** 5666 * Create meter internal drop flow with the original pattern. 5667 * 5668 * @param dev 5669 * Pointer to Ethernet device. 5670 * @param[in] flow 5671 * Parent flow structure pointer. 5672 * @param[in] attr 5673 * Flow rule attributes. 5674 * @param[in] items 5675 * Pattern specification (list terminated by the END pattern item). 5676 * @param[in] flow_split_info 5677 * Pointer to flow split info structure. 5678 * @param[in] fm 5679 * Pointer to flow meter structure. 5680 * @param[out] error 5681 * Perform verbose error reporting if not NULL. 5682 * @return 5683 * 0 on success, negative value otherwise 5684 */ 5685 static uint32_t 5686 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5687 struct rte_flow *flow, 5688 const struct rte_flow_attr *attr, 5689 const struct rte_flow_item items[], 5690 struct mlx5_flow_split_info *flow_split_info, 5691 struct mlx5_flow_meter_info *fm, 5692 struct rte_flow_error *error) 5693 { 5694 struct mlx5_flow *dev_flow = NULL; 5695 struct rte_flow_attr drop_attr = *attr; 5696 struct rte_flow_action drop_actions[3]; 5697 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5698 5699 MLX5_ASSERT(fm->drop_cnt); 5700 drop_actions[0].type = 5701 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5702 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5703 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5704 drop_actions[1].conf = NULL; 5705 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5706 drop_actions[2].conf = NULL; 5707 drop_split_info.external = false; 5708 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5709 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5710 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5711 return flow_create_split_inner(dev, flow, &dev_flow, 5712 &drop_attr, items, drop_actions, 5713 &drop_split_info, error); 5714 } 5715 5716 /** 5717 * The splitting for meter feature. 5718 * 5719 * - The meter flow will be split to two flows as prefix and 5720 * suffix flow. The packets make sense only it pass the prefix 5721 * meter action. 5722 * 5723 * - Reg_C_5 is used for the packet to match betweend prefix and 5724 * suffix flow. 5725 * 5726 * @param dev 5727 * Pointer to Ethernet device. 5728 * @param[in] flow 5729 * Parent flow structure pointer. 5730 * @param[in] attr 5731 * Flow rule attributes. 5732 * @param[in] items 5733 * Pattern specification (list terminated by the END pattern item). 5734 * @param[in] actions 5735 * Associated actions (list terminated by the END action). 5736 * @param[in] flow_split_info 5737 * Pointer to flow split info structure. 5738 * @param[out] error 5739 * Perform verbose error reporting if not NULL. 5740 * @return 5741 * 0 on success, negative value otherwise 5742 */ 5743 static int 5744 flow_create_split_meter(struct rte_eth_dev *dev, 5745 struct rte_flow *flow, 5746 const struct rte_flow_attr *attr, 5747 const struct rte_flow_item items[], 5748 const struct rte_flow_action actions[], 5749 struct mlx5_flow_split_info *flow_split_info, 5750 struct rte_flow_error *error) 5751 { 5752 struct mlx5_priv *priv = dev->data->dev_private; 5753 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5754 struct rte_flow_action *sfx_actions = NULL; 5755 struct rte_flow_action *pre_actions = NULL; 5756 struct rte_flow_item *sfx_items = NULL; 5757 struct mlx5_flow *dev_flow = NULL; 5758 struct rte_flow_attr sfx_attr = *attr; 5759 struct mlx5_flow_meter_info *fm = NULL; 5760 uint8_t skip_scale_restore; 5761 bool has_mtr = false; 5762 bool has_modify = false; 5763 bool set_mtr_reg = true; 5764 bool is_mtr_hierarchy = false; 5765 uint32_t meter_id = 0; 5766 uint32_t mtr_idx = 0; 5767 uint32_t mtr_flow_id = 0; 5768 size_t act_size; 5769 size_t item_size; 5770 int actions_n = 0; 5771 int ret = 0; 5772 5773 if (priv->mtr_en) 5774 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 5775 &has_modify, &meter_id); 5776 if (has_mtr) { 5777 if (flow->meter) { 5778 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 5779 if (!fm) 5780 return rte_flow_error_set(error, EINVAL, 5781 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5782 NULL, "Meter not found."); 5783 } else { 5784 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 5785 if (!fm) 5786 return rte_flow_error_set(error, EINVAL, 5787 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5788 NULL, "Meter not found."); 5789 ret = mlx5_flow_meter_attach(priv, fm, 5790 &sfx_attr, error); 5791 if (ret) 5792 return -rte_errno; 5793 flow->meter = mtr_idx; 5794 } 5795 MLX5_ASSERT(wks); 5796 wks->fm = fm; 5797 if (!fm->def_policy) { 5798 wks->policy = mlx5_flow_meter_policy_find(dev, 5799 fm->policy_id, 5800 NULL); 5801 MLX5_ASSERT(wks->policy); 5802 if (wks->policy->is_hierarchy) { 5803 wks->final_policy = 5804 mlx5_flow_meter_hierarchy_get_final_policy(dev, 5805 wks->policy); 5806 if (!wks->final_policy) 5807 return rte_flow_error_set(error, 5808 EINVAL, 5809 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5810 "Failed to find terminal policy of hierarchy."); 5811 is_mtr_hierarchy = true; 5812 } 5813 } 5814 /* 5815 * If it isn't default-policy Meter, and 5816 * 1. There's no action in flow to change 5817 * packet (modify/encap/decap etc.), OR 5818 * 2. No drop count needed for this meter. 5819 * 3. It's not meter hierarchy. 5820 * Then no need to use regC to save meter id anymore. 5821 */ 5822 if (!fm->def_policy && !is_mtr_hierarchy && 5823 (!has_modify || !fm->drop_cnt)) 5824 set_mtr_reg = false; 5825 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 5826 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 5827 sizeof(struct mlx5_rte_flow_action_set_tag); 5828 /* Suffix items: tag, vlan, port id, end. */ 5829 #define METER_SUFFIX_ITEM 4 5830 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5831 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5832 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5833 0, SOCKET_ID_ANY); 5834 if (!sfx_actions) 5835 return rte_flow_error_set(error, ENOMEM, 5836 RTE_FLOW_ERROR_TYPE_ACTION, 5837 NULL, "no memory to split " 5838 "meter flow"); 5839 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5840 act_size); 5841 /* There's no suffix flow for meter of non-default policy. */ 5842 if (!fm->def_policy) 5843 pre_actions = sfx_actions + 1; 5844 else 5845 pre_actions = sfx_actions + actions_n; 5846 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 5847 items, sfx_items, actions, 5848 sfx_actions, pre_actions, 5849 (set_mtr_reg ? &mtr_flow_id : NULL), 5850 error); 5851 if (ret) { 5852 ret = -rte_errno; 5853 goto exit; 5854 } 5855 /* Add the prefix subflow. */ 5856 flow_split_info->prefix_mark = 0; 5857 skip_scale_restore = flow_split_info->skip_scale; 5858 flow_split_info->skip_scale |= 5859 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5860 ret = flow_create_split_inner(dev, flow, &dev_flow, 5861 attr, items, pre_actions, 5862 flow_split_info, error); 5863 flow_split_info->skip_scale = skip_scale_restore; 5864 if (ret) { 5865 if (mtr_flow_id) 5866 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 5867 ret = -rte_errno; 5868 goto exit; 5869 } 5870 if (mtr_flow_id) { 5871 dev_flow->handle->split_flow_id = mtr_flow_id; 5872 dev_flow->handle->is_meter_flow_id = 1; 5873 } 5874 if (!fm->def_policy) { 5875 if (!set_mtr_reg && fm->drop_cnt) 5876 ret = 5877 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 5878 &sfx_attr, items, 5879 flow_split_info, 5880 fm, error); 5881 goto exit; 5882 } 5883 /* Setting the sfx group atrr. */ 5884 sfx_attr.group = sfx_attr.transfer ? 5885 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5886 MLX5_FLOW_TABLE_LEVEL_METER; 5887 flow_split_info->prefix_layers = 5888 flow_get_prefix_layer_flags(dev_flow); 5889 flow_split_info->prefix_mark = dev_flow->handle->mark; 5890 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 5891 } 5892 /* Add the prefix subflow. */ 5893 ret = flow_create_split_metadata(dev, flow, 5894 &sfx_attr, sfx_items ? 5895 sfx_items : items, 5896 sfx_actions ? sfx_actions : actions, 5897 flow_split_info, error); 5898 exit: 5899 if (sfx_actions) 5900 mlx5_free(sfx_actions); 5901 return ret; 5902 } 5903 5904 /** 5905 * The splitting for sample feature. 5906 * 5907 * Once Sample action is detected in the action list, the flow actions should 5908 * be split into prefix sub flow and suffix sub flow. 5909 * 5910 * The original items remain in the prefix sub flow, all actions preceding the 5911 * sample action and the sample action itself will be copied to the prefix 5912 * sub flow, the actions following the sample action will be copied to the 5913 * suffix sub flow, Queue action always be located in the suffix sub flow. 5914 * 5915 * In order to make the packet from prefix sub flow matches with suffix sub 5916 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5917 * flow uses tag item with the unique flow id. 5918 * 5919 * @param dev 5920 * Pointer to Ethernet device. 5921 * @param[in] flow 5922 * Parent flow structure pointer. 5923 * @param[in] attr 5924 * Flow rule attributes. 5925 * @param[in] items 5926 * Pattern specification (list terminated by the END pattern item). 5927 * @param[in] actions 5928 * Associated actions (list terminated by the END action). 5929 * @param[in] flow_split_info 5930 * Pointer to flow split info structure. 5931 * @param[out] error 5932 * Perform verbose error reporting if not NULL. 5933 * @return 5934 * 0 on success, negative value otherwise 5935 */ 5936 static int 5937 flow_create_split_sample(struct rte_eth_dev *dev, 5938 struct rte_flow *flow, 5939 const struct rte_flow_attr *attr, 5940 const struct rte_flow_item items[], 5941 const struct rte_flow_action actions[], 5942 struct mlx5_flow_split_info *flow_split_info, 5943 struct rte_flow_error *error) 5944 { 5945 struct mlx5_priv *priv = dev->data->dev_private; 5946 struct rte_flow_action *sfx_actions = NULL; 5947 struct rte_flow_action *pre_actions = NULL; 5948 struct rte_flow_item *sfx_items = NULL; 5949 struct mlx5_flow *dev_flow = NULL; 5950 struct rte_flow_attr sfx_attr = *attr; 5951 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5952 struct mlx5_flow_dv_sample_resource *sample_res; 5953 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5954 struct mlx5_flow_tbl_resource *sfx_tbl; 5955 #endif 5956 size_t act_size; 5957 size_t item_size; 5958 uint32_t fdb_tx = 0; 5959 int32_t tag_id = 0; 5960 int actions_n = 0; 5961 int sample_action_pos; 5962 int qrss_action_pos; 5963 int add_tag = 0; 5964 int modify_after_mirror = 0; 5965 uint16_t jump_table = 0; 5966 const uint32_t next_ft_step = 1; 5967 int ret = 0; 5968 5969 if (priv->sampler_en) 5970 actions_n = flow_check_match_action(actions, attr, 5971 RTE_FLOW_ACTION_TYPE_SAMPLE, 5972 &sample_action_pos, &qrss_action_pos, 5973 &modify_after_mirror); 5974 if (actions_n) { 5975 /* The prefix actions must includes sample, tag, end. */ 5976 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5977 + sizeof(struct mlx5_rte_flow_action_set_tag); 5978 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5979 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5980 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5981 item_size), 0, SOCKET_ID_ANY); 5982 if (!sfx_actions) 5983 return rte_flow_error_set(error, ENOMEM, 5984 RTE_FLOW_ERROR_TYPE_ACTION, 5985 NULL, "no memory to split " 5986 "sample flow"); 5987 /* The representor_id is UINT16_MAX for uplink. */ 5988 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 5989 /* 5990 * When reg_c_preserve is set, metadata registers Cx preserve 5991 * their value even through packet duplication. 5992 */ 5993 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 5994 if (add_tag) 5995 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5996 + act_size); 5997 if (modify_after_mirror) 5998 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 5999 next_ft_step; 6000 pre_actions = sfx_actions + actions_n; 6001 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6002 actions, sfx_actions, 6003 pre_actions, actions_n, 6004 sample_action_pos, 6005 qrss_action_pos, jump_table, 6006 error); 6007 if (tag_id < 0 || (add_tag && !tag_id)) { 6008 ret = -rte_errno; 6009 goto exit; 6010 } 6011 if (modify_after_mirror) 6012 flow_split_info->skip_scale = 6013 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6014 /* Add the prefix subflow. */ 6015 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6016 items, pre_actions, 6017 flow_split_info, error); 6018 if (ret) { 6019 ret = -rte_errno; 6020 goto exit; 6021 } 6022 dev_flow->handle->split_flow_id = tag_id; 6023 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6024 if (!modify_after_mirror) { 6025 /* Set the sfx group attr. */ 6026 sample_res = (struct mlx5_flow_dv_sample_resource *) 6027 dev_flow->dv.sample_res; 6028 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6029 sample_res->normal_path_tbl; 6030 sfx_tbl_data = container_of(sfx_tbl, 6031 struct mlx5_flow_tbl_data_entry, 6032 tbl); 6033 sfx_attr.group = sfx_attr.transfer ? 6034 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6035 } else { 6036 MLX5_ASSERT(attr->transfer); 6037 sfx_attr.group = jump_table; 6038 } 6039 flow_split_info->prefix_layers = 6040 flow_get_prefix_layer_flags(dev_flow); 6041 flow_split_info->prefix_mark = dev_flow->handle->mark; 6042 /* Suffix group level already be scaled with factor, set 6043 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6044 * again in translation. 6045 */ 6046 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6047 #endif 6048 } 6049 /* Add the suffix subflow. */ 6050 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6051 sfx_items ? sfx_items : items, 6052 sfx_actions ? sfx_actions : actions, 6053 flow_split_info, error); 6054 exit: 6055 if (sfx_actions) 6056 mlx5_free(sfx_actions); 6057 return ret; 6058 } 6059 6060 /** 6061 * Split the flow to subflow set. The splitters might be linked 6062 * in the chain, like this: 6063 * flow_create_split_outer() calls: 6064 * flow_create_split_meter() calls: 6065 * flow_create_split_metadata(meter_subflow_0) calls: 6066 * flow_create_split_inner(metadata_subflow_0) 6067 * flow_create_split_inner(metadata_subflow_1) 6068 * flow_create_split_inner(metadata_subflow_2) 6069 * flow_create_split_metadata(meter_subflow_1) calls: 6070 * flow_create_split_inner(metadata_subflow_0) 6071 * flow_create_split_inner(metadata_subflow_1) 6072 * flow_create_split_inner(metadata_subflow_2) 6073 * 6074 * This provide flexible way to add new levels of flow splitting. 6075 * The all of successfully created subflows are included to the 6076 * parent flow dev_flow list. 6077 * 6078 * @param dev 6079 * Pointer to Ethernet device. 6080 * @param[in] flow 6081 * Parent flow structure pointer. 6082 * @param[in] attr 6083 * Flow rule attributes. 6084 * @param[in] items 6085 * Pattern specification (list terminated by the END pattern item). 6086 * @param[in] actions 6087 * Associated actions (list terminated by the END action). 6088 * @param[in] flow_split_info 6089 * Pointer to flow split info structure. 6090 * @param[out] error 6091 * Perform verbose error reporting if not NULL. 6092 * @return 6093 * 0 on success, negative value otherwise 6094 */ 6095 static int 6096 flow_create_split_outer(struct rte_eth_dev *dev, 6097 struct rte_flow *flow, 6098 const struct rte_flow_attr *attr, 6099 const struct rte_flow_item items[], 6100 const struct rte_flow_action actions[], 6101 struct mlx5_flow_split_info *flow_split_info, 6102 struct rte_flow_error *error) 6103 { 6104 int ret; 6105 6106 ret = flow_create_split_sample(dev, flow, attr, items, 6107 actions, flow_split_info, error); 6108 MLX5_ASSERT(ret <= 0); 6109 return ret; 6110 } 6111 6112 static inline struct mlx5_flow_tunnel * 6113 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6114 { 6115 struct mlx5_flow_tunnel *tunnel; 6116 6117 #pragma GCC diagnostic push 6118 #pragma GCC diagnostic ignored "-Wcast-qual" 6119 tunnel = (typeof(tunnel))flow->tunnel; 6120 #pragma GCC diagnostic pop 6121 6122 return tunnel; 6123 } 6124 6125 /** 6126 * Adjust flow RSS workspace if needed. 6127 * 6128 * @param wks 6129 * Pointer to thread flow work space. 6130 * @param rss_desc 6131 * Pointer to RSS descriptor. 6132 * @param[in] nrssq_num 6133 * New RSS queue number. 6134 * 6135 * @return 6136 * 0 on success, -1 otherwise and rte_errno is set. 6137 */ 6138 static int 6139 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6140 struct mlx5_flow_rss_desc *rss_desc, 6141 uint32_t nrssq_num) 6142 { 6143 if (likely(nrssq_num <= wks->rssq_num)) 6144 return 0; 6145 rss_desc->queue = realloc(rss_desc->queue, 6146 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6147 if (!rss_desc->queue) { 6148 rte_errno = ENOMEM; 6149 return -1; 6150 } 6151 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6152 return 0; 6153 } 6154 6155 /** 6156 * Create a flow and add it to @p list. 6157 * 6158 * @param dev 6159 * Pointer to Ethernet device. 6160 * @param list 6161 * Pointer to a TAILQ flow list. If this parameter NULL, 6162 * no list insertion occurred, flow is just created, 6163 * this is caller's responsibility to track the 6164 * created flow. 6165 * @param[in] attr 6166 * Flow rule attributes. 6167 * @param[in] items 6168 * Pattern specification (list terminated by the END pattern item). 6169 * @param[in] actions 6170 * Associated actions (list terminated by the END action). 6171 * @param[in] external 6172 * This flow rule is created by request external to PMD. 6173 * @param[out] error 6174 * Perform verbose error reporting if not NULL. 6175 * 6176 * @return 6177 * A flow index on success, 0 otherwise and rte_errno is set. 6178 */ 6179 static uint32_t 6180 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6181 const struct rte_flow_attr *attr, 6182 const struct rte_flow_item items[], 6183 const struct rte_flow_action original_actions[], 6184 bool external, struct rte_flow_error *error) 6185 { 6186 struct mlx5_priv *priv = dev->data->dev_private; 6187 struct rte_flow *flow = NULL; 6188 struct mlx5_flow *dev_flow; 6189 const struct rte_flow_action_rss *rss = NULL; 6190 struct mlx5_translated_action_handle 6191 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6192 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6193 union { 6194 struct mlx5_flow_expand_rss buf; 6195 uint8_t buffer[2048]; 6196 } expand_buffer; 6197 union { 6198 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6199 uint8_t buffer[2048]; 6200 } actions_rx; 6201 union { 6202 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6203 uint8_t buffer[2048]; 6204 } actions_hairpin_tx; 6205 union { 6206 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6207 uint8_t buffer[2048]; 6208 } items_tx; 6209 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6210 struct mlx5_flow_rss_desc *rss_desc; 6211 const struct rte_flow_action *p_actions_rx; 6212 uint32_t i; 6213 uint32_t idx = 0; 6214 int hairpin_flow; 6215 struct rte_flow_attr attr_tx = { .priority = 0 }; 6216 const struct rte_flow_action *actions; 6217 struct rte_flow_action *translated_actions = NULL; 6218 struct mlx5_flow_tunnel *tunnel; 6219 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6220 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6221 struct mlx5_flow_split_info flow_split_info = { 6222 .external = !!external, 6223 .skip_scale = 0, 6224 .flow_idx = 0, 6225 .prefix_mark = 0, 6226 .prefix_layers = 0, 6227 .table_id = 0 6228 }; 6229 int ret; 6230 6231 MLX5_ASSERT(wks); 6232 rss_desc = &wks->rss_desc; 6233 ret = flow_action_handles_translate(dev, original_actions, 6234 indir_actions, 6235 &indir_actions_n, 6236 &translated_actions, error); 6237 if (ret < 0) { 6238 MLX5_ASSERT(translated_actions == NULL); 6239 return 0; 6240 } 6241 actions = translated_actions ? translated_actions : original_actions; 6242 p_actions_rx = actions; 6243 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6244 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6245 external, hairpin_flow, error); 6246 if (ret < 0) 6247 goto error_before_hairpin_split; 6248 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6249 if (!flow) { 6250 rte_errno = ENOMEM; 6251 goto error_before_hairpin_split; 6252 } 6253 if (hairpin_flow > 0) { 6254 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6255 rte_errno = EINVAL; 6256 goto error_before_hairpin_split; 6257 } 6258 flow_hairpin_split(dev, actions, actions_rx.actions, 6259 actions_hairpin_tx.actions, items_tx.items, 6260 idx); 6261 p_actions_rx = actions_rx.actions; 6262 } 6263 flow_split_info.flow_idx = idx; 6264 flow->drv_type = flow_get_drv_type(dev, attr); 6265 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6266 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6267 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6268 /* RSS Action only works on NIC RX domain */ 6269 if (attr->ingress && !attr->transfer) 6270 rss = flow_get_rss_action(dev, p_actions_rx); 6271 if (rss) { 6272 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6273 return 0; 6274 /* 6275 * The following information is required by 6276 * mlx5_flow_hashfields_adjust() in advance. 6277 */ 6278 rss_desc->level = rss->level; 6279 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 6280 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 6281 } 6282 flow->dev_handles = 0; 6283 if (rss && rss->types) { 6284 unsigned int graph_root; 6285 6286 graph_root = find_graph_root(items, rss->level); 6287 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6288 items, rss->types, 6289 mlx5_support_expansion, graph_root); 6290 MLX5_ASSERT(ret > 0 && 6291 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6292 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6293 for (i = 0; i < buf->entries; ++i) 6294 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6295 } 6296 } else { 6297 buf->entries = 1; 6298 buf->entry[0].pattern = (void *)(uintptr_t)items; 6299 } 6300 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6301 indir_actions_n); 6302 for (i = 0; i < buf->entries; ++i) { 6303 /* Initialize flow split data. */ 6304 flow_split_info.prefix_layers = 0; 6305 flow_split_info.prefix_mark = 0; 6306 flow_split_info.skip_scale = 0; 6307 /* 6308 * The splitter may create multiple dev_flows, 6309 * depending on configuration. In the simplest 6310 * case it just creates unmodified original flow. 6311 */ 6312 ret = flow_create_split_outer(dev, flow, attr, 6313 buf->entry[i].pattern, 6314 p_actions_rx, &flow_split_info, 6315 error); 6316 if (ret < 0) 6317 goto error; 6318 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6319 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6320 p_actions_rx, 6321 idx, 6322 wks->flows[0].tunnel, 6323 &default_miss_ctx, 6324 error); 6325 if (ret < 0) { 6326 mlx5_free(default_miss_ctx.queue); 6327 goto error; 6328 } 6329 } 6330 } 6331 /* Create the tx flow. */ 6332 if (hairpin_flow) { 6333 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6334 attr_tx.ingress = 0; 6335 attr_tx.egress = 1; 6336 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6337 actions_hairpin_tx.actions, 6338 idx, error); 6339 if (!dev_flow) 6340 goto error; 6341 dev_flow->flow = flow; 6342 dev_flow->external = 0; 6343 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6344 dev_flow->handle, next); 6345 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6346 items_tx.items, 6347 actions_hairpin_tx.actions, error); 6348 if (ret < 0) 6349 goto error; 6350 } 6351 /* 6352 * Update the metadata register copy table. If extensive 6353 * metadata feature is enabled and registers are supported 6354 * we might create the extra rte_flow for each unique 6355 * MARK/FLAG action ID. 6356 * 6357 * The table is updated for ingress Flows only, because 6358 * the egress Flows belong to the different device and 6359 * copy table should be updated in peer NIC Rx domain. 6360 */ 6361 if (attr->ingress && 6362 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6363 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6364 if (ret) 6365 goto error; 6366 } 6367 /* 6368 * If the flow is external (from application) OR device is started, 6369 * OR mreg discover, then apply immediately. 6370 */ 6371 if (external || dev->data->dev_started || 6372 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6373 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6374 ret = flow_drv_apply(dev, flow, error); 6375 if (ret < 0) 6376 goto error; 6377 } 6378 flow->type = type; 6379 flow_rxq_flags_set(dev, flow); 6380 rte_free(translated_actions); 6381 tunnel = flow_tunnel_from_rule(wks->flows); 6382 if (tunnel) { 6383 flow->tunnel = 1; 6384 flow->tunnel_id = tunnel->tunnel_id; 6385 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6386 mlx5_free(default_miss_ctx.queue); 6387 } 6388 mlx5_flow_pop_thread_workspace(); 6389 return idx; 6390 error: 6391 MLX5_ASSERT(flow); 6392 ret = rte_errno; /* Save rte_errno before cleanup. */ 6393 flow_mreg_del_copy_action(dev, flow); 6394 flow_drv_destroy(dev, flow); 6395 if (rss_desc->shared_rss) 6396 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6397 mlx5_ipool_get 6398 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6399 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6400 mlx5_ipool_free(priv->flows[type], idx); 6401 rte_errno = ret; /* Restore rte_errno. */ 6402 ret = rte_errno; 6403 rte_errno = ret; 6404 mlx5_flow_pop_thread_workspace(); 6405 error_before_hairpin_split: 6406 rte_free(translated_actions); 6407 return 0; 6408 } 6409 6410 /** 6411 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6412 * incoming packets to table 1. 6413 * 6414 * Other flow rules, requested for group n, will be created in 6415 * e-switch table n+1. 6416 * Jump action to e-switch group n will be created to group n+1. 6417 * 6418 * Used when working in switchdev mode, to utilise advantages of table 1 6419 * and above. 6420 * 6421 * @param dev 6422 * Pointer to Ethernet device. 6423 * 6424 * @return 6425 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6426 */ 6427 struct rte_flow * 6428 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6429 { 6430 const struct rte_flow_attr attr = { 6431 .group = 0, 6432 .priority = 0, 6433 .ingress = 1, 6434 .egress = 0, 6435 .transfer = 1, 6436 }; 6437 const struct rte_flow_item pattern = { 6438 .type = RTE_FLOW_ITEM_TYPE_END, 6439 }; 6440 struct rte_flow_action_jump jump = { 6441 .group = 1, 6442 }; 6443 const struct rte_flow_action actions[] = { 6444 { 6445 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6446 .conf = &jump, 6447 }, 6448 { 6449 .type = RTE_FLOW_ACTION_TYPE_END, 6450 }, 6451 }; 6452 struct rte_flow_error error; 6453 6454 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6455 &attr, &pattern, 6456 actions, false, &error); 6457 } 6458 6459 /** 6460 * Validate a flow supported by the NIC. 6461 * 6462 * @see rte_flow_validate() 6463 * @see rte_flow_ops 6464 */ 6465 int 6466 mlx5_flow_validate(struct rte_eth_dev *dev, 6467 const struct rte_flow_attr *attr, 6468 const struct rte_flow_item items[], 6469 const struct rte_flow_action original_actions[], 6470 struct rte_flow_error *error) 6471 { 6472 int hairpin_flow; 6473 struct mlx5_translated_action_handle 6474 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6475 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6476 const struct rte_flow_action *actions; 6477 struct rte_flow_action *translated_actions = NULL; 6478 int ret = flow_action_handles_translate(dev, original_actions, 6479 indir_actions, 6480 &indir_actions_n, 6481 &translated_actions, error); 6482 6483 if (ret) 6484 return ret; 6485 actions = translated_actions ? translated_actions : original_actions; 6486 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6487 ret = flow_drv_validate(dev, attr, items, actions, 6488 true, hairpin_flow, error); 6489 rte_free(translated_actions); 6490 return ret; 6491 } 6492 6493 /** 6494 * Create a flow. 6495 * 6496 * @see rte_flow_create() 6497 * @see rte_flow_ops 6498 */ 6499 struct rte_flow * 6500 mlx5_flow_create(struct rte_eth_dev *dev, 6501 const struct rte_flow_attr *attr, 6502 const struct rte_flow_item items[], 6503 const struct rte_flow_action actions[], 6504 struct rte_flow_error *error) 6505 { 6506 /* 6507 * If the device is not started yet, it is not allowed to created a 6508 * flow from application. PMD default flows and traffic control flows 6509 * are not affected. 6510 */ 6511 if (unlikely(!dev->data->dev_started)) { 6512 DRV_LOG(DEBUG, "port %u is not started when " 6513 "inserting a flow", dev->data->port_id); 6514 rte_flow_error_set(error, ENODEV, 6515 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6516 NULL, 6517 "port not started"); 6518 return NULL; 6519 } 6520 6521 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6522 attr, items, actions, 6523 true, error); 6524 } 6525 6526 /** 6527 * Destroy a flow in a list. 6528 * 6529 * @param dev 6530 * Pointer to Ethernet device. 6531 * @param[in] flow_idx 6532 * Index of flow to destroy. 6533 */ 6534 static void 6535 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6536 uint32_t flow_idx) 6537 { 6538 struct mlx5_priv *priv = dev->data->dev_private; 6539 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6540 6541 if (!flow) 6542 return; 6543 MLX5_ASSERT(flow->type == type); 6544 /* 6545 * Update RX queue flags only if port is started, otherwise it is 6546 * already clean. 6547 */ 6548 if (dev->data->dev_started) 6549 flow_rxq_flags_trim(dev, flow); 6550 flow_drv_destroy(dev, flow); 6551 if (flow->tunnel) { 6552 struct mlx5_flow_tunnel *tunnel; 6553 6554 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6555 RTE_VERIFY(tunnel); 6556 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6557 mlx5_flow_tunnel_free(dev, tunnel); 6558 } 6559 flow_mreg_del_copy_action(dev, flow); 6560 mlx5_ipool_free(priv->flows[type], flow_idx); 6561 } 6562 6563 /** 6564 * Destroy all flows. 6565 * 6566 * @param dev 6567 * Pointer to Ethernet device. 6568 * @param type 6569 * Flow type to be flushed. 6570 * @param active 6571 * If flushing is called avtively. 6572 */ 6573 void 6574 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6575 bool active) 6576 { 6577 struct mlx5_priv *priv = dev->data->dev_private; 6578 uint32_t num_flushed = 0, fidx = 1; 6579 struct rte_flow *flow; 6580 6581 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6582 flow_list_destroy(dev, type, fidx); 6583 num_flushed++; 6584 } 6585 if (active) { 6586 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6587 dev->data->port_id, num_flushed); 6588 } 6589 } 6590 6591 /** 6592 * Stop all default actions for flows. 6593 * 6594 * @param dev 6595 * Pointer to Ethernet device. 6596 */ 6597 void 6598 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6599 { 6600 flow_mreg_del_default_copy_action(dev); 6601 flow_rxq_flags_clear(dev); 6602 } 6603 6604 /** 6605 * Start all default actions for flows. 6606 * 6607 * @param dev 6608 * Pointer to Ethernet device. 6609 * @return 6610 * 0 on success, a negative errno value otherwise and rte_errno is set. 6611 */ 6612 int 6613 mlx5_flow_start_default(struct rte_eth_dev *dev) 6614 { 6615 struct rte_flow_error error; 6616 6617 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6618 return flow_mreg_add_default_copy_action(dev, &error); 6619 } 6620 6621 /** 6622 * Release key of thread specific flow workspace data. 6623 */ 6624 void 6625 flow_release_workspace(void *data) 6626 { 6627 struct mlx5_flow_workspace *wks = data; 6628 struct mlx5_flow_workspace *next; 6629 6630 while (wks) { 6631 next = wks->next; 6632 free(wks->rss_desc.queue); 6633 free(wks); 6634 wks = next; 6635 } 6636 } 6637 6638 /** 6639 * Get thread specific current flow workspace. 6640 * 6641 * @return pointer to thread specific flow workspace data, NULL on error. 6642 */ 6643 struct mlx5_flow_workspace* 6644 mlx5_flow_get_thread_workspace(void) 6645 { 6646 struct mlx5_flow_workspace *data; 6647 6648 data = mlx5_flow_os_get_specific_workspace(); 6649 MLX5_ASSERT(data && data->inuse); 6650 if (!data || !data->inuse) 6651 DRV_LOG(ERR, "flow workspace not initialized."); 6652 return data; 6653 } 6654 6655 /** 6656 * Allocate and init new flow workspace. 6657 * 6658 * @return pointer to flow workspace data, NULL on error. 6659 */ 6660 static struct mlx5_flow_workspace* 6661 flow_alloc_thread_workspace(void) 6662 { 6663 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6664 6665 if (!data) { 6666 DRV_LOG(ERR, "Failed to allocate flow workspace " 6667 "memory."); 6668 return NULL; 6669 } 6670 data->rss_desc.queue = calloc(1, 6671 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6672 if (!data->rss_desc.queue) 6673 goto err; 6674 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6675 return data; 6676 err: 6677 if (data->rss_desc.queue) 6678 free(data->rss_desc.queue); 6679 free(data); 6680 return NULL; 6681 } 6682 6683 /** 6684 * Get new thread specific flow workspace. 6685 * 6686 * If current workspace inuse, create new one and set as current. 6687 * 6688 * @return pointer to thread specific flow workspace data, NULL on error. 6689 */ 6690 static struct mlx5_flow_workspace* 6691 mlx5_flow_push_thread_workspace(void) 6692 { 6693 struct mlx5_flow_workspace *curr; 6694 struct mlx5_flow_workspace *data; 6695 6696 curr = mlx5_flow_os_get_specific_workspace(); 6697 if (!curr) { 6698 data = flow_alloc_thread_workspace(); 6699 if (!data) 6700 return NULL; 6701 } else if (!curr->inuse) { 6702 data = curr; 6703 } else if (curr->next) { 6704 data = curr->next; 6705 } else { 6706 data = flow_alloc_thread_workspace(); 6707 if (!data) 6708 return NULL; 6709 curr->next = data; 6710 data->prev = curr; 6711 } 6712 data->inuse = 1; 6713 data->flow_idx = 0; 6714 /* Set as current workspace */ 6715 if (mlx5_flow_os_set_specific_workspace(data)) 6716 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6717 return data; 6718 } 6719 6720 /** 6721 * Close current thread specific flow workspace. 6722 * 6723 * If previous workspace available, set it as current. 6724 * 6725 * @return pointer to thread specific flow workspace data, NULL on error. 6726 */ 6727 static void 6728 mlx5_flow_pop_thread_workspace(void) 6729 { 6730 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6731 6732 if (!data) 6733 return; 6734 if (!data->inuse) { 6735 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6736 return; 6737 } 6738 data->inuse = 0; 6739 if (!data->prev) 6740 return; 6741 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6742 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6743 } 6744 6745 /** 6746 * Verify the flow list is empty 6747 * 6748 * @param dev 6749 * Pointer to Ethernet device. 6750 * 6751 * @return the number of flows not released. 6752 */ 6753 int 6754 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 6755 { 6756 struct mlx5_priv *priv = dev->data->dev_private; 6757 struct rte_flow *flow; 6758 uint32_t idx = 0; 6759 int ret = 0, i; 6760 6761 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 6762 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 6763 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6764 dev->data->port_id, (void *)flow); 6765 ret++; 6766 } 6767 } 6768 return ret; 6769 } 6770 6771 /** 6772 * Enable default hairpin egress flow. 6773 * 6774 * @param dev 6775 * Pointer to Ethernet device. 6776 * @param queue 6777 * The queue index. 6778 * 6779 * @return 6780 * 0 on success, a negative errno value otherwise and rte_errno is set. 6781 */ 6782 int 6783 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6784 uint32_t queue) 6785 { 6786 const struct rte_flow_attr attr = { 6787 .egress = 1, 6788 .priority = 0, 6789 }; 6790 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6791 .queue = queue, 6792 }; 6793 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6794 .queue = UINT32_MAX, 6795 }; 6796 struct rte_flow_item items[] = { 6797 { 6798 .type = (enum rte_flow_item_type) 6799 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6800 .spec = &queue_spec, 6801 .last = NULL, 6802 .mask = &queue_mask, 6803 }, 6804 { 6805 .type = RTE_FLOW_ITEM_TYPE_END, 6806 }, 6807 }; 6808 struct rte_flow_action_jump jump = { 6809 .group = MLX5_HAIRPIN_TX_TABLE, 6810 }; 6811 struct rte_flow_action actions[2]; 6812 uint32_t flow_idx; 6813 struct rte_flow_error error; 6814 6815 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6816 actions[0].conf = &jump; 6817 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6818 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6819 &attr, items, actions, false, &error); 6820 if (!flow_idx) { 6821 DRV_LOG(DEBUG, 6822 "Failed to create ctrl flow: rte_errno(%d)," 6823 " type(%d), message(%s)", 6824 rte_errno, error.type, 6825 error.message ? error.message : " (no stated reason)"); 6826 return -rte_errno; 6827 } 6828 return 0; 6829 } 6830 6831 /** 6832 * Enable a control flow configured from the control plane. 6833 * 6834 * @param dev 6835 * Pointer to Ethernet device. 6836 * @param eth_spec 6837 * An Ethernet flow spec to apply. 6838 * @param eth_mask 6839 * An Ethernet flow mask to apply. 6840 * @param vlan_spec 6841 * A VLAN flow spec to apply. 6842 * @param vlan_mask 6843 * A VLAN flow mask to apply. 6844 * 6845 * @return 6846 * 0 on success, a negative errno value otherwise and rte_errno is set. 6847 */ 6848 int 6849 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6850 struct rte_flow_item_eth *eth_spec, 6851 struct rte_flow_item_eth *eth_mask, 6852 struct rte_flow_item_vlan *vlan_spec, 6853 struct rte_flow_item_vlan *vlan_mask) 6854 { 6855 struct mlx5_priv *priv = dev->data->dev_private; 6856 const struct rte_flow_attr attr = { 6857 .ingress = 1, 6858 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6859 }; 6860 struct rte_flow_item items[] = { 6861 { 6862 .type = RTE_FLOW_ITEM_TYPE_ETH, 6863 .spec = eth_spec, 6864 .last = NULL, 6865 .mask = eth_mask, 6866 }, 6867 { 6868 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6869 RTE_FLOW_ITEM_TYPE_END, 6870 .spec = vlan_spec, 6871 .last = NULL, 6872 .mask = vlan_mask, 6873 }, 6874 { 6875 .type = RTE_FLOW_ITEM_TYPE_END, 6876 }, 6877 }; 6878 uint16_t queue[priv->reta_idx_n]; 6879 struct rte_flow_action_rss action_rss = { 6880 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6881 .level = 0, 6882 .types = priv->rss_conf.rss_hf, 6883 .key_len = priv->rss_conf.rss_key_len, 6884 .queue_num = priv->reta_idx_n, 6885 .key = priv->rss_conf.rss_key, 6886 .queue = queue, 6887 }; 6888 struct rte_flow_action actions[] = { 6889 { 6890 .type = RTE_FLOW_ACTION_TYPE_RSS, 6891 .conf = &action_rss, 6892 }, 6893 { 6894 .type = RTE_FLOW_ACTION_TYPE_END, 6895 }, 6896 }; 6897 uint32_t flow_idx; 6898 struct rte_flow_error error; 6899 unsigned int i; 6900 6901 if (!priv->reta_idx_n || !priv->rxqs_n) { 6902 return 0; 6903 } 6904 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6905 action_rss.types = 0; 6906 for (i = 0; i != priv->reta_idx_n; ++i) 6907 queue[i] = (*priv->reta_idx)[i]; 6908 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6909 &attr, items, actions, false, &error); 6910 if (!flow_idx) 6911 return -rte_errno; 6912 return 0; 6913 } 6914 6915 /** 6916 * Enable a flow control configured from the control plane. 6917 * 6918 * @param dev 6919 * Pointer to Ethernet device. 6920 * @param eth_spec 6921 * An Ethernet flow spec to apply. 6922 * @param eth_mask 6923 * An Ethernet flow mask to apply. 6924 * 6925 * @return 6926 * 0 on success, a negative errno value otherwise and rte_errno is set. 6927 */ 6928 int 6929 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6930 struct rte_flow_item_eth *eth_spec, 6931 struct rte_flow_item_eth *eth_mask) 6932 { 6933 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6934 } 6935 6936 /** 6937 * Create default miss flow rule matching lacp traffic 6938 * 6939 * @param dev 6940 * Pointer to Ethernet device. 6941 * @param eth_spec 6942 * An Ethernet flow spec to apply. 6943 * 6944 * @return 6945 * 0 on success, a negative errno value otherwise and rte_errno is set. 6946 */ 6947 int 6948 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6949 { 6950 /* 6951 * The LACP matching is done by only using ether type since using 6952 * a multicast dst mac causes kernel to give low priority to this flow. 6953 */ 6954 static const struct rte_flow_item_eth lacp_spec = { 6955 .type = RTE_BE16(0x8809), 6956 }; 6957 static const struct rte_flow_item_eth lacp_mask = { 6958 .type = 0xffff, 6959 }; 6960 const struct rte_flow_attr attr = { 6961 .ingress = 1, 6962 }; 6963 struct rte_flow_item items[] = { 6964 { 6965 .type = RTE_FLOW_ITEM_TYPE_ETH, 6966 .spec = &lacp_spec, 6967 .mask = &lacp_mask, 6968 }, 6969 { 6970 .type = RTE_FLOW_ITEM_TYPE_END, 6971 }, 6972 }; 6973 struct rte_flow_action actions[] = { 6974 { 6975 .type = (enum rte_flow_action_type) 6976 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6977 }, 6978 { 6979 .type = RTE_FLOW_ACTION_TYPE_END, 6980 }, 6981 }; 6982 struct rte_flow_error error; 6983 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6984 &attr, items, actions, 6985 false, &error); 6986 6987 if (!flow_idx) 6988 return -rte_errno; 6989 return 0; 6990 } 6991 6992 /** 6993 * Destroy a flow. 6994 * 6995 * @see rte_flow_destroy() 6996 * @see rte_flow_ops 6997 */ 6998 int 6999 mlx5_flow_destroy(struct rte_eth_dev *dev, 7000 struct rte_flow *flow, 7001 struct rte_flow_error *error __rte_unused) 7002 { 7003 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7004 (uintptr_t)(void *)flow); 7005 return 0; 7006 } 7007 7008 /** 7009 * Destroy all flows. 7010 * 7011 * @see rte_flow_flush() 7012 * @see rte_flow_ops 7013 */ 7014 int 7015 mlx5_flow_flush(struct rte_eth_dev *dev, 7016 struct rte_flow_error *error __rte_unused) 7017 { 7018 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7019 return 0; 7020 } 7021 7022 /** 7023 * Isolated mode. 7024 * 7025 * @see rte_flow_isolate() 7026 * @see rte_flow_ops 7027 */ 7028 int 7029 mlx5_flow_isolate(struct rte_eth_dev *dev, 7030 int enable, 7031 struct rte_flow_error *error) 7032 { 7033 struct mlx5_priv *priv = dev->data->dev_private; 7034 7035 if (dev->data->dev_started) { 7036 rte_flow_error_set(error, EBUSY, 7037 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7038 NULL, 7039 "port must be stopped first"); 7040 return -rte_errno; 7041 } 7042 priv->isolated = !!enable; 7043 if (enable) 7044 dev->dev_ops = &mlx5_dev_ops_isolate; 7045 else 7046 dev->dev_ops = &mlx5_dev_ops; 7047 7048 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7049 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7050 7051 return 0; 7052 } 7053 7054 /** 7055 * Query a flow. 7056 * 7057 * @see rte_flow_query() 7058 * @see rte_flow_ops 7059 */ 7060 static int 7061 flow_drv_query(struct rte_eth_dev *dev, 7062 uint32_t flow_idx, 7063 const struct rte_flow_action *actions, 7064 void *data, 7065 struct rte_flow_error *error) 7066 { 7067 struct mlx5_priv *priv = dev->data->dev_private; 7068 const struct mlx5_flow_driver_ops *fops; 7069 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7070 flow_idx); 7071 enum mlx5_flow_drv_type ftype; 7072 7073 if (!flow) { 7074 return rte_flow_error_set(error, ENOENT, 7075 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7076 NULL, 7077 "invalid flow handle"); 7078 } 7079 ftype = flow->drv_type; 7080 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7081 fops = flow_get_drv_ops(ftype); 7082 7083 return fops->query(dev, flow, actions, data, error); 7084 } 7085 7086 /** 7087 * Query a flow. 7088 * 7089 * @see rte_flow_query() 7090 * @see rte_flow_ops 7091 */ 7092 int 7093 mlx5_flow_query(struct rte_eth_dev *dev, 7094 struct rte_flow *flow, 7095 const struct rte_flow_action *actions, 7096 void *data, 7097 struct rte_flow_error *error) 7098 { 7099 int ret; 7100 7101 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7102 error); 7103 if (ret < 0) 7104 return ret; 7105 return 0; 7106 } 7107 7108 /** 7109 * Get rte_flow callbacks. 7110 * 7111 * @param dev 7112 * Pointer to Ethernet device structure. 7113 * @param ops 7114 * Pointer to operation-specific structure. 7115 * 7116 * @return 0 7117 */ 7118 int 7119 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7120 const struct rte_flow_ops **ops) 7121 { 7122 *ops = &mlx5_flow_ops; 7123 return 0; 7124 } 7125 7126 /** 7127 * Validate meter policy actions. 7128 * Dispatcher for action type specific validation. 7129 * 7130 * @param[in] dev 7131 * Pointer to the Ethernet device structure. 7132 * @param[in] action 7133 * The meter policy action object to validate. 7134 * @param[in] attr 7135 * Attributes of flow to determine steering domain. 7136 * @param[out] is_rss 7137 * Is RSS or not. 7138 * @param[out] domain_bitmap 7139 * Domain bitmap. 7140 * @param[out] is_def_policy 7141 * Is default policy or not. 7142 * @param[out] error 7143 * Perform verbose error reporting if not NULL. Initialized in case of 7144 * error only. 7145 * 7146 * @return 7147 * 0 on success, otherwise negative errno value. 7148 */ 7149 int 7150 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7151 const struct rte_flow_action *actions[RTE_COLORS], 7152 struct rte_flow_attr *attr, 7153 bool *is_rss, 7154 uint8_t *domain_bitmap, 7155 bool *is_def_policy, 7156 struct rte_mtr_error *error) 7157 { 7158 const struct mlx5_flow_driver_ops *fops; 7159 7160 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7161 return fops->validate_mtr_acts(dev, actions, attr, 7162 is_rss, domain_bitmap, is_def_policy, error); 7163 } 7164 7165 /** 7166 * Destroy the meter table set. 7167 * 7168 * @param[in] dev 7169 * Pointer to Ethernet device. 7170 * @param[in] mtr_policy 7171 * Meter policy struct. 7172 */ 7173 void 7174 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7175 struct mlx5_flow_meter_policy *mtr_policy) 7176 { 7177 const struct mlx5_flow_driver_ops *fops; 7178 7179 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7180 fops->destroy_mtr_acts(dev, mtr_policy); 7181 } 7182 7183 /** 7184 * Create policy action, lock free, 7185 * (mutex should be acquired by caller). 7186 * Dispatcher for action type specific call. 7187 * 7188 * @param[in] dev 7189 * Pointer to the Ethernet device structure. 7190 * @param[in] mtr_policy 7191 * Meter policy struct. 7192 * @param[in] action 7193 * Action specification used to create meter actions. 7194 * @param[out] error 7195 * Perform verbose error reporting if not NULL. Initialized in case of 7196 * error only. 7197 * 7198 * @return 7199 * 0 on success, otherwise negative errno value. 7200 */ 7201 int 7202 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7203 struct mlx5_flow_meter_policy *mtr_policy, 7204 const struct rte_flow_action *actions[RTE_COLORS], 7205 struct rte_mtr_error *error) 7206 { 7207 const struct mlx5_flow_driver_ops *fops; 7208 7209 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7210 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7211 } 7212 7213 /** 7214 * Create policy rules, lock free, 7215 * (mutex should be acquired by caller). 7216 * Dispatcher for action type specific call. 7217 * 7218 * @param[in] dev 7219 * Pointer to the Ethernet device structure. 7220 * @param[in] mtr_policy 7221 * Meter policy struct. 7222 * 7223 * @return 7224 * 0 on success, -1 otherwise. 7225 */ 7226 int 7227 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7228 struct mlx5_flow_meter_policy *mtr_policy) 7229 { 7230 const struct mlx5_flow_driver_ops *fops; 7231 7232 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7233 return fops->create_policy_rules(dev, mtr_policy); 7234 } 7235 7236 /** 7237 * Destroy policy rules, lock free, 7238 * (mutex should be acquired by caller). 7239 * Dispatcher for action type specific call. 7240 * 7241 * @param[in] dev 7242 * Pointer to the Ethernet device structure. 7243 * @param[in] mtr_policy 7244 * Meter policy struct. 7245 */ 7246 void 7247 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7248 struct mlx5_flow_meter_policy *mtr_policy) 7249 { 7250 const struct mlx5_flow_driver_ops *fops; 7251 7252 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7253 fops->destroy_policy_rules(dev, mtr_policy); 7254 } 7255 7256 /** 7257 * Destroy the default policy table set. 7258 * 7259 * @param[in] dev 7260 * Pointer to Ethernet device. 7261 */ 7262 void 7263 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7264 { 7265 const struct mlx5_flow_driver_ops *fops; 7266 7267 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7268 fops->destroy_def_policy(dev); 7269 } 7270 7271 /** 7272 * Destroy the default policy table set. 7273 * 7274 * @param[in] dev 7275 * Pointer to Ethernet device. 7276 * 7277 * @return 7278 * 0 on success, -1 otherwise. 7279 */ 7280 int 7281 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7282 { 7283 const struct mlx5_flow_driver_ops *fops; 7284 7285 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7286 return fops->create_def_policy(dev); 7287 } 7288 7289 /** 7290 * Create the needed meter and suffix tables. 7291 * 7292 * @param[in] dev 7293 * Pointer to Ethernet device. 7294 * 7295 * @return 7296 * 0 on success, -1 otherwise. 7297 */ 7298 int 7299 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7300 struct mlx5_flow_meter_info *fm, 7301 uint32_t mtr_idx, 7302 uint8_t domain_bitmap) 7303 { 7304 const struct mlx5_flow_driver_ops *fops; 7305 7306 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7307 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7308 } 7309 7310 /** 7311 * Destroy the meter table set. 7312 * 7313 * @param[in] dev 7314 * Pointer to Ethernet device. 7315 * @param[in] tbl 7316 * Pointer to the meter table set. 7317 */ 7318 void 7319 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7320 struct mlx5_flow_meter_info *fm) 7321 { 7322 const struct mlx5_flow_driver_ops *fops; 7323 7324 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7325 fops->destroy_mtr_tbls(dev, fm); 7326 } 7327 7328 /** 7329 * Destroy the global meter drop table. 7330 * 7331 * @param[in] dev 7332 * Pointer to Ethernet device. 7333 */ 7334 void 7335 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7336 { 7337 const struct mlx5_flow_driver_ops *fops; 7338 7339 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7340 fops->destroy_mtr_drop_tbls(dev); 7341 } 7342 7343 /** 7344 * Destroy the sub policy table with RX queue. 7345 * 7346 * @param[in] dev 7347 * Pointer to Ethernet device. 7348 * @param[in] mtr_policy 7349 * Pointer to meter policy table. 7350 */ 7351 void 7352 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7353 struct mlx5_flow_meter_policy *mtr_policy) 7354 { 7355 const struct mlx5_flow_driver_ops *fops; 7356 7357 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7358 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7359 } 7360 7361 /** 7362 * Allocate the needed aso flow meter id. 7363 * 7364 * @param[in] dev 7365 * Pointer to Ethernet device. 7366 * 7367 * @return 7368 * Index to aso flow meter on success, NULL otherwise. 7369 */ 7370 uint32_t 7371 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7372 { 7373 const struct mlx5_flow_driver_ops *fops; 7374 7375 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7376 return fops->create_meter(dev); 7377 } 7378 7379 /** 7380 * Free the aso flow meter id. 7381 * 7382 * @param[in] dev 7383 * Pointer to Ethernet device. 7384 * @param[in] mtr_idx 7385 * Index to aso flow meter to be free. 7386 * 7387 * @return 7388 * 0 on success. 7389 */ 7390 void 7391 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7392 { 7393 const struct mlx5_flow_driver_ops *fops; 7394 7395 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7396 fops->free_meter(dev, mtr_idx); 7397 } 7398 7399 /** 7400 * Allocate a counter. 7401 * 7402 * @param[in] dev 7403 * Pointer to Ethernet device structure. 7404 * 7405 * @return 7406 * Index to allocated counter on success, 0 otherwise. 7407 */ 7408 uint32_t 7409 mlx5_counter_alloc(struct rte_eth_dev *dev) 7410 { 7411 const struct mlx5_flow_driver_ops *fops; 7412 struct rte_flow_attr attr = { .transfer = 0 }; 7413 7414 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7415 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7416 return fops->counter_alloc(dev); 7417 } 7418 DRV_LOG(ERR, 7419 "port %u counter allocate is not supported.", 7420 dev->data->port_id); 7421 return 0; 7422 } 7423 7424 /** 7425 * Free a counter. 7426 * 7427 * @param[in] dev 7428 * Pointer to Ethernet device structure. 7429 * @param[in] cnt 7430 * Index to counter to be free. 7431 */ 7432 void 7433 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7434 { 7435 const struct mlx5_flow_driver_ops *fops; 7436 struct rte_flow_attr attr = { .transfer = 0 }; 7437 7438 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7439 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7440 fops->counter_free(dev, cnt); 7441 return; 7442 } 7443 DRV_LOG(ERR, 7444 "port %u counter free is not supported.", 7445 dev->data->port_id); 7446 } 7447 7448 /** 7449 * Query counter statistics. 7450 * 7451 * @param[in] dev 7452 * Pointer to Ethernet device structure. 7453 * @param[in] cnt 7454 * Index to counter to query. 7455 * @param[in] clear 7456 * Set to clear counter statistics. 7457 * @param[out] pkts 7458 * The counter hits packets number to save. 7459 * @param[out] bytes 7460 * The counter hits bytes number to save. 7461 * 7462 * @return 7463 * 0 on success, a negative errno value otherwise. 7464 */ 7465 int 7466 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7467 bool clear, uint64_t *pkts, uint64_t *bytes) 7468 { 7469 const struct mlx5_flow_driver_ops *fops; 7470 struct rte_flow_attr attr = { .transfer = 0 }; 7471 7472 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7473 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7474 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7475 } 7476 DRV_LOG(ERR, 7477 "port %u counter query is not supported.", 7478 dev->data->port_id); 7479 return -ENOTSUP; 7480 } 7481 7482 /** 7483 * Allocate a new memory for the counter values wrapped by all the needed 7484 * management. 7485 * 7486 * @param[in] sh 7487 * Pointer to mlx5_dev_ctx_shared object. 7488 * 7489 * @return 7490 * 0 on success, a negative errno value otherwise. 7491 */ 7492 static int 7493 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7494 { 7495 struct mlx5_devx_mkey_attr mkey_attr; 7496 struct mlx5_counter_stats_mem_mng *mem_mng; 7497 volatile struct flow_counter_stats *raw_data; 7498 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7499 int size = (sizeof(struct flow_counter_stats) * 7500 MLX5_COUNTERS_PER_POOL + 7501 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7502 sizeof(struct mlx5_counter_stats_mem_mng); 7503 size_t pgsize = rte_mem_page_size(); 7504 uint8_t *mem; 7505 int i; 7506 7507 if (pgsize == (size_t)-1) { 7508 DRV_LOG(ERR, "Failed to get mem page size"); 7509 rte_errno = ENOMEM; 7510 return -ENOMEM; 7511 } 7512 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7513 if (!mem) { 7514 rte_errno = ENOMEM; 7515 return -ENOMEM; 7516 } 7517 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7518 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7519 mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size, 7520 IBV_ACCESS_LOCAL_WRITE); 7521 if (!mem_mng->umem) { 7522 rte_errno = errno; 7523 mlx5_free(mem); 7524 return -rte_errno; 7525 } 7526 memset(&mkey_attr, 0, sizeof(mkey_attr)); 7527 mkey_attr.addr = (uintptr_t)mem; 7528 mkey_attr.size = size; 7529 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7530 mkey_attr.pd = sh->pdn; 7531 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 7532 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 7533 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7534 if (!mem_mng->dm) { 7535 mlx5_os_umem_dereg(mem_mng->umem); 7536 rte_errno = errno; 7537 mlx5_free(mem); 7538 return -rte_errno; 7539 } 7540 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7541 raw_data = (volatile struct flow_counter_stats *)mem; 7542 for (i = 0; i < raws_n; ++i) { 7543 mem_mng->raws[i].mem_mng = mem_mng; 7544 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7545 } 7546 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7547 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7548 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7549 next); 7550 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7551 sh->cmng.mem_mng = mem_mng; 7552 return 0; 7553 } 7554 7555 /** 7556 * Set the statistic memory to the new counter pool. 7557 * 7558 * @param[in] sh 7559 * Pointer to mlx5_dev_ctx_shared object. 7560 * @param[in] pool 7561 * Pointer to the pool to set the statistic memory. 7562 * 7563 * @return 7564 * 0 on success, a negative errno value otherwise. 7565 */ 7566 static int 7567 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7568 struct mlx5_flow_counter_pool *pool) 7569 { 7570 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7571 /* Resize statistic memory once used out. */ 7572 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7573 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7574 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7575 return -1; 7576 } 7577 rte_spinlock_lock(&pool->sl); 7578 pool->raw = cmng->mem_mng->raws + pool->index % 7579 MLX5_CNT_CONTAINER_RESIZE; 7580 rte_spinlock_unlock(&pool->sl); 7581 pool->raw_hw = NULL; 7582 return 0; 7583 } 7584 7585 #define MLX5_POOL_QUERY_FREQ_US 1000000 7586 7587 /** 7588 * Set the periodic procedure for triggering asynchronous batch queries for all 7589 * the counter pools. 7590 * 7591 * @param[in] sh 7592 * Pointer to mlx5_dev_ctx_shared object. 7593 */ 7594 void 7595 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7596 { 7597 uint32_t pools_n, us; 7598 7599 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7600 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7601 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7602 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7603 sh->cmng.query_thread_on = 0; 7604 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7605 } else { 7606 sh->cmng.query_thread_on = 1; 7607 } 7608 } 7609 7610 /** 7611 * The periodic procedure for triggering asynchronous batch queries for all the 7612 * counter pools. This function is probably called by the host thread. 7613 * 7614 * @param[in] arg 7615 * The parameter for the alarm process. 7616 */ 7617 void 7618 mlx5_flow_query_alarm(void *arg) 7619 { 7620 struct mlx5_dev_ctx_shared *sh = arg; 7621 int ret; 7622 uint16_t pool_index = sh->cmng.pool_index; 7623 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7624 struct mlx5_flow_counter_pool *pool; 7625 uint16_t n_valid; 7626 7627 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7628 goto set_alarm; 7629 rte_spinlock_lock(&cmng->pool_update_sl); 7630 pool = cmng->pools[pool_index]; 7631 n_valid = cmng->n_valid; 7632 rte_spinlock_unlock(&cmng->pool_update_sl); 7633 /* Set the statistic memory to the new created pool. */ 7634 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7635 goto set_alarm; 7636 if (pool->raw_hw) 7637 /* There is a pool query in progress. */ 7638 goto set_alarm; 7639 pool->raw_hw = 7640 LIST_FIRST(&sh->cmng.free_stat_raws); 7641 if (!pool->raw_hw) 7642 /* No free counter statistics raw memory. */ 7643 goto set_alarm; 7644 /* 7645 * Identify the counters released between query trigger and query 7646 * handle more efficiently. The counter released in this gap period 7647 * should wait for a new round of query as the new arrived packets 7648 * will not be taken into account. 7649 */ 7650 pool->query_gen++; 7651 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7652 MLX5_COUNTERS_PER_POOL, 7653 NULL, NULL, 7654 pool->raw_hw->mem_mng->dm->id, 7655 (void *)(uintptr_t) 7656 pool->raw_hw->data, 7657 sh->devx_comp, 7658 (uint64_t)(uintptr_t)pool); 7659 if (ret) { 7660 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7661 " %d", pool->min_dcs->id); 7662 pool->raw_hw = NULL; 7663 goto set_alarm; 7664 } 7665 LIST_REMOVE(pool->raw_hw, next); 7666 sh->cmng.pending_queries++; 7667 pool_index++; 7668 if (pool_index >= n_valid) 7669 pool_index = 0; 7670 set_alarm: 7671 sh->cmng.pool_index = pool_index; 7672 mlx5_set_query_alarm(sh); 7673 } 7674 7675 /** 7676 * Check and callback event for new aged flow in the counter pool 7677 * 7678 * @param[in] sh 7679 * Pointer to mlx5_dev_ctx_shared object. 7680 * @param[in] pool 7681 * Pointer to Current counter pool. 7682 */ 7683 static void 7684 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7685 struct mlx5_flow_counter_pool *pool) 7686 { 7687 struct mlx5_priv *priv; 7688 struct mlx5_flow_counter *cnt; 7689 struct mlx5_age_info *age_info; 7690 struct mlx5_age_param *age_param; 7691 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7692 struct mlx5_counter_stats_raw *prev = pool->raw; 7693 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7694 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7695 uint16_t expected = AGE_CANDIDATE; 7696 uint32_t i; 7697 7698 pool->time_of_last_age_check = curr_time; 7699 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7700 cnt = MLX5_POOL_GET_CNT(pool, i); 7701 age_param = MLX5_CNT_TO_AGE(cnt); 7702 if (__atomic_load_n(&age_param->state, 7703 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7704 continue; 7705 if (cur->data[i].hits != prev->data[i].hits) { 7706 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7707 __ATOMIC_RELAXED); 7708 continue; 7709 } 7710 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7711 time_delta, 7712 __ATOMIC_RELAXED) <= age_param->timeout) 7713 continue; 7714 /** 7715 * Hold the lock first, or if between the 7716 * state AGE_TMOUT and tailq operation the 7717 * release happened, the release procedure 7718 * may delete a non-existent tailq node. 7719 */ 7720 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7721 age_info = GET_PORT_AGE_INFO(priv); 7722 rte_spinlock_lock(&age_info->aged_sl); 7723 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7724 AGE_TMOUT, false, 7725 __ATOMIC_RELAXED, 7726 __ATOMIC_RELAXED)) { 7727 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7728 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7729 } 7730 rte_spinlock_unlock(&age_info->aged_sl); 7731 } 7732 mlx5_age_event_prepare(sh); 7733 } 7734 7735 /** 7736 * Handler for the HW respond about ready values from an asynchronous batch 7737 * query. This function is probably called by the host thread. 7738 * 7739 * @param[in] sh 7740 * The pointer to the shared device context. 7741 * @param[in] async_id 7742 * The Devx async ID. 7743 * @param[in] status 7744 * The status of the completion. 7745 */ 7746 void 7747 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7748 uint64_t async_id, int status) 7749 { 7750 struct mlx5_flow_counter_pool *pool = 7751 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7752 struct mlx5_counter_stats_raw *raw_to_free; 7753 uint8_t query_gen = pool->query_gen ^ 1; 7754 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7755 enum mlx5_counter_type cnt_type = 7756 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7757 MLX5_COUNTER_TYPE_ORIGIN; 7758 7759 if (unlikely(status)) { 7760 raw_to_free = pool->raw_hw; 7761 } else { 7762 raw_to_free = pool->raw; 7763 if (pool->is_aged) 7764 mlx5_flow_aging_check(sh, pool); 7765 rte_spinlock_lock(&pool->sl); 7766 pool->raw = pool->raw_hw; 7767 rte_spinlock_unlock(&pool->sl); 7768 /* Be sure the new raw counters data is updated in memory. */ 7769 rte_io_wmb(); 7770 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7771 rte_spinlock_lock(&cmng->csl[cnt_type]); 7772 TAILQ_CONCAT(&cmng->counters[cnt_type], 7773 &pool->counters[query_gen], next); 7774 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7775 } 7776 } 7777 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7778 pool->raw_hw = NULL; 7779 sh->cmng.pending_queries--; 7780 } 7781 7782 static int 7783 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7784 const struct flow_grp_info *grp_info, 7785 struct rte_flow_error *error) 7786 { 7787 if (grp_info->transfer && grp_info->external && 7788 grp_info->fdb_def_rule) { 7789 if (group == UINT32_MAX) 7790 return rte_flow_error_set 7791 (error, EINVAL, 7792 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7793 NULL, 7794 "group index not supported"); 7795 *table = group + 1; 7796 } else { 7797 *table = group; 7798 } 7799 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7800 return 0; 7801 } 7802 7803 /** 7804 * Translate the rte_flow group index to HW table value. 7805 * 7806 * If tunnel offload is disabled, all group ids converted to flow table 7807 * id using the standard method. 7808 * If tunnel offload is enabled, group id can be converted using the 7809 * standard or tunnel conversion method. Group conversion method 7810 * selection depends on flags in `grp_info` parameter: 7811 * - Internal (grp_info.external == 0) groups conversion uses the 7812 * standard method. 7813 * - Group ids in JUMP action converted with the tunnel conversion. 7814 * - Group id in rule attribute conversion depends on a rule type and 7815 * group id value: 7816 * ** non zero group attributes converted with the tunnel method 7817 * ** zero group attribute in non-tunnel rule is converted using the 7818 * standard method - there's only one root table 7819 * ** zero group attribute in steer tunnel rule is converted with the 7820 * standard method - single root table 7821 * ** zero group attribute in match tunnel rule is a special OvS 7822 * case: that value is used for portability reasons. That group 7823 * id is converted with the tunnel conversion method. 7824 * 7825 * @param[in] dev 7826 * Port device 7827 * @param[in] tunnel 7828 * PMD tunnel offload object 7829 * @param[in] group 7830 * rte_flow group index value. 7831 * @param[out] table 7832 * HW table value. 7833 * @param[in] grp_info 7834 * flags used for conversion 7835 * @param[out] error 7836 * Pointer to error structure. 7837 * 7838 * @return 7839 * 0 on success, a negative errno value otherwise and rte_errno is set. 7840 */ 7841 int 7842 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7843 const struct mlx5_flow_tunnel *tunnel, 7844 uint32_t group, uint32_t *table, 7845 const struct flow_grp_info *grp_info, 7846 struct rte_flow_error *error) 7847 { 7848 int ret; 7849 bool standard_translation; 7850 7851 if (!grp_info->skip_scale && grp_info->external && 7852 group < MLX5_MAX_TABLES_EXTERNAL) 7853 group *= MLX5_FLOW_TABLE_FACTOR; 7854 if (is_tunnel_offload_active(dev)) { 7855 standard_translation = !grp_info->external || 7856 grp_info->std_tbl_fix; 7857 } else { 7858 standard_translation = true; 7859 } 7860 DRV_LOG(DEBUG, 7861 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 7862 dev->data->port_id, group, grp_info->transfer, 7863 grp_info->external, grp_info->fdb_def_rule, 7864 standard_translation ? "STANDARD" : "TUNNEL"); 7865 if (standard_translation) 7866 ret = flow_group_to_table(dev->data->port_id, group, table, 7867 grp_info, error); 7868 else 7869 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7870 table, error); 7871 7872 return ret; 7873 } 7874 7875 /** 7876 * Discover availability of metadata reg_c's. 7877 * 7878 * Iteratively use test flows to check availability. 7879 * 7880 * @param[in] dev 7881 * Pointer to the Ethernet device structure. 7882 * 7883 * @return 7884 * 0 on success, a negative errno value otherwise and rte_errno is set. 7885 */ 7886 int 7887 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7888 { 7889 struct mlx5_priv *priv = dev->data->dev_private; 7890 struct mlx5_dev_config *config = &priv->config; 7891 enum modify_reg idx; 7892 int n = 0; 7893 7894 /* reg_c[0] and reg_c[1] are reserved. */ 7895 config->flow_mreg_c[n++] = REG_C_0; 7896 config->flow_mreg_c[n++] = REG_C_1; 7897 /* Discover availability of other reg_c's. */ 7898 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7899 struct rte_flow_attr attr = { 7900 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7901 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7902 .ingress = 1, 7903 }; 7904 struct rte_flow_item items[] = { 7905 [0] = { 7906 .type = RTE_FLOW_ITEM_TYPE_END, 7907 }, 7908 }; 7909 struct rte_flow_action actions[] = { 7910 [0] = { 7911 .type = (enum rte_flow_action_type) 7912 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7913 .conf = &(struct mlx5_flow_action_copy_mreg){ 7914 .src = REG_C_1, 7915 .dst = idx, 7916 }, 7917 }, 7918 [1] = { 7919 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7920 .conf = &(struct rte_flow_action_jump){ 7921 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7922 }, 7923 }, 7924 [2] = { 7925 .type = RTE_FLOW_ACTION_TYPE_END, 7926 }, 7927 }; 7928 uint32_t flow_idx; 7929 struct rte_flow *flow; 7930 struct rte_flow_error error; 7931 7932 if (!config->dv_flow_en) 7933 break; 7934 /* Create internal flow, validation skips copy action. */ 7935 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 7936 items, actions, false, &error); 7937 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7938 flow_idx); 7939 if (!flow) 7940 continue; 7941 config->flow_mreg_c[n++] = idx; 7942 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 7943 } 7944 for (; n < MLX5_MREG_C_NUM; ++n) 7945 config->flow_mreg_c[n] = REG_NON; 7946 return 0; 7947 } 7948 7949 int 7950 save_dump_file(const uint8_t *data, uint32_t size, 7951 uint32_t type, uint32_t id, void *arg, FILE *file) 7952 { 7953 char line[BUF_SIZE]; 7954 uint32_t out = 0; 7955 uint32_t k; 7956 uint32_t actions_num; 7957 struct rte_flow_query_count *count; 7958 7959 memset(line, 0, BUF_SIZE); 7960 switch (type) { 7961 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 7962 actions_num = *(uint32_t *)(arg); 7963 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,", 7964 type, id, actions_num); 7965 break; 7966 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 7967 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,", 7968 type, id); 7969 break; 7970 case DR_DUMP_REC_TYPE_PMD_COUNTER: 7971 count = (struct rte_flow_query_count *)arg; 7972 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type, 7973 id, count->hits, count->bytes); 7974 return 0; 7975 default: 7976 return -1; 7977 } 7978 7979 for (k = 0; k < size; k++) { 7980 /* Make sure we do not overrun the line buffer length. */ 7981 if (out >= BUF_SIZE - 4) { 7982 line[out] = '\0'; 7983 break; 7984 } 7985 out += snprintf(line + out, BUF_SIZE - out, "%02x", 7986 (data[k]) & 0xff); 7987 } 7988 fprintf(file, "%s\n", line); 7989 return 0; 7990 } 7991 7992 int 7993 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 7994 struct rte_flow_query_count *count, struct rte_flow_error *error) 7995 { 7996 struct rte_flow_action action[2]; 7997 enum mlx5_flow_drv_type ftype; 7998 const struct mlx5_flow_driver_ops *fops; 7999 8000 if (!flow) { 8001 return rte_flow_error_set(error, ENOENT, 8002 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8003 NULL, 8004 "invalid flow handle"); 8005 } 8006 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8007 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8008 if (flow->counter) { 8009 memset(count, 0, sizeof(struct rte_flow_query_count)); 8010 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8011 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8012 ftype < MLX5_FLOW_TYPE_MAX); 8013 fops = flow_get_drv_ops(ftype); 8014 return fops->query(dev, flow, action, count, error); 8015 } 8016 return -1; 8017 } 8018 8019 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8020 /** 8021 * Dump flow ipool data to file 8022 * 8023 * @param[in] dev 8024 * The pointer to Ethernet device. 8025 * @param[in] file 8026 * A pointer to a file for output. 8027 * @param[out] error 8028 * Perform verbose error reporting if not NULL. PMDs initialize this 8029 * structure in case of error only. 8030 * @return 8031 * 0 on success, a negative value otherwise. 8032 */ 8033 int 8034 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8035 struct rte_flow *flow, FILE *file, 8036 struct rte_flow_error *error) 8037 { 8038 struct mlx5_priv *priv = dev->data->dev_private; 8039 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8040 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8041 uint32_t handle_idx; 8042 struct mlx5_flow_handle *dh; 8043 struct rte_flow_query_count count; 8044 uint32_t actions_num; 8045 const uint8_t *data; 8046 size_t size; 8047 uint32_t id; 8048 uint32_t type; 8049 8050 if (!flow) { 8051 return rte_flow_error_set(error, ENOENT, 8052 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8053 NULL, 8054 "invalid flow handle"); 8055 } 8056 handle_idx = flow->dev_handles; 8057 while (handle_idx) { 8058 dh = mlx5_ipool_get(priv->sh->ipool 8059 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8060 if (!dh) 8061 continue; 8062 handle_idx = dh->next.next; 8063 id = (uint32_t)(uintptr_t)dh->drv_flow; 8064 8065 /* query counter */ 8066 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8067 if (!mlx5_flow_query_counter(dev, flow, &count, error)) 8068 save_dump_file(NULL, 0, type, 8069 id, (void *)&count, file); 8070 8071 /* Get modify_hdr and encap_decap buf from ipools. */ 8072 encap_decap = NULL; 8073 modify_hdr = dh->dvh.modify_hdr; 8074 8075 if (dh->dvh.rix_encap_decap) { 8076 encap_decap = mlx5_ipool_get(priv->sh->ipool 8077 [MLX5_IPOOL_DECAP_ENCAP], 8078 dh->dvh.rix_encap_decap); 8079 } 8080 if (modify_hdr) { 8081 data = (const uint8_t *)modify_hdr->actions; 8082 size = (size_t)(modify_hdr->actions_num) * 8; 8083 actions_num = modify_hdr->actions_num; 8084 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8085 save_dump_file(data, size, type, id, 8086 (void *)(&actions_num), file); 8087 } 8088 if (encap_decap) { 8089 data = encap_decap->buf; 8090 size = encap_decap->size; 8091 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8092 save_dump_file(data, size, type, 8093 id, NULL, file); 8094 } 8095 } 8096 return 0; 8097 } 8098 #endif 8099 8100 /** 8101 * Dump flow raw hw data to file 8102 * 8103 * @param[in] dev 8104 * The pointer to Ethernet device. 8105 * @param[in] file 8106 * A pointer to a file for output. 8107 * @param[out] error 8108 * Perform verbose error reporting if not NULL. PMDs initialize this 8109 * structure in case of error only. 8110 * @return 8111 * 0 on success, a nagative value otherwise. 8112 */ 8113 int 8114 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8115 FILE *file, 8116 struct rte_flow_error *error __rte_unused) 8117 { 8118 struct mlx5_priv *priv = dev->data->dev_private; 8119 struct mlx5_dev_ctx_shared *sh = priv->sh; 8120 uint32_t handle_idx; 8121 int ret; 8122 struct mlx5_flow_handle *dh; 8123 struct rte_flow *flow; 8124 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8125 uint32_t idx; 8126 #endif 8127 8128 if (!priv->config.dv_flow_en) { 8129 if (fputs("device dv flow disabled\n", file) <= 0) 8130 return -errno; 8131 return -ENOTSUP; 8132 } 8133 8134 /* dump all */ 8135 if (!flow_idx) { 8136 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8137 MLX5_IPOOL_FOREACH(priv->flows[MLX5_FLOW_TYPE_GEN], idx, flow) 8138 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8139 #endif 8140 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8141 sh->rx_domain, 8142 sh->tx_domain, file); 8143 } 8144 /* dump one */ 8145 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8146 (uintptr_t)(void *)flow_idx); 8147 if (!flow) 8148 return -ENOENT; 8149 8150 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8151 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8152 #endif 8153 handle_idx = flow->dev_handles; 8154 while (handle_idx) { 8155 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8156 handle_idx); 8157 if (!dh) 8158 return -ENOENT; 8159 if (dh->drv_flow) { 8160 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8161 file); 8162 if (ret) 8163 return -ENOENT; 8164 } 8165 handle_idx = dh->next.next; 8166 } 8167 return 0; 8168 } 8169 8170 /** 8171 * Get aged-out flows. 8172 * 8173 * @param[in] dev 8174 * Pointer to the Ethernet device structure. 8175 * @param[in] context 8176 * The address of an array of pointers to the aged-out flows contexts. 8177 * @param[in] nb_countexts 8178 * The length of context array pointers. 8179 * @param[out] error 8180 * Perform verbose error reporting if not NULL. Initialized in case of 8181 * error only. 8182 * 8183 * @return 8184 * how many contexts get in success, otherwise negative errno value. 8185 * if nb_contexts is 0, return the amount of all aged contexts. 8186 * if nb_contexts is not 0 , return the amount of aged flows reported 8187 * in the context array. 8188 */ 8189 int 8190 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8191 uint32_t nb_contexts, struct rte_flow_error *error) 8192 { 8193 const struct mlx5_flow_driver_ops *fops; 8194 struct rte_flow_attr attr = { .transfer = 0 }; 8195 8196 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8197 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8198 return fops->get_aged_flows(dev, contexts, nb_contexts, 8199 error); 8200 } 8201 DRV_LOG(ERR, 8202 "port %u get aged flows is not supported.", 8203 dev->data->port_id); 8204 return -ENOTSUP; 8205 } 8206 8207 /* Wrapper for driver action_validate op callback */ 8208 static int 8209 flow_drv_action_validate(struct rte_eth_dev *dev, 8210 const struct rte_flow_indir_action_conf *conf, 8211 const struct rte_flow_action *action, 8212 const struct mlx5_flow_driver_ops *fops, 8213 struct rte_flow_error *error) 8214 { 8215 static const char err_msg[] = "indirect action validation unsupported"; 8216 8217 if (!fops->action_validate) { 8218 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8219 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8220 NULL, err_msg); 8221 return -rte_errno; 8222 } 8223 return fops->action_validate(dev, conf, action, error); 8224 } 8225 8226 /** 8227 * Destroys the shared action by handle. 8228 * 8229 * @param dev 8230 * Pointer to Ethernet device structure. 8231 * @param[in] handle 8232 * Handle for the indirect action object to be destroyed. 8233 * @param[out] error 8234 * Perform verbose error reporting if not NULL. PMDs initialize this 8235 * structure in case of error only. 8236 * 8237 * @return 8238 * 0 on success, a negative errno value otherwise and rte_errno is set. 8239 * 8240 * @note: wrapper for driver action_create op callback. 8241 */ 8242 static int 8243 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8244 struct rte_flow_action_handle *handle, 8245 struct rte_flow_error *error) 8246 { 8247 static const char err_msg[] = "indirect action destruction unsupported"; 8248 struct rte_flow_attr attr = { .transfer = 0 }; 8249 const struct mlx5_flow_driver_ops *fops = 8250 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8251 8252 if (!fops->action_destroy) { 8253 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8254 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8255 NULL, err_msg); 8256 return -rte_errno; 8257 } 8258 return fops->action_destroy(dev, handle, error); 8259 } 8260 8261 /* Wrapper for driver action_destroy op callback */ 8262 static int 8263 flow_drv_action_update(struct rte_eth_dev *dev, 8264 struct rte_flow_action_handle *handle, 8265 const void *update, 8266 const struct mlx5_flow_driver_ops *fops, 8267 struct rte_flow_error *error) 8268 { 8269 static const char err_msg[] = "indirect action update unsupported"; 8270 8271 if (!fops->action_update) { 8272 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8273 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8274 NULL, err_msg); 8275 return -rte_errno; 8276 } 8277 return fops->action_update(dev, handle, update, error); 8278 } 8279 8280 /* Wrapper for driver action_destroy op callback */ 8281 static int 8282 flow_drv_action_query(struct rte_eth_dev *dev, 8283 const struct rte_flow_action_handle *handle, 8284 void *data, 8285 const struct mlx5_flow_driver_ops *fops, 8286 struct rte_flow_error *error) 8287 { 8288 static const char err_msg[] = "indirect action query unsupported"; 8289 8290 if (!fops->action_query) { 8291 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8292 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8293 NULL, err_msg); 8294 return -rte_errno; 8295 } 8296 return fops->action_query(dev, handle, data, error); 8297 } 8298 8299 /** 8300 * Create indirect action for reuse in multiple flow rules. 8301 * 8302 * @param dev 8303 * Pointer to Ethernet device structure. 8304 * @param conf 8305 * Pointer to indirect action object configuration. 8306 * @param[in] action 8307 * Action configuration for indirect action object creation. 8308 * @param[out] error 8309 * Perform verbose error reporting if not NULL. PMDs initialize this 8310 * structure in case of error only. 8311 * @return 8312 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8313 */ 8314 static struct rte_flow_action_handle * 8315 mlx5_action_handle_create(struct rte_eth_dev *dev, 8316 const struct rte_flow_indir_action_conf *conf, 8317 const struct rte_flow_action *action, 8318 struct rte_flow_error *error) 8319 { 8320 static const char err_msg[] = "indirect action creation unsupported"; 8321 struct rte_flow_attr attr = { .transfer = 0 }; 8322 const struct mlx5_flow_driver_ops *fops = 8323 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8324 8325 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8326 return NULL; 8327 if (!fops->action_create) { 8328 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8329 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8330 NULL, err_msg); 8331 return NULL; 8332 } 8333 return fops->action_create(dev, conf, action, error); 8334 } 8335 8336 /** 8337 * Updates inplace the indirect action configuration pointed by *handle* 8338 * with the configuration provided as *update* argument. 8339 * The update of the indirect action configuration effects all flow rules 8340 * reusing the action via handle. 8341 * 8342 * @param dev 8343 * Pointer to Ethernet device structure. 8344 * @param[in] handle 8345 * Handle for the indirect action to be updated. 8346 * @param[in] update 8347 * Action specification used to modify the action pointed by handle. 8348 * *update* could be of same type with the action pointed by the *handle* 8349 * handle argument, or some other structures like a wrapper, depending on 8350 * the indirect action type. 8351 * @param[out] error 8352 * Perform verbose error reporting if not NULL. PMDs initialize this 8353 * structure in case of error only. 8354 * 8355 * @return 8356 * 0 on success, a negative errno value otherwise and rte_errno is set. 8357 */ 8358 static int 8359 mlx5_action_handle_update(struct rte_eth_dev *dev, 8360 struct rte_flow_action_handle *handle, 8361 const void *update, 8362 struct rte_flow_error *error) 8363 { 8364 struct rte_flow_attr attr = { .transfer = 0 }; 8365 const struct mlx5_flow_driver_ops *fops = 8366 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8367 int ret; 8368 8369 ret = flow_drv_action_validate(dev, NULL, 8370 (const struct rte_flow_action *)update, fops, error); 8371 if (ret) 8372 return ret; 8373 return flow_drv_action_update(dev, handle, update, fops, 8374 error); 8375 } 8376 8377 /** 8378 * Query the indirect action by handle. 8379 * 8380 * This function allows retrieving action-specific data such as counters. 8381 * Data is gathered by special action which may be present/referenced in 8382 * more than one flow rule definition. 8383 * 8384 * see @RTE_FLOW_ACTION_TYPE_COUNT 8385 * 8386 * @param dev 8387 * Pointer to Ethernet device structure. 8388 * @param[in] handle 8389 * Handle for the indirect action to query. 8390 * @param[in, out] data 8391 * Pointer to storage for the associated query data type. 8392 * @param[out] error 8393 * Perform verbose error reporting if not NULL. PMDs initialize this 8394 * structure in case of error only. 8395 * 8396 * @return 8397 * 0 on success, a negative errno value otherwise and rte_errno is set. 8398 */ 8399 static int 8400 mlx5_action_handle_query(struct rte_eth_dev *dev, 8401 const struct rte_flow_action_handle *handle, 8402 void *data, 8403 struct rte_flow_error *error) 8404 { 8405 struct rte_flow_attr attr = { .transfer = 0 }; 8406 const struct mlx5_flow_driver_ops *fops = 8407 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8408 8409 return flow_drv_action_query(dev, handle, data, fops, error); 8410 } 8411 8412 /** 8413 * Destroy all indirect actions (shared RSS). 8414 * 8415 * @param dev 8416 * Pointer to Ethernet device. 8417 * 8418 * @return 8419 * 0 on success, a negative errno value otherwise and rte_errno is set. 8420 */ 8421 int 8422 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8423 { 8424 struct rte_flow_error error; 8425 struct mlx5_priv *priv = dev->data->dev_private; 8426 struct mlx5_shared_action_rss *shared_rss; 8427 int ret = 0; 8428 uint32_t idx; 8429 8430 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8431 priv->rss_shared_actions, idx, shared_rss, next) { 8432 ret |= mlx5_action_handle_destroy(dev, 8433 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8434 } 8435 return ret; 8436 } 8437 8438 #ifndef HAVE_MLX5DV_DR 8439 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8440 #else 8441 #define MLX5_DOMAIN_SYNC_FLOW \ 8442 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8443 #endif 8444 8445 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8446 { 8447 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8448 const struct mlx5_flow_driver_ops *fops; 8449 int ret; 8450 struct rte_flow_attr attr = { .transfer = 0 }; 8451 8452 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8453 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8454 if (ret > 0) 8455 ret = -ret; 8456 return ret; 8457 } 8458 8459 const struct mlx5_flow_tunnel * 8460 mlx5_get_tof(const struct rte_flow_item *item, 8461 const struct rte_flow_action *action, 8462 enum mlx5_tof_rule_type *rule_type) 8463 { 8464 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 8465 if (item->type == (typeof(item->type)) 8466 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 8467 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 8468 return flow_items_to_tunnel(item); 8469 } 8470 } 8471 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 8472 if (action->type == (typeof(action->type)) 8473 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 8474 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 8475 return flow_actions_to_tunnel(action); 8476 } 8477 } 8478 return NULL; 8479 } 8480 8481 /** 8482 * tunnel offload functionalilty is defined for DV environment only 8483 */ 8484 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8485 __extension__ 8486 union tunnel_offload_mark { 8487 uint32_t val; 8488 struct { 8489 uint32_t app_reserve:8; 8490 uint32_t table_id:15; 8491 uint32_t transfer:1; 8492 uint32_t _unused_:8; 8493 }; 8494 }; 8495 8496 static bool 8497 mlx5_access_tunnel_offload_db 8498 (struct rte_eth_dev *dev, 8499 bool (*match)(struct rte_eth_dev *, 8500 struct mlx5_flow_tunnel *, const void *), 8501 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8502 void (*miss)(struct rte_eth_dev *, void *), 8503 void *ctx, bool lock_op); 8504 8505 static int 8506 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 8507 struct rte_flow *flow, 8508 const struct rte_flow_attr *attr, 8509 const struct rte_flow_action *app_actions, 8510 uint32_t flow_idx, 8511 const struct mlx5_flow_tunnel *tunnel, 8512 struct tunnel_default_miss_ctx *ctx, 8513 struct rte_flow_error *error) 8514 { 8515 struct mlx5_priv *priv = dev->data->dev_private; 8516 struct mlx5_flow *dev_flow; 8517 struct rte_flow_attr miss_attr = *attr; 8518 const struct rte_flow_item miss_items[2] = { 8519 { 8520 .type = RTE_FLOW_ITEM_TYPE_ETH, 8521 .spec = NULL, 8522 .last = NULL, 8523 .mask = NULL 8524 }, 8525 { 8526 .type = RTE_FLOW_ITEM_TYPE_END, 8527 .spec = NULL, 8528 .last = NULL, 8529 .mask = NULL 8530 } 8531 }; 8532 union tunnel_offload_mark mark_id; 8533 struct rte_flow_action_mark miss_mark; 8534 struct rte_flow_action miss_actions[3] = { 8535 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 8536 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 8537 }; 8538 const struct rte_flow_action_jump *jump_data; 8539 uint32_t i, flow_table = 0; /* prevent compilation warning */ 8540 struct flow_grp_info grp_info = { 8541 .external = 1, 8542 .transfer = attr->transfer, 8543 .fdb_def_rule = !!priv->fdb_def_rule, 8544 .std_tbl_fix = 0, 8545 }; 8546 int ret; 8547 8548 if (!attr->transfer) { 8549 uint32_t q_size; 8550 8551 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 8552 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 8553 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 8554 0, SOCKET_ID_ANY); 8555 if (!ctx->queue) 8556 return rte_flow_error_set 8557 (error, ENOMEM, 8558 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8559 NULL, "invalid default miss RSS"); 8560 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8561 ctx->action_rss.level = 0, 8562 ctx->action_rss.types = priv->rss_conf.rss_hf, 8563 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 8564 ctx->action_rss.queue_num = priv->reta_idx_n, 8565 ctx->action_rss.key = priv->rss_conf.rss_key, 8566 ctx->action_rss.queue = ctx->queue; 8567 if (!priv->reta_idx_n || !priv->rxqs_n) 8568 return rte_flow_error_set 8569 (error, EINVAL, 8570 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8571 NULL, "invalid port configuration"); 8572 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 8573 ctx->action_rss.types = 0; 8574 for (i = 0; i != priv->reta_idx_n; ++i) 8575 ctx->queue[i] = (*priv->reta_idx)[i]; 8576 } else { 8577 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 8578 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 8579 } 8580 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 8581 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 8582 jump_data = app_actions->conf; 8583 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 8584 miss_attr.group = jump_data->group; 8585 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 8586 &flow_table, &grp_info, error); 8587 if (ret) 8588 return rte_flow_error_set(error, EINVAL, 8589 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8590 NULL, "invalid tunnel id"); 8591 mark_id.app_reserve = 0; 8592 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 8593 mark_id.transfer = !!attr->transfer; 8594 mark_id._unused_ = 0; 8595 miss_mark.id = mark_id.val; 8596 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 8597 miss_items, miss_actions, flow_idx, error); 8598 if (!dev_flow) 8599 return -rte_errno; 8600 dev_flow->flow = flow; 8601 dev_flow->external = true; 8602 dev_flow->tunnel = tunnel; 8603 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 8604 /* Subflow object was created, we must include one in the list. */ 8605 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 8606 dev_flow->handle, next); 8607 DRV_LOG(DEBUG, 8608 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 8609 dev->data->port_id, tunnel->app_tunnel.type, 8610 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 8611 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 8612 miss_actions, error); 8613 if (!ret) 8614 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 8615 error); 8616 8617 return ret; 8618 } 8619 8620 static const struct mlx5_flow_tbl_data_entry * 8621 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 8622 { 8623 struct mlx5_priv *priv = dev->data->dev_private; 8624 struct mlx5_dev_ctx_shared *sh = priv->sh; 8625 struct mlx5_hlist_entry *he; 8626 union tunnel_offload_mark mbits = { .val = mark }; 8627 union mlx5_flow_tbl_key table_key = { 8628 { 8629 .level = tunnel_id_to_flow_tbl(mbits.table_id), 8630 .id = 0, 8631 .reserved = 0, 8632 .dummy = 0, 8633 .is_fdb = !!mbits.transfer, 8634 .is_egress = 0, 8635 } 8636 }; 8637 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 8638 return he ? 8639 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 8640 } 8641 8642 static void 8643 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 8644 struct mlx5_hlist_entry *entry) 8645 { 8646 struct mlx5_dev_ctx_shared *sh = list->ctx; 8647 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8648 8649 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8650 tunnel_flow_tbl_to_id(tte->flow_table)); 8651 mlx5_free(tte); 8652 } 8653 8654 static int 8655 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused, 8656 struct mlx5_hlist_entry *entry, 8657 uint64_t key, void *cb_ctx __rte_unused) 8658 { 8659 union tunnel_tbl_key tbl = { 8660 .val = key, 8661 }; 8662 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8663 8664 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 8665 } 8666 8667 static struct mlx5_hlist_entry * 8668 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key, 8669 void *ctx __rte_unused) 8670 { 8671 struct mlx5_dev_ctx_shared *sh = list->ctx; 8672 struct tunnel_tbl_entry *tte; 8673 union tunnel_tbl_key tbl = { 8674 .val = key, 8675 }; 8676 8677 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 8678 sizeof(*tte), 0, 8679 SOCKET_ID_ANY); 8680 if (!tte) 8681 goto err; 8682 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8683 &tte->flow_table); 8684 if (tte->flow_table >= MLX5_MAX_TABLES) { 8685 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 8686 tte->flow_table); 8687 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8688 tte->flow_table); 8689 goto err; 8690 } else if (!tte->flow_table) { 8691 goto err; 8692 } 8693 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 8694 tte->tunnel_id = tbl.tunnel_id; 8695 tte->group = tbl.group; 8696 return &tte->hash; 8697 err: 8698 if (tte) 8699 mlx5_free(tte); 8700 return NULL; 8701 } 8702 8703 static uint32_t 8704 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 8705 const struct mlx5_flow_tunnel *tunnel, 8706 uint32_t group, uint32_t *table, 8707 struct rte_flow_error *error) 8708 { 8709 struct mlx5_hlist_entry *he; 8710 struct tunnel_tbl_entry *tte; 8711 union tunnel_tbl_key key = { 8712 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 8713 .group = group 8714 }; 8715 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8716 struct mlx5_hlist *group_hash; 8717 8718 group_hash = tunnel ? tunnel->groups : thub->groups; 8719 he = mlx5_hlist_register(group_hash, key.val, NULL); 8720 if (!he) 8721 return rte_flow_error_set(error, EINVAL, 8722 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8723 NULL, 8724 "tunnel group index not supported"); 8725 tte = container_of(he, typeof(*tte), hash); 8726 *table = tte->flow_table; 8727 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 8728 dev->data->port_id, key.tunnel_id, group, *table); 8729 return 0; 8730 } 8731 8732 static void 8733 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8734 struct mlx5_flow_tunnel *tunnel) 8735 { 8736 struct mlx5_priv *priv = dev->data->dev_private; 8737 struct mlx5_indexed_pool *ipool; 8738 8739 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8740 dev->data->port_id, tunnel->tunnel_id); 8741 LIST_REMOVE(tunnel, chain); 8742 mlx5_hlist_destroy(tunnel->groups); 8743 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8744 mlx5_ipool_free(ipool, tunnel->tunnel_id); 8745 } 8746 8747 static bool 8748 mlx5_access_tunnel_offload_db 8749 (struct rte_eth_dev *dev, 8750 bool (*match)(struct rte_eth_dev *, 8751 struct mlx5_flow_tunnel *, const void *), 8752 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8753 void (*miss)(struct rte_eth_dev *, void *), 8754 void *ctx, bool lock_op) 8755 { 8756 bool verdict = false; 8757 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8758 struct mlx5_flow_tunnel *tunnel; 8759 8760 rte_spinlock_lock(&thub->sl); 8761 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 8762 verdict = match(dev, tunnel, (const void *)ctx); 8763 if (verdict) 8764 break; 8765 } 8766 if (!lock_op) 8767 rte_spinlock_unlock(&thub->sl); 8768 if (verdict && hit) 8769 hit(dev, tunnel, ctx); 8770 if (!verdict && miss) 8771 miss(dev, ctx); 8772 if (lock_op) 8773 rte_spinlock_unlock(&thub->sl); 8774 8775 return verdict; 8776 } 8777 8778 struct tunnel_db_find_tunnel_id_ctx { 8779 uint32_t tunnel_id; 8780 struct mlx5_flow_tunnel *tunnel; 8781 }; 8782 8783 static bool 8784 find_tunnel_id_match(struct rte_eth_dev *dev, 8785 struct mlx5_flow_tunnel *tunnel, const void *x) 8786 { 8787 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8788 8789 RTE_SET_USED(dev); 8790 return tunnel->tunnel_id == ctx->tunnel_id; 8791 } 8792 8793 static void 8794 find_tunnel_id_hit(struct rte_eth_dev *dev, 8795 struct mlx5_flow_tunnel *tunnel, void *x) 8796 { 8797 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8798 RTE_SET_USED(dev); 8799 ctx->tunnel = tunnel; 8800 } 8801 8802 static struct mlx5_flow_tunnel * 8803 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 8804 { 8805 struct tunnel_db_find_tunnel_id_ctx ctx = { 8806 .tunnel_id = id, 8807 }; 8808 8809 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 8810 find_tunnel_id_hit, NULL, &ctx, true); 8811 8812 return ctx.tunnel; 8813 } 8814 8815 static struct mlx5_flow_tunnel * 8816 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 8817 const struct rte_flow_tunnel *app_tunnel) 8818 { 8819 struct mlx5_priv *priv = dev->data->dev_private; 8820 struct mlx5_indexed_pool *ipool; 8821 struct mlx5_flow_tunnel *tunnel; 8822 uint32_t id; 8823 8824 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8825 tunnel = mlx5_ipool_zmalloc(ipool, &id); 8826 if (!tunnel) 8827 return NULL; 8828 if (id >= MLX5_MAX_TUNNELS) { 8829 mlx5_ipool_free(ipool, id); 8830 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 8831 return NULL; 8832 } 8833 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 8834 mlx5_flow_tunnel_grp2tbl_create_cb, 8835 mlx5_flow_tunnel_grp2tbl_match_cb, 8836 mlx5_flow_tunnel_grp2tbl_remove_cb); 8837 if (!tunnel->groups) { 8838 mlx5_ipool_free(ipool, id); 8839 return NULL; 8840 } 8841 tunnel->groups->ctx = priv->sh; 8842 /* initiate new PMD tunnel */ 8843 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8844 tunnel->tunnel_id = id; 8845 tunnel->action.type = (typeof(tunnel->action.type)) 8846 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8847 tunnel->action.conf = tunnel; 8848 tunnel->item.type = (typeof(tunnel->item.type)) 8849 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8850 tunnel->item.spec = tunnel; 8851 tunnel->item.last = NULL; 8852 tunnel->item.mask = NULL; 8853 8854 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8855 dev->data->port_id, tunnel->tunnel_id); 8856 8857 return tunnel; 8858 } 8859 8860 struct tunnel_db_get_tunnel_ctx { 8861 const struct rte_flow_tunnel *app_tunnel; 8862 struct mlx5_flow_tunnel *tunnel; 8863 }; 8864 8865 static bool get_tunnel_match(struct rte_eth_dev *dev, 8866 struct mlx5_flow_tunnel *tunnel, const void *x) 8867 { 8868 const struct tunnel_db_get_tunnel_ctx *ctx = x; 8869 8870 RTE_SET_USED(dev); 8871 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 8872 sizeof(*ctx->app_tunnel)); 8873 } 8874 8875 static void get_tunnel_hit(struct rte_eth_dev *dev, 8876 struct mlx5_flow_tunnel *tunnel, void *x) 8877 { 8878 /* called under tunnel spinlock protection */ 8879 struct tunnel_db_get_tunnel_ctx *ctx = x; 8880 8881 RTE_SET_USED(dev); 8882 tunnel->refctn++; 8883 ctx->tunnel = tunnel; 8884 } 8885 8886 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 8887 { 8888 /* called under tunnel spinlock protection */ 8889 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8890 struct tunnel_db_get_tunnel_ctx *ctx = x; 8891 8892 rte_spinlock_unlock(&thub->sl); 8893 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 8894 rte_spinlock_lock(&thub->sl); 8895 if (ctx->tunnel) { 8896 ctx->tunnel->refctn = 1; 8897 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 8898 } 8899 } 8900 8901 8902 static int 8903 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 8904 const struct rte_flow_tunnel *app_tunnel, 8905 struct mlx5_flow_tunnel **tunnel) 8906 { 8907 struct tunnel_db_get_tunnel_ctx ctx = { 8908 .app_tunnel = app_tunnel, 8909 }; 8910 8911 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 8912 get_tunnel_miss, &ctx, true); 8913 *tunnel = ctx.tunnel; 8914 return ctx.tunnel ? 0 : -ENOMEM; 8915 } 8916 8917 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 8918 { 8919 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 8920 8921 if (!thub) 8922 return; 8923 if (!LIST_EMPTY(&thub->tunnels)) 8924 DRV_LOG(WARNING, "port %u tunnels present", port_id); 8925 mlx5_hlist_destroy(thub->groups); 8926 mlx5_free(thub); 8927 } 8928 8929 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 8930 { 8931 int err; 8932 struct mlx5_flow_tunnel_hub *thub; 8933 8934 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 8935 0, SOCKET_ID_ANY); 8936 if (!thub) 8937 return -ENOMEM; 8938 LIST_INIT(&thub->tunnels); 8939 rte_spinlock_init(&thub->sl); 8940 thub->groups = mlx5_hlist_create("flow groups", 8941 rte_align32pow2(MLX5_MAX_TABLES), 0, 8942 0, mlx5_flow_tunnel_grp2tbl_create_cb, 8943 mlx5_flow_tunnel_grp2tbl_match_cb, 8944 mlx5_flow_tunnel_grp2tbl_remove_cb); 8945 if (!thub->groups) { 8946 err = -rte_errno; 8947 goto err; 8948 } 8949 thub->groups->ctx = sh; 8950 sh->tunnel_hub = thub; 8951 8952 return 0; 8953 8954 err: 8955 if (thub->groups) 8956 mlx5_hlist_destroy(thub->groups); 8957 if (thub) 8958 mlx5_free(thub); 8959 return err; 8960 } 8961 8962 static inline bool 8963 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 8964 struct rte_flow_tunnel *tunnel, 8965 const char *err_msg) 8966 { 8967 err_msg = NULL; 8968 if (!is_tunnel_offload_active(dev)) { 8969 err_msg = "tunnel offload was not activated"; 8970 goto out; 8971 } else if (!tunnel) { 8972 err_msg = "no application tunnel"; 8973 goto out; 8974 } 8975 8976 switch (tunnel->type) { 8977 default: 8978 err_msg = "unsupported tunnel type"; 8979 goto out; 8980 case RTE_FLOW_ITEM_TYPE_VXLAN: 8981 break; 8982 } 8983 8984 out: 8985 return !err_msg; 8986 } 8987 8988 static int 8989 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 8990 struct rte_flow_tunnel *app_tunnel, 8991 struct rte_flow_action **actions, 8992 uint32_t *num_of_actions, 8993 struct rte_flow_error *error) 8994 { 8995 int ret; 8996 struct mlx5_flow_tunnel *tunnel; 8997 const char *err_msg = NULL; 8998 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8999 9000 if (!verdict) 9001 return rte_flow_error_set(error, EINVAL, 9002 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9003 err_msg); 9004 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9005 if (ret < 0) { 9006 return rte_flow_error_set(error, ret, 9007 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9008 "failed to initialize pmd tunnel"); 9009 } 9010 *actions = &tunnel->action; 9011 *num_of_actions = 1; 9012 return 0; 9013 } 9014 9015 static int 9016 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9017 struct rte_flow_tunnel *app_tunnel, 9018 struct rte_flow_item **items, 9019 uint32_t *num_of_items, 9020 struct rte_flow_error *error) 9021 { 9022 int ret; 9023 struct mlx5_flow_tunnel *tunnel; 9024 const char *err_msg = NULL; 9025 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 9026 9027 if (!verdict) 9028 return rte_flow_error_set(error, EINVAL, 9029 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9030 err_msg); 9031 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9032 if (ret < 0) { 9033 return rte_flow_error_set(error, ret, 9034 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9035 "failed to initialize pmd tunnel"); 9036 } 9037 *items = &tunnel->item; 9038 *num_of_items = 1; 9039 return 0; 9040 } 9041 9042 struct tunnel_db_element_release_ctx { 9043 struct rte_flow_item *items; 9044 struct rte_flow_action *actions; 9045 uint32_t num_elements; 9046 struct rte_flow_error *error; 9047 int ret; 9048 }; 9049 9050 static bool 9051 tunnel_element_release_match(struct rte_eth_dev *dev, 9052 struct mlx5_flow_tunnel *tunnel, const void *x) 9053 { 9054 const struct tunnel_db_element_release_ctx *ctx = x; 9055 9056 RTE_SET_USED(dev); 9057 if (ctx->num_elements != 1) 9058 return false; 9059 else if (ctx->items) 9060 return ctx->items == &tunnel->item; 9061 else if (ctx->actions) 9062 return ctx->actions == &tunnel->action; 9063 9064 return false; 9065 } 9066 9067 static void 9068 tunnel_element_release_hit(struct rte_eth_dev *dev, 9069 struct mlx5_flow_tunnel *tunnel, void *x) 9070 { 9071 struct tunnel_db_element_release_ctx *ctx = x; 9072 ctx->ret = 0; 9073 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9074 mlx5_flow_tunnel_free(dev, tunnel); 9075 } 9076 9077 static void 9078 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9079 { 9080 struct tunnel_db_element_release_ctx *ctx = x; 9081 RTE_SET_USED(dev); 9082 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9083 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9084 "invalid argument"); 9085 } 9086 9087 static int 9088 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9089 struct rte_flow_item *pmd_items, 9090 uint32_t num_items, struct rte_flow_error *err) 9091 { 9092 struct tunnel_db_element_release_ctx ctx = { 9093 .items = pmd_items, 9094 .actions = NULL, 9095 .num_elements = num_items, 9096 .error = err, 9097 }; 9098 9099 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9100 tunnel_element_release_hit, 9101 tunnel_element_release_miss, &ctx, false); 9102 9103 return ctx.ret; 9104 } 9105 9106 static int 9107 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9108 struct rte_flow_action *pmd_actions, 9109 uint32_t num_actions, struct rte_flow_error *err) 9110 { 9111 struct tunnel_db_element_release_ctx ctx = { 9112 .items = NULL, 9113 .actions = pmd_actions, 9114 .num_elements = num_actions, 9115 .error = err, 9116 }; 9117 9118 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9119 tunnel_element_release_hit, 9120 tunnel_element_release_miss, &ctx, false); 9121 9122 return ctx.ret; 9123 } 9124 9125 static int 9126 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9127 struct rte_mbuf *m, 9128 struct rte_flow_restore_info *info, 9129 struct rte_flow_error *err) 9130 { 9131 uint64_t ol_flags = m->ol_flags; 9132 const struct mlx5_flow_tbl_data_entry *tble; 9133 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 9134 9135 if (!is_tunnel_offload_active(dev)) { 9136 info->flags = 0; 9137 return 0; 9138 } 9139 9140 if ((ol_flags & mask) != mask) 9141 goto err; 9142 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9143 if (!tble) { 9144 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9145 dev->data->port_id, m->hash.fdir.hi); 9146 goto err; 9147 } 9148 MLX5_ASSERT(tble->tunnel); 9149 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9150 info->group_id = tble->group_id; 9151 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9152 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9153 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9154 9155 return 0; 9156 9157 err: 9158 return rte_flow_error_set(err, EINVAL, 9159 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9160 "failed to get restore info"); 9161 } 9162 9163 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9164 static int 9165 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9166 __rte_unused struct rte_flow_tunnel *app_tunnel, 9167 __rte_unused struct rte_flow_action **actions, 9168 __rte_unused uint32_t *num_of_actions, 9169 __rte_unused struct rte_flow_error *error) 9170 { 9171 return -ENOTSUP; 9172 } 9173 9174 static int 9175 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9176 __rte_unused struct rte_flow_tunnel *app_tunnel, 9177 __rte_unused struct rte_flow_item **items, 9178 __rte_unused uint32_t *num_of_items, 9179 __rte_unused struct rte_flow_error *error) 9180 { 9181 return -ENOTSUP; 9182 } 9183 9184 static int 9185 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9186 __rte_unused struct rte_flow_item *pmd_items, 9187 __rte_unused uint32_t num_items, 9188 __rte_unused struct rte_flow_error *err) 9189 { 9190 return -ENOTSUP; 9191 } 9192 9193 static int 9194 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9195 __rte_unused struct rte_flow_action *pmd_action, 9196 __rte_unused uint32_t num_actions, 9197 __rte_unused struct rte_flow_error *err) 9198 { 9199 return -ENOTSUP; 9200 } 9201 9202 static int 9203 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9204 __rte_unused struct rte_mbuf *m, 9205 __rte_unused struct rte_flow_restore_info *i, 9206 __rte_unused struct rte_flow_error *err) 9207 { 9208 return -ENOTSUP; 9209 } 9210 9211 static int 9212 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9213 __rte_unused struct rte_flow *flow, 9214 __rte_unused const struct rte_flow_attr *attr, 9215 __rte_unused const struct rte_flow_action *actions, 9216 __rte_unused uint32_t flow_idx, 9217 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9218 __rte_unused struct tunnel_default_miss_ctx *ctx, 9219 __rte_unused struct rte_flow_error *error) 9220 { 9221 return -ENOTSUP; 9222 } 9223 9224 static struct mlx5_flow_tunnel * 9225 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9226 __rte_unused uint32_t id) 9227 { 9228 return NULL; 9229 } 9230 9231 static void 9232 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9233 __rte_unused struct mlx5_flow_tunnel *tunnel) 9234 { 9235 } 9236 9237 static uint32_t 9238 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9239 __rte_unused const struct mlx5_flow_tunnel *t, 9240 __rte_unused uint32_t group, 9241 __rte_unused uint32_t *table, 9242 struct rte_flow_error *error) 9243 { 9244 return rte_flow_error_set(error, ENOTSUP, 9245 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9246 "tunnel offload requires DV support"); 9247 } 9248 9249 void 9250 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9251 __rte_unused uint16_t port_id) 9252 { 9253 } 9254 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9255 9256 static void 9257 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9258 { 9259 int ret; 9260 struct rte_flow_error error; 9261 9262 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9263 char *item_name; 9264 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9265 sizeof(item_name), 9266 (void *)(uintptr_t)item->type, &error); 9267 if (ret > 0) 9268 printf("%s ", item_name); 9269 else 9270 printf("%d\n", (int)item->type); 9271 } 9272 printf("END\n"); 9273 } 9274