1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see ETH_RSS_* definitions). 102 */ 103 uint8_t optional; 104 /**< optional expand field. Default 0 to expand, 1 not go deeper. */ 105 }; 106 107 /** Object returned by mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_rss { 109 uint32_t entries; 110 /**< Number of entries @p patterns and @p priorities. */ 111 struct { 112 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 113 uint32_t priority; /**< Priority offset for each expansion. */ 114 } entry[]; 115 }; 116 117 static void 118 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 119 120 static bool 121 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 122 { 123 switch (item->type) { 124 case RTE_FLOW_ITEM_TYPE_ETH: 125 case RTE_FLOW_ITEM_TYPE_VLAN: 126 case RTE_FLOW_ITEM_TYPE_IPV4: 127 case RTE_FLOW_ITEM_TYPE_IPV6: 128 case RTE_FLOW_ITEM_TYPE_UDP: 129 case RTE_FLOW_ITEM_TYPE_TCP: 130 case RTE_FLOW_ITEM_TYPE_VXLAN: 131 case RTE_FLOW_ITEM_TYPE_NVGRE: 132 case RTE_FLOW_ITEM_TYPE_GRE: 133 case RTE_FLOW_ITEM_TYPE_GENEVE: 134 return true; 135 default: 136 break; 137 } 138 return false; 139 } 140 141 static enum rte_flow_item_type 142 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 143 { 144 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 145 uint16_t ether_type = 0; 146 uint16_t ether_type_m; 147 uint8_t ip_next_proto = 0; 148 uint8_t ip_next_proto_m; 149 150 if (item == NULL || item->spec == NULL) 151 return ret; 152 switch (item->type) { 153 case RTE_FLOW_ITEM_TYPE_ETH: 154 if (item->mask) 155 ether_type_m = ((const struct rte_flow_item_eth *) 156 (item->mask))->type; 157 else 158 ether_type_m = rte_flow_item_eth_mask.type; 159 if (ether_type_m != RTE_BE16(0xFFFF)) 160 break; 161 ether_type = ((const struct rte_flow_item_eth *) 162 (item->spec))->type; 163 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 164 ret = RTE_FLOW_ITEM_TYPE_IPV4; 165 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 166 ret = RTE_FLOW_ITEM_TYPE_IPV6; 167 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 168 ret = RTE_FLOW_ITEM_TYPE_VLAN; 169 else 170 ret = RTE_FLOW_ITEM_TYPE_END; 171 break; 172 case RTE_FLOW_ITEM_TYPE_VLAN: 173 if (item->mask) 174 ether_type_m = ((const struct rte_flow_item_vlan *) 175 (item->mask))->inner_type; 176 else 177 ether_type_m = rte_flow_item_vlan_mask.inner_type; 178 if (ether_type_m != RTE_BE16(0xFFFF)) 179 break; 180 ether_type = ((const struct rte_flow_item_vlan *) 181 (item->spec))->inner_type; 182 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 183 ret = RTE_FLOW_ITEM_TYPE_IPV4; 184 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 185 ret = RTE_FLOW_ITEM_TYPE_IPV6; 186 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 187 ret = RTE_FLOW_ITEM_TYPE_VLAN; 188 else 189 ret = RTE_FLOW_ITEM_TYPE_END; 190 break; 191 case RTE_FLOW_ITEM_TYPE_IPV4: 192 if (item->mask) 193 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 194 (item->mask))->hdr.next_proto_id; 195 else 196 ip_next_proto_m = 197 rte_flow_item_ipv4_mask.hdr.next_proto_id; 198 if (ip_next_proto_m != 0xFF) 199 break; 200 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 201 (item->spec))->hdr.next_proto_id; 202 if (ip_next_proto == IPPROTO_UDP) 203 ret = RTE_FLOW_ITEM_TYPE_UDP; 204 else if (ip_next_proto == IPPROTO_TCP) 205 ret = RTE_FLOW_ITEM_TYPE_TCP; 206 else if (ip_next_proto == IPPROTO_IP) 207 ret = RTE_FLOW_ITEM_TYPE_IPV4; 208 else if (ip_next_proto == IPPROTO_IPV6) 209 ret = RTE_FLOW_ITEM_TYPE_IPV6; 210 else 211 ret = RTE_FLOW_ITEM_TYPE_END; 212 break; 213 case RTE_FLOW_ITEM_TYPE_IPV6: 214 if (item->mask) 215 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 216 (item->mask))->hdr.proto; 217 else 218 ip_next_proto_m = 219 rte_flow_item_ipv6_mask.hdr.proto; 220 if (ip_next_proto_m != 0xFF) 221 break; 222 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 223 (item->spec))->hdr.proto; 224 if (ip_next_proto == IPPROTO_UDP) 225 ret = RTE_FLOW_ITEM_TYPE_UDP; 226 else if (ip_next_proto == IPPROTO_TCP) 227 ret = RTE_FLOW_ITEM_TYPE_TCP; 228 else if (ip_next_proto == IPPROTO_IP) 229 ret = RTE_FLOW_ITEM_TYPE_IPV4; 230 else if (ip_next_proto == IPPROTO_IPV6) 231 ret = RTE_FLOW_ITEM_TYPE_IPV6; 232 else 233 ret = RTE_FLOW_ITEM_TYPE_END; 234 break; 235 default: 236 ret = RTE_FLOW_ITEM_TYPE_VOID; 237 break; 238 } 239 return ret; 240 } 241 242 #define MLX5_RSS_EXP_ELT_N 16 243 244 /** 245 * Expand RSS flows into several possible flows according to the RSS hash 246 * fields requested and the driver capabilities. 247 * 248 * @param[out] buf 249 * Buffer to store the result expansion. 250 * @param[in] size 251 * Buffer size in bytes. If 0, @p buf can be NULL. 252 * @param[in] pattern 253 * User flow pattern. 254 * @param[in] types 255 * RSS types to expand (see ETH_RSS_* definitions). 256 * @param[in] graph 257 * Input graph to expand @p pattern according to @p types. 258 * @param[in] graph_root_index 259 * Index of root node in @p graph, typically 0. 260 * 261 * @return 262 * A positive value representing the size of @p buf in bytes regardless of 263 * @p size on success, a negative errno value otherwise and rte_errno is 264 * set, the following errors are defined: 265 * 266 * -E2BIG: graph-depth @p graph is too deep. 267 * -EINVAL: @p size has not enough space for expanded pattern. 268 */ 269 static int 270 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 271 const struct rte_flow_item *pattern, uint64_t types, 272 const struct mlx5_flow_expand_node graph[], 273 int graph_root_index) 274 { 275 const struct rte_flow_item *item; 276 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 277 const int *next_node; 278 const int *stack[MLX5_RSS_EXP_ELT_N]; 279 int stack_pos = 0; 280 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 281 unsigned int i; 282 size_t lsize; 283 size_t user_pattern_size = 0; 284 void *addr = NULL; 285 const struct mlx5_flow_expand_node *next = NULL; 286 struct rte_flow_item missed_item; 287 int missed = 0; 288 int elt = 0; 289 const struct rte_flow_item *last_item = NULL; 290 291 memset(&missed_item, 0, sizeof(missed_item)); 292 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 293 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 294 if (lsize > size) 295 return -EINVAL; 296 buf->entry[0].priority = 0; 297 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 298 buf->entries = 0; 299 addr = buf->entry[0].pattern; 300 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 301 if (!mlx5_flow_is_rss_expandable_item(item)) { 302 user_pattern_size += sizeof(*item); 303 continue; 304 } 305 last_item = item; 306 for (i = 0; node->next && node->next[i]; ++i) { 307 next = &graph[node->next[i]]; 308 if (next->type == item->type) 309 break; 310 } 311 if (next) 312 node = next; 313 user_pattern_size += sizeof(*item); 314 } 315 user_pattern_size += sizeof(*item); /* Handle END item. */ 316 lsize += user_pattern_size; 317 if (lsize > size) 318 return -EINVAL; 319 /* Copy the user pattern in the first entry of the buffer. */ 320 rte_memcpy(addr, pattern, user_pattern_size); 321 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 322 buf->entries = 1; 323 /* Start expanding. */ 324 memset(flow_items, 0, sizeof(flow_items)); 325 user_pattern_size -= sizeof(*item); 326 /* 327 * Check if the last valid item has spec set, need complete pattern, 328 * and the pattern can be used for expansion. 329 */ 330 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 331 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 332 /* Item type END indicates expansion is not required. */ 333 return lsize; 334 } 335 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 336 next = NULL; 337 missed = 1; 338 for (i = 0; node->next && node->next[i]; ++i) { 339 next = &graph[node->next[i]]; 340 if (next->type == missed_item.type) { 341 flow_items[0].type = missed_item.type; 342 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 343 break; 344 } 345 next = NULL; 346 } 347 } 348 if (next && missed) { 349 elt = 2; /* missed item + item end. */ 350 node = next; 351 lsize += elt * sizeof(*item) + user_pattern_size; 352 if (lsize > size) 353 return -EINVAL; 354 if (node->rss_types & types) { 355 buf->entry[buf->entries].priority = 1; 356 buf->entry[buf->entries].pattern = addr; 357 buf->entries++; 358 rte_memcpy(addr, buf->entry[0].pattern, 359 user_pattern_size); 360 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 361 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 362 addr = (void *)(((uintptr_t)addr) + 363 elt * sizeof(*item)); 364 } 365 } 366 memset(flow_items, 0, sizeof(flow_items)); 367 next_node = node->next; 368 stack[stack_pos] = next_node; 369 node = next_node ? &graph[*next_node] : NULL; 370 while (node) { 371 flow_items[stack_pos].type = node->type; 372 if (node->rss_types & types) { 373 size_t n; 374 /* 375 * compute the number of items to copy from the 376 * expansion and copy it. 377 * When the stack_pos is 0, there are 1 element in it, 378 * plus the addition END item. 379 */ 380 elt = stack_pos + 2; 381 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 382 lsize += elt * sizeof(*item) + user_pattern_size; 383 if (lsize > size) 384 return -EINVAL; 385 n = elt * sizeof(*item); 386 buf->entry[buf->entries].priority = 387 stack_pos + 1 + missed; 388 buf->entry[buf->entries].pattern = addr; 389 buf->entries++; 390 rte_memcpy(addr, buf->entry[0].pattern, 391 user_pattern_size); 392 addr = (void *)(((uintptr_t)addr) + 393 user_pattern_size); 394 rte_memcpy(addr, &missed_item, 395 missed * sizeof(*item)); 396 addr = (void *)(((uintptr_t)addr) + 397 missed * sizeof(*item)); 398 rte_memcpy(addr, flow_items, n); 399 addr = (void *)(((uintptr_t)addr) + n); 400 } 401 /* Go deeper. */ 402 if (!node->optional && node->next) { 403 next_node = node->next; 404 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 405 rte_errno = E2BIG; 406 return -rte_errno; 407 } 408 stack[stack_pos] = next_node; 409 } else if (*(next_node + 1)) { 410 /* Follow up with the next possibility. */ 411 ++next_node; 412 } else { 413 /* Move to the next path. */ 414 if (stack_pos) 415 next_node = stack[--stack_pos]; 416 next_node++; 417 stack[stack_pos] = next_node; 418 } 419 node = *next_node ? &graph[*next_node] : NULL; 420 }; 421 return lsize; 422 } 423 424 enum mlx5_expansion { 425 MLX5_EXPANSION_ROOT, 426 MLX5_EXPANSION_ROOT_OUTER, 427 MLX5_EXPANSION_ROOT_ETH_VLAN, 428 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 429 MLX5_EXPANSION_OUTER_ETH, 430 MLX5_EXPANSION_OUTER_ETH_VLAN, 431 MLX5_EXPANSION_OUTER_VLAN, 432 MLX5_EXPANSION_OUTER_IPV4, 433 MLX5_EXPANSION_OUTER_IPV4_UDP, 434 MLX5_EXPANSION_OUTER_IPV4_TCP, 435 MLX5_EXPANSION_OUTER_IPV6, 436 MLX5_EXPANSION_OUTER_IPV6_UDP, 437 MLX5_EXPANSION_OUTER_IPV6_TCP, 438 MLX5_EXPANSION_VXLAN, 439 MLX5_EXPANSION_VXLAN_GPE, 440 MLX5_EXPANSION_GRE, 441 MLX5_EXPANSION_NVGRE, 442 MLX5_EXPANSION_GRE_KEY, 443 MLX5_EXPANSION_MPLS, 444 MLX5_EXPANSION_ETH, 445 MLX5_EXPANSION_ETH_VLAN, 446 MLX5_EXPANSION_VLAN, 447 MLX5_EXPANSION_IPV4, 448 MLX5_EXPANSION_IPV4_UDP, 449 MLX5_EXPANSION_IPV4_TCP, 450 MLX5_EXPANSION_IPV6, 451 MLX5_EXPANSION_IPV6_UDP, 452 MLX5_EXPANSION_IPV6_TCP, 453 }; 454 455 /** Supported expansion of items. */ 456 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 457 [MLX5_EXPANSION_ROOT] = { 458 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 459 MLX5_EXPANSION_IPV4, 460 MLX5_EXPANSION_IPV6), 461 .type = RTE_FLOW_ITEM_TYPE_END, 462 }, 463 [MLX5_EXPANSION_ROOT_OUTER] = { 464 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 465 MLX5_EXPANSION_OUTER_IPV4, 466 MLX5_EXPANSION_OUTER_IPV6), 467 .type = RTE_FLOW_ITEM_TYPE_END, 468 }, 469 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 470 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 471 .type = RTE_FLOW_ITEM_TYPE_END, 472 }, 473 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 474 .next = MLX5_FLOW_EXPAND_RSS_NEXT 475 (MLX5_EXPANSION_OUTER_ETH_VLAN), 476 .type = RTE_FLOW_ITEM_TYPE_END, 477 }, 478 [MLX5_EXPANSION_OUTER_ETH] = { 479 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 480 MLX5_EXPANSION_OUTER_IPV6, 481 MLX5_EXPANSION_MPLS), 482 .type = RTE_FLOW_ITEM_TYPE_ETH, 483 .rss_types = 0, 484 }, 485 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 486 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 487 .type = RTE_FLOW_ITEM_TYPE_ETH, 488 .rss_types = 0, 489 }, 490 [MLX5_EXPANSION_OUTER_VLAN] = { 491 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 492 MLX5_EXPANSION_OUTER_IPV6), 493 .type = RTE_FLOW_ITEM_TYPE_VLAN, 494 }, 495 [MLX5_EXPANSION_OUTER_IPV4] = { 496 .next = MLX5_FLOW_EXPAND_RSS_NEXT 497 (MLX5_EXPANSION_OUTER_IPV4_UDP, 498 MLX5_EXPANSION_OUTER_IPV4_TCP, 499 MLX5_EXPANSION_GRE, 500 MLX5_EXPANSION_NVGRE, 501 MLX5_EXPANSION_IPV4, 502 MLX5_EXPANSION_IPV6), 503 .type = RTE_FLOW_ITEM_TYPE_IPV4, 504 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 505 ETH_RSS_NONFRAG_IPV4_OTHER, 506 }, 507 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 508 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 509 MLX5_EXPANSION_VXLAN_GPE), 510 .type = RTE_FLOW_ITEM_TYPE_UDP, 511 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 512 }, 513 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 514 .type = RTE_FLOW_ITEM_TYPE_TCP, 515 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 516 }, 517 [MLX5_EXPANSION_OUTER_IPV6] = { 518 .next = MLX5_FLOW_EXPAND_RSS_NEXT 519 (MLX5_EXPANSION_OUTER_IPV6_UDP, 520 MLX5_EXPANSION_OUTER_IPV6_TCP, 521 MLX5_EXPANSION_IPV4, 522 MLX5_EXPANSION_IPV6, 523 MLX5_EXPANSION_GRE, 524 MLX5_EXPANSION_NVGRE), 525 .type = RTE_FLOW_ITEM_TYPE_IPV6, 526 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 527 ETH_RSS_NONFRAG_IPV6_OTHER, 528 }, 529 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 530 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 531 MLX5_EXPANSION_VXLAN_GPE), 532 .type = RTE_FLOW_ITEM_TYPE_UDP, 533 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 534 }, 535 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 536 .type = RTE_FLOW_ITEM_TYPE_TCP, 537 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 538 }, 539 [MLX5_EXPANSION_VXLAN] = { 540 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 541 MLX5_EXPANSION_IPV4, 542 MLX5_EXPANSION_IPV6), 543 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 544 }, 545 [MLX5_EXPANSION_VXLAN_GPE] = { 546 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 547 MLX5_EXPANSION_IPV4, 548 MLX5_EXPANSION_IPV6), 549 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 550 }, 551 [MLX5_EXPANSION_GRE] = { 552 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 553 MLX5_EXPANSION_IPV6, 554 MLX5_EXPANSION_GRE_KEY), 555 .type = RTE_FLOW_ITEM_TYPE_GRE, 556 }, 557 [MLX5_EXPANSION_GRE_KEY] = { 558 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 559 MLX5_EXPANSION_IPV6), 560 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 561 .optional = 1, 562 }, 563 [MLX5_EXPANSION_NVGRE] = { 564 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 565 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 566 }, 567 [MLX5_EXPANSION_MPLS] = { 568 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 569 MLX5_EXPANSION_IPV6), 570 .type = RTE_FLOW_ITEM_TYPE_MPLS, 571 }, 572 [MLX5_EXPANSION_ETH] = { 573 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 574 MLX5_EXPANSION_IPV6), 575 .type = RTE_FLOW_ITEM_TYPE_ETH, 576 }, 577 [MLX5_EXPANSION_ETH_VLAN] = { 578 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 579 .type = RTE_FLOW_ITEM_TYPE_ETH, 580 }, 581 [MLX5_EXPANSION_VLAN] = { 582 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 583 MLX5_EXPANSION_IPV6), 584 .type = RTE_FLOW_ITEM_TYPE_VLAN, 585 }, 586 [MLX5_EXPANSION_IPV4] = { 587 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 588 MLX5_EXPANSION_IPV4_TCP), 589 .type = RTE_FLOW_ITEM_TYPE_IPV4, 590 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 591 ETH_RSS_NONFRAG_IPV4_OTHER, 592 }, 593 [MLX5_EXPANSION_IPV4_UDP] = { 594 .type = RTE_FLOW_ITEM_TYPE_UDP, 595 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 596 }, 597 [MLX5_EXPANSION_IPV4_TCP] = { 598 .type = RTE_FLOW_ITEM_TYPE_TCP, 599 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 600 }, 601 [MLX5_EXPANSION_IPV6] = { 602 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 603 MLX5_EXPANSION_IPV6_TCP), 604 .type = RTE_FLOW_ITEM_TYPE_IPV6, 605 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 606 ETH_RSS_NONFRAG_IPV6_OTHER, 607 }, 608 [MLX5_EXPANSION_IPV6_UDP] = { 609 .type = RTE_FLOW_ITEM_TYPE_UDP, 610 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 611 }, 612 [MLX5_EXPANSION_IPV6_TCP] = { 613 .type = RTE_FLOW_ITEM_TYPE_TCP, 614 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 615 }, 616 }; 617 618 static struct rte_flow_action_handle * 619 mlx5_action_handle_create(struct rte_eth_dev *dev, 620 const struct rte_flow_indir_action_conf *conf, 621 const struct rte_flow_action *action, 622 struct rte_flow_error *error); 623 static int mlx5_action_handle_destroy 624 (struct rte_eth_dev *dev, 625 struct rte_flow_action_handle *handle, 626 struct rte_flow_error *error); 627 static int mlx5_action_handle_update 628 (struct rte_eth_dev *dev, 629 struct rte_flow_action_handle *handle, 630 const void *update, 631 struct rte_flow_error *error); 632 static int mlx5_action_handle_query 633 (struct rte_eth_dev *dev, 634 const struct rte_flow_action_handle *handle, 635 void *data, 636 struct rte_flow_error *error); 637 static int 638 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 639 struct rte_flow_tunnel *app_tunnel, 640 struct rte_flow_action **actions, 641 uint32_t *num_of_actions, 642 struct rte_flow_error *error); 643 static int 644 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 645 struct rte_flow_tunnel *app_tunnel, 646 struct rte_flow_item **items, 647 uint32_t *num_of_items, 648 struct rte_flow_error *error); 649 static int 650 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 651 struct rte_flow_item *pmd_items, 652 uint32_t num_items, struct rte_flow_error *err); 653 static int 654 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 655 struct rte_flow_action *pmd_actions, 656 uint32_t num_actions, 657 struct rte_flow_error *err); 658 static int 659 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 660 struct rte_mbuf *m, 661 struct rte_flow_restore_info *info, 662 struct rte_flow_error *err); 663 664 static const struct rte_flow_ops mlx5_flow_ops = { 665 .validate = mlx5_flow_validate, 666 .create = mlx5_flow_create, 667 .destroy = mlx5_flow_destroy, 668 .flush = mlx5_flow_flush, 669 .isolate = mlx5_flow_isolate, 670 .query = mlx5_flow_query, 671 .dev_dump = mlx5_flow_dev_dump, 672 .get_aged_flows = mlx5_flow_get_aged_flows, 673 .action_handle_create = mlx5_action_handle_create, 674 .action_handle_destroy = mlx5_action_handle_destroy, 675 .action_handle_update = mlx5_action_handle_update, 676 .action_handle_query = mlx5_action_handle_query, 677 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 678 .tunnel_match = mlx5_flow_tunnel_match, 679 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 680 .tunnel_item_release = mlx5_flow_tunnel_item_release, 681 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 682 }; 683 684 /* Tunnel information. */ 685 struct mlx5_flow_tunnel_info { 686 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 687 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 688 }; 689 690 static struct mlx5_flow_tunnel_info tunnels_info[] = { 691 { 692 .tunnel = MLX5_FLOW_LAYER_VXLAN, 693 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 694 }, 695 { 696 .tunnel = MLX5_FLOW_LAYER_GENEVE, 697 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 698 }, 699 { 700 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 701 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 702 }, 703 { 704 .tunnel = MLX5_FLOW_LAYER_GRE, 705 .ptype = RTE_PTYPE_TUNNEL_GRE, 706 }, 707 { 708 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 709 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 710 }, 711 { 712 .tunnel = MLX5_FLOW_LAYER_MPLS, 713 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 714 }, 715 { 716 .tunnel = MLX5_FLOW_LAYER_NVGRE, 717 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 718 }, 719 { 720 .tunnel = MLX5_FLOW_LAYER_IPIP, 721 .ptype = RTE_PTYPE_TUNNEL_IP, 722 }, 723 { 724 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 725 .ptype = RTE_PTYPE_TUNNEL_IP, 726 }, 727 { 728 .tunnel = MLX5_FLOW_LAYER_GTP, 729 .ptype = RTE_PTYPE_TUNNEL_GTPU, 730 }, 731 }; 732 733 734 735 /** 736 * Translate tag ID to register. 737 * 738 * @param[in] dev 739 * Pointer to the Ethernet device structure. 740 * @param[in] feature 741 * The feature that request the register. 742 * @param[in] id 743 * The request register ID. 744 * @param[out] error 745 * Error description in case of any. 746 * 747 * @return 748 * The request register on success, a negative errno 749 * value otherwise and rte_errno is set. 750 */ 751 int 752 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 753 enum mlx5_feature_name feature, 754 uint32_t id, 755 struct rte_flow_error *error) 756 { 757 struct mlx5_priv *priv = dev->data->dev_private; 758 struct mlx5_dev_config *config = &priv->config; 759 enum modify_reg start_reg; 760 bool skip_mtr_reg = false; 761 762 switch (feature) { 763 case MLX5_HAIRPIN_RX: 764 return REG_B; 765 case MLX5_HAIRPIN_TX: 766 return REG_A; 767 case MLX5_METADATA_RX: 768 switch (config->dv_xmeta_en) { 769 case MLX5_XMETA_MODE_LEGACY: 770 return REG_B; 771 case MLX5_XMETA_MODE_META16: 772 return REG_C_0; 773 case MLX5_XMETA_MODE_META32: 774 return REG_C_1; 775 } 776 break; 777 case MLX5_METADATA_TX: 778 return REG_A; 779 case MLX5_METADATA_FDB: 780 switch (config->dv_xmeta_en) { 781 case MLX5_XMETA_MODE_LEGACY: 782 return REG_NON; 783 case MLX5_XMETA_MODE_META16: 784 return REG_C_0; 785 case MLX5_XMETA_MODE_META32: 786 return REG_C_1; 787 } 788 break; 789 case MLX5_FLOW_MARK: 790 switch (config->dv_xmeta_en) { 791 case MLX5_XMETA_MODE_LEGACY: 792 return REG_NON; 793 case MLX5_XMETA_MODE_META16: 794 return REG_C_1; 795 case MLX5_XMETA_MODE_META32: 796 return REG_C_0; 797 } 798 break; 799 case MLX5_MTR_ID: 800 /* 801 * If meter color and meter id share one register, flow match 802 * should use the meter color register for match. 803 */ 804 if (priv->mtr_reg_share) 805 return priv->mtr_color_reg; 806 else 807 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 808 REG_C_3; 809 case MLX5_MTR_COLOR: 810 case MLX5_ASO_FLOW_HIT: 811 case MLX5_ASO_CONNTRACK: 812 /* All features use the same REG_C. */ 813 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 814 return priv->mtr_color_reg; 815 case MLX5_COPY_MARK: 816 /* 817 * Metadata COPY_MARK register using is in meter suffix sub 818 * flow while with meter. It's safe to share the same register. 819 */ 820 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 821 case MLX5_APP_TAG: 822 /* 823 * If meter is enable, it will engage the register for color 824 * match and flow match. If meter color match is not using the 825 * REG_C_2, need to skip the REG_C_x be used by meter color 826 * match. 827 * If meter is disable, free to use all available registers. 828 */ 829 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 830 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 831 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 832 if (id > (uint32_t)(REG_C_7 - start_reg)) 833 return rte_flow_error_set(error, EINVAL, 834 RTE_FLOW_ERROR_TYPE_ITEM, 835 NULL, "invalid tag id"); 836 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 837 return rte_flow_error_set(error, ENOTSUP, 838 RTE_FLOW_ERROR_TYPE_ITEM, 839 NULL, "unsupported tag id"); 840 /* 841 * This case means meter is using the REG_C_x great than 2. 842 * Take care not to conflict with meter color REG_C_x. 843 * If the available index REG_C_y >= REG_C_x, skip the 844 * color register. 845 */ 846 if (skip_mtr_reg && config->flow_mreg_c 847 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 848 if (id >= (uint32_t)(REG_C_7 - start_reg)) 849 return rte_flow_error_set(error, EINVAL, 850 RTE_FLOW_ERROR_TYPE_ITEM, 851 NULL, "invalid tag id"); 852 if (config->flow_mreg_c 853 [id + 1 + start_reg - REG_C_0] != REG_NON) 854 return config->flow_mreg_c 855 [id + 1 + start_reg - REG_C_0]; 856 return rte_flow_error_set(error, ENOTSUP, 857 RTE_FLOW_ERROR_TYPE_ITEM, 858 NULL, "unsupported tag id"); 859 } 860 return config->flow_mreg_c[id + start_reg - REG_C_0]; 861 } 862 MLX5_ASSERT(false); 863 return rte_flow_error_set(error, EINVAL, 864 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 865 NULL, "invalid feature name"); 866 } 867 868 /** 869 * Check extensive flow metadata register support. 870 * 871 * @param dev 872 * Pointer to rte_eth_dev structure. 873 * 874 * @return 875 * True if device supports extensive flow metadata register, otherwise false. 876 */ 877 bool 878 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 879 { 880 struct mlx5_priv *priv = dev->data->dev_private; 881 struct mlx5_dev_config *config = &priv->config; 882 883 /* 884 * Having available reg_c can be regarded inclusively as supporting 885 * extensive flow metadata register, which could mean, 886 * - metadata register copy action by modify header. 887 * - 16 modify header actions is supported. 888 * - reg_c's are preserved across different domain (FDB and NIC) on 889 * packet loopback by flow lookup miss. 890 */ 891 return config->flow_mreg_c[2] != REG_NON; 892 } 893 894 /** 895 * Get the lowest priority. 896 * 897 * @param[in] dev 898 * Pointer to the Ethernet device structure. 899 * @param[in] attributes 900 * Pointer to device flow rule attributes. 901 * 902 * @return 903 * The value of lowest priority of flow. 904 */ 905 uint32_t 906 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 907 const struct rte_flow_attr *attr) 908 { 909 struct mlx5_priv *priv = dev->data->dev_private; 910 911 if (!attr->group && !attr->transfer) 912 return priv->config.flow_prio - 2; 913 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 914 } 915 916 /** 917 * Calculate matcher priority of the flow. 918 * 919 * @param[in] dev 920 * Pointer to the Ethernet device structure. 921 * @param[in] attr 922 * Pointer to device flow rule attributes. 923 * @param[in] subpriority 924 * The priority based on the items. 925 * @return 926 * The matcher priority of the flow. 927 */ 928 uint16_t 929 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 930 const struct rte_flow_attr *attr, 931 uint32_t subpriority) 932 { 933 uint16_t priority = (uint16_t)attr->priority; 934 struct mlx5_priv *priv = dev->data->dev_private; 935 936 if (!attr->group && !attr->transfer) { 937 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 938 priority = priv->config.flow_prio - 1; 939 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 940 } 941 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 942 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 943 return priority * 3 + subpriority; 944 } 945 946 /** 947 * Verify the @p item specifications (spec, last, mask) are compatible with the 948 * NIC capabilities. 949 * 950 * @param[in] item 951 * Item specification. 952 * @param[in] mask 953 * @p item->mask or flow default bit-masks. 954 * @param[in] nic_mask 955 * Bit-masks covering supported fields by the NIC to compare with user mask. 956 * @param[in] size 957 * Bit-masks size in bytes. 958 * @param[in] range_accepted 959 * True if range of values is accepted for specific fields, false otherwise. 960 * @param[out] error 961 * Pointer to error structure. 962 * 963 * @return 964 * 0 on success, a negative errno value otherwise and rte_errno is set. 965 */ 966 int 967 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 968 const uint8_t *mask, 969 const uint8_t *nic_mask, 970 unsigned int size, 971 bool range_accepted, 972 struct rte_flow_error *error) 973 { 974 unsigned int i; 975 976 MLX5_ASSERT(nic_mask); 977 for (i = 0; i < size; ++i) 978 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 979 return rte_flow_error_set(error, ENOTSUP, 980 RTE_FLOW_ERROR_TYPE_ITEM, 981 item, 982 "mask enables non supported" 983 " bits"); 984 if (!item->spec && (item->mask || item->last)) 985 return rte_flow_error_set(error, EINVAL, 986 RTE_FLOW_ERROR_TYPE_ITEM, item, 987 "mask/last without a spec is not" 988 " supported"); 989 if (item->spec && item->last && !range_accepted) { 990 uint8_t spec[size]; 991 uint8_t last[size]; 992 unsigned int i; 993 int ret; 994 995 for (i = 0; i < size; ++i) { 996 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 997 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 998 } 999 ret = memcmp(spec, last, size); 1000 if (ret != 0) 1001 return rte_flow_error_set(error, EINVAL, 1002 RTE_FLOW_ERROR_TYPE_ITEM, 1003 item, 1004 "range is not valid"); 1005 } 1006 return 0; 1007 } 1008 1009 /** 1010 * Adjust the hash fields according to the @p flow information. 1011 * 1012 * @param[in] dev_flow. 1013 * Pointer to the mlx5_flow. 1014 * @param[in] tunnel 1015 * 1 when the hash field is for a tunnel item. 1016 * @param[in] layer_types 1017 * ETH_RSS_* types. 1018 * @param[in] hash_fields 1019 * Item hash fields. 1020 * 1021 * @return 1022 * The hash fields that should be used. 1023 */ 1024 uint64_t 1025 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1026 int tunnel __rte_unused, uint64_t layer_types, 1027 uint64_t hash_fields) 1028 { 1029 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1030 int rss_request_inner = rss_desc->level >= 2; 1031 1032 /* Check RSS hash level for tunnel. */ 1033 if (tunnel && rss_request_inner) 1034 hash_fields |= IBV_RX_HASH_INNER; 1035 else if (tunnel || rss_request_inner) 1036 return 0; 1037 #endif 1038 /* Check if requested layer matches RSS hash fields. */ 1039 if (!(rss_desc->types & layer_types)) 1040 return 0; 1041 return hash_fields; 1042 } 1043 1044 /** 1045 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1046 * if several tunnel rules are used on this queue, the tunnel ptype will be 1047 * cleared. 1048 * 1049 * @param rxq_ctrl 1050 * Rx queue to update. 1051 */ 1052 static void 1053 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1054 { 1055 unsigned int i; 1056 uint32_t tunnel_ptype = 0; 1057 1058 /* Look up for the ptype to use. */ 1059 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1060 if (!rxq_ctrl->flow_tunnels_n[i]) 1061 continue; 1062 if (!tunnel_ptype) { 1063 tunnel_ptype = tunnels_info[i].ptype; 1064 } else { 1065 tunnel_ptype = 0; 1066 break; 1067 } 1068 } 1069 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1070 } 1071 1072 /** 1073 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1074 * flow. 1075 * 1076 * @param[in] dev 1077 * Pointer to the Ethernet device structure. 1078 * @param[in] dev_handle 1079 * Pointer to device flow handle structure. 1080 */ 1081 void 1082 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1083 struct mlx5_flow_handle *dev_handle) 1084 { 1085 struct mlx5_priv *priv = dev->data->dev_private; 1086 const int mark = dev_handle->mark; 1087 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1088 struct mlx5_ind_table_obj *ind_tbl = NULL; 1089 unsigned int i; 1090 1091 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1092 struct mlx5_hrxq *hrxq; 1093 1094 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1095 dev_handle->rix_hrxq); 1096 if (hrxq) 1097 ind_tbl = hrxq->ind_table; 1098 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1099 struct mlx5_shared_action_rss *shared_rss; 1100 1101 shared_rss = mlx5_ipool_get 1102 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1103 dev_handle->rix_srss); 1104 if (shared_rss) 1105 ind_tbl = shared_rss->ind_tbl; 1106 } 1107 if (!ind_tbl) 1108 return; 1109 for (i = 0; i != ind_tbl->queues_n; ++i) { 1110 int idx = ind_tbl->queues[i]; 1111 struct mlx5_rxq_ctrl *rxq_ctrl = 1112 container_of((*priv->rxqs)[idx], 1113 struct mlx5_rxq_ctrl, rxq); 1114 1115 /* 1116 * To support metadata register copy on Tx loopback, 1117 * this must be always enabled (metadata may arive 1118 * from other port - not from local flows only. 1119 */ 1120 if (priv->config.dv_flow_en && 1121 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1122 mlx5_flow_ext_mreg_supported(dev)) { 1123 rxq_ctrl->rxq.mark = 1; 1124 rxq_ctrl->flow_mark_n = 1; 1125 } else if (mark) { 1126 rxq_ctrl->rxq.mark = 1; 1127 rxq_ctrl->flow_mark_n++; 1128 } 1129 if (tunnel) { 1130 unsigned int j; 1131 1132 /* Increase the counter matching the flow. */ 1133 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1134 if ((tunnels_info[j].tunnel & 1135 dev_handle->layers) == 1136 tunnels_info[j].tunnel) { 1137 rxq_ctrl->flow_tunnels_n[j]++; 1138 break; 1139 } 1140 } 1141 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1142 } 1143 } 1144 } 1145 1146 /** 1147 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1148 * 1149 * @param[in] dev 1150 * Pointer to the Ethernet device structure. 1151 * @param[in] flow 1152 * Pointer to flow structure. 1153 */ 1154 static void 1155 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1156 { 1157 struct mlx5_priv *priv = dev->data->dev_private; 1158 uint32_t handle_idx; 1159 struct mlx5_flow_handle *dev_handle; 1160 1161 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1162 handle_idx, dev_handle, next) 1163 flow_drv_rxq_flags_set(dev, dev_handle); 1164 } 1165 1166 /** 1167 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1168 * device flow if no other flow uses it with the same kind of request. 1169 * 1170 * @param dev 1171 * Pointer to Ethernet device. 1172 * @param[in] dev_handle 1173 * Pointer to the device flow handle structure. 1174 */ 1175 static void 1176 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1177 struct mlx5_flow_handle *dev_handle) 1178 { 1179 struct mlx5_priv *priv = dev->data->dev_private; 1180 const int mark = dev_handle->mark; 1181 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1182 struct mlx5_ind_table_obj *ind_tbl = NULL; 1183 unsigned int i; 1184 1185 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1186 struct mlx5_hrxq *hrxq; 1187 1188 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1189 dev_handle->rix_hrxq); 1190 if (hrxq) 1191 ind_tbl = hrxq->ind_table; 1192 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1193 struct mlx5_shared_action_rss *shared_rss; 1194 1195 shared_rss = mlx5_ipool_get 1196 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1197 dev_handle->rix_srss); 1198 if (shared_rss) 1199 ind_tbl = shared_rss->ind_tbl; 1200 } 1201 if (!ind_tbl) 1202 return; 1203 MLX5_ASSERT(dev->data->dev_started); 1204 for (i = 0; i != ind_tbl->queues_n; ++i) { 1205 int idx = ind_tbl->queues[i]; 1206 struct mlx5_rxq_ctrl *rxq_ctrl = 1207 container_of((*priv->rxqs)[idx], 1208 struct mlx5_rxq_ctrl, rxq); 1209 1210 if (priv->config.dv_flow_en && 1211 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1212 mlx5_flow_ext_mreg_supported(dev)) { 1213 rxq_ctrl->rxq.mark = 1; 1214 rxq_ctrl->flow_mark_n = 1; 1215 } else if (mark) { 1216 rxq_ctrl->flow_mark_n--; 1217 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1218 } 1219 if (tunnel) { 1220 unsigned int j; 1221 1222 /* Decrease the counter matching the flow. */ 1223 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1224 if ((tunnels_info[j].tunnel & 1225 dev_handle->layers) == 1226 tunnels_info[j].tunnel) { 1227 rxq_ctrl->flow_tunnels_n[j]--; 1228 break; 1229 } 1230 } 1231 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1232 } 1233 } 1234 } 1235 1236 /** 1237 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1238 * @p flow if no other flow uses it with the same kind of request. 1239 * 1240 * @param dev 1241 * Pointer to Ethernet device. 1242 * @param[in] flow 1243 * Pointer to the flow. 1244 */ 1245 static void 1246 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1247 { 1248 struct mlx5_priv *priv = dev->data->dev_private; 1249 uint32_t handle_idx; 1250 struct mlx5_flow_handle *dev_handle; 1251 1252 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1253 handle_idx, dev_handle, next) 1254 flow_drv_rxq_flags_trim(dev, dev_handle); 1255 } 1256 1257 /** 1258 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1259 * 1260 * @param dev 1261 * Pointer to Ethernet device. 1262 */ 1263 static void 1264 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1265 { 1266 struct mlx5_priv *priv = dev->data->dev_private; 1267 unsigned int i; 1268 1269 for (i = 0; i != priv->rxqs_n; ++i) { 1270 struct mlx5_rxq_ctrl *rxq_ctrl; 1271 unsigned int j; 1272 1273 if (!(*priv->rxqs)[i]) 1274 continue; 1275 rxq_ctrl = container_of((*priv->rxqs)[i], 1276 struct mlx5_rxq_ctrl, rxq); 1277 rxq_ctrl->flow_mark_n = 0; 1278 rxq_ctrl->rxq.mark = 0; 1279 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1280 rxq_ctrl->flow_tunnels_n[j] = 0; 1281 rxq_ctrl->rxq.tunnel = 0; 1282 } 1283 } 1284 1285 /** 1286 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1287 * 1288 * @param[in] dev 1289 * Pointer to the Ethernet device structure. 1290 */ 1291 void 1292 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1293 { 1294 struct mlx5_priv *priv = dev->data->dev_private; 1295 struct mlx5_rxq_data *data; 1296 unsigned int i; 1297 1298 for (i = 0; i != priv->rxqs_n; ++i) { 1299 if (!(*priv->rxqs)[i]) 1300 continue; 1301 data = (*priv->rxqs)[i]; 1302 if (!rte_flow_dynf_metadata_avail()) { 1303 data->dynf_meta = 0; 1304 data->flow_meta_mask = 0; 1305 data->flow_meta_offset = -1; 1306 data->flow_meta_port_mask = 0; 1307 } else { 1308 data->dynf_meta = 1; 1309 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1310 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1311 data->flow_meta_port_mask = (uint32_t)~0; 1312 if (priv->config.dv_xmeta_en == MLX5_XMETA_MODE_META16) 1313 data->flow_meta_port_mask >>= 16; 1314 } 1315 } 1316 } 1317 1318 /* 1319 * return a pointer to the desired action in the list of actions. 1320 * 1321 * @param[in] actions 1322 * The list of actions to search the action in. 1323 * @param[in] action 1324 * The action to find. 1325 * 1326 * @return 1327 * Pointer to the action in the list, if found. NULL otherwise. 1328 */ 1329 const struct rte_flow_action * 1330 mlx5_flow_find_action(const struct rte_flow_action *actions, 1331 enum rte_flow_action_type action) 1332 { 1333 if (actions == NULL) 1334 return NULL; 1335 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1336 if (actions->type == action) 1337 return actions; 1338 return NULL; 1339 } 1340 1341 /* 1342 * Validate the flag action. 1343 * 1344 * @param[in] action_flags 1345 * Bit-fields that holds the actions detected until now. 1346 * @param[in] attr 1347 * Attributes of flow that includes this action. 1348 * @param[out] error 1349 * Pointer to error structure. 1350 * 1351 * @return 1352 * 0 on success, a negative errno value otherwise and rte_errno is set. 1353 */ 1354 int 1355 mlx5_flow_validate_action_flag(uint64_t action_flags, 1356 const struct rte_flow_attr *attr, 1357 struct rte_flow_error *error) 1358 { 1359 if (action_flags & MLX5_FLOW_ACTION_MARK) 1360 return rte_flow_error_set(error, EINVAL, 1361 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1362 "can't mark and flag in same flow"); 1363 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1364 return rte_flow_error_set(error, EINVAL, 1365 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1366 "can't have 2 flag" 1367 " actions in same flow"); 1368 if (attr->egress) 1369 return rte_flow_error_set(error, ENOTSUP, 1370 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1371 "flag action not supported for " 1372 "egress"); 1373 return 0; 1374 } 1375 1376 /* 1377 * Validate the mark action. 1378 * 1379 * @param[in] action 1380 * Pointer to the queue action. 1381 * @param[in] action_flags 1382 * Bit-fields that holds the actions detected until now. 1383 * @param[in] attr 1384 * Attributes of flow that includes this action. 1385 * @param[out] error 1386 * Pointer to error structure. 1387 * 1388 * @return 1389 * 0 on success, a negative errno value otherwise and rte_errno is set. 1390 */ 1391 int 1392 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1393 uint64_t action_flags, 1394 const struct rte_flow_attr *attr, 1395 struct rte_flow_error *error) 1396 { 1397 const struct rte_flow_action_mark *mark = action->conf; 1398 1399 if (!mark) 1400 return rte_flow_error_set(error, EINVAL, 1401 RTE_FLOW_ERROR_TYPE_ACTION, 1402 action, 1403 "configuration cannot be null"); 1404 if (mark->id >= MLX5_FLOW_MARK_MAX) 1405 return rte_flow_error_set(error, EINVAL, 1406 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1407 &mark->id, 1408 "mark id must in 0 <= id < " 1409 RTE_STR(MLX5_FLOW_MARK_MAX)); 1410 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1411 return rte_flow_error_set(error, EINVAL, 1412 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1413 "can't flag and mark in same flow"); 1414 if (action_flags & MLX5_FLOW_ACTION_MARK) 1415 return rte_flow_error_set(error, EINVAL, 1416 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1417 "can't have 2 mark actions in same" 1418 " flow"); 1419 if (attr->egress) 1420 return rte_flow_error_set(error, ENOTSUP, 1421 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1422 "mark action not supported for " 1423 "egress"); 1424 return 0; 1425 } 1426 1427 /* 1428 * Validate the drop action. 1429 * 1430 * @param[in] action_flags 1431 * Bit-fields that holds the actions detected until now. 1432 * @param[in] attr 1433 * Attributes of flow that includes this action. 1434 * @param[out] error 1435 * Pointer to error structure. 1436 * 1437 * @return 1438 * 0 on success, a negative errno value otherwise and rte_errno is set. 1439 */ 1440 int 1441 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1442 const struct rte_flow_attr *attr, 1443 struct rte_flow_error *error) 1444 { 1445 if (attr->egress) 1446 return rte_flow_error_set(error, ENOTSUP, 1447 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1448 "drop action not supported for " 1449 "egress"); 1450 return 0; 1451 } 1452 1453 /* 1454 * Validate the queue action. 1455 * 1456 * @param[in] action 1457 * Pointer to the queue action. 1458 * @param[in] action_flags 1459 * Bit-fields that holds the actions detected until now. 1460 * @param[in] dev 1461 * Pointer to the Ethernet device structure. 1462 * @param[in] attr 1463 * Attributes of flow that includes this action. 1464 * @param[out] error 1465 * Pointer to error structure. 1466 * 1467 * @return 1468 * 0 on success, a negative errno value otherwise and rte_errno is set. 1469 */ 1470 int 1471 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1472 uint64_t action_flags, 1473 struct rte_eth_dev *dev, 1474 const struct rte_flow_attr *attr, 1475 struct rte_flow_error *error) 1476 { 1477 struct mlx5_priv *priv = dev->data->dev_private; 1478 const struct rte_flow_action_queue *queue = action->conf; 1479 1480 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1481 return rte_flow_error_set(error, EINVAL, 1482 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1483 "can't have 2 fate actions in" 1484 " same flow"); 1485 if (!priv->rxqs_n) 1486 return rte_flow_error_set(error, EINVAL, 1487 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1488 NULL, "No Rx queues configured"); 1489 if (queue->index >= priv->rxqs_n) 1490 return rte_flow_error_set(error, EINVAL, 1491 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1492 &queue->index, 1493 "queue index out of range"); 1494 if (!(*priv->rxqs)[queue->index]) 1495 return rte_flow_error_set(error, EINVAL, 1496 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1497 &queue->index, 1498 "queue is not configured"); 1499 if (attr->egress) 1500 return rte_flow_error_set(error, ENOTSUP, 1501 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1502 "queue action not supported for " 1503 "egress"); 1504 return 0; 1505 } 1506 1507 /* 1508 * Validate the rss action. 1509 * 1510 * @param[in] dev 1511 * Pointer to the Ethernet device structure. 1512 * @param[in] action 1513 * Pointer to the queue action. 1514 * @param[out] error 1515 * Pointer to error structure. 1516 * 1517 * @return 1518 * 0 on success, a negative errno value otherwise and rte_errno is set. 1519 */ 1520 int 1521 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1522 const struct rte_flow_action *action, 1523 struct rte_flow_error *error) 1524 { 1525 struct mlx5_priv *priv = dev->data->dev_private; 1526 const struct rte_flow_action_rss *rss = action->conf; 1527 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1528 unsigned int i; 1529 1530 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1531 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1532 return rte_flow_error_set(error, ENOTSUP, 1533 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1534 &rss->func, 1535 "RSS hash function not supported"); 1536 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1537 if (rss->level > 2) 1538 #else 1539 if (rss->level > 1) 1540 #endif 1541 return rte_flow_error_set(error, ENOTSUP, 1542 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1543 &rss->level, 1544 "tunnel RSS is not supported"); 1545 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1546 if (rss->key_len == 0 && rss->key != NULL) 1547 return rte_flow_error_set(error, ENOTSUP, 1548 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1549 &rss->key_len, 1550 "RSS hash key length 0"); 1551 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1552 return rte_flow_error_set(error, ENOTSUP, 1553 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1554 &rss->key_len, 1555 "RSS hash key too small"); 1556 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1557 return rte_flow_error_set(error, ENOTSUP, 1558 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1559 &rss->key_len, 1560 "RSS hash key too large"); 1561 if (rss->queue_num > priv->config.ind_table_max_size) 1562 return rte_flow_error_set(error, ENOTSUP, 1563 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1564 &rss->queue_num, 1565 "number of queues too large"); 1566 if (rss->types & MLX5_RSS_HF_MASK) 1567 return rte_flow_error_set(error, ENOTSUP, 1568 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1569 &rss->types, 1570 "some RSS protocols are not" 1571 " supported"); 1572 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1573 !(rss->types & ETH_RSS_IP)) 1574 return rte_flow_error_set(error, EINVAL, 1575 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1576 "L3 partial RSS requested but L3 RSS" 1577 " type not specified"); 1578 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1579 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1580 return rte_flow_error_set(error, EINVAL, 1581 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1582 "L4 partial RSS requested but L4 RSS" 1583 " type not specified"); 1584 if (!priv->rxqs_n) 1585 return rte_flow_error_set(error, EINVAL, 1586 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1587 NULL, "No Rx queues configured"); 1588 if (!rss->queue_num) 1589 return rte_flow_error_set(error, EINVAL, 1590 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1591 NULL, "No queues configured"); 1592 for (i = 0; i != rss->queue_num; ++i) { 1593 struct mlx5_rxq_ctrl *rxq_ctrl; 1594 1595 if (rss->queue[i] >= priv->rxqs_n) 1596 return rte_flow_error_set 1597 (error, EINVAL, 1598 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1599 &rss->queue[i], "queue index out of range"); 1600 if (!(*priv->rxqs)[rss->queue[i]]) 1601 return rte_flow_error_set 1602 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1603 &rss->queue[i], "queue is not configured"); 1604 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1605 struct mlx5_rxq_ctrl, rxq); 1606 if (i == 0) 1607 rxq_type = rxq_ctrl->type; 1608 if (rxq_type != rxq_ctrl->type) 1609 return rte_flow_error_set 1610 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1611 &rss->queue[i], 1612 "combining hairpin and regular RSS queues is not supported"); 1613 } 1614 return 0; 1615 } 1616 1617 /* 1618 * Validate the rss action. 1619 * 1620 * @param[in] action 1621 * Pointer to the queue action. 1622 * @param[in] action_flags 1623 * Bit-fields that holds the actions detected until now. 1624 * @param[in] dev 1625 * Pointer to the Ethernet device structure. 1626 * @param[in] attr 1627 * Attributes of flow that includes this action. 1628 * @param[in] item_flags 1629 * Items that were detected. 1630 * @param[out] error 1631 * Pointer to error structure. 1632 * 1633 * @return 1634 * 0 on success, a negative errno value otherwise and rte_errno is set. 1635 */ 1636 int 1637 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1638 uint64_t action_flags, 1639 struct rte_eth_dev *dev, 1640 const struct rte_flow_attr *attr, 1641 uint64_t item_flags, 1642 struct rte_flow_error *error) 1643 { 1644 const struct rte_flow_action_rss *rss = action->conf; 1645 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1646 int ret; 1647 1648 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1649 return rte_flow_error_set(error, EINVAL, 1650 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1651 "can't have 2 fate actions" 1652 " in same flow"); 1653 ret = mlx5_validate_action_rss(dev, action, error); 1654 if (ret) 1655 return ret; 1656 if (attr->egress) 1657 return rte_flow_error_set(error, ENOTSUP, 1658 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1659 "rss action not supported for " 1660 "egress"); 1661 if (rss->level > 1 && !tunnel) 1662 return rte_flow_error_set(error, EINVAL, 1663 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1664 "inner RSS is not supported for " 1665 "non-tunnel flows"); 1666 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1667 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1668 return rte_flow_error_set(error, EINVAL, 1669 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1670 "RSS on eCPRI is not supported now"); 1671 } 1672 return 0; 1673 } 1674 1675 /* 1676 * Validate the default miss action. 1677 * 1678 * @param[in] action_flags 1679 * Bit-fields that holds the actions detected until now. 1680 * @param[out] error 1681 * Pointer to error structure. 1682 * 1683 * @return 1684 * 0 on success, a negative errno value otherwise and rte_errno is set. 1685 */ 1686 int 1687 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1688 const struct rte_flow_attr *attr, 1689 struct rte_flow_error *error) 1690 { 1691 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1692 return rte_flow_error_set(error, EINVAL, 1693 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1694 "can't have 2 fate actions in" 1695 " same flow"); 1696 if (attr->egress) 1697 return rte_flow_error_set(error, ENOTSUP, 1698 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1699 "default miss action not supported " 1700 "for egress"); 1701 if (attr->group) 1702 return rte_flow_error_set(error, ENOTSUP, 1703 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1704 "only group 0 is supported"); 1705 if (attr->transfer) 1706 return rte_flow_error_set(error, ENOTSUP, 1707 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1708 NULL, "transfer is not supported"); 1709 return 0; 1710 } 1711 1712 /* 1713 * Validate the count action. 1714 * 1715 * @param[in] dev 1716 * Pointer to the Ethernet device structure. 1717 * @param[in] attr 1718 * Attributes of flow that includes this action. 1719 * @param[out] error 1720 * Pointer to error structure. 1721 * 1722 * @return 1723 * 0 on success, a negative errno value otherwise and rte_errno is set. 1724 */ 1725 int 1726 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1727 const struct rte_flow_attr *attr, 1728 struct rte_flow_error *error) 1729 { 1730 if (attr->egress) 1731 return rte_flow_error_set(error, ENOTSUP, 1732 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1733 "count action not supported for " 1734 "egress"); 1735 return 0; 1736 } 1737 1738 /* 1739 * Validate the ASO CT action. 1740 * 1741 * @param[in] dev 1742 * Pointer to the Ethernet device structure. 1743 * @param[in] conntrack 1744 * Pointer to the CT action profile. 1745 * @param[out] error 1746 * Pointer to error structure. 1747 * 1748 * @return 1749 * 0 on success, a negative errno value otherwise and rte_errno is set. 1750 */ 1751 int 1752 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1753 const struct rte_flow_action_conntrack *conntrack, 1754 struct rte_flow_error *error) 1755 { 1756 RTE_SET_USED(dev); 1757 1758 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1759 return rte_flow_error_set(error, EINVAL, 1760 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1761 "Invalid CT state"); 1762 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1763 return rte_flow_error_set(error, EINVAL, 1764 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1765 "Invalid last TCP packet flag"); 1766 return 0; 1767 } 1768 1769 /** 1770 * Verify the @p attributes will be correctly understood by the NIC and store 1771 * them in the @p flow if everything is correct. 1772 * 1773 * @param[in] dev 1774 * Pointer to the Ethernet device structure. 1775 * @param[in] attributes 1776 * Pointer to flow attributes 1777 * @param[out] error 1778 * Pointer to error structure. 1779 * 1780 * @return 1781 * 0 on success, a negative errno value otherwise and rte_errno is set. 1782 */ 1783 int 1784 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1785 const struct rte_flow_attr *attributes, 1786 struct rte_flow_error *error) 1787 { 1788 struct mlx5_priv *priv = dev->data->dev_private; 1789 uint32_t priority_max = priv->config.flow_prio - 1; 1790 1791 if (attributes->group) 1792 return rte_flow_error_set(error, ENOTSUP, 1793 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1794 NULL, "groups is not supported"); 1795 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1796 attributes->priority >= priority_max) 1797 return rte_flow_error_set(error, ENOTSUP, 1798 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1799 NULL, "priority out of range"); 1800 if (attributes->egress) 1801 return rte_flow_error_set(error, ENOTSUP, 1802 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1803 "egress is not supported"); 1804 if (attributes->transfer && !priv->config.dv_esw_en) 1805 return rte_flow_error_set(error, ENOTSUP, 1806 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1807 NULL, "transfer is not supported"); 1808 if (!attributes->ingress) 1809 return rte_flow_error_set(error, EINVAL, 1810 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1811 NULL, 1812 "ingress attribute is mandatory"); 1813 return 0; 1814 } 1815 1816 /** 1817 * Validate ICMP6 item. 1818 * 1819 * @param[in] item 1820 * Item specification. 1821 * @param[in] item_flags 1822 * Bit-fields that holds the items detected until now. 1823 * @param[in] ext_vlan_sup 1824 * Whether extended VLAN features are supported or not. 1825 * @param[out] error 1826 * Pointer to error structure. 1827 * 1828 * @return 1829 * 0 on success, a negative errno value otherwise and rte_errno is set. 1830 */ 1831 int 1832 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1833 uint64_t item_flags, 1834 uint8_t target_protocol, 1835 struct rte_flow_error *error) 1836 { 1837 const struct rte_flow_item_icmp6 *mask = item->mask; 1838 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1839 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1840 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1841 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1842 MLX5_FLOW_LAYER_OUTER_L4; 1843 int ret; 1844 1845 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1846 return rte_flow_error_set(error, EINVAL, 1847 RTE_FLOW_ERROR_TYPE_ITEM, item, 1848 "protocol filtering not compatible" 1849 " with ICMP6 layer"); 1850 if (!(item_flags & l3m)) 1851 return rte_flow_error_set(error, EINVAL, 1852 RTE_FLOW_ERROR_TYPE_ITEM, item, 1853 "IPv6 is mandatory to filter on" 1854 " ICMP6"); 1855 if (item_flags & l4m) 1856 return rte_flow_error_set(error, EINVAL, 1857 RTE_FLOW_ERROR_TYPE_ITEM, item, 1858 "multiple L4 layers not supported"); 1859 if (!mask) 1860 mask = &rte_flow_item_icmp6_mask; 1861 ret = mlx5_flow_item_acceptable 1862 (item, (const uint8_t *)mask, 1863 (const uint8_t *)&rte_flow_item_icmp6_mask, 1864 sizeof(struct rte_flow_item_icmp6), 1865 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1866 if (ret < 0) 1867 return ret; 1868 return 0; 1869 } 1870 1871 /** 1872 * Validate ICMP item. 1873 * 1874 * @param[in] item 1875 * Item specification. 1876 * @param[in] item_flags 1877 * Bit-fields that holds the items detected until now. 1878 * @param[out] error 1879 * Pointer to error structure. 1880 * 1881 * @return 1882 * 0 on success, a negative errno value otherwise and rte_errno is set. 1883 */ 1884 int 1885 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1886 uint64_t item_flags, 1887 uint8_t target_protocol, 1888 struct rte_flow_error *error) 1889 { 1890 const struct rte_flow_item_icmp *mask = item->mask; 1891 const struct rte_flow_item_icmp nic_mask = { 1892 .hdr.icmp_type = 0xff, 1893 .hdr.icmp_code = 0xff, 1894 .hdr.icmp_ident = RTE_BE16(0xffff), 1895 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1896 }; 1897 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1898 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1899 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1900 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1901 MLX5_FLOW_LAYER_OUTER_L4; 1902 int ret; 1903 1904 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1905 return rte_flow_error_set(error, EINVAL, 1906 RTE_FLOW_ERROR_TYPE_ITEM, item, 1907 "protocol filtering not compatible" 1908 " with ICMP layer"); 1909 if (!(item_flags & l3m)) 1910 return rte_flow_error_set(error, EINVAL, 1911 RTE_FLOW_ERROR_TYPE_ITEM, item, 1912 "IPv4 is mandatory to filter" 1913 " on ICMP"); 1914 if (item_flags & l4m) 1915 return rte_flow_error_set(error, EINVAL, 1916 RTE_FLOW_ERROR_TYPE_ITEM, item, 1917 "multiple L4 layers not supported"); 1918 if (!mask) 1919 mask = &nic_mask; 1920 ret = mlx5_flow_item_acceptable 1921 (item, (const uint8_t *)mask, 1922 (const uint8_t *)&nic_mask, 1923 sizeof(struct rte_flow_item_icmp), 1924 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1925 if (ret < 0) 1926 return ret; 1927 return 0; 1928 } 1929 1930 /** 1931 * Validate Ethernet item. 1932 * 1933 * @param[in] item 1934 * Item specification. 1935 * @param[in] item_flags 1936 * Bit-fields that holds the items detected until now. 1937 * @param[out] error 1938 * Pointer to error structure. 1939 * 1940 * @return 1941 * 0 on success, a negative errno value otherwise and rte_errno is set. 1942 */ 1943 int 1944 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1945 uint64_t item_flags, bool ext_vlan_sup, 1946 struct rte_flow_error *error) 1947 { 1948 const struct rte_flow_item_eth *mask = item->mask; 1949 const struct rte_flow_item_eth nic_mask = { 1950 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1951 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1952 .type = RTE_BE16(0xffff), 1953 .has_vlan = ext_vlan_sup ? 1 : 0, 1954 }; 1955 int ret; 1956 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1957 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1958 MLX5_FLOW_LAYER_OUTER_L2; 1959 1960 if (item_flags & ethm) 1961 return rte_flow_error_set(error, ENOTSUP, 1962 RTE_FLOW_ERROR_TYPE_ITEM, item, 1963 "multiple L2 layers not supported"); 1964 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1965 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1966 return rte_flow_error_set(error, EINVAL, 1967 RTE_FLOW_ERROR_TYPE_ITEM, item, 1968 "L2 layer should not follow " 1969 "L3 layers"); 1970 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1971 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1972 return rte_flow_error_set(error, EINVAL, 1973 RTE_FLOW_ERROR_TYPE_ITEM, item, 1974 "L2 layer should not follow VLAN"); 1975 if (!mask) 1976 mask = &rte_flow_item_eth_mask; 1977 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1978 (const uint8_t *)&nic_mask, 1979 sizeof(struct rte_flow_item_eth), 1980 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1981 return ret; 1982 } 1983 1984 /** 1985 * Validate VLAN item. 1986 * 1987 * @param[in] item 1988 * Item specification. 1989 * @param[in] item_flags 1990 * Bit-fields that holds the items detected until now. 1991 * @param[in] dev 1992 * Ethernet device flow is being created on. 1993 * @param[out] error 1994 * Pointer to error structure. 1995 * 1996 * @return 1997 * 0 on success, a negative errno value otherwise and rte_errno is set. 1998 */ 1999 int 2000 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2001 uint64_t item_flags, 2002 struct rte_eth_dev *dev, 2003 struct rte_flow_error *error) 2004 { 2005 const struct rte_flow_item_vlan *spec = item->spec; 2006 const struct rte_flow_item_vlan *mask = item->mask; 2007 const struct rte_flow_item_vlan nic_mask = { 2008 .tci = RTE_BE16(UINT16_MAX), 2009 .inner_type = RTE_BE16(UINT16_MAX), 2010 }; 2011 uint16_t vlan_tag = 0; 2012 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2013 int ret; 2014 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2015 MLX5_FLOW_LAYER_INNER_L4) : 2016 (MLX5_FLOW_LAYER_OUTER_L3 | 2017 MLX5_FLOW_LAYER_OUTER_L4); 2018 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2019 MLX5_FLOW_LAYER_OUTER_VLAN; 2020 2021 if (item_flags & vlanm) 2022 return rte_flow_error_set(error, EINVAL, 2023 RTE_FLOW_ERROR_TYPE_ITEM, item, 2024 "multiple VLAN layers not supported"); 2025 else if ((item_flags & l34m) != 0) 2026 return rte_flow_error_set(error, EINVAL, 2027 RTE_FLOW_ERROR_TYPE_ITEM, item, 2028 "VLAN cannot follow L3/L4 layer"); 2029 if (!mask) 2030 mask = &rte_flow_item_vlan_mask; 2031 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2032 (const uint8_t *)&nic_mask, 2033 sizeof(struct rte_flow_item_vlan), 2034 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2035 if (ret) 2036 return ret; 2037 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2038 struct mlx5_priv *priv = dev->data->dev_private; 2039 2040 if (priv->vmwa_context) { 2041 /* 2042 * Non-NULL context means we have a virtual machine 2043 * and SR-IOV enabled, we have to create VLAN interface 2044 * to make hypervisor to setup E-Switch vport 2045 * context correctly. We avoid creating the multiple 2046 * VLAN interfaces, so we cannot support VLAN tag mask. 2047 */ 2048 return rte_flow_error_set(error, EINVAL, 2049 RTE_FLOW_ERROR_TYPE_ITEM, 2050 item, 2051 "VLAN tag mask is not" 2052 " supported in virtual" 2053 " environment"); 2054 } 2055 } 2056 if (spec) { 2057 vlan_tag = spec->tci; 2058 vlan_tag &= mask->tci; 2059 } 2060 /* 2061 * From verbs perspective an empty VLAN is equivalent 2062 * to a packet without VLAN layer. 2063 */ 2064 if (!vlan_tag) 2065 return rte_flow_error_set(error, EINVAL, 2066 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2067 item->spec, 2068 "VLAN cannot be empty"); 2069 return 0; 2070 } 2071 2072 /** 2073 * Validate IPV4 item. 2074 * 2075 * @param[in] item 2076 * Item specification. 2077 * @param[in] item_flags 2078 * Bit-fields that holds the items detected until now. 2079 * @param[in] last_item 2080 * Previous validated item in the pattern items. 2081 * @param[in] ether_type 2082 * Type in the ethernet layer header (including dot1q). 2083 * @param[in] acc_mask 2084 * Acceptable mask, if NULL default internal default mask 2085 * will be used to check whether item fields are supported. 2086 * @param[in] range_accepted 2087 * True if range of values is accepted for specific fields, false otherwise. 2088 * @param[out] error 2089 * Pointer to error structure. 2090 * 2091 * @return 2092 * 0 on success, a negative errno value otherwise and rte_errno is set. 2093 */ 2094 int 2095 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2096 uint64_t item_flags, 2097 uint64_t last_item, 2098 uint16_t ether_type, 2099 const struct rte_flow_item_ipv4 *acc_mask, 2100 bool range_accepted, 2101 struct rte_flow_error *error) 2102 { 2103 const struct rte_flow_item_ipv4 *mask = item->mask; 2104 const struct rte_flow_item_ipv4 *spec = item->spec; 2105 const struct rte_flow_item_ipv4 nic_mask = { 2106 .hdr = { 2107 .src_addr = RTE_BE32(0xffffffff), 2108 .dst_addr = RTE_BE32(0xffffffff), 2109 .type_of_service = 0xff, 2110 .next_proto_id = 0xff, 2111 }, 2112 }; 2113 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2114 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2115 MLX5_FLOW_LAYER_OUTER_L3; 2116 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2117 MLX5_FLOW_LAYER_OUTER_L4; 2118 int ret; 2119 uint8_t next_proto = 0xFF; 2120 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2121 MLX5_FLOW_LAYER_OUTER_VLAN | 2122 MLX5_FLOW_LAYER_INNER_VLAN); 2123 2124 if ((last_item & l2_vlan) && ether_type && 2125 ether_type != RTE_ETHER_TYPE_IPV4) 2126 return rte_flow_error_set(error, EINVAL, 2127 RTE_FLOW_ERROR_TYPE_ITEM, item, 2128 "IPv4 cannot follow L2/VLAN layer " 2129 "which ether type is not IPv4"); 2130 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2131 if (mask && spec) 2132 next_proto = mask->hdr.next_proto_id & 2133 spec->hdr.next_proto_id; 2134 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2135 return rte_flow_error_set(error, EINVAL, 2136 RTE_FLOW_ERROR_TYPE_ITEM, 2137 item, 2138 "multiple tunnel " 2139 "not supported"); 2140 } 2141 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2142 return rte_flow_error_set(error, EINVAL, 2143 RTE_FLOW_ERROR_TYPE_ITEM, item, 2144 "wrong tunnel type - IPv6 specified " 2145 "but IPv4 item provided"); 2146 if (item_flags & l3m) 2147 return rte_flow_error_set(error, ENOTSUP, 2148 RTE_FLOW_ERROR_TYPE_ITEM, item, 2149 "multiple L3 layers not supported"); 2150 else if (item_flags & l4m) 2151 return rte_flow_error_set(error, EINVAL, 2152 RTE_FLOW_ERROR_TYPE_ITEM, item, 2153 "L3 cannot follow an L4 layer."); 2154 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2155 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2156 return rte_flow_error_set(error, EINVAL, 2157 RTE_FLOW_ERROR_TYPE_ITEM, item, 2158 "L3 cannot follow an NVGRE layer."); 2159 if (!mask) 2160 mask = &rte_flow_item_ipv4_mask; 2161 else if (mask->hdr.next_proto_id != 0 && 2162 mask->hdr.next_proto_id != 0xff) 2163 return rte_flow_error_set(error, EINVAL, 2164 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2165 "partial mask is not supported" 2166 " for protocol"); 2167 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2168 acc_mask ? (const uint8_t *)acc_mask 2169 : (const uint8_t *)&nic_mask, 2170 sizeof(struct rte_flow_item_ipv4), 2171 range_accepted, error); 2172 if (ret < 0) 2173 return ret; 2174 return 0; 2175 } 2176 2177 /** 2178 * Validate IPV6 item. 2179 * 2180 * @param[in] item 2181 * Item specification. 2182 * @param[in] item_flags 2183 * Bit-fields that holds the items detected until now. 2184 * @param[in] last_item 2185 * Previous validated item in the pattern items. 2186 * @param[in] ether_type 2187 * Type in the ethernet layer header (including dot1q). 2188 * @param[in] acc_mask 2189 * Acceptable mask, if NULL default internal default mask 2190 * will be used to check whether item fields are supported. 2191 * @param[out] error 2192 * Pointer to error structure. 2193 * 2194 * @return 2195 * 0 on success, a negative errno value otherwise and rte_errno is set. 2196 */ 2197 int 2198 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2199 uint64_t item_flags, 2200 uint64_t last_item, 2201 uint16_t ether_type, 2202 const struct rte_flow_item_ipv6 *acc_mask, 2203 struct rte_flow_error *error) 2204 { 2205 const struct rte_flow_item_ipv6 *mask = item->mask; 2206 const struct rte_flow_item_ipv6 *spec = item->spec; 2207 const struct rte_flow_item_ipv6 nic_mask = { 2208 .hdr = { 2209 .src_addr = 2210 "\xff\xff\xff\xff\xff\xff\xff\xff" 2211 "\xff\xff\xff\xff\xff\xff\xff\xff", 2212 .dst_addr = 2213 "\xff\xff\xff\xff\xff\xff\xff\xff" 2214 "\xff\xff\xff\xff\xff\xff\xff\xff", 2215 .vtc_flow = RTE_BE32(0xffffffff), 2216 .proto = 0xff, 2217 }, 2218 }; 2219 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2220 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2221 MLX5_FLOW_LAYER_OUTER_L3; 2222 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2223 MLX5_FLOW_LAYER_OUTER_L4; 2224 int ret; 2225 uint8_t next_proto = 0xFF; 2226 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2227 MLX5_FLOW_LAYER_OUTER_VLAN | 2228 MLX5_FLOW_LAYER_INNER_VLAN); 2229 2230 if ((last_item & l2_vlan) && ether_type && 2231 ether_type != RTE_ETHER_TYPE_IPV6) 2232 return rte_flow_error_set(error, EINVAL, 2233 RTE_FLOW_ERROR_TYPE_ITEM, item, 2234 "IPv6 cannot follow L2/VLAN layer " 2235 "which ether type is not IPv6"); 2236 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2237 next_proto = spec->hdr.proto; 2238 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2239 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2240 return rte_flow_error_set(error, EINVAL, 2241 RTE_FLOW_ERROR_TYPE_ITEM, 2242 item, 2243 "multiple tunnel " 2244 "not supported"); 2245 } 2246 if (next_proto == IPPROTO_HOPOPTS || 2247 next_proto == IPPROTO_ROUTING || 2248 next_proto == IPPROTO_FRAGMENT || 2249 next_proto == IPPROTO_ESP || 2250 next_proto == IPPROTO_AH || 2251 next_proto == IPPROTO_DSTOPTS) 2252 return rte_flow_error_set(error, EINVAL, 2253 RTE_FLOW_ERROR_TYPE_ITEM, item, 2254 "IPv6 proto (next header) should " 2255 "not be set as extension header"); 2256 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2257 return rte_flow_error_set(error, EINVAL, 2258 RTE_FLOW_ERROR_TYPE_ITEM, item, 2259 "wrong tunnel type - IPv4 specified " 2260 "but IPv6 item provided"); 2261 if (item_flags & l3m) 2262 return rte_flow_error_set(error, ENOTSUP, 2263 RTE_FLOW_ERROR_TYPE_ITEM, item, 2264 "multiple L3 layers not supported"); 2265 else if (item_flags & l4m) 2266 return rte_flow_error_set(error, EINVAL, 2267 RTE_FLOW_ERROR_TYPE_ITEM, item, 2268 "L3 cannot follow an L4 layer."); 2269 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2270 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2271 return rte_flow_error_set(error, EINVAL, 2272 RTE_FLOW_ERROR_TYPE_ITEM, item, 2273 "L3 cannot follow an NVGRE layer."); 2274 if (!mask) 2275 mask = &rte_flow_item_ipv6_mask; 2276 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2277 acc_mask ? (const uint8_t *)acc_mask 2278 : (const uint8_t *)&nic_mask, 2279 sizeof(struct rte_flow_item_ipv6), 2280 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2281 if (ret < 0) 2282 return ret; 2283 return 0; 2284 } 2285 2286 /** 2287 * Validate UDP item. 2288 * 2289 * @param[in] item 2290 * Item specification. 2291 * @param[in] item_flags 2292 * Bit-fields that holds the items detected until now. 2293 * @param[in] target_protocol 2294 * The next protocol in the previous item. 2295 * @param[in] flow_mask 2296 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2297 * @param[out] error 2298 * Pointer to error structure. 2299 * 2300 * @return 2301 * 0 on success, a negative errno value otherwise and rte_errno is set. 2302 */ 2303 int 2304 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2305 uint64_t item_flags, 2306 uint8_t target_protocol, 2307 struct rte_flow_error *error) 2308 { 2309 const struct rte_flow_item_udp *mask = item->mask; 2310 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2311 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2312 MLX5_FLOW_LAYER_OUTER_L3; 2313 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2314 MLX5_FLOW_LAYER_OUTER_L4; 2315 int ret; 2316 2317 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2318 return rte_flow_error_set(error, EINVAL, 2319 RTE_FLOW_ERROR_TYPE_ITEM, item, 2320 "protocol filtering not compatible" 2321 " with UDP layer"); 2322 if (!(item_flags & l3m)) 2323 return rte_flow_error_set(error, EINVAL, 2324 RTE_FLOW_ERROR_TYPE_ITEM, item, 2325 "L3 is mandatory to filter on L4"); 2326 if (item_flags & l4m) 2327 return rte_flow_error_set(error, EINVAL, 2328 RTE_FLOW_ERROR_TYPE_ITEM, item, 2329 "multiple L4 layers not supported"); 2330 if (!mask) 2331 mask = &rte_flow_item_udp_mask; 2332 ret = mlx5_flow_item_acceptable 2333 (item, (const uint8_t *)mask, 2334 (const uint8_t *)&rte_flow_item_udp_mask, 2335 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2336 error); 2337 if (ret < 0) 2338 return ret; 2339 return 0; 2340 } 2341 2342 /** 2343 * Validate TCP item. 2344 * 2345 * @param[in] item 2346 * Item specification. 2347 * @param[in] item_flags 2348 * Bit-fields that holds the items detected until now. 2349 * @param[in] target_protocol 2350 * The next protocol in the previous item. 2351 * @param[out] error 2352 * Pointer to error structure. 2353 * 2354 * @return 2355 * 0 on success, a negative errno value otherwise and rte_errno is set. 2356 */ 2357 int 2358 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2359 uint64_t item_flags, 2360 uint8_t target_protocol, 2361 const struct rte_flow_item_tcp *flow_mask, 2362 struct rte_flow_error *error) 2363 { 2364 const struct rte_flow_item_tcp *mask = item->mask; 2365 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2366 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2367 MLX5_FLOW_LAYER_OUTER_L3; 2368 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2369 MLX5_FLOW_LAYER_OUTER_L4; 2370 int ret; 2371 2372 MLX5_ASSERT(flow_mask); 2373 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2374 return rte_flow_error_set(error, EINVAL, 2375 RTE_FLOW_ERROR_TYPE_ITEM, item, 2376 "protocol filtering not compatible" 2377 " with TCP layer"); 2378 if (!(item_flags & l3m)) 2379 return rte_flow_error_set(error, EINVAL, 2380 RTE_FLOW_ERROR_TYPE_ITEM, item, 2381 "L3 is mandatory to filter on L4"); 2382 if (item_flags & l4m) 2383 return rte_flow_error_set(error, EINVAL, 2384 RTE_FLOW_ERROR_TYPE_ITEM, item, 2385 "multiple L4 layers not supported"); 2386 if (!mask) 2387 mask = &rte_flow_item_tcp_mask; 2388 ret = mlx5_flow_item_acceptable 2389 (item, (const uint8_t *)mask, 2390 (const uint8_t *)flow_mask, 2391 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2392 error); 2393 if (ret < 0) 2394 return ret; 2395 return 0; 2396 } 2397 2398 /** 2399 * Validate VXLAN item. 2400 * 2401 * @param[in] item 2402 * Item specification. 2403 * @param[in] item_flags 2404 * Bit-fields that holds the items detected until now. 2405 * @param[in] target_protocol 2406 * The next protocol in the previous item. 2407 * @param[out] error 2408 * Pointer to error structure. 2409 * 2410 * @return 2411 * 0 on success, a negative errno value otherwise and rte_errno is set. 2412 */ 2413 int 2414 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2415 uint64_t item_flags, 2416 struct rte_flow_error *error) 2417 { 2418 const struct rte_flow_item_vxlan *spec = item->spec; 2419 const struct rte_flow_item_vxlan *mask = item->mask; 2420 int ret; 2421 union vni { 2422 uint32_t vlan_id; 2423 uint8_t vni[4]; 2424 } id = { .vlan_id = 0, }; 2425 2426 2427 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2428 return rte_flow_error_set(error, ENOTSUP, 2429 RTE_FLOW_ERROR_TYPE_ITEM, item, 2430 "multiple tunnel layers not" 2431 " supported"); 2432 /* 2433 * Verify only UDPv4 is present as defined in 2434 * https://tools.ietf.org/html/rfc7348 2435 */ 2436 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2437 return rte_flow_error_set(error, EINVAL, 2438 RTE_FLOW_ERROR_TYPE_ITEM, item, 2439 "no outer UDP layer found"); 2440 if (!mask) 2441 mask = &rte_flow_item_vxlan_mask; 2442 ret = mlx5_flow_item_acceptable 2443 (item, (const uint8_t *)mask, 2444 (const uint8_t *)&rte_flow_item_vxlan_mask, 2445 sizeof(struct rte_flow_item_vxlan), 2446 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2447 if (ret < 0) 2448 return ret; 2449 if (spec) { 2450 memcpy(&id.vni[1], spec->vni, 3); 2451 memcpy(&id.vni[1], mask->vni, 3); 2452 } 2453 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2454 return rte_flow_error_set(error, ENOTSUP, 2455 RTE_FLOW_ERROR_TYPE_ITEM, item, 2456 "VXLAN tunnel must be fully defined"); 2457 return 0; 2458 } 2459 2460 /** 2461 * Validate VXLAN_GPE item. 2462 * 2463 * @param[in] item 2464 * Item specification. 2465 * @param[in] item_flags 2466 * Bit-fields that holds the items detected until now. 2467 * @param[in] priv 2468 * Pointer to the private data structure. 2469 * @param[in] target_protocol 2470 * The next protocol in the previous item. 2471 * @param[out] error 2472 * Pointer to error structure. 2473 * 2474 * @return 2475 * 0 on success, a negative errno value otherwise and rte_errno is set. 2476 */ 2477 int 2478 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2479 uint64_t item_flags, 2480 struct rte_eth_dev *dev, 2481 struct rte_flow_error *error) 2482 { 2483 struct mlx5_priv *priv = dev->data->dev_private; 2484 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2485 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2486 int ret; 2487 union vni { 2488 uint32_t vlan_id; 2489 uint8_t vni[4]; 2490 } id = { .vlan_id = 0, }; 2491 2492 if (!priv->config.l3_vxlan_en) 2493 return rte_flow_error_set(error, ENOTSUP, 2494 RTE_FLOW_ERROR_TYPE_ITEM, item, 2495 "L3 VXLAN is not enabled by device" 2496 " parameter and/or not configured in" 2497 " firmware"); 2498 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2499 return rte_flow_error_set(error, ENOTSUP, 2500 RTE_FLOW_ERROR_TYPE_ITEM, item, 2501 "multiple tunnel layers not" 2502 " supported"); 2503 /* 2504 * Verify only UDPv4 is present as defined in 2505 * https://tools.ietf.org/html/rfc7348 2506 */ 2507 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2508 return rte_flow_error_set(error, EINVAL, 2509 RTE_FLOW_ERROR_TYPE_ITEM, item, 2510 "no outer UDP layer found"); 2511 if (!mask) 2512 mask = &rte_flow_item_vxlan_gpe_mask; 2513 ret = mlx5_flow_item_acceptable 2514 (item, (const uint8_t *)mask, 2515 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2516 sizeof(struct rte_flow_item_vxlan_gpe), 2517 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2518 if (ret < 0) 2519 return ret; 2520 if (spec) { 2521 if (spec->protocol) 2522 return rte_flow_error_set(error, ENOTSUP, 2523 RTE_FLOW_ERROR_TYPE_ITEM, 2524 item, 2525 "VxLAN-GPE protocol" 2526 " not supported"); 2527 memcpy(&id.vni[1], spec->vni, 3); 2528 memcpy(&id.vni[1], mask->vni, 3); 2529 } 2530 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2531 return rte_flow_error_set(error, ENOTSUP, 2532 RTE_FLOW_ERROR_TYPE_ITEM, item, 2533 "VXLAN-GPE tunnel must be fully" 2534 " defined"); 2535 return 0; 2536 } 2537 /** 2538 * Validate GRE Key item. 2539 * 2540 * @param[in] item 2541 * Item specification. 2542 * @param[in] item_flags 2543 * Bit flags to mark detected items. 2544 * @param[in] gre_item 2545 * Pointer to gre_item 2546 * @param[out] error 2547 * Pointer to error structure. 2548 * 2549 * @return 2550 * 0 on success, a negative errno value otherwise and rte_errno is set. 2551 */ 2552 int 2553 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2554 uint64_t item_flags, 2555 const struct rte_flow_item *gre_item, 2556 struct rte_flow_error *error) 2557 { 2558 const rte_be32_t *mask = item->mask; 2559 int ret = 0; 2560 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2561 const struct rte_flow_item_gre *gre_spec; 2562 const struct rte_flow_item_gre *gre_mask; 2563 2564 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2565 return rte_flow_error_set(error, ENOTSUP, 2566 RTE_FLOW_ERROR_TYPE_ITEM, item, 2567 "Multiple GRE key not support"); 2568 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2569 return rte_flow_error_set(error, ENOTSUP, 2570 RTE_FLOW_ERROR_TYPE_ITEM, item, 2571 "No preceding GRE header"); 2572 if (item_flags & MLX5_FLOW_LAYER_INNER) 2573 return rte_flow_error_set(error, ENOTSUP, 2574 RTE_FLOW_ERROR_TYPE_ITEM, item, 2575 "GRE key following a wrong item"); 2576 gre_mask = gre_item->mask; 2577 if (!gre_mask) 2578 gre_mask = &rte_flow_item_gre_mask; 2579 gre_spec = gre_item->spec; 2580 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2581 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2582 return rte_flow_error_set(error, EINVAL, 2583 RTE_FLOW_ERROR_TYPE_ITEM, item, 2584 "Key bit must be on"); 2585 2586 if (!mask) 2587 mask = &gre_key_default_mask; 2588 ret = mlx5_flow_item_acceptable 2589 (item, (const uint8_t *)mask, 2590 (const uint8_t *)&gre_key_default_mask, 2591 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2592 return ret; 2593 } 2594 2595 /** 2596 * Validate GRE item. 2597 * 2598 * @param[in] item 2599 * Item specification. 2600 * @param[in] item_flags 2601 * Bit flags to mark detected items. 2602 * @param[in] target_protocol 2603 * The next protocol in the previous item. 2604 * @param[out] error 2605 * Pointer to error structure. 2606 * 2607 * @return 2608 * 0 on success, a negative errno value otherwise and rte_errno is set. 2609 */ 2610 int 2611 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2612 uint64_t item_flags, 2613 uint8_t target_protocol, 2614 struct rte_flow_error *error) 2615 { 2616 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2617 const struct rte_flow_item_gre *mask = item->mask; 2618 int ret; 2619 const struct rte_flow_item_gre nic_mask = { 2620 .c_rsvd0_ver = RTE_BE16(0xB000), 2621 .protocol = RTE_BE16(UINT16_MAX), 2622 }; 2623 2624 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2625 return rte_flow_error_set(error, EINVAL, 2626 RTE_FLOW_ERROR_TYPE_ITEM, item, 2627 "protocol filtering not compatible" 2628 " with this GRE layer"); 2629 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2630 return rte_flow_error_set(error, ENOTSUP, 2631 RTE_FLOW_ERROR_TYPE_ITEM, item, 2632 "multiple tunnel layers not" 2633 " supported"); 2634 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2635 return rte_flow_error_set(error, ENOTSUP, 2636 RTE_FLOW_ERROR_TYPE_ITEM, item, 2637 "L3 Layer is missing"); 2638 if (!mask) 2639 mask = &rte_flow_item_gre_mask; 2640 ret = mlx5_flow_item_acceptable 2641 (item, (const uint8_t *)mask, 2642 (const uint8_t *)&nic_mask, 2643 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2644 error); 2645 if (ret < 0) 2646 return ret; 2647 #ifndef HAVE_MLX5DV_DR 2648 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2649 if (spec && (spec->protocol & mask->protocol)) 2650 return rte_flow_error_set(error, ENOTSUP, 2651 RTE_FLOW_ERROR_TYPE_ITEM, item, 2652 "without MPLS support the" 2653 " specification cannot be used for" 2654 " filtering"); 2655 #endif 2656 #endif 2657 return 0; 2658 } 2659 2660 /** 2661 * Validate Geneve item. 2662 * 2663 * @param[in] item 2664 * Item specification. 2665 * @param[in] itemFlags 2666 * Bit-fields that holds the items detected until now. 2667 * @param[in] enPriv 2668 * Pointer to the private data structure. 2669 * @param[out] error 2670 * Pointer to error structure. 2671 * 2672 * @return 2673 * 0 on success, a negative errno value otherwise and rte_errno is set. 2674 */ 2675 2676 int 2677 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2678 uint64_t item_flags, 2679 struct rte_eth_dev *dev, 2680 struct rte_flow_error *error) 2681 { 2682 struct mlx5_priv *priv = dev->data->dev_private; 2683 const struct rte_flow_item_geneve *spec = item->spec; 2684 const struct rte_flow_item_geneve *mask = item->mask; 2685 int ret; 2686 uint16_t gbhdr; 2687 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2688 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2689 const struct rte_flow_item_geneve nic_mask = { 2690 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2691 .vni = "\xff\xff\xff", 2692 .protocol = RTE_BE16(UINT16_MAX), 2693 }; 2694 2695 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2696 return rte_flow_error_set(error, ENOTSUP, 2697 RTE_FLOW_ERROR_TYPE_ITEM, item, 2698 "L3 Geneve is not enabled by device" 2699 " parameter and/or not configured in" 2700 " firmware"); 2701 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2702 return rte_flow_error_set(error, ENOTSUP, 2703 RTE_FLOW_ERROR_TYPE_ITEM, item, 2704 "multiple tunnel layers not" 2705 " supported"); 2706 /* 2707 * Verify only UDPv4 is present as defined in 2708 * https://tools.ietf.org/html/rfc7348 2709 */ 2710 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2711 return rte_flow_error_set(error, EINVAL, 2712 RTE_FLOW_ERROR_TYPE_ITEM, item, 2713 "no outer UDP layer found"); 2714 if (!mask) 2715 mask = &rte_flow_item_geneve_mask; 2716 ret = mlx5_flow_item_acceptable 2717 (item, (const uint8_t *)mask, 2718 (const uint8_t *)&nic_mask, 2719 sizeof(struct rte_flow_item_geneve), 2720 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2721 if (ret) 2722 return ret; 2723 if (spec) { 2724 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2725 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2726 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2727 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2728 return rte_flow_error_set(error, ENOTSUP, 2729 RTE_FLOW_ERROR_TYPE_ITEM, 2730 item, 2731 "Geneve protocol unsupported" 2732 " fields are being used"); 2733 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2734 return rte_flow_error_set 2735 (error, ENOTSUP, 2736 RTE_FLOW_ERROR_TYPE_ITEM, 2737 item, 2738 "Unsupported Geneve options length"); 2739 } 2740 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2741 return rte_flow_error_set 2742 (error, ENOTSUP, 2743 RTE_FLOW_ERROR_TYPE_ITEM, item, 2744 "Geneve tunnel must be fully defined"); 2745 return 0; 2746 } 2747 2748 /** 2749 * Validate Geneve TLV option item. 2750 * 2751 * @param[in] item 2752 * Item specification. 2753 * @param[in] last_item 2754 * Previous validated item in the pattern items. 2755 * @param[in] geneve_item 2756 * Previous GENEVE item specification. 2757 * @param[in] dev 2758 * Pointer to the rte_eth_dev structure. 2759 * @param[out] error 2760 * Pointer to error structure. 2761 * 2762 * @return 2763 * 0 on success, a negative errno value otherwise and rte_errno is set. 2764 */ 2765 int 2766 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2767 uint64_t last_item, 2768 const struct rte_flow_item *geneve_item, 2769 struct rte_eth_dev *dev, 2770 struct rte_flow_error *error) 2771 { 2772 struct mlx5_priv *priv = dev->data->dev_private; 2773 struct mlx5_dev_ctx_shared *sh = priv->sh; 2774 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2775 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2776 uint8_t data_max_supported = 2777 hca_attr->max_geneve_tlv_option_data_len * 4; 2778 struct mlx5_dev_config *config = &priv->config; 2779 const struct rte_flow_item_geneve *geneve_spec; 2780 const struct rte_flow_item_geneve *geneve_mask; 2781 const struct rte_flow_item_geneve_opt *spec = item->spec; 2782 const struct rte_flow_item_geneve_opt *mask = item->mask; 2783 unsigned int i; 2784 unsigned int data_len; 2785 uint8_t tlv_option_len; 2786 uint16_t optlen_m, optlen_v; 2787 const struct rte_flow_item_geneve_opt full_mask = { 2788 .option_class = RTE_BE16(0xffff), 2789 .option_type = 0xff, 2790 .option_len = 0x1f, 2791 }; 2792 2793 if (!mask) 2794 mask = &rte_flow_item_geneve_opt_mask; 2795 if (!spec) 2796 return rte_flow_error_set 2797 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2798 "Geneve TLV opt class/type/length must be specified"); 2799 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2800 return rte_flow_error_set 2801 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2802 "Geneve TLV opt length exceeeds the limit (31)"); 2803 /* Check if class type and length masks are full. */ 2804 if (full_mask.option_class != mask->option_class || 2805 full_mask.option_type != mask->option_type || 2806 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2807 return rte_flow_error_set 2808 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2809 "Geneve TLV opt class/type/length masks must be full"); 2810 /* Check if length is supported */ 2811 if ((uint32_t)spec->option_len > 2812 config->hca_attr.max_geneve_tlv_option_data_len) 2813 return rte_flow_error_set 2814 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2815 "Geneve TLV opt length not supported"); 2816 if (config->hca_attr.max_geneve_tlv_options > 1) 2817 DRV_LOG(DEBUG, 2818 "max_geneve_tlv_options supports more than 1 option"); 2819 /* Check GENEVE item preceding. */ 2820 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2821 return rte_flow_error_set 2822 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2823 "Geneve opt item must be preceded with Geneve item"); 2824 geneve_spec = geneve_item->spec; 2825 geneve_mask = geneve_item->mask ? geneve_item->mask : 2826 &rte_flow_item_geneve_mask; 2827 /* Check if GENEVE TLV option size doesn't exceed option length */ 2828 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2829 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2830 tlv_option_len = spec->option_len & mask->option_len; 2831 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2832 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2833 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2834 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2835 if ((optlen_v & optlen_m) <= tlv_option_len) 2836 return rte_flow_error_set 2837 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2838 "GENEVE TLV option length exceeds optlen"); 2839 } 2840 /* Check if length is 0 or data is 0. */ 2841 if (spec->data == NULL || spec->option_len == 0) 2842 return rte_flow_error_set 2843 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2844 "Geneve TLV opt with zero data/length not supported"); 2845 /* Check not all data & mask are 0. */ 2846 data_len = spec->option_len * 4; 2847 if (mask->data == NULL) { 2848 for (i = 0; i < data_len; i++) 2849 if (spec->data[i]) 2850 break; 2851 if (i == data_len) 2852 return rte_flow_error_set(error, ENOTSUP, 2853 RTE_FLOW_ERROR_TYPE_ITEM, item, 2854 "Can't match on Geneve option data 0"); 2855 } else { 2856 for (i = 0; i < data_len; i++) 2857 if (spec->data[i] & mask->data[i]) 2858 break; 2859 if (i == data_len) 2860 return rte_flow_error_set(error, ENOTSUP, 2861 RTE_FLOW_ERROR_TYPE_ITEM, item, 2862 "Can't match on Geneve option data and mask 0"); 2863 /* Check data mask supported. */ 2864 for (i = data_max_supported; i < data_len ; i++) 2865 if (mask->data[i]) 2866 return rte_flow_error_set(error, ENOTSUP, 2867 RTE_FLOW_ERROR_TYPE_ITEM, item, 2868 "Data mask is of unsupported size"); 2869 } 2870 /* Check GENEVE option is supported in NIC. */ 2871 if (!config->hca_attr.geneve_tlv_opt) 2872 return rte_flow_error_set 2873 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2874 "Geneve TLV opt not supported"); 2875 /* Check if we already have geneve option with different type/class. */ 2876 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2877 geneve_opt_resource = sh->geneve_tlv_option_resource; 2878 if (geneve_opt_resource != NULL) 2879 if (geneve_opt_resource->option_class != spec->option_class || 2880 geneve_opt_resource->option_type != spec->option_type || 2881 geneve_opt_resource->length != spec->option_len) { 2882 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2883 return rte_flow_error_set(error, ENOTSUP, 2884 RTE_FLOW_ERROR_TYPE_ITEM, item, 2885 "Only one Geneve TLV option supported"); 2886 } 2887 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2888 return 0; 2889 } 2890 2891 /** 2892 * Validate MPLS item. 2893 * 2894 * @param[in] dev 2895 * Pointer to the rte_eth_dev structure. 2896 * @param[in] item 2897 * Item specification. 2898 * @param[in] item_flags 2899 * Bit-fields that holds the items detected until now. 2900 * @param[in] prev_layer 2901 * The protocol layer indicated in previous item. 2902 * @param[out] error 2903 * Pointer to error structure. 2904 * 2905 * @return 2906 * 0 on success, a negative errno value otherwise and rte_errno is set. 2907 */ 2908 int 2909 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2910 const struct rte_flow_item *item __rte_unused, 2911 uint64_t item_flags __rte_unused, 2912 uint64_t prev_layer __rte_unused, 2913 struct rte_flow_error *error) 2914 { 2915 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2916 const struct rte_flow_item_mpls *mask = item->mask; 2917 struct mlx5_priv *priv = dev->data->dev_private; 2918 int ret; 2919 2920 if (!priv->config.mpls_en) 2921 return rte_flow_error_set(error, ENOTSUP, 2922 RTE_FLOW_ERROR_TYPE_ITEM, item, 2923 "MPLS not supported or" 2924 " disabled in firmware" 2925 " configuration."); 2926 /* MPLS over IP, UDP, GRE is allowed */ 2927 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2928 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2929 MLX5_FLOW_LAYER_GRE | 2930 MLX5_FLOW_LAYER_GRE_KEY))) 2931 return rte_flow_error_set(error, EINVAL, 2932 RTE_FLOW_ERROR_TYPE_ITEM, item, 2933 "protocol filtering not compatible" 2934 " with MPLS layer"); 2935 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2936 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2937 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2938 return rte_flow_error_set(error, ENOTSUP, 2939 RTE_FLOW_ERROR_TYPE_ITEM, item, 2940 "multiple tunnel layers not" 2941 " supported"); 2942 if (!mask) 2943 mask = &rte_flow_item_mpls_mask; 2944 ret = mlx5_flow_item_acceptable 2945 (item, (const uint8_t *)mask, 2946 (const uint8_t *)&rte_flow_item_mpls_mask, 2947 sizeof(struct rte_flow_item_mpls), 2948 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2949 if (ret < 0) 2950 return ret; 2951 return 0; 2952 #else 2953 return rte_flow_error_set(error, ENOTSUP, 2954 RTE_FLOW_ERROR_TYPE_ITEM, item, 2955 "MPLS is not supported by Verbs, please" 2956 " update."); 2957 #endif 2958 } 2959 2960 /** 2961 * Validate NVGRE item. 2962 * 2963 * @param[in] item 2964 * Item specification. 2965 * @param[in] item_flags 2966 * Bit flags to mark detected items. 2967 * @param[in] target_protocol 2968 * The next protocol in the previous item. 2969 * @param[out] error 2970 * Pointer to error structure. 2971 * 2972 * @return 2973 * 0 on success, a negative errno value otherwise and rte_errno is set. 2974 */ 2975 int 2976 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2977 uint64_t item_flags, 2978 uint8_t target_protocol, 2979 struct rte_flow_error *error) 2980 { 2981 const struct rte_flow_item_nvgre *mask = item->mask; 2982 int ret; 2983 2984 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2985 return rte_flow_error_set(error, EINVAL, 2986 RTE_FLOW_ERROR_TYPE_ITEM, item, 2987 "protocol filtering not compatible" 2988 " with this GRE layer"); 2989 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2990 return rte_flow_error_set(error, ENOTSUP, 2991 RTE_FLOW_ERROR_TYPE_ITEM, item, 2992 "multiple tunnel layers not" 2993 " supported"); 2994 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2995 return rte_flow_error_set(error, ENOTSUP, 2996 RTE_FLOW_ERROR_TYPE_ITEM, item, 2997 "L3 Layer is missing"); 2998 if (!mask) 2999 mask = &rte_flow_item_nvgre_mask; 3000 ret = mlx5_flow_item_acceptable 3001 (item, (const uint8_t *)mask, 3002 (const uint8_t *)&rte_flow_item_nvgre_mask, 3003 sizeof(struct rte_flow_item_nvgre), 3004 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3005 if (ret < 0) 3006 return ret; 3007 return 0; 3008 } 3009 3010 /** 3011 * Validate eCPRI item. 3012 * 3013 * @param[in] item 3014 * Item specification. 3015 * @param[in] item_flags 3016 * Bit-fields that holds the items detected until now. 3017 * @param[in] last_item 3018 * Previous validated item in the pattern items. 3019 * @param[in] ether_type 3020 * Type in the ethernet layer header (including dot1q). 3021 * @param[in] acc_mask 3022 * Acceptable mask, if NULL default internal default mask 3023 * will be used to check whether item fields are supported. 3024 * @param[out] error 3025 * Pointer to error structure. 3026 * 3027 * @return 3028 * 0 on success, a negative errno value otherwise and rte_errno is set. 3029 */ 3030 int 3031 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3032 uint64_t item_flags, 3033 uint64_t last_item, 3034 uint16_t ether_type, 3035 const struct rte_flow_item_ecpri *acc_mask, 3036 struct rte_flow_error *error) 3037 { 3038 const struct rte_flow_item_ecpri *mask = item->mask; 3039 const struct rte_flow_item_ecpri nic_mask = { 3040 .hdr = { 3041 .common = { 3042 .u32 = 3043 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3044 .type = 0xFF, 3045 }).u32), 3046 }, 3047 .dummy[0] = 0xFFFFFFFF, 3048 }, 3049 }; 3050 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3051 MLX5_FLOW_LAYER_OUTER_VLAN); 3052 struct rte_flow_item_ecpri mask_lo; 3053 3054 if (!(last_item & outer_l2_vlan) && 3055 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3056 return rte_flow_error_set(error, EINVAL, 3057 RTE_FLOW_ERROR_TYPE_ITEM, item, 3058 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3059 if ((last_item & outer_l2_vlan) && ether_type && 3060 ether_type != RTE_ETHER_TYPE_ECPRI) 3061 return rte_flow_error_set(error, EINVAL, 3062 RTE_FLOW_ERROR_TYPE_ITEM, item, 3063 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3064 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3065 return rte_flow_error_set(error, EINVAL, 3066 RTE_FLOW_ERROR_TYPE_ITEM, item, 3067 "eCPRI with tunnel is not supported right now"); 3068 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3069 return rte_flow_error_set(error, ENOTSUP, 3070 RTE_FLOW_ERROR_TYPE_ITEM, item, 3071 "multiple L3 layers not supported"); 3072 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3073 return rte_flow_error_set(error, EINVAL, 3074 RTE_FLOW_ERROR_TYPE_ITEM, item, 3075 "eCPRI cannot coexist with a TCP layer"); 3076 /* In specification, eCPRI could be over UDP layer. */ 3077 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3078 return rte_flow_error_set(error, EINVAL, 3079 RTE_FLOW_ERROR_TYPE_ITEM, item, 3080 "eCPRI over UDP layer is not yet supported right now"); 3081 /* Mask for type field in common header could be zero. */ 3082 if (!mask) 3083 mask = &rte_flow_item_ecpri_mask; 3084 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3085 /* Input mask is in big-endian format. */ 3086 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3087 return rte_flow_error_set(error, EINVAL, 3088 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3089 "partial mask is not supported for protocol"); 3090 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3091 return rte_flow_error_set(error, EINVAL, 3092 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3093 "message header mask must be after a type mask"); 3094 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3095 acc_mask ? (const uint8_t *)acc_mask 3096 : (const uint8_t *)&nic_mask, 3097 sizeof(struct rte_flow_item_ecpri), 3098 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3099 } 3100 3101 /** 3102 * Release resource related QUEUE/RSS action split. 3103 * 3104 * @param dev 3105 * Pointer to Ethernet device. 3106 * @param flow 3107 * Flow to release id's from. 3108 */ 3109 static void 3110 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 3111 struct rte_flow *flow) 3112 { 3113 struct mlx5_priv *priv = dev->data->dev_private; 3114 uint32_t handle_idx; 3115 struct mlx5_flow_handle *dev_handle; 3116 3117 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 3118 handle_idx, dev_handle, next) 3119 if (dev_handle->split_flow_id && 3120 !dev_handle->is_meter_flow_id) 3121 mlx5_ipool_free(priv->sh->ipool 3122 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 3123 dev_handle->split_flow_id); 3124 } 3125 3126 static int 3127 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3128 const struct rte_flow_attr *attr __rte_unused, 3129 const struct rte_flow_item items[] __rte_unused, 3130 const struct rte_flow_action actions[] __rte_unused, 3131 bool external __rte_unused, 3132 int hairpin __rte_unused, 3133 struct rte_flow_error *error) 3134 { 3135 return rte_flow_error_set(error, ENOTSUP, 3136 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3137 } 3138 3139 static struct mlx5_flow * 3140 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3141 const struct rte_flow_attr *attr __rte_unused, 3142 const struct rte_flow_item items[] __rte_unused, 3143 const struct rte_flow_action actions[] __rte_unused, 3144 struct rte_flow_error *error) 3145 { 3146 rte_flow_error_set(error, ENOTSUP, 3147 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3148 return NULL; 3149 } 3150 3151 static int 3152 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3153 struct mlx5_flow *dev_flow __rte_unused, 3154 const struct rte_flow_attr *attr __rte_unused, 3155 const struct rte_flow_item items[] __rte_unused, 3156 const struct rte_flow_action actions[] __rte_unused, 3157 struct rte_flow_error *error) 3158 { 3159 return rte_flow_error_set(error, ENOTSUP, 3160 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3161 } 3162 3163 static int 3164 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3165 struct rte_flow *flow __rte_unused, 3166 struct rte_flow_error *error) 3167 { 3168 return rte_flow_error_set(error, ENOTSUP, 3169 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3170 } 3171 3172 static void 3173 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3174 struct rte_flow *flow __rte_unused) 3175 { 3176 } 3177 3178 static void 3179 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3180 struct rte_flow *flow __rte_unused) 3181 { 3182 } 3183 3184 static int 3185 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3186 struct rte_flow *flow __rte_unused, 3187 const struct rte_flow_action *actions __rte_unused, 3188 void *data __rte_unused, 3189 struct rte_flow_error *error) 3190 { 3191 return rte_flow_error_set(error, ENOTSUP, 3192 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3193 } 3194 3195 static int 3196 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3197 uint32_t domains __rte_unused, 3198 uint32_t flags __rte_unused) 3199 { 3200 return 0; 3201 } 3202 3203 /* Void driver to protect from null pointer reference. */ 3204 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3205 .validate = flow_null_validate, 3206 .prepare = flow_null_prepare, 3207 .translate = flow_null_translate, 3208 .apply = flow_null_apply, 3209 .remove = flow_null_remove, 3210 .destroy = flow_null_destroy, 3211 .query = flow_null_query, 3212 .sync_domain = flow_null_sync_domain, 3213 }; 3214 3215 /** 3216 * Select flow driver type according to flow attributes and device 3217 * configuration. 3218 * 3219 * @param[in] dev 3220 * Pointer to the dev structure. 3221 * @param[in] attr 3222 * Pointer to the flow attributes. 3223 * 3224 * @return 3225 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3226 */ 3227 static enum mlx5_flow_drv_type 3228 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3229 { 3230 struct mlx5_priv *priv = dev->data->dev_private; 3231 /* The OS can determine first a specific flow type (DV, VERBS) */ 3232 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3233 3234 if (type != MLX5_FLOW_TYPE_MAX) 3235 return type; 3236 /* If no OS specific type - continue with DV/VERBS selection */ 3237 if (attr->transfer && priv->config.dv_esw_en) 3238 type = MLX5_FLOW_TYPE_DV; 3239 if (!attr->transfer) 3240 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3241 MLX5_FLOW_TYPE_VERBS; 3242 return type; 3243 } 3244 3245 #define flow_get_drv_ops(type) flow_drv_ops[type] 3246 3247 /** 3248 * Flow driver validation API. This abstracts calling driver specific functions. 3249 * The type of flow driver is determined according to flow attributes. 3250 * 3251 * @param[in] dev 3252 * Pointer to the dev structure. 3253 * @param[in] attr 3254 * Pointer to the flow attributes. 3255 * @param[in] items 3256 * Pointer to the list of items. 3257 * @param[in] actions 3258 * Pointer to the list of actions. 3259 * @param[in] external 3260 * This flow rule is created by request external to PMD. 3261 * @param[in] hairpin 3262 * Number of hairpin TX actions, 0 means classic flow. 3263 * @param[out] error 3264 * Pointer to the error structure. 3265 * 3266 * @return 3267 * 0 on success, a negative errno value otherwise and rte_errno is set. 3268 */ 3269 static inline int 3270 flow_drv_validate(struct rte_eth_dev *dev, 3271 const struct rte_flow_attr *attr, 3272 const struct rte_flow_item items[], 3273 const struct rte_flow_action actions[], 3274 bool external, int hairpin, struct rte_flow_error *error) 3275 { 3276 const struct mlx5_flow_driver_ops *fops; 3277 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3278 3279 fops = flow_get_drv_ops(type); 3280 return fops->validate(dev, attr, items, actions, external, 3281 hairpin, error); 3282 } 3283 3284 /** 3285 * Flow driver preparation API. This abstracts calling driver specific 3286 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3287 * calculates the size of memory required for device flow, allocates the memory, 3288 * initializes the device flow and returns the pointer. 3289 * 3290 * @note 3291 * This function initializes device flow structure such as dv or verbs in 3292 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3293 * rest. For example, adding returning device flow to flow->dev_flow list and 3294 * setting backward reference to the flow should be done out of this function. 3295 * layers field is not filled either. 3296 * 3297 * @param[in] dev 3298 * Pointer to the dev structure. 3299 * @param[in] attr 3300 * Pointer to the flow attributes. 3301 * @param[in] items 3302 * Pointer to the list of items. 3303 * @param[in] actions 3304 * Pointer to the list of actions. 3305 * @param[in] flow_idx 3306 * This memory pool index to the flow. 3307 * @param[out] error 3308 * Pointer to the error structure. 3309 * 3310 * @return 3311 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3312 */ 3313 static inline struct mlx5_flow * 3314 flow_drv_prepare(struct rte_eth_dev *dev, 3315 const struct rte_flow *flow, 3316 const struct rte_flow_attr *attr, 3317 const struct rte_flow_item items[], 3318 const struct rte_flow_action actions[], 3319 uint32_t flow_idx, 3320 struct rte_flow_error *error) 3321 { 3322 const struct mlx5_flow_driver_ops *fops; 3323 enum mlx5_flow_drv_type type = flow->drv_type; 3324 struct mlx5_flow *mlx5_flow = NULL; 3325 3326 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3327 fops = flow_get_drv_ops(type); 3328 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3329 if (mlx5_flow) 3330 mlx5_flow->flow_idx = flow_idx; 3331 return mlx5_flow; 3332 } 3333 3334 /** 3335 * Flow driver translation API. This abstracts calling driver specific 3336 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3337 * translates a generic flow into a driver flow. flow_drv_prepare() must 3338 * precede. 3339 * 3340 * @note 3341 * dev_flow->layers could be filled as a result of parsing during translation 3342 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3343 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3344 * flow->actions could be overwritten even though all the expanded dev_flows 3345 * have the same actions. 3346 * 3347 * @param[in] dev 3348 * Pointer to the rte dev structure. 3349 * @param[in, out] dev_flow 3350 * Pointer to the mlx5 flow. 3351 * @param[in] attr 3352 * Pointer to the flow attributes. 3353 * @param[in] items 3354 * Pointer to the list of items. 3355 * @param[in] actions 3356 * Pointer to the list of actions. 3357 * @param[out] error 3358 * Pointer to the error structure. 3359 * 3360 * @return 3361 * 0 on success, a negative errno value otherwise and rte_errno is set. 3362 */ 3363 static inline int 3364 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3365 const struct rte_flow_attr *attr, 3366 const struct rte_flow_item items[], 3367 const struct rte_flow_action actions[], 3368 struct rte_flow_error *error) 3369 { 3370 const struct mlx5_flow_driver_ops *fops; 3371 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3372 3373 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3374 fops = flow_get_drv_ops(type); 3375 return fops->translate(dev, dev_flow, attr, items, actions, error); 3376 } 3377 3378 /** 3379 * Flow driver apply API. This abstracts calling driver specific functions. 3380 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3381 * translated driver flows on to device. flow_drv_translate() must precede. 3382 * 3383 * @param[in] dev 3384 * Pointer to Ethernet device structure. 3385 * @param[in, out] flow 3386 * Pointer to flow structure. 3387 * @param[out] error 3388 * Pointer to error structure. 3389 * 3390 * @return 3391 * 0 on success, a negative errno value otherwise and rte_errno is set. 3392 */ 3393 static inline int 3394 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3395 struct rte_flow_error *error) 3396 { 3397 const struct mlx5_flow_driver_ops *fops; 3398 enum mlx5_flow_drv_type type = flow->drv_type; 3399 3400 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3401 fops = flow_get_drv_ops(type); 3402 return fops->apply(dev, flow, error); 3403 } 3404 3405 /** 3406 * Flow driver destroy API. This abstracts calling driver specific functions. 3407 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3408 * on device and releases resources of the flow. 3409 * 3410 * @param[in] dev 3411 * Pointer to Ethernet device. 3412 * @param[in, out] flow 3413 * Pointer to flow structure. 3414 */ 3415 static inline void 3416 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3417 { 3418 const struct mlx5_flow_driver_ops *fops; 3419 enum mlx5_flow_drv_type type = flow->drv_type; 3420 3421 flow_mreg_split_qrss_release(dev, flow); 3422 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3423 fops = flow_get_drv_ops(type); 3424 fops->destroy(dev, flow); 3425 } 3426 3427 /** 3428 * Flow driver find RSS policy tbl API. This abstracts calling driver 3429 * specific functions. Parent flow (rte_flow) should have driver 3430 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3431 * 3432 * @param[in] dev 3433 * Pointer to Ethernet device. 3434 * @param[in, out] flow 3435 * Pointer to flow structure. 3436 * @param[in] policy 3437 * Pointer to meter policy table. 3438 * @param[in] rss_desc 3439 * Pointer to rss_desc 3440 */ 3441 static struct mlx5_flow_meter_sub_policy * 3442 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3443 struct rte_flow *flow, 3444 struct mlx5_flow_meter_policy *policy, 3445 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3446 { 3447 const struct mlx5_flow_driver_ops *fops; 3448 enum mlx5_flow_drv_type type = flow->drv_type; 3449 3450 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3451 fops = flow_get_drv_ops(type); 3452 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3453 } 3454 3455 /** 3456 * Get RSS action from the action list. 3457 * 3458 * @param[in] dev 3459 * Pointer to Ethernet device. 3460 * @param[in] actions 3461 * Pointer to the list of actions. 3462 * @param[in] flow 3463 * Parent flow structure pointer. 3464 * 3465 * @return 3466 * Pointer to the RSS action if exist, else return NULL. 3467 */ 3468 static const struct rte_flow_action_rss* 3469 flow_get_rss_action(struct rte_eth_dev *dev, 3470 const struct rte_flow_action actions[]) 3471 { 3472 struct mlx5_priv *priv = dev->data->dev_private; 3473 const struct rte_flow_action_rss *rss = NULL; 3474 3475 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3476 switch (actions->type) { 3477 case RTE_FLOW_ACTION_TYPE_RSS: 3478 rss = actions->conf; 3479 break; 3480 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3481 { 3482 const struct rte_flow_action_sample *sample = 3483 actions->conf; 3484 const struct rte_flow_action *act = sample->actions; 3485 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3486 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3487 rss = act->conf; 3488 break; 3489 } 3490 case RTE_FLOW_ACTION_TYPE_METER: 3491 { 3492 uint32_t mtr_idx; 3493 struct mlx5_flow_meter_info *fm; 3494 struct mlx5_flow_meter_policy *policy; 3495 const struct rte_flow_action_meter *mtr = actions->conf; 3496 3497 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3498 if (fm) { 3499 policy = mlx5_flow_meter_policy_find(dev, 3500 fm->policy_id, NULL); 3501 if (policy && policy->is_rss) 3502 rss = 3503 policy->act_cnt[RTE_COLOR_GREEN].rss->conf; 3504 } 3505 break; 3506 } 3507 default: 3508 break; 3509 } 3510 } 3511 return rss; 3512 } 3513 3514 /** 3515 * Get ASO age action by index. 3516 * 3517 * @param[in] dev 3518 * Pointer to the Ethernet device structure. 3519 * @param[in] age_idx 3520 * Index to the ASO age action. 3521 * 3522 * @return 3523 * The specified ASO age action. 3524 */ 3525 struct mlx5_aso_age_action* 3526 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3527 { 3528 uint16_t pool_idx = age_idx & UINT16_MAX; 3529 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3530 struct mlx5_priv *priv = dev->data->dev_private; 3531 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3532 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3533 3534 return &pool->actions[offset - 1]; 3535 } 3536 3537 /* maps indirect action to translated direct in some actions array */ 3538 struct mlx5_translated_action_handle { 3539 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3540 int index; /**< Index in related array of rte_flow_action. */ 3541 }; 3542 3543 /** 3544 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3545 * direct action if translation possible. 3546 * This functionality used to run same execution path for both direct and 3547 * indirect actions on flow create. All necessary preparations for indirect 3548 * action handling should be performed on *handle* actions list returned 3549 * from this call. 3550 * 3551 * @param[in] dev 3552 * Pointer to Ethernet device. 3553 * @param[in] actions 3554 * List of actions to translate. 3555 * @param[out] handle 3556 * List to store translated indirect action object handles. 3557 * @param[in, out] indir_n 3558 * Size of *handle* array. On return should be updated with number of 3559 * indirect actions retrieved from the *actions* list. 3560 * @param[out] translated_actions 3561 * List of actions where all indirect actions were translated to direct 3562 * if possible. NULL if no translation took place. 3563 * @param[out] error 3564 * Pointer to the error structure. 3565 * 3566 * @return 3567 * 0 on success, a negative errno value otherwise and rte_errno is set. 3568 */ 3569 static int 3570 flow_action_handles_translate(struct rte_eth_dev *dev, 3571 const struct rte_flow_action actions[], 3572 struct mlx5_translated_action_handle *handle, 3573 int *indir_n, 3574 struct rte_flow_action **translated_actions, 3575 struct rte_flow_error *error) 3576 { 3577 struct mlx5_priv *priv = dev->data->dev_private; 3578 struct rte_flow_action *translated = NULL; 3579 size_t actions_size; 3580 int n; 3581 int copied_n = 0; 3582 struct mlx5_translated_action_handle *handle_end = NULL; 3583 3584 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3585 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3586 continue; 3587 if (copied_n == *indir_n) { 3588 return rte_flow_error_set 3589 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3590 NULL, "too many shared actions"); 3591 } 3592 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3593 sizeof(actions[n].conf)); 3594 handle[copied_n].index = n; 3595 copied_n++; 3596 } 3597 n++; 3598 *indir_n = copied_n; 3599 if (!copied_n) 3600 return 0; 3601 actions_size = sizeof(struct rte_flow_action) * n; 3602 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3603 if (!translated) { 3604 rte_errno = ENOMEM; 3605 return -ENOMEM; 3606 } 3607 memcpy(translated, actions, actions_size); 3608 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3609 struct mlx5_shared_action_rss *shared_rss; 3610 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3611 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3612 uint32_t idx = act_idx & 3613 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3614 3615 switch (type) { 3616 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3617 shared_rss = mlx5_ipool_get 3618 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3619 translated[handle->index].type = 3620 RTE_FLOW_ACTION_TYPE_RSS; 3621 translated[handle->index].conf = 3622 &shared_rss->origin; 3623 break; 3624 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3625 translated[handle->index].type = 3626 (enum rte_flow_action_type) 3627 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3628 translated[handle->index].conf = (void *)(uintptr_t)idx; 3629 break; 3630 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3631 if (priv->sh->flow_hit_aso_en) { 3632 translated[handle->index].type = 3633 (enum rte_flow_action_type) 3634 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3635 translated[handle->index].conf = 3636 (void *)(uintptr_t)idx; 3637 break; 3638 } 3639 /* Fall-through */ 3640 case MLX5_INDIRECT_ACTION_TYPE_CT: 3641 if (priv->sh->ct_aso_en) { 3642 translated[handle->index].type = 3643 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3644 translated[handle->index].conf = 3645 (void *)(uintptr_t)idx; 3646 break; 3647 } 3648 /* Fall-through */ 3649 default: 3650 mlx5_free(translated); 3651 return rte_flow_error_set 3652 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3653 NULL, "invalid indirect action type"); 3654 } 3655 } 3656 *translated_actions = translated; 3657 return 0; 3658 } 3659 3660 /** 3661 * Get Shared RSS action from the action list. 3662 * 3663 * @param[in] dev 3664 * Pointer to Ethernet device. 3665 * @param[in] shared 3666 * Pointer to the list of actions. 3667 * @param[in] shared_n 3668 * Actions list length. 3669 * 3670 * @return 3671 * The MLX5 RSS action ID if exists, otherwise return 0. 3672 */ 3673 static uint32_t 3674 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3675 struct mlx5_translated_action_handle *handle, 3676 int shared_n) 3677 { 3678 struct mlx5_translated_action_handle *handle_end; 3679 struct mlx5_priv *priv = dev->data->dev_private; 3680 struct mlx5_shared_action_rss *shared_rss; 3681 3682 3683 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3684 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3685 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3686 uint32_t idx = act_idx & 3687 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3688 switch (type) { 3689 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3690 shared_rss = mlx5_ipool_get 3691 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3692 idx); 3693 __atomic_add_fetch(&shared_rss->refcnt, 1, 3694 __ATOMIC_RELAXED); 3695 return idx; 3696 default: 3697 break; 3698 } 3699 } 3700 return 0; 3701 } 3702 3703 static unsigned int 3704 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3705 { 3706 const struct rte_flow_item *item; 3707 unsigned int has_vlan = 0; 3708 3709 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3710 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3711 has_vlan = 1; 3712 break; 3713 } 3714 } 3715 if (has_vlan) 3716 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3717 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3718 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3719 MLX5_EXPANSION_ROOT_OUTER; 3720 } 3721 3722 /** 3723 * Get layer flags from the prefix flow. 3724 * 3725 * Some flows may be split to several subflows, the prefix subflow gets the 3726 * match items and the suffix sub flow gets the actions. 3727 * Some actions need the user defined match item flags to get the detail for 3728 * the action. 3729 * This function helps the suffix flow to get the item layer flags from prefix 3730 * subflow. 3731 * 3732 * @param[in] dev_flow 3733 * Pointer the created preifx subflow. 3734 * 3735 * @return 3736 * The layers get from prefix subflow. 3737 */ 3738 static inline uint64_t 3739 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3740 { 3741 uint64_t layers = 0; 3742 3743 /* 3744 * Layers bits could be localization, but usually the compiler will 3745 * help to do the optimization work for source code. 3746 * If no decap actions, use the layers directly. 3747 */ 3748 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3749 return dev_flow->handle->layers; 3750 /* Convert L3 layers with decap action. */ 3751 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3752 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3753 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3754 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3755 /* Convert L4 layers with decap action. */ 3756 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3757 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3758 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3759 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3760 return layers; 3761 } 3762 3763 /** 3764 * Get metadata split action information. 3765 * 3766 * @param[in] actions 3767 * Pointer to the list of actions. 3768 * @param[out] qrss 3769 * Pointer to the return pointer. 3770 * @param[out] qrss_type 3771 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3772 * if no QUEUE/RSS is found. 3773 * @param[out] encap_idx 3774 * Pointer to the index of the encap action if exists, otherwise the last 3775 * action index. 3776 * 3777 * @return 3778 * Total number of actions. 3779 */ 3780 static int 3781 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3782 const struct rte_flow_action **qrss, 3783 int *encap_idx) 3784 { 3785 const struct rte_flow_action_raw_encap *raw_encap; 3786 int actions_n = 0; 3787 int raw_decap_idx = -1; 3788 3789 *encap_idx = -1; 3790 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3791 switch (actions->type) { 3792 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3793 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3794 *encap_idx = actions_n; 3795 break; 3796 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3797 raw_decap_idx = actions_n; 3798 break; 3799 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3800 raw_encap = actions->conf; 3801 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3802 *encap_idx = raw_decap_idx != -1 ? 3803 raw_decap_idx : actions_n; 3804 break; 3805 case RTE_FLOW_ACTION_TYPE_QUEUE: 3806 case RTE_FLOW_ACTION_TYPE_RSS: 3807 *qrss = actions; 3808 break; 3809 default: 3810 break; 3811 } 3812 actions_n++; 3813 } 3814 if (*encap_idx == -1) 3815 *encap_idx = actions_n; 3816 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3817 return actions_n + 1; 3818 } 3819 3820 /** 3821 * Check if the action will change packet. 3822 * 3823 * @param dev 3824 * Pointer to Ethernet device. 3825 * @param[in] type 3826 * action type. 3827 * 3828 * @return 3829 * true if action will change packet, false otherwise. 3830 */ 3831 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 3832 enum rte_flow_action_type type) 3833 { 3834 struct mlx5_priv *priv = dev->data->dev_private; 3835 3836 switch (type) { 3837 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3838 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3839 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3840 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3841 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3842 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3843 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3844 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3845 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3846 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3847 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3848 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3849 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3850 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3851 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 3852 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 3853 case RTE_FLOW_ACTION_TYPE_SET_META: 3854 case RTE_FLOW_ACTION_TYPE_SET_TAG: 3855 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 3856 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3857 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3858 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3859 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3860 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3861 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3862 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3863 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3864 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3865 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 3866 return true; 3867 case RTE_FLOW_ACTION_TYPE_FLAG: 3868 case RTE_FLOW_ACTION_TYPE_MARK: 3869 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 3870 return true; 3871 else 3872 return false; 3873 default: 3874 return false; 3875 } 3876 } 3877 3878 /** 3879 * Check meter action from the action list. 3880 * 3881 * @param dev 3882 * Pointer to Ethernet device. 3883 * @param[in] actions 3884 * Pointer to the list of actions. 3885 * @param[out] has_mtr 3886 * Pointer to the meter exist flag. 3887 * @param[out] has_modify 3888 * Pointer to the flag showing there's packet change action. 3889 * @param[out] meter_id 3890 * Pointer to the meter id. 3891 * 3892 * @return 3893 * Total number of actions. 3894 */ 3895 static int 3896 flow_check_meter_action(struct rte_eth_dev *dev, 3897 const struct rte_flow_action actions[], 3898 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 3899 { 3900 const struct rte_flow_action_meter *mtr = NULL; 3901 int actions_n = 0; 3902 3903 MLX5_ASSERT(has_mtr); 3904 *has_mtr = false; 3905 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3906 switch (actions->type) { 3907 case RTE_FLOW_ACTION_TYPE_METER: 3908 mtr = actions->conf; 3909 *meter_id = mtr->mtr_id; 3910 *has_mtr = true; 3911 break; 3912 default: 3913 break; 3914 } 3915 if (!*has_mtr) 3916 *has_modify |= flow_check_modify_action_type(dev, 3917 actions->type); 3918 actions_n++; 3919 } 3920 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3921 return actions_n + 1; 3922 } 3923 3924 /** 3925 * Check if the flow should be split due to hairpin. 3926 * The reason for the split is that in current HW we can't 3927 * support encap and push-vlan on Rx, so if a flow contains 3928 * these actions we move it to Tx. 3929 * 3930 * @param dev 3931 * Pointer to Ethernet device. 3932 * @param[in] attr 3933 * Flow rule attributes. 3934 * @param[in] actions 3935 * Associated actions (list terminated by the END action). 3936 * 3937 * @return 3938 * > 0 the number of actions and the flow should be split, 3939 * 0 when no split required. 3940 */ 3941 static int 3942 flow_check_hairpin_split(struct rte_eth_dev *dev, 3943 const struct rte_flow_attr *attr, 3944 const struct rte_flow_action actions[]) 3945 { 3946 int queue_action = 0; 3947 int action_n = 0; 3948 int split = 0; 3949 const struct rte_flow_action_queue *queue; 3950 const struct rte_flow_action_rss *rss; 3951 const struct rte_flow_action_raw_encap *raw_encap; 3952 const struct rte_eth_hairpin_conf *conf; 3953 3954 if (!attr->ingress) 3955 return 0; 3956 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3957 switch (actions->type) { 3958 case RTE_FLOW_ACTION_TYPE_QUEUE: 3959 queue = actions->conf; 3960 if (queue == NULL) 3961 return 0; 3962 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3963 if (conf == NULL || conf->tx_explicit != 0) 3964 return 0; 3965 queue_action = 1; 3966 action_n++; 3967 break; 3968 case RTE_FLOW_ACTION_TYPE_RSS: 3969 rss = actions->conf; 3970 if (rss == NULL || rss->queue_num == 0) 3971 return 0; 3972 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3973 if (conf == NULL || conf->tx_explicit != 0) 3974 return 0; 3975 queue_action = 1; 3976 action_n++; 3977 break; 3978 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3979 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3980 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3981 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3982 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3983 split++; 3984 action_n++; 3985 break; 3986 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3987 raw_encap = actions->conf; 3988 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3989 split++; 3990 action_n++; 3991 break; 3992 default: 3993 action_n++; 3994 break; 3995 } 3996 } 3997 if (split && queue_action) 3998 return action_n; 3999 return 0; 4000 } 4001 4002 /* Declare flow create/destroy prototype in advance. */ 4003 static uint32_t 4004 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 4005 const struct rte_flow_attr *attr, 4006 const struct rte_flow_item items[], 4007 const struct rte_flow_action actions[], 4008 bool external, struct rte_flow_error *error); 4009 4010 static void 4011 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 4012 uint32_t flow_idx); 4013 4014 int 4015 flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused, 4016 struct mlx5_hlist_entry *entry, 4017 uint64_t key, void *cb_ctx __rte_unused) 4018 { 4019 struct mlx5_flow_mreg_copy_resource *mcp_res = 4020 container_of(entry, typeof(*mcp_res), hlist_ent); 4021 4022 return mcp_res->mark_id != key; 4023 } 4024 4025 struct mlx5_hlist_entry * 4026 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 4027 void *cb_ctx) 4028 { 4029 struct rte_eth_dev *dev = list->ctx; 4030 struct mlx5_priv *priv = dev->data->dev_private; 4031 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4032 struct mlx5_flow_mreg_copy_resource *mcp_res; 4033 struct rte_flow_error *error = ctx->error; 4034 uint32_t idx = 0; 4035 int ret; 4036 uint32_t mark_id = key; 4037 struct rte_flow_attr attr = { 4038 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4039 .ingress = 1, 4040 }; 4041 struct mlx5_rte_flow_item_tag tag_spec = { 4042 .data = mark_id, 4043 }; 4044 struct rte_flow_item items[] = { 4045 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4046 }; 4047 struct rte_flow_action_mark ftag = { 4048 .id = mark_id, 4049 }; 4050 struct mlx5_flow_action_copy_mreg cp_mreg = { 4051 .dst = REG_B, 4052 .src = REG_NON, 4053 }; 4054 struct rte_flow_action_jump jump = { 4055 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4056 }; 4057 struct rte_flow_action actions[] = { 4058 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4059 }; 4060 4061 /* Fill the register fileds in the flow. */ 4062 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4063 if (ret < 0) 4064 return NULL; 4065 tag_spec.id = ret; 4066 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4067 if (ret < 0) 4068 return NULL; 4069 cp_mreg.src = ret; 4070 /* Provide the full width of FLAG specific value. */ 4071 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4072 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4073 /* Build a new flow. */ 4074 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4075 items[0] = (struct rte_flow_item){ 4076 .type = (enum rte_flow_item_type) 4077 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4078 .spec = &tag_spec, 4079 }; 4080 items[1] = (struct rte_flow_item){ 4081 .type = RTE_FLOW_ITEM_TYPE_END, 4082 }; 4083 actions[0] = (struct rte_flow_action){ 4084 .type = (enum rte_flow_action_type) 4085 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4086 .conf = &ftag, 4087 }; 4088 actions[1] = (struct rte_flow_action){ 4089 .type = (enum rte_flow_action_type) 4090 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4091 .conf = &cp_mreg, 4092 }; 4093 actions[2] = (struct rte_flow_action){ 4094 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4095 .conf = &jump, 4096 }; 4097 actions[3] = (struct rte_flow_action){ 4098 .type = RTE_FLOW_ACTION_TYPE_END, 4099 }; 4100 } else { 4101 /* Default rule, wildcard match. */ 4102 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4103 items[0] = (struct rte_flow_item){ 4104 .type = RTE_FLOW_ITEM_TYPE_END, 4105 }; 4106 actions[0] = (struct rte_flow_action){ 4107 .type = (enum rte_flow_action_type) 4108 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4109 .conf = &cp_mreg, 4110 }; 4111 actions[1] = (struct rte_flow_action){ 4112 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4113 .conf = &jump, 4114 }; 4115 actions[2] = (struct rte_flow_action){ 4116 .type = RTE_FLOW_ACTION_TYPE_END, 4117 }; 4118 } 4119 /* Build a new entry. */ 4120 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4121 if (!mcp_res) { 4122 rte_errno = ENOMEM; 4123 return NULL; 4124 } 4125 mcp_res->idx = idx; 4126 mcp_res->mark_id = mark_id; 4127 /* 4128 * The copy Flows are not included in any list. There 4129 * ones are referenced from other Flows and can not 4130 * be applied, removed, deleted in ardbitrary order 4131 * by list traversing. 4132 */ 4133 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 4134 actions, false, error); 4135 if (!mcp_res->rix_flow) { 4136 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4137 return NULL; 4138 } 4139 return &mcp_res->hlist_ent; 4140 } 4141 4142 /** 4143 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4144 * 4145 * As mark_id is unique, if there's already a registered flow for the mark_id, 4146 * return by increasing the reference counter of the resource. Otherwise, create 4147 * the resource (mcp_res) and flow. 4148 * 4149 * Flow looks like, 4150 * - If ingress port is ANY and reg_c[1] is mark_id, 4151 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4152 * 4153 * For default flow (zero mark_id), flow is like, 4154 * - If ingress port is ANY, 4155 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4156 * 4157 * @param dev 4158 * Pointer to Ethernet device. 4159 * @param mark_id 4160 * ID of MARK action, zero means default flow for META. 4161 * @param[out] error 4162 * Perform verbose error reporting if not NULL. 4163 * 4164 * @return 4165 * Associated resource on success, NULL otherwise and rte_errno is set. 4166 */ 4167 static struct mlx5_flow_mreg_copy_resource * 4168 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4169 struct rte_flow_error *error) 4170 { 4171 struct mlx5_priv *priv = dev->data->dev_private; 4172 struct mlx5_hlist_entry *entry; 4173 struct mlx5_flow_cb_ctx ctx = { 4174 .dev = dev, 4175 .error = error, 4176 }; 4177 4178 /* Check if already registered. */ 4179 MLX5_ASSERT(priv->mreg_cp_tbl); 4180 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4181 if (!entry) 4182 return NULL; 4183 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4184 hlist_ent); 4185 } 4186 4187 void 4188 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 4189 { 4190 struct mlx5_flow_mreg_copy_resource *mcp_res = 4191 container_of(entry, typeof(*mcp_res), hlist_ent); 4192 struct rte_eth_dev *dev = list->ctx; 4193 struct mlx5_priv *priv = dev->data->dev_private; 4194 4195 MLX5_ASSERT(mcp_res->rix_flow); 4196 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 4197 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4198 } 4199 4200 /** 4201 * Release flow in RX_CP_TBL. 4202 * 4203 * @param dev 4204 * Pointer to Ethernet device. 4205 * @flow 4206 * Parent flow for wich copying is provided. 4207 */ 4208 static void 4209 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4210 struct rte_flow *flow) 4211 { 4212 struct mlx5_flow_mreg_copy_resource *mcp_res; 4213 struct mlx5_priv *priv = dev->data->dev_private; 4214 4215 if (!flow->rix_mreg_copy) 4216 return; 4217 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4218 flow->rix_mreg_copy); 4219 if (!mcp_res || !priv->mreg_cp_tbl) 4220 return; 4221 MLX5_ASSERT(mcp_res->rix_flow); 4222 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4223 flow->rix_mreg_copy = 0; 4224 } 4225 4226 /** 4227 * Remove the default copy action from RX_CP_TBL. 4228 * 4229 * This functions is called in the mlx5_dev_start(). No thread safe 4230 * is guaranteed. 4231 * 4232 * @param dev 4233 * Pointer to Ethernet device. 4234 */ 4235 static void 4236 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4237 { 4238 struct mlx5_hlist_entry *entry; 4239 struct mlx5_priv *priv = dev->data->dev_private; 4240 4241 /* Check if default flow is registered. */ 4242 if (!priv->mreg_cp_tbl) 4243 return; 4244 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 4245 MLX5_DEFAULT_COPY_ID, NULL); 4246 if (!entry) 4247 return; 4248 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4249 } 4250 4251 /** 4252 * Add the default copy action in in RX_CP_TBL. 4253 * 4254 * This functions is called in the mlx5_dev_start(). No thread safe 4255 * is guaranteed. 4256 * 4257 * @param dev 4258 * Pointer to Ethernet device. 4259 * @param[out] error 4260 * Perform verbose error reporting if not NULL. 4261 * 4262 * @return 4263 * 0 for success, negative value otherwise and rte_errno is set. 4264 */ 4265 static int 4266 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4267 struct rte_flow_error *error) 4268 { 4269 struct mlx5_priv *priv = dev->data->dev_private; 4270 struct mlx5_flow_mreg_copy_resource *mcp_res; 4271 4272 /* Check whether extensive metadata feature is engaged. */ 4273 if (!priv->config.dv_flow_en || 4274 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4275 !mlx5_flow_ext_mreg_supported(dev) || 4276 !priv->sh->dv_regc0_mask) 4277 return 0; 4278 /* 4279 * Add default mreg copy flow may be called multiple time, but 4280 * only be called once in stop. Avoid register it twice. 4281 */ 4282 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 4283 return 0; 4284 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 4285 if (!mcp_res) 4286 return -rte_errno; 4287 return 0; 4288 } 4289 4290 /** 4291 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4292 * 4293 * All the flow having Q/RSS action should be split by 4294 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4295 * performs the following, 4296 * - CQE->flow_tag := reg_c[1] (MARK) 4297 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4298 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4299 * but there should be a flow per each MARK ID set by MARK action. 4300 * 4301 * For the aforementioned reason, if there's a MARK action in flow's action 4302 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4303 * the MARK ID to CQE's flow_tag like, 4304 * - If reg_c[1] is mark_id, 4305 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4306 * 4307 * For SET_META action which stores value in reg_c[0], as the destination is 4308 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4309 * MARK ID means the default flow. The default flow looks like, 4310 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4311 * 4312 * @param dev 4313 * Pointer to Ethernet device. 4314 * @param flow 4315 * Pointer to flow structure. 4316 * @param[in] actions 4317 * Pointer to the list of actions. 4318 * @param[out] error 4319 * Perform verbose error reporting if not NULL. 4320 * 4321 * @return 4322 * 0 on success, negative value otherwise and rte_errno is set. 4323 */ 4324 static int 4325 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4326 struct rte_flow *flow, 4327 const struct rte_flow_action *actions, 4328 struct rte_flow_error *error) 4329 { 4330 struct mlx5_priv *priv = dev->data->dev_private; 4331 struct mlx5_dev_config *config = &priv->config; 4332 struct mlx5_flow_mreg_copy_resource *mcp_res; 4333 const struct rte_flow_action_mark *mark; 4334 4335 /* Check whether extensive metadata feature is engaged. */ 4336 if (!config->dv_flow_en || 4337 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4338 !mlx5_flow_ext_mreg_supported(dev) || 4339 !priv->sh->dv_regc0_mask) 4340 return 0; 4341 /* Find MARK action. */ 4342 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4343 switch (actions->type) { 4344 case RTE_FLOW_ACTION_TYPE_FLAG: 4345 mcp_res = flow_mreg_add_copy_action 4346 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4347 if (!mcp_res) 4348 return -rte_errno; 4349 flow->rix_mreg_copy = mcp_res->idx; 4350 return 0; 4351 case RTE_FLOW_ACTION_TYPE_MARK: 4352 mark = (const struct rte_flow_action_mark *) 4353 actions->conf; 4354 mcp_res = 4355 flow_mreg_add_copy_action(dev, mark->id, error); 4356 if (!mcp_res) 4357 return -rte_errno; 4358 flow->rix_mreg_copy = mcp_res->idx; 4359 return 0; 4360 default: 4361 break; 4362 } 4363 } 4364 return 0; 4365 } 4366 4367 #define MLX5_MAX_SPLIT_ACTIONS 24 4368 #define MLX5_MAX_SPLIT_ITEMS 24 4369 4370 /** 4371 * Split the hairpin flow. 4372 * Since HW can't support encap and push-vlan on Rx, we move these 4373 * actions to Tx. 4374 * If the count action is after the encap then we also 4375 * move the count action. in this case the count will also measure 4376 * the outer bytes. 4377 * 4378 * @param dev 4379 * Pointer to Ethernet device. 4380 * @param[in] actions 4381 * Associated actions (list terminated by the END action). 4382 * @param[out] actions_rx 4383 * Rx flow actions. 4384 * @param[out] actions_tx 4385 * Tx flow actions.. 4386 * @param[out] pattern_tx 4387 * The pattern items for the Tx flow. 4388 * @param[out] flow_id 4389 * The flow ID connected to this flow. 4390 * 4391 * @return 4392 * 0 on success. 4393 */ 4394 static int 4395 flow_hairpin_split(struct rte_eth_dev *dev, 4396 const struct rte_flow_action actions[], 4397 struct rte_flow_action actions_rx[], 4398 struct rte_flow_action actions_tx[], 4399 struct rte_flow_item pattern_tx[], 4400 uint32_t flow_id) 4401 { 4402 const struct rte_flow_action_raw_encap *raw_encap; 4403 const struct rte_flow_action_raw_decap *raw_decap; 4404 struct mlx5_rte_flow_action_set_tag *set_tag; 4405 struct rte_flow_action *tag_action; 4406 struct mlx5_rte_flow_item_tag *tag_item; 4407 struct rte_flow_item *item; 4408 char *addr; 4409 int encap = 0; 4410 4411 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4412 switch (actions->type) { 4413 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4414 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4415 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4416 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4417 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4418 rte_memcpy(actions_tx, actions, 4419 sizeof(struct rte_flow_action)); 4420 actions_tx++; 4421 break; 4422 case RTE_FLOW_ACTION_TYPE_COUNT: 4423 if (encap) { 4424 rte_memcpy(actions_tx, actions, 4425 sizeof(struct rte_flow_action)); 4426 actions_tx++; 4427 } else { 4428 rte_memcpy(actions_rx, actions, 4429 sizeof(struct rte_flow_action)); 4430 actions_rx++; 4431 } 4432 break; 4433 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4434 raw_encap = actions->conf; 4435 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4436 memcpy(actions_tx, actions, 4437 sizeof(struct rte_flow_action)); 4438 actions_tx++; 4439 encap = 1; 4440 } else { 4441 rte_memcpy(actions_rx, actions, 4442 sizeof(struct rte_flow_action)); 4443 actions_rx++; 4444 } 4445 break; 4446 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4447 raw_decap = actions->conf; 4448 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4449 memcpy(actions_tx, actions, 4450 sizeof(struct rte_flow_action)); 4451 actions_tx++; 4452 } else { 4453 rte_memcpy(actions_rx, actions, 4454 sizeof(struct rte_flow_action)); 4455 actions_rx++; 4456 } 4457 break; 4458 default: 4459 rte_memcpy(actions_rx, actions, 4460 sizeof(struct rte_flow_action)); 4461 actions_rx++; 4462 break; 4463 } 4464 } 4465 /* Add set meta action and end action for the Rx flow. */ 4466 tag_action = actions_rx; 4467 tag_action->type = (enum rte_flow_action_type) 4468 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4469 actions_rx++; 4470 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4471 actions_rx++; 4472 set_tag = (void *)actions_rx; 4473 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4474 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4475 .data = flow_id, 4476 }; 4477 MLX5_ASSERT(set_tag->id > REG_NON); 4478 tag_action->conf = set_tag; 4479 /* Create Tx item list. */ 4480 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4481 addr = (void *)&pattern_tx[2]; 4482 item = pattern_tx; 4483 item->type = (enum rte_flow_item_type) 4484 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4485 tag_item = (void *)addr; 4486 tag_item->data = flow_id; 4487 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4488 MLX5_ASSERT(set_tag->id > REG_NON); 4489 item->spec = tag_item; 4490 addr += sizeof(struct mlx5_rte_flow_item_tag); 4491 tag_item = (void *)addr; 4492 tag_item->data = UINT32_MAX; 4493 tag_item->id = UINT16_MAX; 4494 item->mask = tag_item; 4495 item->last = NULL; 4496 item++; 4497 item->type = RTE_FLOW_ITEM_TYPE_END; 4498 return 0; 4499 } 4500 4501 /** 4502 * The last stage of splitting chain, just creates the subflow 4503 * without any modification. 4504 * 4505 * @param[in] dev 4506 * Pointer to Ethernet device. 4507 * @param[in] flow 4508 * Parent flow structure pointer. 4509 * @param[in, out] sub_flow 4510 * Pointer to return the created subflow, may be NULL. 4511 * @param[in] attr 4512 * Flow rule attributes. 4513 * @param[in] items 4514 * Pattern specification (list terminated by the END pattern item). 4515 * @param[in] actions 4516 * Associated actions (list terminated by the END action). 4517 * @param[in] flow_split_info 4518 * Pointer to flow split info structure. 4519 * @param[out] error 4520 * Perform verbose error reporting if not NULL. 4521 * @return 4522 * 0 on success, negative value otherwise 4523 */ 4524 static int 4525 flow_create_split_inner(struct rte_eth_dev *dev, 4526 struct rte_flow *flow, 4527 struct mlx5_flow **sub_flow, 4528 const struct rte_flow_attr *attr, 4529 const struct rte_flow_item items[], 4530 const struct rte_flow_action actions[], 4531 struct mlx5_flow_split_info *flow_split_info, 4532 struct rte_flow_error *error) 4533 { 4534 struct mlx5_flow *dev_flow; 4535 4536 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4537 flow_split_info->flow_idx, error); 4538 if (!dev_flow) 4539 return -rte_errno; 4540 dev_flow->flow = flow; 4541 dev_flow->external = flow_split_info->external; 4542 dev_flow->skip_scale = flow_split_info->skip_scale; 4543 /* Subflow object was created, we must include one in the list. */ 4544 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4545 dev_flow->handle, next); 4546 /* 4547 * If dev_flow is as one of the suffix flow, some actions in suffix 4548 * flow may need some user defined item layer flags, and pass the 4549 * Metadate rxq mark flag to suffix flow as well. 4550 */ 4551 if (flow_split_info->prefix_layers) 4552 dev_flow->handle->layers = flow_split_info->prefix_layers; 4553 if (flow_split_info->prefix_mark) 4554 dev_flow->handle->mark = 1; 4555 if (sub_flow) 4556 *sub_flow = dev_flow; 4557 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4558 dev_flow->dv.table_id = flow_split_info->table_id; 4559 #endif 4560 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4561 } 4562 4563 /** 4564 * Get the sub policy of a meter. 4565 * 4566 * @param[in] dev 4567 * Pointer to Ethernet device. 4568 * @param[in] flow 4569 * Parent flow structure pointer. 4570 * @param[in] policy_id; 4571 * Meter Policy id. 4572 * @param[in] attr 4573 * Flow rule attributes. 4574 * @param[in] items 4575 * Pattern specification (list terminated by the END pattern item). 4576 * @param[out] error 4577 * Perform verbose error reporting if not NULL. 4578 * 4579 * @return 4580 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4581 */ 4582 static struct mlx5_flow_meter_sub_policy * 4583 get_meter_sub_policy(struct rte_eth_dev *dev, 4584 struct rte_flow *flow, 4585 uint32_t policy_id, 4586 const struct rte_flow_attr *attr, 4587 const struct rte_flow_item items[], 4588 struct rte_flow_error *error) 4589 { 4590 struct mlx5_flow_meter_policy *policy; 4591 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4592 4593 policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 4594 if (!policy) { 4595 rte_flow_error_set(error, EINVAL, 4596 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4597 "Failed to find Meter Policy."); 4598 goto exit; 4599 } 4600 if (policy->is_rss || 4601 (policy->is_queue && 4602 !policy->sub_policys[MLX5_MTR_DOMAIN_INGRESS][0]->rix_hrxq[0])) { 4603 struct mlx5_flow_workspace *wks = 4604 mlx5_flow_get_thread_workspace(); 4605 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4606 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4607 uint32_t i; 4608 4609 MLX5_ASSERT(wks); 4610 /** 4611 * This is a tmp dev_flow, 4612 * no need to register any matcher for it in translate. 4613 */ 4614 wks->skip_matcher_reg = 1; 4615 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4616 struct mlx5_flow dev_flow = {0}; 4617 struct mlx5_flow_handle dev_handle = { {0} }; 4618 4619 if (policy->is_rss) { 4620 const void *rss_act = 4621 policy->act_cnt[i].rss->conf; 4622 struct rte_flow_action rss_actions[2] = { 4623 [0] = { 4624 .type = RTE_FLOW_ACTION_TYPE_RSS, 4625 .conf = rss_act 4626 }, 4627 [1] = { 4628 .type = RTE_FLOW_ACTION_TYPE_END, 4629 .conf = NULL 4630 } 4631 }; 4632 4633 dev_flow.handle = &dev_handle; 4634 dev_flow.ingress = attr->ingress; 4635 dev_flow.flow = flow; 4636 dev_flow.external = 0; 4637 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4638 dev_flow.dv.transfer = attr->transfer; 4639 #endif 4640 /** 4641 * Translate RSS action to get rss hash fields. 4642 */ 4643 if (flow_drv_translate(dev, &dev_flow, attr, 4644 items, rss_actions, error)) 4645 goto exit; 4646 rss_desc_v[i] = wks->rss_desc; 4647 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4648 rss_desc_v[i].hash_fields = 4649 dev_flow.hash_fields; 4650 rss_desc_v[i].queue_num = 4651 rss_desc_v[i].hash_fields ? 4652 rss_desc_v[i].queue_num : 1; 4653 rss_desc_v[i].tunnel = 4654 !!(dev_flow.handle->layers & 4655 MLX5_FLOW_LAYER_TUNNEL); 4656 } else { 4657 /* This is queue action. */ 4658 rss_desc_v[i] = wks->rss_desc; 4659 rss_desc_v[i].key_len = 0; 4660 rss_desc_v[i].hash_fields = 0; 4661 rss_desc_v[i].queue = 4662 &policy->act_cnt[i].queue; 4663 rss_desc_v[i].queue_num = 1; 4664 } 4665 rss_desc[i] = &rss_desc_v[i]; 4666 } 4667 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4668 flow, policy, rss_desc); 4669 } else { 4670 enum mlx5_meter_domain mtr_domain = 4671 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4672 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4673 MLX5_MTR_DOMAIN_INGRESS; 4674 sub_policy = policy->sub_policys[mtr_domain][0]; 4675 } 4676 if (!sub_policy) { 4677 rte_flow_error_set(error, EINVAL, 4678 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4679 "Failed to get meter sub-policy."); 4680 goto exit; 4681 } 4682 exit: 4683 return sub_policy; 4684 } 4685 4686 /** 4687 * Split the meter flow. 4688 * 4689 * As meter flow will split to three sub flow, other than meter 4690 * action, the other actions make sense to only meter accepts 4691 * the packet. If it need to be dropped, no other additional 4692 * actions should be take. 4693 * 4694 * One kind of special action which decapsulates the L3 tunnel 4695 * header will be in the prefix sub flow, as not to take the 4696 * L3 tunnel header into account. 4697 * 4698 * @param[in] dev 4699 * Pointer to Ethernet device. 4700 * @param[in] flow 4701 * Parent flow structure pointer. 4702 * @param[in] fm 4703 * Pointer to flow meter structure. 4704 * @param[in] attr 4705 * Flow rule attributes. 4706 * @param[in] items 4707 * Pattern specification (list terminated by the END pattern item). 4708 * @param[out] sfx_items 4709 * Suffix flow match items (list terminated by the END pattern item). 4710 * @param[in] actions 4711 * Associated actions (list terminated by the END action). 4712 * @param[out] actions_sfx 4713 * Suffix flow actions. 4714 * @param[out] actions_pre 4715 * Prefix flow actions. 4716 * @param[out] mtr_flow_id 4717 * Pointer to meter flow id. 4718 * @param[out] error 4719 * Perform verbose error reporting if not NULL. 4720 * 4721 * @return 4722 * 0 on success, a negative errno value otherwise and rte_errno is set. 4723 */ 4724 static int 4725 flow_meter_split_prep(struct rte_eth_dev *dev, 4726 struct rte_flow *flow, 4727 struct mlx5_flow_meter_info *fm, 4728 const struct rte_flow_attr *attr, 4729 const struct rte_flow_item items[], 4730 struct rte_flow_item sfx_items[], 4731 const struct rte_flow_action actions[], 4732 struct rte_flow_action actions_sfx[], 4733 struct rte_flow_action actions_pre[], 4734 uint32_t *mtr_flow_id, 4735 struct rte_flow_error *error) 4736 { 4737 struct mlx5_priv *priv = dev->data->dev_private; 4738 struct rte_flow_action *tag_action = NULL; 4739 struct rte_flow_item *tag_item; 4740 struct mlx5_rte_flow_action_set_tag *set_tag; 4741 const struct rte_flow_action_raw_encap *raw_encap; 4742 const struct rte_flow_action_raw_decap *raw_decap; 4743 struct mlx5_rte_flow_item_tag *tag_item_spec; 4744 struct mlx5_rte_flow_item_tag *tag_item_mask; 4745 uint32_t tag_id = 0; 4746 struct rte_flow_item *vlan_item_dst = NULL; 4747 const struct rte_flow_item *vlan_item_src = NULL; 4748 struct rte_flow_action *hw_mtr_action; 4749 struct rte_flow_action *action_pre_head = NULL; 4750 int32_t flow_src_port = priv->representor_id; 4751 bool mtr_first; 4752 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 4753 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 4754 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 4755 uint32_t flow_id = 0; 4756 uint32_t flow_id_reversed = 0; 4757 uint8_t flow_id_bits = 0; 4758 int shift; 4759 4760 /* Prepare the suffix subflow items. */ 4761 tag_item = sfx_items++; 4762 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4763 struct mlx5_priv *port_priv; 4764 const struct rte_flow_item_port_id *pid_v; 4765 int item_type = items->type; 4766 4767 switch (item_type) { 4768 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4769 pid_v = items->spec; 4770 MLX5_ASSERT(pid_v); 4771 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 4772 if (!port_priv) 4773 return rte_flow_error_set(error, 4774 rte_errno, 4775 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 4776 pid_v, 4777 "Failed to get port info."); 4778 flow_src_port = port_priv->representor_id; 4779 memcpy(sfx_items, items, sizeof(*sfx_items)); 4780 sfx_items++; 4781 break; 4782 case RTE_FLOW_ITEM_TYPE_VLAN: 4783 /* Determine if copy vlan item below. */ 4784 vlan_item_src = items; 4785 vlan_item_dst = sfx_items++; 4786 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 4787 break; 4788 default: 4789 break; 4790 } 4791 } 4792 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4793 sfx_items++; 4794 mtr_first = priv->sh->meter_aso_en && 4795 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 4796 /* For ASO meter, meter must be before tag in TX direction. */ 4797 if (mtr_first) { 4798 action_pre_head = actions_pre++; 4799 /* Leave space for tag action. */ 4800 tag_action = actions_pre++; 4801 } 4802 /* Prepare the actions for prefix and suffix flow. */ 4803 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4804 struct rte_flow_action *action_cur = NULL; 4805 4806 switch (actions->type) { 4807 case RTE_FLOW_ACTION_TYPE_METER: 4808 if (mtr_first) { 4809 action_cur = action_pre_head; 4810 } else { 4811 /* Leave space for tag action. */ 4812 tag_action = actions_pre++; 4813 action_cur = actions_pre++; 4814 } 4815 break; 4816 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4817 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4818 action_cur = actions_pre++; 4819 break; 4820 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4821 raw_encap = actions->conf; 4822 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4823 action_cur = actions_pre++; 4824 break; 4825 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4826 raw_decap = actions->conf; 4827 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4828 action_cur = actions_pre++; 4829 break; 4830 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4831 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4832 if (vlan_item_dst && vlan_item_src) { 4833 memcpy(vlan_item_dst, vlan_item_src, 4834 sizeof(*vlan_item_dst)); 4835 /* 4836 * Convert to internal match item, it is used 4837 * for vlan push and set vid. 4838 */ 4839 vlan_item_dst->type = (enum rte_flow_item_type) 4840 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4841 } 4842 break; 4843 default: 4844 break; 4845 } 4846 if (!action_cur) 4847 action_cur = (fm->def_policy) ? 4848 actions_sfx++ : actions_pre++; 4849 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 4850 } 4851 /* Add end action to the actions. */ 4852 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4853 if (priv->sh->meter_aso_en) { 4854 /** 4855 * For ASO meter, need to add an extra jump action explicitly, 4856 * to jump from meter to policer table. 4857 */ 4858 struct mlx5_flow_meter_sub_policy *sub_policy; 4859 struct mlx5_flow_tbl_data_entry *tbl_data; 4860 4861 if (!fm->def_policy) { 4862 sub_policy = get_meter_sub_policy(dev, flow, 4863 fm->policy_id, attr, 4864 items, error); 4865 if (!sub_policy) 4866 return -rte_errno; 4867 } else { 4868 enum mlx5_meter_domain mtr_domain = 4869 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4870 attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4871 MLX5_MTR_DOMAIN_INGRESS; 4872 4873 sub_policy = 4874 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 4875 } 4876 tbl_data = container_of(sub_policy->tbl_rsc, 4877 struct mlx5_flow_tbl_data_entry, tbl); 4878 hw_mtr_action = actions_pre++; 4879 hw_mtr_action->type = (enum rte_flow_action_type) 4880 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 4881 hw_mtr_action->conf = tbl_data->jump.action; 4882 } 4883 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4884 actions_pre++; 4885 if (!tag_action) 4886 return rte_flow_error_set(error, ENOMEM, 4887 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4888 "No tag action space."); 4889 if (!mtr_flow_id) { 4890 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 4891 goto exit; 4892 } 4893 /* Only default-policy Meter creates mtr flow id. */ 4894 if (fm->def_policy) { 4895 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 4896 if (!tag_id) 4897 return rte_flow_error_set(error, ENOMEM, 4898 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4899 "Failed to allocate meter flow id."); 4900 flow_id = tag_id - 1; 4901 flow_id_bits = (!flow_id) ? 1 : 4902 (MLX5_REG_BITS - __builtin_clz(flow_id)); 4903 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 4904 mtr_reg_bits) { 4905 mlx5_ipool_free(fm->flow_ipool, tag_id); 4906 return rte_flow_error_set(error, EINVAL, 4907 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4908 "Meter flow id exceeds max limit."); 4909 } 4910 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 4911 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 4912 } 4913 /* Build tag actions and items for meter_id/meter flow_id. */ 4914 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 4915 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4916 tag_item_mask = tag_item_spec + 1; 4917 /* Both flow_id and meter_id share the same register. */ 4918 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4919 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 4920 0, error), 4921 .offset = mtr_id_offset, 4922 .length = mtr_reg_bits, 4923 .data = flow->meter, 4924 }; 4925 /* 4926 * The color Reg bits used by flow_id are growing from 4927 * msb to lsb, so must do bit reverse for flow_id val in RegC. 4928 */ 4929 for (shift = 0; shift < flow_id_bits; shift++) 4930 flow_id_reversed = (flow_id_reversed << 1) | 4931 ((flow_id >> shift) & 0x1); 4932 set_tag->data |= 4933 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 4934 tag_item_spec->id = set_tag->id; 4935 tag_item_spec->data = set_tag->data << mtr_id_offset; 4936 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 4937 tag_action->type = (enum rte_flow_action_type) 4938 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4939 tag_action->conf = set_tag; 4940 tag_item->type = (enum rte_flow_item_type) 4941 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4942 tag_item->spec = tag_item_spec; 4943 tag_item->last = NULL; 4944 tag_item->mask = tag_item_mask; 4945 exit: 4946 if (mtr_flow_id) 4947 *mtr_flow_id = tag_id; 4948 return 0; 4949 } 4950 4951 /** 4952 * Split action list having QUEUE/RSS for metadata register copy. 4953 * 4954 * Once Q/RSS action is detected in user's action list, the flow action 4955 * should be split in order to copy metadata registers, which will happen in 4956 * RX_CP_TBL like, 4957 * - CQE->flow_tag := reg_c[1] (MARK) 4958 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4959 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4960 * This is because the last action of each flow must be a terminal action 4961 * (QUEUE, RSS or DROP). 4962 * 4963 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4964 * stored and kept in the mlx5_flow structure per each sub_flow. 4965 * 4966 * The Q/RSS action is replaced with, 4967 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4968 * And the following JUMP action is added at the end, 4969 * - JUMP, to RX_CP_TBL. 4970 * 4971 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4972 * flow_create_split_metadata() routine. The flow will look like, 4973 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4974 * 4975 * @param dev 4976 * Pointer to Ethernet device. 4977 * @param[out] split_actions 4978 * Pointer to store split actions to jump to CP_TBL. 4979 * @param[in] actions 4980 * Pointer to the list of original flow actions. 4981 * @param[in] qrss 4982 * Pointer to the Q/RSS action. 4983 * @param[in] actions_n 4984 * Number of original actions. 4985 * @param[out] error 4986 * Perform verbose error reporting if not NULL. 4987 * 4988 * @return 4989 * non-zero unique flow_id on success, otherwise 0 and 4990 * error/rte_error are set. 4991 */ 4992 static uint32_t 4993 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4994 struct rte_flow_action *split_actions, 4995 const struct rte_flow_action *actions, 4996 const struct rte_flow_action *qrss, 4997 int actions_n, struct rte_flow_error *error) 4998 { 4999 struct mlx5_priv *priv = dev->data->dev_private; 5000 struct mlx5_rte_flow_action_set_tag *set_tag; 5001 struct rte_flow_action_jump *jump; 5002 const int qrss_idx = qrss - actions; 5003 uint32_t flow_id = 0; 5004 int ret = 0; 5005 5006 /* 5007 * Given actions will be split 5008 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5009 * - Add jump to mreg CP_TBL. 5010 * As a result, there will be one more action. 5011 */ 5012 ++actions_n; 5013 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5014 set_tag = (void *)(split_actions + actions_n); 5015 /* 5016 * If tag action is not set to void(it means we are not the meter 5017 * suffix flow), add the tag action. Since meter suffix flow already 5018 * has the tag added. 5019 */ 5020 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 5021 /* 5022 * Allocate the new subflow ID. This one is unique within 5023 * device and not shared with representors. Otherwise, 5024 * we would have to resolve multi-thread access synch 5025 * issue. Each flow on the shared device is appended 5026 * with source vport identifier, so the resulting 5027 * flows will be unique in the shared (by master and 5028 * representors) domain even if they have coinciding 5029 * IDs. 5030 */ 5031 mlx5_ipool_malloc(priv->sh->ipool 5032 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5033 if (!flow_id) 5034 return rte_flow_error_set(error, ENOMEM, 5035 RTE_FLOW_ERROR_TYPE_ACTION, 5036 NULL, "can't allocate id " 5037 "for split Q/RSS subflow"); 5038 /* Internal SET_TAG action to set flow ID. */ 5039 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5040 .data = flow_id, 5041 }; 5042 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5043 if (ret < 0) 5044 return ret; 5045 set_tag->id = ret; 5046 /* Construct new actions array. */ 5047 /* Replace QUEUE/RSS action. */ 5048 split_actions[qrss_idx] = (struct rte_flow_action){ 5049 .type = (enum rte_flow_action_type) 5050 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5051 .conf = set_tag, 5052 }; 5053 } 5054 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5055 jump = (void *)(set_tag + 1); 5056 *jump = (struct rte_flow_action_jump){ 5057 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5058 }; 5059 split_actions[actions_n - 2] = (struct rte_flow_action){ 5060 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5061 .conf = jump, 5062 }; 5063 split_actions[actions_n - 1] = (struct rte_flow_action){ 5064 .type = RTE_FLOW_ACTION_TYPE_END, 5065 }; 5066 return flow_id; 5067 } 5068 5069 /** 5070 * Extend the given action list for Tx metadata copy. 5071 * 5072 * Copy the given action list to the ext_actions and add flow metadata register 5073 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5074 * 5075 * @param[out] ext_actions 5076 * Pointer to the extended action list. 5077 * @param[in] actions 5078 * Pointer to the list of actions. 5079 * @param[in] actions_n 5080 * Number of actions in the list. 5081 * @param[out] error 5082 * Perform verbose error reporting if not NULL. 5083 * @param[in] encap_idx 5084 * The encap action inndex. 5085 * 5086 * @return 5087 * 0 on success, negative value otherwise 5088 */ 5089 static int 5090 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5091 struct rte_flow_action *ext_actions, 5092 const struct rte_flow_action *actions, 5093 int actions_n, struct rte_flow_error *error, 5094 int encap_idx) 5095 { 5096 struct mlx5_flow_action_copy_mreg *cp_mreg = 5097 (struct mlx5_flow_action_copy_mreg *) 5098 (ext_actions + actions_n + 1); 5099 int ret; 5100 5101 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5102 if (ret < 0) 5103 return ret; 5104 cp_mreg->dst = ret; 5105 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5106 if (ret < 0) 5107 return ret; 5108 cp_mreg->src = ret; 5109 if (encap_idx != 0) 5110 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5111 if (encap_idx == actions_n - 1) { 5112 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5113 .type = (enum rte_flow_action_type) 5114 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5115 .conf = cp_mreg, 5116 }; 5117 ext_actions[actions_n] = (struct rte_flow_action){ 5118 .type = RTE_FLOW_ACTION_TYPE_END, 5119 }; 5120 } else { 5121 ext_actions[encap_idx] = (struct rte_flow_action){ 5122 .type = (enum rte_flow_action_type) 5123 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5124 .conf = cp_mreg, 5125 }; 5126 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5127 sizeof(*ext_actions) * (actions_n - encap_idx)); 5128 } 5129 return 0; 5130 } 5131 5132 /** 5133 * Check the match action from the action list. 5134 * 5135 * @param[in] actions 5136 * Pointer to the list of actions. 5137 * @param[in] attr 5138 * Flow rule attributes. 5139 * @param[in] action 5140 * The action to be check if exist. 5141 * @param[out] match_action_pos 5142 * Pointer to the position of the matched action if exists, otherwise is -1. 5143 * @param[out] qrss_action_pos 5144 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5145 * @param[out] modify_after_mirror 5146 * Pointer to the flag of modify action after FDB mirroring. 5147 * 5148 * @return 5149 * > 0 the total number of actions. 5150 * 0 if not found match action in action list. 5151 */ 5152 static int 5153 flow_check_match_action(const struct rte_flow_action actions[], 5154 const struct rte_flow_attr *attr, 5155 enum rte_flow_action_type action, 5156 int *match_action_pos, int *qrss_action_pos, 5157 int *modify_after_mirror) 5158 { 5159 const struct rte_flow_action_sample *sample; 5160 int actions_n = 0; 5161 uint32_t ratio = 0; 5162 int sub_type = 0; 5163 int flag = 0; 5164 int fdb_mirror = 0; 5165 5166 *match_action_pos = -1; 5167 *qrss_action_pos = -1; 5168 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5169 if (actions->type == action) { 5170 flag = 1; 5171 *match_action_pos = actions_n; 5172 } 5173 switch (actions->type) { 5174 case RTE_FLOW_ACTION_TYPE_QUEUE: 5175 case RTE_FLOW_ACTION_TYPE_RSS: 5176 *qrss_action_pos = actions_n; 5177 break; 5178 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5179 sample = actions->conf; 5180 ratio = sample->ratio; 5181 sub_type = ((const struct rte_flow_action *) 5182 (sample->actions))->type; 5183 if (ratio == 1 && attr->transfer) 5184 fdb_mirror = 1; 5185 break; 5186 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5187 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5188 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5189 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5190 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5191 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5192 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5193 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5194 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5195 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5196 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5197 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5198 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5199 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5200 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5201 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5202 case RTE_FLOW_ACTION_TYPE_FLAG: 5203 case RTE_FLOW_ACTION_TYPE_MARK: 5204 case RTE_FLOW_ACTION_TYPE_SET_META: 5205 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5206 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5207 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5208 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5209 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5210 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5211 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5212 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5213 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5214 case RTE_FLOW_ACTION_TYPE_METER: 5215 if (fdb_mirror) 5216 *modify_after_mirror = 1; 5217 break; 5218 default: 5219 break; 5220 } 5221 actions_n++; 5222 } 5223 if (flag && fdb_mirror && !*modify_after_mirror) { 5224 /* FDB mirroring uses the destination array to implement 5225 * instead of FLOW_SAMPLER object. 5226 */ 5227 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5228 flag = 0; 5229 } 5230 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5231 return flag ? actions_n + 1 : 0; 5232 } 5233 5234 #define SAMPLE_SUFFIX_ITEM 2 5235 5236 /** 5237 * Split the sample flow. 5238 * 5239 * As sample flow will split to two sub flow, sample flow with 5240 * sample action, the other actions will move to new suffix flow. 5241 * 5242 * Also add unique tag id with tag action in the sample flow, 5243 * the same tag id will be as match in the suffix flow. 5244 * 5245 * @param dev 5246 * Pointer to Ethernet device. 5247 * @param[in] add_tag 5248 * Add extra tag action flag. 5249 * @param[out] sfx_items 5250 * Suffix flow match items (list terminated by the END pattern item). 5251 * @param[in] actions 5252 * Associated actions (list terminated by the END action). 5253 * @param[out] actions_sfx 5254 * Suffix flow actions. 5255 * @param[out] actions_pre 5256 * Prefix flow actions. 5257 * @param[in] actions_n 5258 * The total number of actions. 5259 * @param[in] sample_action_pos 5260 * The sample action position. 5261 * @param[in] qrss_action_pos 5262 * The Queue/RSS action position. 5263 * @param[in] jump_table 5264 * Add extra jump action flag. 5265 * @param[out] error 5266 * Perform verbose error reporting if not NULL. 5267 * 5268 * @return 5269 * 0 on success, or unique flow_id, a negative errno value 5270 * otherwise and rte_errno is set. 5271 */ 5272 static int 5273 flow_sample_split_prep(struct rte_eth_dev *dev, 5274 int add_tag, 5275 struct rte_flow_item sfx_items[], 5276 const struct rte_flow_action actions[], 5277 struct rte_flow_action actions_sfx[], 5278 struct rte_flow_action actions_pre[], 5279 int actions_n, 5280 int sample_action_pos, 5281 int qrss_action_pos, 5282 int jump_table, 5283 struct rte_flow_error *error) 5284 { 5285 struct mlx5_priv *priv = dev->data->dev_private; 5286 struct mlx5_rte_flow_action_set_tag *set_tag; 5287 struct mlx5_rte_flow_item_tag *tag_spec; 5288 struct mlx5_rte_flow_item_tag *tag_mask; 5289 struct rte_flow_action_jump *jump_action; 5290 uint32_t tag_id = 0; 5291 int index; 5292 int append_index = 0; 5293 int ret; 5294 5295 if (sample_action_pos < 0) 5296 return rte_flow_error_set(error, EINVAL, 5297 RTE_FLOW_ERROR_TYPE_ACTION, 5298 NULL, "invalid position of sample " 5299 "action in list"); 5300 /* Prepare the actions for prefix and suffix flow. */ 5301 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5302 index = qrss_action_pos; 5303 /* Put the preceding the Queue/RSS action into prefix flow. */ 5304 if (index != 0) 5305 memcpy(actions_pre, actions, 5306 sizeof(struct rte_flow_action) * index); 5307 /* Put others preceding the sample action into prefix flow. */ 5308 if (sample_action_pos > index + 1) 5309 memcpy(actions_pre + index, actions + index + 1, 5310 sizeof(struct rte_flow_action) * 5311 (sample_action_pos - index - 1)); 5312 index = sample_action_pos - 1; 5313 /* Put Queue/RSS action into Suffix flow. */ 5314 memcpy(actions_sfx, actions + qrss_action_pos, 5315 sizeof(struct rte_flow_action)); 5316 actions_sfx++; 5317 } else { 5318 index = sample_action_pos; 5319 if (index != 0) 5320 memcpy(actions_pre, actions, 5321 sizeof(struct rte_flow_action) * index); 5322 } 5323 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5324 * For CX6DX and above, metadata registers Cx preserve their value, 5325 * add an extra tag action for NIC-RX and E-Switch Domain. 5326 */ 5327 if (add_tag) { 5328 /* Prepare the prefix tag action. */ 5329 append_index++; 5330 set_tag = (void *)(actions_pre + actions_n + append_index); 5331 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5332 if (ret < 0) 5333 return ret; 5334 mlx5_ipool_malloc(priv->sh->ipool 5335 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5336 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5337 .id = ret, 5338 .data = tag_id, 5339 }; 5340 /* Prepare the suffix subflow items. */ 5341 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5342 tag_spec->data = tag_id; 5343 tag_spec->id = set_tag->id; 5344 tag_mask = tag_spec + 1; 5345 tag_mask->data = UINT32_MAX; 5346 sfx_items[0] = (struct rte_flow_item){ 5347 .type = (enum rte_flow_item_type) 5348 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5349 .spec = tag_spec, 5350 .last = NULL, 5351 .mask = tag_mask, 5352 }; 5353 sfx_items[1] = (struct rte_flow_item){ 5354 .type = (enum rte_flow_item_type) 5355 RTE_FLOW_ITEM_TYPE_END, 5356 }; 5357 /* Prepare the tag action in prefix subflow. */ 5358 actions_pre[index++] = 5359 (struct rte_flow_action){ 5360 .type = (enum rte_flow_action_type) 5361 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5362 .conf = set_tag, 5363 }; 5364 } 5365 memcpy(actions_pre + index, actions + sample_action_pos, 5366 sizeof(struct rte_flow_action)); 5367 index += 1; 5368 /* For the modify action after the sample action in E-Switch mirroring, 5369 * Add the extra jump action in prefix subflow and jump into the next 5370 * table, then do the modify action in the new table. 5371 */ 5372 if (jump_table) { 5373 /* Prepare the prefix jump action. */ 5374 append_index++; 5375 jump_action = (void *)(actions_pre + actions_n + append_index); 5376 jump_action->group = jump_table; 5377 actions_pre[index++] = 5378 (struct rte_flow_action){ 5379 .type = (enum rte_flow_action_type) 5380 RTE_FLOW_ACTION_TYPE_JUMP, 5381 .conf = jump_action, 5382 }; 5383 } 5384 actions_pre[index] = (struct rte_flow_action){ 5385 .type = (enum rte_flow_action_type) 5386 RTE_FLOW_ACTION_TYPE_END, 5387 }; 5388 /* Put the actions after sample into Suffix flow. */ 5389 memcpy(actions_sfx, actions + sample_action_pos + 1, 5390 sizeof(struct rte_flow_action) * 5391 (actions_n - sample_action_pos - 1)); 5392 return tag_id; 5393 } 5394 5395 /** 5396 * The splitting for metadata feature. 5397 * 5398 * - Q/RSS action on NIC Rx should be split in order to pass by 5399 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5400 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5401 * 5402 * - All the actions on NIC Tx should have a mreg copy action to 5403 * copy reg_a from WQE to reg_c[0]. 5404 * 5405 * @param dev 5406 * Pointer to Ethernet device. 5407 * @param[in] flow 5408 * Parent flow structure pointer. 5409 * @param[in] attr 5410 * Flow rule attributes. 5411 * @param[in] items 5412 * Pattern specification (list terminated by the END pattern item). 5413 * @param[in] actions 5414 * Associated actions (list terminated by the END action). 5415 * @param[in] flow_split_info 5416 * Pointer to flow split info structure. 5417 * @param[out] error 5418 * Perform verbose error reporting if not NULL. 5419 * @return 5420 * 0 on success, negative value otherwise 5421 */ 5422 static int 5423 flow_create_split_metadata(struct rte_eth_dev *dev, 5424 struct rte_flow *flow, 5425 const struct rte_flow_attr *attr, 5426 const struct rte_flow_item items[], 5427 const struct rte_flow_action actions[], 5428 struct mlx5_flow_split_info *flow_split_info, 5429 struct rte_flow_error *error) 5430 { 5431 struct mlx5_priv *priv = dev->data->dev_private; 5432 struct mlx5_dev_config *config = &priv->config; 5433 const struct rte_flow_action *qrss = NULL; 5434 struct rte_flow_action *ext_actions = NULL; 5435 struct mlx5_flow *dev_flow = NULL; 5436 uint32_t qrss_id = 0; 5437 int mtr_sfx = 0; 5438 size_t act_size; 5439 int actions_n; 5440 int encap_idx; 5441 int ret; 5442 5443 /* Check whether extensive metadata feature is engaged. */ 5444 if (!config->dv_flow_en || 5445 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5446 !mlx5_flow_ext_mreg_supported(dev)) 5447 return flow_create_split_inner(dev, flow, NULL, attr, items, 5448 actions, flow_split_info, error); 5449 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5450 &encap_idx); 5451 if (qrss) { 5452 /* Exclude hairpin flows from splitting. */ 5453 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5454 const struct rte_flow_action_queue *queue; 5455 5456 queue = qrss->conf; 5457 if (mlx5_rxq_get_type(dev, queue->index) == 5458 MLX5_RXQ_TYPE_HAIRPIN) 5459 qrss = NULL; 5460 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5461 const struct rte_flow_action_rss *rss; 5462 5463 rss = qrss->conf; 5464 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5465 MLX5_RXQ_TYPE_HAIRPIN) 5466 qrss = NULL; 5467 } 5468 } 5469 if (qrss) { 5470 /* Check if it is in meter suffix table. */ 5471 mtr_sfx = attr->group == (attr->transfer ? 5472 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5473 MLX5_FLOW_TABLE_LEVEL_METER); 5474 /* 5475 * Q/RSS action on NIC Rx should be split in order to pass by 5476 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5477 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5478 */ 5479 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5480 sizeof(struct rte_flow_action_set_tag) + 5481 sizeof(struct rte_flow_action_jump); 5482 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5483 SOCKET_ID_ANY); 5484 if (!ext_actions) 5485 return rte_flow_error_set(error, ENOMEM, 5486 RTE_FLOW_ERROR_TYPE_ACTION, 5487 NULL, "no memory to split " 5488 "metadata flow"); 5489 /* 5490 * If we are the suffix flow of meter, tag already exist. 5491 * Set the tag action to void. 5492 */ 5493 if (mtr_sfx) 5494 ext_actions[qrss - actions].type = 5495 RTE_FLOW_ACTION_TYPE_VOID; 5496 else 5497 ext_actions[qrss - actions].type = 5498 (enum rte_flow_action_type) 5499 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5500 /* 5501 * Create the new actions list with removed Q/RSS action 5502 * and appended set tag and jump to register copy table 5503 * (RX_CP_TBL). We should preallocate unique tag ID here 5504 * in advance, because it is needed for set tag action. 5505 */ 5506 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5507 qrss, actions_n, error); 5508 if (!mtr_sfx && !qrss_id) { 5509 ret = -rte_errno; 5510 goto exit; 5511 } 5512 } else if (attr->egress && !attr->transfer) { 5513 /* 5514 * All the actions on NIC Tx should have a metadata register 5515 * copy action to copy reg_a from WQE to reg_c[meta] 5516 */ 5517 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5518 sizeof(struct mlx5_flow_action_copy_mreg); 5519 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5520 SOCKET_ID_ANY); 5521 if (!ext_actions) 5522 return rte_flow_error_set(error, ENOMEM, 5523 RTE_FLOW_ERROR_TYPE_ACTION, 5524 NULL, "no memory to split " 5525 "metadata flow"); 5526 /* Create the action list appended with copy register. */ 5527 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5528 actions_n, error, encap_idx); 5529 if (ret < 0) 5530 goto exit; 5531 } 5532 /* Add the unmodified original or prefix subflow. */ 5533 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5534 items, ext_actions ? ext_actions : 5535 actions, flow_split_info, error); 5536 if (ret < 0) 5537 goto exit; 5538 MLX5_ASSERT(dev_flow); 5539 if (qrss) { 5540 const struct rte_flow_attr q_attr = { 5541 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5542 .ingress = 1, 5543 }; 5544 /* Internal PMD action to set register. */ 5545 struct mlx5_rte_flow_item_tag q_tag_spec = { 5546 .data = qrss_id, 5547 .id = REG_NON, 5548 }; 5549 struct rte_flow_item q_items[] = { 5550 { 5551 .type = (enum rte_flow_item_type) 5552 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5553 .spec = &q_tag_spec, 5554 .last = NULL, 5555 .mask = NULL, 5556 }, 5557 { 5558 .type = RTE_FLOW_ITEM_TYPE_END, 5559 }, 5560 }; 5561 struct rte_flow_action q_actions[] = { 5562 { 5563 .type = qrss->type, 5564 .conf = qrss->conf, 5565 }, 5566 { 5567 .type = RTE_FLOW_ACTION_TYPE_END, 5568 }, 5569 }; 5570 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5571 5572 /* 5573 * Configure the tag item only if there is no meter subflow. 5574 * Since tag is already marked in the meter suffix subflow 5575 * we can just use the meter suffix items as is. 5576 */ 5577 if (qrss_id) { 5578 /* Not meter subflow. */ 5579 MLX5_ASSERT(!mtr_sfx); 5580 /* 5581 * Put unique id in prefix flow due to it is destroyed 5582 * after suffix flow and id will be freed after there 5583 * is no actual flows with this id and identifier 5584 * reallocation becomes possible (for example, for 5585 * other flows in other threads). 5586 */ 5587 dev_flow->handle->split_flow_id = qrss_id; 5588 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5589 error); 5590 if (ret < 0) 5591 goto exit; 5592 q_tag_spec.id = ret; 5593 } 5594 dev_flow = NULL; 5595 /* Add suffix subflow to execute Q/RSS. */ 5596 flow_split_info->prefix_layers = layers; 5597 flow_split_info->prefix_mark = 0; 5598 ret = flow_create_split_inner(dev, flow, &dev_flow, 5599 &q_attr, mtr_sfx ? items : 5600 q_items, q_actions, 5601 flow_split_info, error); 5602 if (ret < 0) 5603 goto exit; 5604 /* qrss ID should be freed if failed. */ 5605 qrss_id = 0; 5606 MLX5_ASSERT(dev_flow); 5607 } 5608 5609 exit: 5610 /* 5611 * We do not destroy the partially created sub_flows in case of error. 5612 * These ones are included into parent flow list and will be destroyed 5613 * by flow_drv_destroy. 5614 */ 5615 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5616 qrss_id); 5617 mlx5_free(ext_actions); 5618 return ret; 5619 } 5620 5621 /** 5622 * Create meter internal drop flow with the original pattern. 5623 * 5624 * @param dev 5625 * Pointer to Ethernet device. 5626 * @param[in] flow 5627 * Parent flow structure pointer. 5628 * @param[in] attr 5629 * Flow rule attributes. 5630 * @param[in] items 5631 * Pattern specification (list terminated by the END pattern item). 5632 * @param[in] flow_split_info 5633 * Pointer to flow split info structure. 5634 * @param[in] fm 5635 * Pointer to flow meter structure. 5636 * @param[out] error 5637 * Perform verbose error reporting if not NULL. 5638 * @return 5639 * 0 on success, negative value otherwise 5640 */ 5641 static uint32_t 5642 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5643 struct rte_flow *flow, 5644 const struct rte_flow_attr *attr, 5645 const struct rte_flow_item items[], 5646 struct mlx5_flow_split_info *flow_split_info, 5647 struct mlx5_flow_meter_info *fm, 5648 struct rte_flow_error *error) 5649 { 5650 struct mlx5_flow *dev_flow = NULL; 5651 struct rte_flow_attr drop_attr = *attr; 5652 struct rte_flow_action drop_actions[3]; 5653 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5654 5655 MLX5_ASSERT(fm->drop_cnt); 5656 drop_actions[0].type = 5657 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5658 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5659 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5660 drop_actions[1].conf = NULL; 5661 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5662 drop_actions[2].conf = NULL; 5663 drop_split_info.external = false; 5664 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5665 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5666 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5667 return flow_create_split_inner(dev, flow, &dev_flow, 5668 &drop_attr, items, drop_actions, 5669 &drop_split_info, error); 5670 } 5671 5672 /** 5673 * The splitting for meter feature. 5674 * 5675 * - The meter flow will be split to two flows as prefix and 5676 * suffix flow. The packets make sense only it pass the prefix 5677 * meter action. 5678 * 5679 * - Reg_C_5 is used for the packet to match betweend prefix and 5680 * suffix flow. 5681 * 5682 * @param dev 5683 * Pointer to Ethernet device. 5684 * @param[in] flow 5685 * Parent flow structure pointer. 5686 * @param[in] attr 5687 * Flow rule attributes. 5688 * @param[in] items 5689 * Pattern specification (list terminated by the END pattern item). 5690 * @param[in] actions 5691 * Associated actions (list terminated by the END action). 5692 * @param[in] flow_split_info 5693 * Pointer to flow split info structure. 5694 * @param[out] error 5695 * Perform verbose error reporting if not NULL. 5696 * @return 5697 * 0 on success, negative value otherwise 5698 */ 5699 static int 5700 flow_create_split_meter(struct rte_eth_dev *dev, 5701 struct rte_flow *flow, 5702 const struct rte_flow_attr *attr, 5703 const struct rte_flow_item items[], 5704 const struct rte_flow_action actions[], 5705 struct mlx5_flow_split_info *flow_split_info, 5706 struct rte_flow_error *error) 5707 { 5708 struct mlx5_priv *priv = dev->data->dev_private; 5709 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5710 struct rte_flow_action *sfx_actions = NULL; 5711 struct rte_flow_action *pre_actions = NULL; 5712 struct rte_flow_item *sfx_items = NULL; 5713 struct mlx5_flow *dev_flow = NULL; 5714 struct rte_flow_attr sfx_attr = *attr; 5715 struct mlx5_flow_meter_info *fm = NULL; 5716 uint8_t skip_scale_restore; 5717 bool has_mtr = false; 5718 bool has_modify = false; 5719 bool set_mtr_reg = true; 5720 uint32_t meter_id = 0; 5721 uint32_t mtr_idx = 0; 5722 uint32_t mtr_flow_id = 0; 5723 size_t act_size; 5724 size_t item_size; 5725 int actions_n = 0; 5726 int ret = 0; 5727 5728 if (priv->mtr_en) 5729 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 5730 &has_modify, &meter_id); 5731 if (has_mtr) { 5732 if (flow->meter) { 5733 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 5734 if (!fm) 5735 return rte_flow_error_set(error, EINVAL, 5736 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5737 NULL, "Meter not found."); 5738 } else { 5739 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 5740 if (!fm) 5741 return rte_flow_error_set(error, EINVAL, 5742 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5743 NULL, "Meter not found."); 5744 ret = mlx5_flow_meter_attach(priv, fm, 5745 &sfx_attr, error); 5746 if (ret) 5747 return -rte_errno; 5748 flow->meter = mtr_idx; 5749 } 5750 MLX5_ASSERT(wks); 5751 wks->fm = fm; 5752 /* 5753 * If it isn't default-policy Meter, and 5754 * 1. There's no action in flow to change 5755 * packet (modify/encap/decap etc.), OR 5756 * 2. No drop count needed for this meter. 5757 * no need to use regC to save meter id anymore. 5758 */ 5759 if (!fm->def_policy && (!has_modify || !fm->drop_cnt)) 5760 set_mtr_reg = false; 5761 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 5762 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 5763 sizeof(struct mlx5_rte_flow_action_set_tag); 5764 /* Suffix items: tag, vlan, port id, end. */ 5765 #define METER_SUFFIX_ITEM 4 5766 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5767 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5768 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5769 0, SOCKET_ID_ANY); 5770 if (!sfx_actions) 5771 return rte_flow_error_set(error, ENOMEM, 5772 RTE_FLOW_ERROR_TYPE_ACTION, 5773 NULL, "no memory to split " 5774 "meter flow"); 5775 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5776 act_size); 5777 /* There's no suffix flow for meter of non-default policy. */ 5778 if (!fm->def_policy) 5779 pre_actions = sfx_actions + 1; 5780 else 5781 pre_actions = sfx_actions + actions_n; 5782 ret = flow_meter_split_prep(dev, flow, fm, &sfx_attr, 5783 items, sfx_items, actions, 5784 sfx_actions, pre_actions, 5785 (set_mtr_reg ? &mtr_flow_id : NULL), 5786 error); 5787 if (ret) { 5788 ret = -rte_errno; 5789 goto exit; 5790 } 5791 /* Add the prefix subflow. */ 5792 flow_split_info->prefix_mark = 0; 5793 skip_scale_restore = flow_split_info->skip_scale; 5794 flow_split_info->skip_scale |= 5795 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5796 ret = flow_create_split_inner(dev, flow, &dev_flow, 5797 attr, items, pre_actions, 5798 flow_split_info, error); 5799 flow_split_info->skip_scale = skip_scale_restore; 5800 if (ret) { 5801 if (mtr_flow_id) 5802 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 5803 ret = -rte_errno; 5804 goto exit; 5805 } 5806 if (mtr_flow_id) { 5807 dev_flow->handle->split_flow_id = mtr_flow_id; 5808 dev_flow->handle->is_meter_flow_id = 1; 5809 } 5810 if (!fm->def_policy) { 5811 if (!set_mtr_reg && fm->drop_cnt) 5812 ret = 5813 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 5814 &sfx_attr, items, 5815 flow_split_info, 5816 fm, error); 5817 goto exit; 5818 } 5819 /* Setting the sfx group atrr. */ 5820 sfx_attr.group = sfx_attr.transfer ? 5821 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5822 MLX5_FLOW_TABLE_LEVEL_METER; 5823 flow_split_info->prefix_layers = 5824 flow_get_prefix_layer_flags(dev_flow); 5825 flow_split_info->prefix_mark = dev_flow->handle->mark; 5826 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 5827 } 5828 /* Add the prefix subflow. */ 5829 ret = flow_create_split_metadata(dev, flow, 5830 &sfx_attr, sfx_items ? 5831 sfx_items : items, 5832 sfx_actions ? sfx_actions : actions, 5833 flow_split_info, error); 5834 exit: 5835 if (sfx_actions) 5836 mlx5_free(sfx_actions); 5837 return ret; 5838 } 5839 5840 /** 5841 * The splitting for sample feature. 5842 * 5843 * Once Sample action is detected in the action list, the flow actions should 5844 * be split into prefix sub flow and suffix sub flow. 5845 * 5846 * The original items remain in the prefix sub flow, all actions preceding the 5847 * sample action and the sample action itself will be copied to the prefix 5848 * sub flow, the actions following the sample action will be copied to the 5849 * suffix sub flow, Queue action always be located in the suffix sub flow. 5850 * 5851 * In order to make the packet from prefix sub flow matches with suffix sub 5852 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5853 * flow uses tag item with the unique flow id. 5854 * 5855 * @param dev 5856 * Pointer to Ethernet device. 5857 * @param[in] flow 5858 * Parent flow structure pointer. 5859 * @param[in] attr 5860 * Flow rule attributes. 5861 * @param[in] items 5862 * Pattern specification (list terminated by the END pattern item). 5863 * @param[in] actions 5864 * Associated actions (list terminated by the END action). 5865 * @param[in] flow_split_info 5866 * Pointer to flow split info structure. 5867 * @param[out] error 5868 * Perform verbose error reporting if not NULL. 5869 * @return 5870 * 0 on success, negative value otherwise 5871 */ 5872 static int 5873 flow_create_split_sample(struct rte_eth_dev *dev, 5874 struct rte_flow *flow, 5875 const struct rte_flow_attr *attr, 5876 const struct rte_flow_item items[], 5877 const struct rte_flow_action actions[], 5878 struct mlx5_flow_split_info *flow_split_info, 5879 struct rte_flow_error *error) 5880 { 5881 struct mlx5_priv *priv = dev->data->dev_private; 5882 struct rte_flow_action *sfx_actions = NULL; 5883 struct rte_flow_action *pre_actions = NULL; 5884 struct rte_flow_item *sfx_items = NULL; 5885 struct mlx5_flow *dev_flow = NULL; 5886 struct rte_flow_attr sfx_attr = *attr; 5887 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5888 struct mlx5_flow_dv_sample_resource *sample_res; 5889 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5890 struct mlx5_flow_tbl_resource *sfx_tbl; 5891 #endif 5892 size_t act_size; 5893 size_t item_size; 5894 uint32_t fdb_tx = 0; 5895 int32_t tag_id = 0; 5896 int actions_n = 0; 5897 int sample_action_pos; 5898 int qrss_action_pos; 5899 int add_tag = 0; 5900 int modify_after_mirror = 0; 5901 uint16_t jump_table = 0; 5902 const uint32_t next_ft_step = 1; 5903 int ret = 0; 5904 5905 if (priv->sampler_en) 5906 actions_n = flow_check_match_action(actions, attr, 5907 RTE_FLOW_ACTION_TYPE_SAMPLE, 5908 &sample_action_pos, &qrss_action_pos, 5909 &modify_after_mirror); 5910 if (actions_n) { 5911 /* The prefix actions must includes sample, tag, end. */ 5912 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5913 + sizeof(struct mlx5_rte_flow_action_set_tag); 5914 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5915 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5916 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5917 item_size), 0, SOCKET_ID_ANY); 5918 if (!sfx_actions) 5919 return rte_flow_error_set(error, ENOMEM, 5920 RTE_FLOW_ERROR_TYPE_ACTION, 5921 NULL, "no memory to split " 5922 "sample flow"); 5923 /* The representor_id is UINT16_MAX for uplink. */ 5924 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 5925 /* 5926 * When reg_c_preserve is set, metadata registers Cx preserve 5927 * their value even through packet duplication. 5928 */ 5929 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 5930 if (add_tag) 5931 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5932 + act_size); 5933 if (modify_after_mirror) 5934 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 5935 next_ft_step; 5936 pre_actions = sfx_actions + actions_n; 5937 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 5938 actions, sfx_actions, 5939 pre_actions, actions_n, 5940 sample_action_pos, 5941 qrss_action_pos, jump_table, 5942 error); 5943 if (tag_id < 0 || (add_tag && !tag_id)) { 5944 ret = -rte_errno; 5945 goto exit; 5946 } 5947 if (modify_after_mirror) 5948 flow_split_info->skip_scale = 5949 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5950 /* Add the prefix subflow. */ 5951 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5952 items, pre_actions, 5953 flow_split_info, error); 5954 if (ret) { 5955 ret = -rte_errno; 5956 goto exit; 5957 } 5958 dev_flow->handle->split_flow_id = tag_id; 5959 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5960 if (!modify_after_mirror) { 5961 /* Set the sfx group attr. */ 5962 sample_res = (struct mlx5_flow_dv_sample_resource *) 5963 dev_flow->dv.sample_res; 5964 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5965 sample_res->normal_path_tbl; 5966 sfx_tbl_data = container_of(sfx_tbl, 5967 struct mlx5_flow_tbl_data_entry, 5968 tbl); 5969 sfx_attr.group = sfx_attr.transfer ? 5970 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 5971 } else { 5972 MLX5_ASSERT(attr->transfer); 5973 sfx_attr.group = jump_table; 5974 } 5975 flow_split_info->prefix_layers = 5976 flow_get_prefix_layer_flags(dev_flow); 5977 flow_split_info->prefix_mark = dev_flow->handle->mark; 5978 /* Suffix group level already be scaled with factor, set 5979 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 5980 * again in translation. 5981 */ 5982 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5983 #endif 5984 } 5985 /* Add the suffix subflow. */ 5986 ret = flow_create_split_meter(dev, flow, &sfx_attr, 5987 sfx_items ? sfx_items : items, 5988 sfx_actions ? sfx_actions : actions, 5989 flow_split_info, error); 5990 exit: 5991 if (sfx_actions) 5992 mlx5_free(sfx_actions); 5993 return ret; 5994 } 5995 5996 /** 5997 * Split the flow to subflow set. The splitters might be linked 5998 * in the chain, like this: 5999 * flow_create_split_outer() calls: 6000 * flow_create_split_meter() calls: 6001 * flow_create_split_metadata(meter_subflow_0) calls: 6002 * flow_create_split_inner(metadata_subflow_0) 6003 * flow_create_split_inner(metadata_subflow_1) 6004 * flow_create_split_inner(metadata_subflow_2) 6005 * flow_create_split_metadata(meter_subflow_1) calls: 6006 * flow_create_split_inner(metadata_subflow_0) 6007 * flow_create_split_inner(metadata_subflow_1) 6008 * flow_create_split_inner(metadata_subflow_2) 6009 * 6010 * This provide flexible way to add new levels of flow splitting. 6011 * The all of successfully created subflows are included to the 6012 * parent flow dev_flow list. 6013 * 6014 * @param dev 6015 * Pointer to Ethernet device. 6016 * @param[in] flow 6017 * Parent flow structure pointer. 6018 * @param[in] attr 6019 * Flow rule attributes. 6020 * @param[in] items 6021 * Pattern specification (list terminated by the END pattern item). 6022 * @param[in] actions 6023 * Associated actions (list terminated by the END action). 6024 * @param[in] flow_split_info 6025 * Pointer to flow split info structure. 6026 * @param[out] error 6027 * Perform verbose error reporting if not NULL. 6028 * @return 6029 * 0 on success, negative value otherwise 6030 */ 6031 static int 6032 flow_create_split_outer(struct rte_eth_dev *dev, 6033 struct rte_flow *flow, 6034 const struct rte_flow_attr *attr, 6035 const struct rte_flow_item items[], 6036 const struct rte_flow_action actions[], 6037 struct mlx5_flow_split_info *flow_split_info, 6038 struct rte_flow_error *error) 6039 { 6040 int ret; 6041 6042 ret = flow_create_split_sample(dev, flow, attr, items, 6043 actions, flow_split_info, error); 6044 MLX5_ASSERT(ret <= 0); 6045 return ret; 6046 } 6047 6048 static inline struct mlx5_flow_tunnel * 6049 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6050 { 6051 struct mlx5_flow_tunnel *tunnel; 6052 6053 #pragma GCC diagnostic push 6054 #pragma GCC diagnostic ignored "-Wcast-qual" 6055 tunnel = (typeof(tunnel))flow->tunnel; 6056 #pragma GCC diagnostic pop 6057 6058 return tunnel; 6059 } 6060 6061 /** 6062 * Adjust flow RSS workspace if needed. 6063 * 6064 * @param wks 6065 * Pointer to thread flow work space. 6066 * @param rss_desc 6067 * Pointer to RSS descriptor. 6068 * @param[in] nrssq_num 6069 * New RSS queue number. 6070 * 6071 * @return 6072 * 0 on success, -1 otherwise and rte_errno is set. 6073 */ 6074 static int 6075 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6076 struct mlx5_flow_rss_desc *rss_desc, 6077 uint32_t nrssq_num) 6078 { 6079 if (likely(nrssq_num <= wks->rssq_num)) 6080 return 0; 6081 rss_desc->queue = realloc(rss_desc->queue, 6082 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6083 if (!rss_desc->queue) { 6084 rte_errno = ENOMEM; 6085 return -1; 6086 } 6087 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6088 return 0; 6089 } 6090 6091 /** 6092 * Create a flow and add it to @p list. 6093 * 6094 * @param dev 6095 * Pointer to Ethernet device. 6096 * @param list 6097 * Pointer to a TAILQ flow list. If this parameter NULL, 6098 * no list insertion occurred, flow is just created, 6099 * this is caller's responsibility to track the 6100 * created flow. 6101 * @param[in] attr 6102 * Flow rule attributes. 6103 * @param[in] items 6104 * Pattern specification (list terminated by the END pattern item). 6105 * @param[in] actions 6106 * Associated actions (list terminated by the END action). 6107 * @param[in] external 6108 * This flow rule is created by request external to PMD. 6109 * @param[out] error 6110 * Perform verbose error reporting if not NULL. 6111 * 6112 * @return 6113 * A flow index on success, 0 otherwise and rte_errno is set. 6114 */ 6115 static uint32_t 6116 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 6117 const struct rte_flow_attr *attr, 6118 const struct rte_flow_item items[], 6119 const struct rte_flow_action original_actions[], 6120 bool external, struct rte_flow_error *error) 6121 { 6122 struct mlx5_priv *priv = dev->data->dev_private; 6123 struct rte_flow *flow = NULL; 6124 struct mlx5_flow *dev_flow; 6125 const struct rte_flow_action_rss *rss = NULL; 6126 struct mlx5_translated_action_handle 6127 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6128 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6129 union { 6130 struct mlx5_flow_expand_rss buf; 6131 uint8_t buffer[2048]; 6132 } expand_buffer; 6133 union { 6134 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6135 uint8_t buffer[2048]; 6136 } actions_rx; 6137 union { 6138 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6139 uint8_t buffer[2048]; 6140 } actions_hairpin_tx; 6141 union { 6142 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6143 uint8_t buffer[2048]; 6144 } items_tx; 6145 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6146 struct mlx5_flow_rss_desc *rss_desc; 6147 const struct rte_flow_action *p_actions_rx; 6148 uint32_t i; 6149 uint32_t idx = 0; 6150 int hairpin_flow; 6151 struct rte_flow_attr attr_tx = { .priority = 0 }; 6152 const struct rte_flow_action *actions; 6153 struct rte_flow_action *translated_actions = NULL; 6154 struct mlx5_flow_tunnel *tunnel; 6155 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6156 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6157 struct mlx5_flow_split_info flow_split_info = { 6158 .external = !!external, 6159 .skip_scale = 0, 6160 .flow_idx = 0, 6161 .prefix_mark = 0, 6162 .prefix_layers = 0, 6163 .table_id = 0 6164 }; 6165 int ret; 6166 6167 MLX5_ASSERT(wks); 6168 rss_desc = &wks->rss_desc; 6169 ret = flow_action_handles_translate(dev, original_actions, 6170 indir_actions, 6171 &indir_actions_n, 6172 &translated_actions, error); 6173 if (ret < 0) { 6174 MLX5_ASSERT(translated_actions == NULL); 6175 return 0; 6176 } 6177 actions = translated_actions ? translated_actions : original_actions; 6178 p_actions_rx = actions; 6179 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6180 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6181 external, hairpin_flow, error); 6182 if (ret < 0) 6183 goto error_before_hairpin_split; 6184 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 6185 if (!flow) { 6186 rte_errno = ENOMEM; 6187 goto error_before_hairpin_split; 6188 } 6189 if (hairpin_flow > 0) { 6190 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6191 rte_errno = EINVAL; 6192 goto error_before_hairpin_split; 6193 } 6194 flow_hairpin_split(dev, actions, actions_rx.actions, 6195 actions_hairpin_tx.actions, items_tx.items, 6196 idx); 6197 p_actions_rx = actions_rx.actions; 6198 } 6199 flow_split_info.flow_idx = idx; 6200 flow->drv_type = flow_get_drv_type(dev, attr); 6201 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6202 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6203 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6204 /* RSS Action only works on NIC RX domain */ 6205 if (attr->ingress && !attr->transfer) 6206 rss = flow_get_rss_action(dev, p_actions_rx); 6207 if (rss) { 6208 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6209 return 0; 6210 /* 6211 * The following information is required by 6212 * mlx5_flow_hashfields_adjust() in advance. 6213 */ 6214 rss_desc->level = rss->level; 6215 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 6216 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 6217 } 6218 flow->dev_handles = 0; 6219 if (rss && rss->types) { 6220 unsigned int graph_root; 6221 6222 graph_root = find_graph_root(items, rss->level); 6223 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6224 items, rss->types, 6225 mlx5_support_expansion, graph_root); 6226 MLX5_ASSERT(ret > 0 && 6227 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6228 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6229 for (i = 0; i < buf->entries; ++i) 6230 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6231 } 6232 } else { 6233 buf->entries = 1; 6234 buf->entry[0].pattern = (void *)(uintptr_t)items; 6235 } 6236 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6237 indir_actions_n); 6238 for (i = 0; i < buf->entries; ++i) { 6239 /* Initialize flow split data. */ 6240 flow_split_info.prefix_layers = 0; 6241 flow_split_info.prefix_mark = 0; 6242 flow_split_info.skip_scale = 0; 6243 /* 6244 * The splitter may create multiple dev_flows, 6245 * depending on configuration. In the simplest 6246 * case it just creates unmodified original flow. 6247 */ 6248 ret = flow_create_split_outer(dev, flow, attr, 6249 buf->entry[i].pattern, 6250 p_actions_rx, &flow_split_info, 6251 error); 6252 if (ret < 0) 6253 goto error; 6254 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6255 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6256 p_actions_rx, 6257 idx, 6258 wks->flows[0].tunnel, 6259 &default_miss_ctx, 6260 error); 6261 if (ret < 0) { 6262 mlx5_free(default_miss_ctx.queue); 6263 goto error; 6264 } 6265 } 6266 } 6267 /* Create the tx flow. */ 6268 if (hairpin_flow) { 6269 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6270 attr_tx.ingress = 0; 6271 attr_tx.egress = 1; 6272 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6273 actions_hairpin_tx.actions, 6274 idx, error); 6275 if (!dev_flow) 6276 goto error; 6277 dev_flow->flow = flow; 6278 dev_flow->external = 0; 6279 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6280 dev_flow->handle, next); 6281 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6282 items_tx.items, 6283 actions_hairpin_tx.actions, error); 6284 if (ret < 0) 6285 goto error; 6286 } 6287 /* 6288 * Update the metadata register copy table. If extensive 6289 * metadata feature is enabled and registers are supported 6290 * we might create the extra rte_flow for each unique 6291 * MARK/FLAG action ID. 6292 * 6293 * The table is updated for ingress Flows only, because 6294 * the egress Flows belong to the different device and 6295 * copy table should be updated in peer NIC Rx domain. 6296 */ 6297 if (attr->ingress && 6298 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6299 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6300 if (ret) 6301 goto error; 6302 } 6303 /* 6304 * If the flow is external (from application) OR device is started, 6305 * OR mreg discover, then apply immediately. 6306 */ 6307 if (external || dev->data->dev_started || 6308 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6309 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6310 ret = flow_drv_apply(dev, flow, error); 6311 if (ret < 0) 6312 goto error; 6313 } 6314 if (list) { 6315 rte_spinlock_lock(&priv->flow_list_lock); 6316 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 6317 flow, next); 6318 rte_spinlock_unlock(&priv->flow_list_lock); 6319 } 6320 flow_rxq_flags_set(dev, flow); 6321 rte_free(translated_actions); 6322 tunnel = flow_tunnel_from_rule(wks->flows); 6323 if (tunnel) { 6324 flow->tunnel = 1; 6325 flow->tunnel_id = tunnel->tunnel_id; 6326 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6327 mlx5_free(default_miss_ctx.queue); 6328 } 6329 mlx5_flow_pop_thread_workspace(); 6330 return idx; 6331 error: 6332 MLX5_ASSERT(flow); 6333 ret = rte_errno; /* Save rte_errno before cleanup. */ 6334 flow_mreg_del_copy_action(dev, flow); 6335 flow_drv_destroy(dev, flow); 6336 if (rss_desc->shared_rss) 6337 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6338 mlx5_ipool_get 6339 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6340 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6341 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 6342 rte_errno = ret; /* Restore rte_errno. */ 6343 ret = rte_errno; 6344 rte_errno = ret; 6345 mlx5_flow_pop_thread_workspace(); 6346 error_before_hairpin_split: 6347 rte_free(translated_actions); 6348 return 0; 6349 } 6350 6351 /** 6352 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6353 * incoming packets to table 1. 6354 * 6355 * Other flow rules, requested for group n, will be created in 6356 * e-switch table n+1. 6357 * Jump action to e-switch group n will be created to group n+1. 6358 * 6359 * Used when working in switchdev mode, to utilise advantages of table 1 6360 * and above. 6361 * 6362 * @param dev 6363 * Pointer to Ethernet device. 6364 * 6365 * @return 6366 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6367 */ 6368 struct rte_flow * 6369 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6370 { 6371 const struct rte_flow_attr attr = { 6372 .group = 0, 6373 .priority = 0, 6374 .ingress = 1, 6375 .egress = 0, 6376 .transfer = 1, 6377 }; 6378 const struct rte_flow_item pattern = { 6379 .type = RTE_FLOW_ITEM_TYPE_END, 6380 }; 6381 struct rte_flow_action_jump jump = { 6382 .group = 1, 6383 }; 6384 const struct rte_flow_action actions[] = { 6385 { 6386 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6387 .conf = &jump, 6388 }, 6389 { 6390 .type = RTE_FLOW_ACTION_TYPE_END, 6391 }, 6392 }; 6393 struct mlx5_priv *priv = dev->data->dev_private; 6394 struct rte_flow_error error; 6395 6396 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 6397 &attr, &pattern, 6398 actions, false, &error); 6399 } 6400 6401 /** 6402 * Validate a flow supported by the NIC. 6403 * 6404 * @see rte_flow_validate() 6405 * @see rte_flow_ops 6406 */ 6407 int 6408 mlx5_flow_validate(struct rte_eth_dev *dev, 6409 const struct rte_flow_attr *attr, 6410 const struct rte_flow_item items[], 6411 const struct rte_flow_action original_actions[], 6412 struct rte_flow_error *error) 6413 { 6414 int hairpin_flow; 6415 struct mlx5_translated_action_handle 6416 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6417 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6418 const struct rte_flow_action *actions; 6419 struct rte_flow_action *translated_actions = NULL; 6420 int ret = flow_action_handles_translate(dev, original_actions, 6421 indir_actions, 6422 &indir_actions_n, 6423 &translated_actions, error); 6424 6425 if (ret) 6426 return ret; 6427 actions = translated_actions ? translated_actions : original_actions; 6428 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6429 ret = flow_drv_validate(dev, attr, items, actions, 6430 true, hairpin_flow, error); 6431 rte_free(translated_actions); 6432 return ret; 6433 } 6434 6435 /** 6436 * Create a flow. 6437 * 6438 * @see rte_flow_create() 6439 * @see rte_flow_ops 6440 */ 6441 struct rte_flow * 6442 mlx5_flow_create(struct rte_eth_dev *dev, 6443 const struct rte_flow_attr *attr, 6444 const struct rte_flow_item items[], 6445 const struct rte_flow_action actions[], 6446 struct rte_flow_error *error) 6447 { 6448 struct mlx5_priv *priv = dev->data->dev_private; 6449 6450 /* 6451 * If the device is not started yet, it is not allowed to created a 6452 * flow from application. PMD default flows and traffic control flows 6453 * are not affected. 6454 */ 6455 if (unlikely(!dev->data->dev_started)) { 6456 DRV_LOG(DEBUG, "port %u is not started when " 6457 "inserting a flow", dev->data->port_id); 6458 rte_flow_error_set(error, ENODEV, 6459 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6460 NULL, 6461 "port not started"); 6462 return NULL; 6463 } 6464 6465 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 6466 attr, items, actions, true, error); 6467 } 6468 6469 /** 6470 * Destroy a flow in a list. 6471 * 6472 * @param dev 6473 * Pointer to Ethernet device. 6474 * @param list 6475 * Pointer to the Indexed flow list. If this parameter NULL, 6476 * there is no flow removal from the list. Be noted that as 6477 * flow is add to the indexed list, memory of the indexed 6478 * list points to maybe changed as flow destroyed. 6479 * @param[in] flow_idx 6480 * Index of flow to destroy. 6481 */ 6482 static void 6483 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 6484 uint32_t flow_idx) 6485 { 6486 struct mlx5_priv *priv = dev->data->dev_private; 6487 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6488 [MLX5_IPOOL_RTE_FLOW], flow_idx); 6489 6490 if (!flow) 6491 return; 6492 /* 6493 * Update RX queue flags only if port is started, otherwise it is 6494 * already clean. 6495 */ 6496 if (dev->data->dev_started) 6497 flow_rxq_flags_trim(dev, flow); 6498 flow_drv_destroy(dev, flow); 6499 if (list) { 6500 rte_spinlock_lock(&priv->flow_list_lock); 6501 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 6502 flow_idx, flow, next); 6503 rte_spinlock_unlock(&priv->flow_list_lock); 6504 } 6505 if (flow->tunnel) { 6506 struct mlx5_flow_tunnel *tunnel; 6507 6508 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6509 RTE_VERIFY(tunnel); 6510 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6511 mlx5_flow_tunnel_free(dev, tunnel); 6512 } 6513 flow_mreg_del_copy_action(dev, flow); 6514 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 6515 } 6516 6517 /** 6518 * Destroy all flows. 6519 * 6520 * @param dev 6521 * Pointer to Ethernet device. 6522 * @param list 6523 * Pointer to the Indexed flow list. 6524 * @param active 6525 * If flushing is called avtively. 6526 */ 6527 void 6528 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 6529 { 6530 uint32_t num_flushed = 0; 6531 6532 while (*list) { 6533 flow_list_destroy(dev, list, *list); 6534 num_flushed++; 6535 } 6536 if (active) { 6537 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6538 dev->data->port_id, num_flushed); 6539 } 6540 } 6541 6542 /** 6543 * Stop all default actions for flows. 6544 * 6545 * @param dev 6546 * Pointer to Ethernet device. 6547 */ 6548 void 6549 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6550 { 6551 flow_mreg_del_default_copy_action(dev); 6552 flow_rxq_flags_clear(dev); 6553 } 6554 6555 /** 6556 * Start all default actions for flows. 6557 * 6558 * @param dev 6559 * Pointer to Ethernet device. 6560 * @return 6561 * 0 on success, a negative errno value otherwise and rte_errno is set. 6562 */ 6563 int 6564 mlx5_flow_start_default(struct rte_eth_dev *dev) 6565 { 6566 struct rte_flow_error error; 6567 6568 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6569 return flow_mreg_add_default_copy_action(dev, &error); 6570 } 6571 6572 /** 6573 * Release key of thread specific flow workspace data. 6574 */ 6575 void 6576 flow_release_workspace(void *data) 6577 { 6578 struct mlx5_flow_workspace *wks = data; 6579 struct mlx5_flow_workspace *next; 6580 6581 while (wks) { 6582 next = wks->next; 6583 free(wks->rss_desc.queue); 6584 free(wks); 6585 wks = next; 6586 } 6587 } 6588 6589 /** 6590 * Get thread specific current flow workspace. 6591 * 6592 * @return pointer to thread specific flow workspace data, NULL on error. 6593 */ 6594 struct mlx5_flow_workspace* 6595 mlx5_flow_get_thread_workspace(void) 6596 { 6597 struct mlx5_flow_workspace *data; 6598 6599 data = mlx5_flow_os_get_specific_workspace(); 6600 MLX5_ASSERT(data && data->inuse); 6601 if (!data || !data->inuse) 6602 DRV_LOG(ERR, "flow workspace not initialized."); 6603 return data; 6604 } 6605 6606 /** 6607 * Allocate and init new flow workspace. 6608 * 6609 * @return pointer to flow workspace data, NULL on error. 6610 */ 6611 static struct mlx5_flow_workspace* 6612 flow_alloc_thread_workspace(void) 6613 { 6614 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6615 6616 if (!data) { 6617 DRV_LOG(ERR, "Failed to allocate flow workspace " 6618 "memory."); 6619 return NULL; 6620 } 6621 data->rss_desc.queue = calloc(1, 6622 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6623 if (!data->rss_desc.queue) 6624 goto err; 6625 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6626 return data; 6627 err: 6628 if (data->rss_desc.queue) 6629 free(data->rss_desc.queue); 6630 free(data); 6631 return NULL; 6632 } 6633 6634 /** 6635 * Get new thread specific flow workspace. 6636 * 6637 * If current workspace inuse, create new one and set as current. 6638 * 6639 * @return pointer to thread specific flow workspace data, NULL on error. 6640 */ 6641 static struct mlx5_flow_workspace* 6642 mlx5_flow_push_thread_workspace(void) 6643 { 6644 struct mlx5_flow_workspace *curr; 6645 struct mlx5_flow_workspace *data; 6646 6647 curr = mlx5_flow_os_get_specific_workspace(); 6648 if (!curr) { 6649 data = flow_alloc_thread_workspace(); 6650 if (!data) 6651 return NULL; 6652 } else if (!curr->inuse) { 6653 data = curr; 6654 } else if (curr->next) { 6655 data = curr->next; 6656 } else { 6657 data = flow_alloc_thread_workspace(); 6658 if (!data) 6659 return NULL; 6660 curr->next = data; 6661 data->prev = curr; 6662 } 6663 data->inuse = 1; 6664 data->flow_idx = 0; 6665 /* Set as current workspace */ 6666 if (mlx5_flow_os_set_specific_workspace(data)) 6667 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6668 return data; 6669 } 6670 6671 /** 6672 * Close current thread specific flow workspace. 6673 * 6674 * If previous workspace available, set it as current. 6675 * 6676 * @return pointer to thread specific flow workspace data, NULL on error. 6677 */ 6678 static void 6679 mlx5_flow_pop_thread_workspace(void) 6680 { 6681 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6682 6683 if (!data) 6684 return; 6685 if (!data->inuse) { 6686 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6687 return; 6688 } 6689 data->inuse = 0; 6690 if (!data->prev) 6691 return; 6692 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6693 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6694 } 6695 6696 /** 6697 * Verify the flow list is empty 6698 * 6699 * @param dev 6700 * Pointer to Ethernet device. 6701 * 6702 * @return the number of flows not released. 6703 */ 6704 int 6705 mlx5_flow_verify(struct rte_eth_dev *dev) 6706 { 6707 struct mlx5_priv *priv = dev->data->dev_private; 6708 struct rte_flow *flow; 6709 uint32_t idx; 6710 int ret = 0; 6711 6712 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 6713 flow, next) { 6714 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6715 dev->data->port_id, (void *)flow); 6716 ++ret; 6717 } 6718 return ret; 6719 } 6720 6721 /** 6722 * Enable default hairpin egress flow. 6723 * 6724 * @param dev 6725 * Pointer to Ethernet device. 6726 * @param queue 6727 * The queue index. 6728 * 6729 * @return 6730 * 0 on success, a negative errno value otherwise and rte_errno is set. 6731 */ 6732 int 6733 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6734 uint32_t queue) 6735 { 6736 struct mlx5_priv *priv = dev->data->dev_private; 6737 const struct rte_flow_attr attr = { 6738 .egress = 1, 6739 .priority = 0, 6740 }; 6741 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6742 .queue = queue, 6743 }; 6744 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6745 .queue = UINT32_MAX, 6746 }; 6747 struct rte_flow_item items[] = { 6748 { 6749 .type = (enum rte_flow_item_type) 6750 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6751 .spec = &queue_spec, 6752 .last = NULL, 6753 .mask = &queue_mask, 6754 }, 6755 { 6756 .type = RTE_FLOW_ITEM_TYPE_END, 6757 }, 6758 }; 6759 struct rte_flow_action_jump jump = { 6760 .group = MLX5_HAIRPIN_TX_TABLE, 6761 }; 6762 struct rte_flow_action actions[2]; 6763 uint32_t flow_idx; 6764 struct rte_flow_error error; 6765 6766 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6767 actions[0].conf = &jump; 6768 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6769 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6770 &attr, items, actions, false, &error); 6771 if (!flow_idx) { 6772 DRV_LOG(DEBUG, 6773 "Failed to create ctrl flow: rte_errno(%d)," 6774 " type(%d), message(%s)", 6775 rte_errno, error.type, 6776 error.message ? error.message : " (no stated reason)"); 6777 return -rte_errno; 6778 } 6779 return 0; 6780 } 6781 6782 /** 6783 * Enable a control flow configured from the control plane. 6784 * 6785 * @param dev 6786 * Pointer to Ethernet device. 6787 * @param eth_spec 6788 * An Ethernet flow spec to apply. 6789 * @param eth_mask 6790 * An Ethernet flow mask to apply. 6791 * @param vlan_spec 6792 * A VLAN flow spec to apply. 6793 * @param vlan_mask 6794 * A VLAN flow mask to apply. 6795 * 6796 * @return 6797 * 0 on success, a negative errno value otherwise and rte_errno is set. 6798 */ 6799 int 6800 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6801 struct rte_flow_item_eth *eth_spec, 6802 struct rte_flow_item_eth *eth_mask, 6803 struct rte_flow_item_vlan *vlan_spec, 6804 struct rte_flow_item_vlan *vlan_mask) 6805 { 6806 struct mlx5_priv *priv = dev->data->dev_private; 6807 const struct rte_flow_attr attr = { 6808 .ingress = 1, 6809 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6810 }; 6811 struct rte_flow_item items[] = { 6812 { 6813 .type = RTE_FLOW_ITEM_TYPE_ETH, 6814 .spec = eth_spec, 6815 .last = NULL, 6816 .mask = eth_mask, 6817 }, 6818 { 6819 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6820 RTE_FLOW_ITEM_TYPE_END, 6821 .spec = vlan_spec, 6822 .last = NULL, 6823 .mask = vlan_mask, 6824 }, 6825 { 6826 .type = RTE_FLOW_ITEM_TYPE_END, 6827 }, 6828 }; 6829 uint16_t queue[priv->reta_idx_n]; 6830 struct rte_flow_action_rss action_rss = { 6831 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6832 .level = 0, 6833 .types = priv->rss_conf.rss_hf, 6834 .key_len = priv->rss_conf.rss_key_len, 6835 .queue_num = priv->reta_idx_n, 6836 .key = priv->rss_conf.rss_key, 6837 .queue = queue, 6838 }; 6839 struct rte_flow_action actions[] = { 6840 { 6841 .type = RTE_FLOW_ACTION_TYPE_RSS, 6842 .conf = &action_rss, 6843 }, 6844 { 6845 .type = RTE_FLOW_ACTION_TYPE_END, 6846 }, 6847 }; 6848 uint32_t flow_idx; 6849 struct rte_flow_error error; 6850 unsigned int i; 6851 6852 if (!priv->reta_idx_n || !priv->rxqs_n) { 6853 return 0; 6854 } 6855 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6856 action_rss.types = 0; 6857 for (i = 0; i != priv->reta_idx_n; ++i) 6858 queue[i] = (*priv->reta_idx)[i]; 6859 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6860 &attr, items, actions, false, &error); 6861 if (!flow_idx) 6862 return -rte_errno; 6863 return 0; 6864 } 6865 6866 /** 6867 * Enable a flow control configured from the control plane. 6868 * 6869 * @param dev 6870 * Pointer to Ethernet device. 6871 * @param eth_spec 6872 * An Ethernet flow spec to apply. 6873 * @param eth_mask 6874 * An Ethernet flow mask to apply. 6875 * 6876 * @return 6877 * 0 on success, a negative errno value otherwise and rte_errno is set. 6878 */ 6879 int 6880 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6881 struct rte_flow_item_eth *eth_spec, 6882 struct rte_flow_item_eth *eth_mask) 6883 { 6884 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6885 } 6886 6887 /** 6888 * Create default miss flow rule matching lacp traffic 6889 * 6890 * @param dev 6891 * Pointer to Ethernet device. 6892 * @param eth_spec 6893 * An Ethernet flow spec to apply. 6894 * 6895 * @return 6896 * 0 on success, a negative errno value otherwise and rte_errno is set. 6897 */ 6898 int 6899 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6900 { 6901 struct mlx5_priv *priv = dev->data->dev_private; 6902 /* 6903 * The LACP matching is done by only using ether type since using 6904 * a multicast dst mac causes kernel to give low priority to this flow. 6905 */ 6906 static const struct rte_flow_item_eth lacp_spec = { 6907 .type = RTE_BE16(0x8809), 6908 }; 6909 static const struct rte_flow_item_eth lacp_mask = { 6910 .type = 0xffff, 6911 }; 6912 const struct rte_flow_attr attr = { 6913 .ingress = 1, 6914 }; 6915 struct rte_flow_item items[] = { 6916 { 6917 .type = RTE_FLOW_ITEM_TYPE_ETH, 6918 .spec = &lacp_spec, 6919 .mask = &lacp_mask, 6920 }, 6921 { 6922 .type = RTE_FLOW_ITEM_TYPE_END, 6923 }, 6924 }; 6925 struct rte_flow_action actions[] = { 6926 { 6927 .type = (enum rte_flow_action_type) 6928 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6929 }, 6930 { 6931 .type = RTE_FLOW_ACTION_TYPE_END, 6932 }, 6933 }; 6934 struct rte_flow_error error; 6935 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6936 &attr, items, actions, false, &error); 6937 6938 if (!flow_idx) 6939 return -rte_errno; 6940 return 0; 6941 } 6942 6943 /** 6944 * Destroy a flow. 6945 * 6946 * @see rte_flow_destroy() 6947 * @see rte_flow_ops 6948 */ 6949 int 6950 mlx5_flow_destroy(struct rte_eth_dev *dev, 6951 struct rte_flow *flow, 6952 struct rte_flow_error *error __rte_unused) 6953 { 6954 struct mlx5_priv *priv = dev->data->dev_private; 6955 6956 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6957 return 0; 6958 } 6959 6960 /** 6961 * Destroy all flows. 6962 * 6963 * @see rte_flow_flush() 6964 * @see rte_flow_ops 6965 */ 6966 int 6967 mlx5_flow_flush(struct rte_eth_dev *dev, 6968 struct rte_flow_error *error __rte_unused) 6969 { 6970 struct mlx5_priv *priv = dev->data->dev_private; 6971 6972 mlx5_flow_list_flush(dev, &priv->flows, false); 6973 return 0; 6974 } 6975 6976 /** 6977 * Isolated mode. 6978 * 6979 * @see rte_flow_isolate() 6980 * @see rte_flow_ops 6981 */ 6982 int 6983 mlx5_flow_isolate(struct rte_eth_dev *dev, 6984 int enable, 6985 struct rte_flow_error *error) 6986 { 6987 struct mlx5_priv *priv = dev->data->dev_private; 6988 6989 if (dev->data->dev_started) { 6990 rte_flow_error_set(error, EBUSY, 6991 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6992 NULL, 6993 "port must be stopped first"); 6994 return -rte_errno; 6995 } 6996 priv->isolated = !!enable; 6997 if (enable) 6998 dev->dev_ops = &mlx5_dev_ops_isolate; 6999 else 7000 dev->dev_ops = &mlx5_dev_ops; 7001 7002 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7003 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7004 7005 return 0; 7006 } 7007 7008 /** 7009 * Query a flow. 7010 * 7011 * @see rte_flow_query() 7012 * @see rte_flow_ops 7013 */ 7014 static int 7015 flow_drv_query(struct rte_eth_dev *dev, 7016 uint32_t flow_idx, 7017 const struct rte_flow_action *actions, 7018 void *data, 7019 struct rte_flow_error *error) 7020 { 7021 struct mlx5_priv *priv = dev->data->dev_private; 7022 const struct mlx5_flow_driver_ops *fops; 7023 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 7024 [MLX5_IPOOL_RTE_FLOW], 7025 flow_idx); 7026 enum mlx5_flow_drv_type ftype; 7027 7028 if (!flow) { 7029 return rte_flow_error_set(error, ENOENT, 7030 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7031 NULL, 7032 "invalid flow handle"); 7033 } 7034 ftype = flow->drv_type; 7035 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7036 fops = flow_get_drv_ops(ftype); 7037 7038 return fops->query(dev, flow, actions, data, error); 7039 } 7040 7041 /** 7042 * Query a flow. 7043 * 7044 * @see rte_flow_query() 7045 * @see rte_flow_ops 7046 */ 7047 int 7048 mlx5_flow_query(struct rte_eth_dev *dev, 7049 struct rte_flow *flow, 7050 const struct rte_flow_action *actions, 7051 void *data, 7052 struct rte_flow_error *error) 7053 { 7054 int ret; 7055 7056 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7057 error); 7058 if (ret < 0) 7059 return ret; 7060 return 0; 7061 } 7062 7063 /** 7064 * Get rte_flow callbacks. 7065 * 7066 * @param dev 7067 * Pointer to Ethernet device structure. 7068 * @param ops 7069 * Pointer to operation-specific structure. 7070 * 7071 * @return 0 7072 */ 7073 int 7074 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7075 const struct rte_flow_ops **ops) 7076 { 7077 *ops = &mlx5_flow_ops; 7078 return 0; 7079 } 7080 7081 /** 7082 * Validate meter policy actions. 7083 * Dispatcher for action type specific validation. 7084 * 7085 * @param[in] dev 7086 * Pointer to the Ethernet device structure. 7087 * @param[in] action 7088 * The meter policy action object to validate. 7089 * @param[in] attr 7090 * Attributes of flow to determine steering domain. 7091 * @param[out] is_rss 7092 * Is RSS or not. 7093 * @param[out] domain_bitmap 7094 * Domain bitmap. 7095 * @param[out] is_def_policy 7096 * Is default policy or not. 7097 * @param[out] error 7098 * Perform verbose error reporting if not NULL. Initialized in case of 7099 * error only. 7100 * 7101 * @return 7102 * 0 on success, otherwise negative errno value. 7103 */ 7104 int 7105 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7106 const struct rte_flow_action *actions[RTE_COLORS], 7107 struct rte_flow_attr *attr, 7108 bool *is_rss, 7109 uint8_t *domain_bitmap, 7110 bool *is_def_policy, 7111 struct rte_mtr_error *error) 7112 { 7113 const struct mlx5_flow_driver_ops *fops; 7114 7115 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7116 return fops->validate_mtr_acts(dev, actions, attr, 7117 is_rss, domain_bitmap, is_def_policy, error); 7118 } 7119 7120 /** 7121 * Destroy the meter table set. 7122 * 7123 * @param[in] dev 7124 * Pointer to Ethernet device. 7125 * @param[in] mtr_policy 7126 * Meter policy struct. 7127 */ 7128 void 7129 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7130 struct mlx5_flow_meter_policy *mtr_policy) 7131 { 7132 const struct mlx5_flow_driver_ops *fops; 7133 7134 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7135 fops->destroy_mtr_acts(dev, mtr_policy); 7136 } 7137 7138 /** 7139 * Create policy action, lock free, 7140 * (mutex should be acquired by caller). 7141 * Dispatcher for action type specific call. 7142 * 7143 * @param[in] dev 7144 * Pointer to the Ethernet device structure. 7145 * @param[in] mtr_policy 7146 * Meter policy struct. 7147 * @param[in] action 7148 * Action specification used to create meter actions. 7149 * @param[out] error 7150 * Perform verbose error reporting if not NULL. Initialized in case of 7151 * error only. 7152 * 7153 * @return 7154 * 0 on success, otherwise negative errno value. 7155 */ 7156 int 7157 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7158 struct mlx5_flow_meter_policy *mtr_policy, 7159 const struct rte_flow_action *actions[RTE_COLORS], 7160 struct rte_mtr_error *error) 7161 { 7162 const struct mlx5_flow_driver_ops *fops; 7163 7164 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7165 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7166 } 7167 7168 /** 7169 * Create policy rules, lock free, 7170 * (mutex should be acquired by caller). 7171 * Dispatcher for action type specific call. 7172 * 7173 * @param[in] dev 7174 * Pointer to the Ethernet device structure. 7175 * @param[in] mtr_policy 7176 * Meter policy struct. 7177 * 7178 * @return 7179 * 0 on success, -1 otherwise. 7180 */ 7181 int 7182 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7183 struct mlx5_flow_meter_policy *mtr_policy) 7184 { 7185 const struct mlx5_flow_driver_ops *fops; 7186 7187 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7188 return fops->create_policy_rules(dev, mtr_policy); 7189 } 7190 7191 /** 7192 * Destroy policy rules, lock free, 7193 * (mutex should be acquired by caller). 7194 * Dispatcher for action type specific call. 7195 * 7196 * @param[in] dev 7197 * Pointer to the Ethernet device structure. 7198 * @param[in] mtr_policy 7199 * Meter policy struct. 7200 */ 7201 void 7202 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7203 struct mlx5_flow_meter_policy *mtr_policy) 7204 { 7205 const struct mlx5_flow_driver_ops *fops; 7206 7207 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7208 fops->destroy_policy_rules(dev, mtr_policy); 7209 } 7210 7211 /** 7212 * Destroy the default policy table set. 7213 * 7214 * @param[in] dev 7215 * Pointer to Ethernet device. 7216 */ 7217 void 7218 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7219 { 7220 const struct mlx5_flow_driver_ops *fops; 7221 7222 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7223 fops->destroy_def_policy(dev); 7224 } 7225 7226 /** 7227 * Destroy the default policy table set. 7228 * 7229 * @param[in] dev 7230 * Pointer to Ethernet device. 7231 * 7232 * @return 7233 * 0 on success, -1 otherwise. 7234 */ 7235 int 7236 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7237 { 7238 const struct mlx5_flow_driver_ops *fops; 7239 7240 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7241 return fops->create_def_policy(dev); 7242 } 7243 7244 /** 7245 * Create the needed meter and suffix tables. 7246 * 7247 * @param[in] dev 7248 * Pointer to Ethernet device. 7249 * 7250 * @return 7251 * 0 on success, -1 otherwise. 7252 */ 7253 int 7254 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7255 struct mlx5_flow_meter_info *fm, 7256 uint32_t mtr_idx, 7257 uint8_t domain_bitmap) 7258 { 7259 const struct mlx5_flow_driver_ops *fops; 7260 7261 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7262 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7263 } 7264 7265 /** 7266 * Destroy the meter table set. 7267 * 7268 * @param[in] dev 7269 * Pointer to Ethernet device. 7270 * @param[in] tbl 7271 * Pointer to the meter table set. 7272 */ 7273 void 7274 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7275 struct mlx5_flow_meter_info *fm) 7276 { 7277 const struct mlx5_flow_driver_ops *fops; 7278 7279 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7280 fops->destroy_mtr_tbls(dev, fm); 7281 } 7282 7283 /** 7284 * Destroy the global meter drop table. 7285 * 7286 * @param[in] dev 7287 * Pointer to Ethernet device. 7288 */ 7289 void 7290 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7291 { 7292 const struct mlx5_flow_driver_ops *fops; 7293 7294 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7295 fops->destroy_mtr_drop_tbls(dev); 7296 } 7297 7298 /** 7299 * Destroy the sub policy table with RX queue. 7300 * 7301 * @param[in] dev 7302 * Pointer to Ethernet device. 7303 * @param[in] mtr_policy 7304 * Pointer to meter policy table. 7305 */ 7306 void 7307 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7308 struct mlx5_flow_meter_policy *mtr_policy) 7309 { 7310 const struct mlx5_flow_driver_ops *fops; 7311 7312 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7313 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7314 } 7315 7316 /** 7317 * Allocate the needed aso flow meter id. 7318 * 7319 * @param[in] dev 7320 * Pointer to Ethernet device. 7321 * 7322 * @return 7323 * Index to aso flow meter on success, NULL otherwise. 7324 */ 7325 uint32_t 7326 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7327 { 7328 const struct mlx5_flow_driver_ops *fops; 7329 7330 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7331 return fops->create_meter(dev); 7332 } 7333 7334 /** 7335 * Free the aso flow meter id. 7336 * 7337 * @param[in] dev 7338 * Pointer to Ethernet device. 7339 * @param[in] mtr_idx 7340 * Index to aso flow meter to be free. 7341 * 7342 * @return 7343 * 0 on success. 7344 */ 7345 void 7346 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7347 { 7348 const struct mlx5_flow_driver_ops *fops; 7349 7350 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7351 fops->free_meter(dev, mtr_idx); 7352 } 7353 7354 /** 7355 * Allocate a counter. 7356 * 7357 * @param[in] dev 7358 * Pointer to Ethernet device structure. 7359 * 7360 * @return 7361 * Index to allocated counter on success, 0 otherwise. 7362 */ 7363 uint32_t 7364 mlx5_counter_alloc(struct rte_eth_dev *dev) 7365 { 7366 const struct mlx5_flow_driver_ops *fops; 7367 struct rte_flow_attr attr = { .transfer = 0 }; 7368 7369 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7370 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7371 return fops->counter_alloc(dev); 7372 } 7373 DRV_LOG(ERR, 7374 "port %u counter allocate is not supported.", 7375 dev->data->port_id); 7376 return 0; 7377 } 7378 7379 /** 7380 * Free a counter. 7381 * 7382 * @param[in] dev 7383 * Pointer to Ethernet device structure. 7384 * @param[in] cnt 7385 * Index to counter to be free. 7386 */ 7387 void 7388 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7389 { 7390 const struct mlx5_flow_driver_ops *fops; 7391 struct rte_flow_attr attr = { .transfer = 0 }; 7392 7393 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7394 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7395 fops->counter_free(dev, cnt); 7396 return; 7397 } 7398 DRV_LOG(ERR, 7399 "port %u counter free is not supported.", 7400 dev->data->port_id); 7401 } 7402 7403 /** 7404 * Query counter statistics. 7405 * 7406 * @param[in] dev 7407 * Pointer to Ethernet device structure. 7408 * @param[in] cnt 7409 * Index to counter to query. 7410 * @param[in] clear 7411 * Set to clear counter statistics. 7412 * @param[out] pkts 7413 * The counter hits packets number to save. 7414 * @param[out] bytes 7415 * The counter hits bytes number to save. 7416 * 7417 * @return 7418 * 0 on success, a negative errno value otherwise. 7419 */ 7420 int 7421 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7422 bool clear, uint64_t *pkts, uint64_t *bytes) 7423 { 7424 const struct mlx5_flow_driver_ops *fops; 7425 struct rte_flow_attr attr = { .transfer = 0 }; 7426 7427 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7428 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7429 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7430 } 7431 DRV_LOG(ERR, 7432 "port %u counter query is not supported.", 7433 dev->data->port_id); 7434 return -ENOTSUP; 7435 } 7436 7437 /** 7438 * Allocate a new memory for the counter values wrapped by all the needed 7439 * management. 7440 * 7441 * @param[in] sh 7442 * Pointer to mlx5_dev_ctx_shared object. 7443 * 7444 * @return 7445 * 0 on success, a negative errno value otherwise. 7446 */ 7447 static int 7448 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7449 { 7450 struct mlx5_devx_mkey_attr mkey_attr; 7451 struct mlx5_counter_stats_mem_mng *mem_mng; 7452 volatile struct flow_counter_stats *raw_data; 7453 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7454 int size = (sizeof(struct flow_counter_stats) * 7455 MLX5_COUNTERS_PER_POOL + 7456 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7457 sizeof(struct mlx5_counter_stats_mem_mng); 7458 size_t pgsize = rte_mem_page_size(); 7459 uint8_t *mem; 7460 int i; 7461 7462 if (pgsize == (size_t)-1) { 7463 DRV_LOG(ERR, "Failed to get mem page size"); 7464 rte_errno = ENOMEM; 7465 return -ENOMEM; 7466 } 7467 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7468 if (!mem) { 7469 rte_errno = ENOMEM; 7470 return -ENOMEM; 7471 } 7472 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7473 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7474 mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size, 7475 IBV_ACCESS_LOCAL_WRITE); 7476 if (!mem_mng->umem) { 7477 rte_errno = errno; 7478 mlx5_free(mem); 7479 return -rte_errno; 7480 } 7481 memset(&mkey_attr, 0, sizeof(mkey_attr)); 7482 mkey_attr.addr = (uintptr_t)mem; 7483 mkey_attr.size = size; 7484 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7485 mkey_attr.pd = sh->pdn; 7486 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 7487 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 7488 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7489 if (!mem_mng->dm) { 7490 mlx5_os_umem_dereg(mem_mng->umem); 7491 rte_errno = errno; 7492 mlx5_free(mem); 7493 return -rte_errno; 7494 } 7495 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7496 raw_data = (volatile struct flow_counter_stats *)mem; 7497 for (i = 0; i < raws_n; ++i) { 7498 mem_mng->raws[i].mem_mng = mem_mng; 7499 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7500 } 7501 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7502 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7503 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7504 next); 7505 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7506 sh->cmng.mem_mng = mem_mng; 7507 return 0; 7508 } 7509 7510 /** 7511 * Set the statistic memory to the new counter pool. 7512 * 7513 * @param[in] sh 7514 * Pointer to mlx5_dev_ctx_shared object. 7515 * @param[in] pool 7516 * Pointer to the pool to set the statistic memory. 7517 * 7518 * @return 7519 * 0 on success, a negative errno value otherwise. 7520 */ 7521 static int 7522 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7523 struct mlx5_flow_counter_pool *pool) 7524 { 7525 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7526 /* Resize statistic memory once used out. */ 7527 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7528 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7529 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7530 return -1; 7531 } 7532 rte_spinlock_lock(&pool->sl); 7533 pool->raw = cmng->mem_mng->raws + pool->index % 7534 MLX5_CNT_CONTAINER_RESIZE; 7535 rte_spinlock_unlock(&pool->sl); 7536 pool->raw_hw = NULL; 7537 return 0; 7538 } 7539 7540 #define MLX5_POOL_QUERY_FREQ_US 1000000 7541 7542 /** 7543 * Set the periodic procedure for triggering asynchronous batch queries for all 7544 * the counter pools. 7545 * 7546 * @param[in] sh 7547 * Pointer to mlx5_dev_ctx_shared object. 7548 */ 7549 void 7550 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7551 { 7552 uint32_t pools_n, us; 7553 7554 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7555 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7556 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7557 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7558 sh->cmng.query_thread_on = 0; 7559 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7560 } else { 7561 sh->cmng.query_thread_on = 1; 7562 } 7563 } 7564 7565 /** 7566 * The periodic procedure for triggering asynchronous batch queries for all the 7567 * counter pools. This function is probably called by the host thread. 7568 * 7569 * @param[in] arg 7570 * The parameter for the alarm process. 7571 */ 7572 void 7573 mlx5_flow_query_alarm(void *arg) 7574 { 7575 struct mlx5_dev_ctx_shared *sh = arg; 7576 int ret; 7577 uint16_t pool_index = sh->cmng.pool_index; 7578 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7579 struct mlx5_flow_counter_pool *pool; 7580 uint16_t n_valid; 7581 7582 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7583 goto set_alarm; 7584 rte_spinlock_lock(&cmng->pool_update_sl); 7585 pool = cmng->pools[pool_index]; 7586 n_valid = cmng->n_valid; 7587 rte_spinlock_unlock(&cmng->pool_update_sl); 7588 /* Set the statistic memory to the new created pool. */ 7589 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7590 goto set_alarm; 7591 if (pool->raw_hw) 7592 /* There is a pool query in progress. */ 7593 goto set_alarm; 7594 pool->raw_hw = 7595 LIST_FIRST(&sh->cmng.free_stat_raws); 7596 if (!pool->raw_hw) 7597 /* No free counter statistics raw memory. */ 7598 goto set_alarm; 7599 /* 7600 * Identify the counters released between query trigger and query 7601 * handle more efficiently. The counter released in this gap period 7602 * should wait for a new round of query as the new arrived packets 7603 * will not be taken into account. 7604 */ 7605 pool->query_gen++; 7606 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7607 MLX5_COUNTERS_PER_POOL, 7608 NULL, NULL, 7609 pool->raw_hw->mem_mng->dm->id, 7610 (void *)(uintptr_t) 7611 pool->raw_hw->data, 7612 sh->devx_comp, 7613 (uint64_t)(uintptr_t)pool); 7614 if (ret) { 7615 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7616 " %d", pool->min_dcs->id); 7617 pool->raw_hw = NULL; 7618 goto set_alarm; 7619 } 7620 LIST_REMOVE(pool->raw_hw, next); 7621 sh->cmng.pending_queries++; 7622 pool_index++; 7623 if (pool_index >= n_valid) 7624 pool_index = 0; 7625 set_alarm: 7626 sh->cmng.pool_index = pool_index; 7627 mlx5_set_query_alarm(sh); 7628 } 7629 7630 /** 7631 * Check and callback event for new aged flow in the counter pool 7632 * 7633 * @param[in] sh 7634 * Pointer to mlx5_dev_ctx_shared object. 7635 * @param[in] pool 7636 * Pointer to Current counter pool. 7637 */ 7638 static void 7639 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7640 struct mlx5_flow_counter_pool *pool) 7641 { 7642 struct mlx5_priv *priv; 7643 struct mlx5_flow_counter *cnt; 7644 struct mlx5_age_info *age_info; 7645 struct mlx5_age_param *age_param; 7646 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7647 struct mlx5_counter_stats_raw *prev = pool->raw; 7648 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7649 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7650 uint16_t expected = AGE_CANDIDATE; 7651 uint32_t i; 7652 7653 pool->time_of_last_age_check = curr_time; 7654 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7655 cnt = MLX5_POOL_GET_CNT(pool, i); 7656 age_param = MLX5_CNT_TO_AGE(cnt); 7657 if (__atomic_load_n(&age_param->state, 7658 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7659 continue; 7660 if (cur->data[i].hits != prev->data[i].hits) { 7661 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7662 __ATOMIC_RELAXED); 7663 continue; 7664 } 7665 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7666 time_delta, 7667 __ATOMIC_RELAXED) <= age_param->timeout) 7668 continue; 7669 /** 7670 * Hold the lock first, or if between the 7671 * state AGE_TMOUT and tailq operation the 7672 * release happened, the release procedure 7673 * may delete a non-existent tailq node. 7674 */ 7675 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7676 age_info = GET_PORT_AGE_INFO(priv); 7677 rte_spinlock_lock(&age_info->aged_sl); 7678 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7679 AGE_TMOUT, false, 7680 __ATOMIC_RELAXED, 7681 __ATOMIC_RELAXED)) { 7682 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7683 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7684 } 7685 rte_spinlock_unlock(&age_info->aged_sl); 7686 } 7687 mlx5_age_event_prepare(sh); 7688 } 7689 7690 /** 7691 * Handler for the HW respond about ready values from an asynchronous batch 7692 * query. This function is probably called by the host thread. 7693 * 7694 * @param[in] sh 7695 * The pointer to the shared device context. 7696 * @param[in] async_id 7697 * The Devx async ID. 7698 * @param[in] status 7699 * The status of the completion. 7700 */ 7701 void 7702 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7703 uint64_t async_id, int status) 7704 { 7705 struct mlx5_flow_counter_pool *pool = 7706 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7707 struct mlx5_counter_stats_raw *raw_to_free; 7708 uint8_t query_gen = pool->query_gen ^ 1; 7709 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7710 enum mlx5_counter_type cnt_type = 7711 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7712 MLX5_COUNTER_TYPE_ORIGIN; 7713 7714 if (unlikely(status)) { 7715 raw_to_free = pool->raw_hw; 7716 } else { 7717 raw_to_free = pool->raw; 7718 if (pool->is_aged) 7719 mlx5_flow_aging_check(sh, pool); 7720 rte_spinlock_lock(&pool->sl); 7721 pool->raw = pool->raw_hw; 7722 rte_spinlock_unlock(&pool->sl); 7723 /* Be sure the new raw counters data is updated in memory. */ 7724 rte_io_wmb(); 7725 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7726 rte_spinlock_lock(&cmng->csl[cnt_type]); 7727 TAILQ_CONCAT(&cmng->counters[cnt_type], 7728 &pool->counters[query_gen], next); 7729 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7730 } 7731 } 7732 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7733 pool->raw_hw = NULL; 7734 sh->cmng.pending_queries--; 7735 } 7736 7737 static int 7738 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7739 const struct flow_grp_info *grp_info, 7740 struct rte_flow_error *error) 7741 { 7742 if (grp_info->transfer && grp_info->external && 7743 grp_info->fdb_def_rule) { 7744 if (group == UINT32_MAX) 7745 return rte_flow_error_set 7746 (error, EINVAL, 7747 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7748 NULL, 7749 "group index not supported"); 7750 *table = group + 1; 7751 } else { 7752 *table = group; 7753 } 7754 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7755 return 0; 7756 } 7757 7758 /** 7759 * Translate the rte_flow group index to HW table value. 7760 * 7761 * If tunnel offload is disabled, all group ids converted to flow table 7762 * id using the standard method. 7763 * If tunnel offload is enabled, group id can be converted using the 7764 * standard or tunnel conversion method. Group conversion method 7765 * selection depends on flags in `grp_info` parameter: 7766 * - Internal (grp_info.external == 0) groups conversion uses the 7767 * standard method. 7768 * - Group ids in JUMP action converted with the tunnel conversion. 7769 * - Group id in rule attribute conversion depends on a rule type and 7770 * group id value: 7771 * ** non zero group attributes converted with the tunnel method 7772 * ** zero group attribute in non-tunnel rule is converted using the 7773 * standard method - there's only one root table 7774 * ** zero group attribute in steer tunnel rule is converted with the 7775 * standard method - single root table 7776 * ** zero group attribute in match tunnel rule is a special OvS 7777 * case: that value is used for portability reasons. That group 7778 * id is converted with the tunnel conversion method. 7779 * 7780 * @param[in] dev 7781 * Port device 7782 * @param[in] tunnel 7783 * PMD tunnel offload object 7784 * @param[in] group 7785 * rte_flow group index value. 7786 * @param[out] table 7787 * HW table value. 7788 * @param[in] grp_info 7789 * flags used for conversion 7790 * @param[out] error 7791 * Pointer to error structure. 7792 * 7793 * @return 7794 * 0 on success, a negative errno value otherwise and rte_errno is set. 7795 */ 7796 int 7797 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7798 const struct mlx5_flow_tunnel *tunnel, 7799 uint32_t group, uint32_t *table, 7800 const struct flow_grp_info *grp_info, 7801 struct rte_flow_error *error) 7802 { 7803 int ret; 7804 bool standard_translation; 7805 7806 if (!grp_info->skip_scale && grp_info->external && 7807 group < MLX5_MAX_TABLES_EXTERNAL) 7808 group *= MLX5_FLOW_TABLE_FACTOR; 7809 if (is_tunnel_offload_active(dev)) { 7810 standard_translation = !grp_info->external || 7811 grp_info->std_tbl_fix; 7812 } else { 7813 standard_translation = true; 7814 } 7815 DRV_LOG(DEBUG, 7816 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 7817 dev->data->port_id, group, grp_info->transfer, 7818 grp_info->external, grp_info->fdb_def_rule, 7819 standard_translation ? "STANDARD" : "TUNNEL"); 7820 if (standard_translation) 7821 ret = flow_group_to_table(dev->data->port_id, group, table, 7822 grp_info, error); 7823 else 7824 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7825 table, error); 7826 7827 return ret; 7828 } 7829 7830 /** 7831 * Discover availability of metadata reg_c's. 7832 * 7833 * Iteratively use test flows to check availability. 7834 * 7835 * @param[in] dev 7836 * Pointer to the Ethernet device structure. 7837 * 7838 * @return 7839 * 0 on success, a negative errno value otherwise and rte_errno is set. 7840 */ 7841 int 7842 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7843 { 7844 struct mlx5_priv *priv = dev->data->dev_private; 7845 struct mlx5_dev_config *config = &priv->config; 7846 enum modify_reg idx; 7847 int n = 0; 7848 7849 /* reg_c[0] and reg_c[1] are reserved. */ 7850 config->flow_mreg_c[n++] = REG_C_0; 7851 config->flow_mreg_c[n++] = REG_C_1; 7852 /* Discover availability of other reg_c's. */ 7853 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7854 struct rte_flow_attr attr = { 7855 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7856 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7857 .ingress = 1, 7858 }; 7859 struct rte_flow_item items[] = { 7860 [0] = { 7861 .type = RTE_FLOW_ITEM_TYPE_END, 7862 }, 7863 }; 7864 struct rte_flow_action actions[] = { 7865 [0] = { 7866 .type = (enum rte_flow_action_type) 7867 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7868 .conf = &(struct mlx5_flow_action_copy_mreg){ 7869 .src = REG_C_1, 7870 .dst = idx, 7871 }, 7872 }, 7873 [1] = { 7874 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7875 .conf = &(struct rte_flow_action_jump){ 7876 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7877 }, 7878 }, 7879 [2] = { 7880 .type = RTE_FLOW_ACTION_TYPE_END, 7881 }, 7882 }; 7883 uint32_t flow_idx; 7884 struct rte_flow *flow; 7885 struct rte_flow_error error; 7886 7887 if (!config->dv_flow_en) 7888 break; 7889 /* Create internal flow, validation skips copy action. */ 7890 flow_idx = flow_list_create(dev, NULL, &attr, items, 7891 actions, false, &error); 7892 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 7893 flow_idx); 7894 if (!flow) 7895 continue; 7896 config->flow_mreg_c[n++] = idx; 7897 flow_list_destroy(dev, NULL, flow_idx); 7898 } 7899 for (; n < MLX5_MREG_C_NUM; ++n) 7900 config->flow_mreg_c[n] = REG_NON; 7901 return 0; 7902 } 7903 7904 int 7905 save_dump_file(const uint8_t *data, uint32_t size, 7906 uint32_t type, uint32_t id, void *arg, FILE *file) 7907 { 7908 char line[BUF_SIZE]; 7909 uint32_t out = 0; 7910 uint32_t k; 7911 uint32_t actions_num; 7912 struct rte_flow_query_count *count; 7913 7914 memset(line, 0, BUF_SIZE); 7915 switch (type) { 7916 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 7917 actions_num = *(uint32_t *)(arg); 7918 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,", 7919 type, id, actions_num); 7920 break; 7921 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 7922 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,", 7923 type, id); 7924 break; 7925 case DR_DUMP_REC_TYPE_PMD_COUNTER: 7926 count = (struct rte_flow_query_count *)arg; 7927 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type, 7928 id, count->hits, count->bytes); 7929 return 0; 7930 default: 7931 return -1; 7932 } 7933 7934 for (k = 0; k < size; k++) { 7935 /* Make sure we do not overrun the line buffer length. */ 7936 if (out >= BUF_SIZE - 4) { 7937 line[out] = '\0'; 7938 break; 7939 } 7940 out += snprintf(line + out, BUF_SIZE - out, "%02x", 7941 (data[k]) & 0xff); 7942 } 7943 fprintf(file, "%s\n", line); 7944 return 0; 7945 } 7946 7947 int 7948 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 7949 struct rte_flow_query_count *count, struct rte_flow_error *error) 7950 { 7951 struct rte_flow_action action[2]; 7952 enum mlx5_flow_drv_type ftype; 7953 const struct mlx5_flow_driver_ops *fops; 7954 7955 if (!flow) { 7956 return rte_flow_error_set(error, ENOENT, 7957 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7958 NULL, 7959 "invalid flow handle"); 7960 } 7961 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 7962 action[1].type = RTE_FLOW_ACTION_TYPE_END; 7963 if (flow->counter) { 7964 memset(count, 0, sizeof(struct rte_flow_query_count)); 7965 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 7966 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 7967 ftype < MLX5_FLOW_TYPE_MAX); 7968 fops = flow_get_drv_ops(ftype); 7969 return fops->query(dev, flow, action, count, error); 7970 } 7971 return -1; 7972 } 7973 7974 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7975 /** 7976 * Dump flow ipool data to file 7977 * 7978 * @param[in] dev 7979 * The pointer to Ethernet device. 7980 * @param[in] file 7981 * A pointer to a file for output. 7982 * @param[out] error 7983 * Perform verbose error reporting if not NULL. PMDs initialize this 7984 * structure in case of error only. 7985 * @return 7986 * 0 on success, a negative value otherwise. 7987 */ 7988 int 7989 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 7990 struct rte_flow *flow, FILE *file, 7991 struct rte_flow_error *error) 7992 { 7993 struct mlx5_priv *priv = dev->data->dev_private; 7994 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 7995 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 7996 uint32_t handle_idx; 7997 struct mlx5_flow_handle *dh; 7998 struct rte_flow_query_count count; 7999 uint32_t actions_num; 8000 const uint8_t *data; 8001 size_t size; 8002 uint32_t id; 8003 uint32_t type; 8004 8005 if (!flow) { 8006 return rte_flow_error_set(error, ENOENT, 8007 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8008 NULL, 8009 "invalid flow handle"); 8010 } 8011 handle_idx = flow->dev_handles; 8012 while (handle_idx) { 8013 dh = mlx5_ipool_get(priv->sh->ipool 8014 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8015 if (!dh) 8016 continue; 8017 handle_idx = dh->next.next; 8018 id = (uint32_t)(uintptr_t)dh->drv_flow; 8019 8020 /* query counter */ 8021 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8022 if (!mlx5_flow_query_counter(dev, flow, &count, error)) 8023 save_dump_file(NULL, 0, type, 8024 id, (void *)&count, file); 8025 8026 /* Get modify_hdr and encap_decap buf from ipools. */ 8027 encap_decap = NULL; 8028 modify_hdr = dh->dvh.modify_hdr; 8029 8030 if (dh->dvh.rix_encap_decap) { 8031 encap_decap = mlx5_ipool_get(priv->sh->ipool 8032 [MLX5_IPOOL_DECAP_ENCAP], 8033 dh->dvh.rix_encap_decap); 8034 } 8035 if (modify_hdr) { 8036 data = (const uint8_t *)modify_hdr->actions; 8037 size = (size_t)(modify_hdr->actions_num) * 8; 8038 actions_num = modify_hdr->actions_num; 8039 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8040 save_dump_file(data, size, type, id, 8041 (void *)(&actions_num), file); 8042 } 8043 if (encap_decap) { 8044 data = encap_decap->buf; 8045 size = encap_decap->size; 8046 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8047 save_dump_file(data, size, type, 8048 id, NULL, file); 8049 } 8050 } 8051 return 0; 8052 } 8053 #endif 8054 8055 /** 8056 * Dump flow raw hw data to file 8057 * 8058 * @param[in] dev 8059 * The pointer to Ethernet device. 8060 * @param[in] file 8061 * A pointer to a file for output. 8062 * @param[out] error 8063 * Perform verbose error reporting if not NULL. PMDs initialize this 8064 * structure in case of error only. 8065 * @return 8066 * 0 on success, a nagative value otherwise. 8067 */ 8068 int 8069 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8070 FILE *file, 8071 struct rte_flow_error *error __rte_unused) 8072 { 8073 struct mlx5_priv *priv = dev->data->dev_private; 8074 struct mlx5_dev_ctx_shared *sh = priv->sh; 8075 uint32_t handle_idx; 8076 int ret; 8077 struct mlx5_flow_handle *dh; 8078 struct rte_flow *flow; 8079 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8080 uint32_t idx; 8081 #endif 8082 8083 if (!priv->config.dv_flow_en) { 8084 if (fputs("device dv flow disabled\n", file) <= 0) 8085 return -errno; 8086 return -ENOTSUP; 8087 } 8088 8089 /* dump all */ 8090 if (!flow_idx) { 8091 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8092 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 8093 priv->flows, idx, flow, next) 8094 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8095 #endif 8096 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8097 sh->rx_domain, 8098 sh->tx_domain, file); 8099 } 8100 /* dump one */ 8101 flow = mlx5_ipool_get(priv->sh->ipool 8102 [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx); 8103 if (!flow) 8104 return -ENOENT; 8105 8106 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8107 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8108 #endif 8109 handle_idx = flow->dev_handles; 8110 while (handle_idx) { 8111 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8112 handle_idx); 8113 if (!dh) 8114 return -ENOENT; 8115 if (dh->drv_flow) { 8116 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8117 file); 8118 if (ret) 8119 return -ENOENT; 8120 } 8121 handle_idx = dh->next.next; 8122 } 8123 return 0; 8124 } 8125 8126 /** 8127 * Get aged-out flows. 8128 * 8129 * @param[in] dev 8130 * Pointer to the Ethernet device structure. 8131 * @param[in] context 8132 * The address of an array of pointers to the aged-out flows contexts. 8133 * @param[in] nb_countexts 8134 * The length of context array pointers. 8135 * @param[out] error 8136 * Perform verbose error reporting if not NULL. Initialized in case of 8137 * error only. 8138 * 8139 * @return 8140 * how many contexts get in success, otherwise negative errno value. 8141 * if nb_contexts is 0, return the amount of all aged contexts. 8142 * if nb_contexts is not 0 , return the amount of aged flows reported 8143 * in the context array. 8144 */ 8145 int 8146 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8147 uint32_t nb_contexts, struct rte_flow_error *error) 8148 { 8149 const struct mlx5_flow_driver_ops *fops; 8150 struct rte_flow_attr attr = { .transfer = 0 }; 8151 8152 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8153 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8154 return fops->get_aged_flows(dev, contexts, nb_contexts, 8155 error); 8156 } 8157 DRV_LOG(ERR, 8158 "port %u get aged flows is not supported.", 8159 dev->data->port_id); 8160 return -ENOTSUP; 8161 } 8162 8163 /* Wrapper for driver action_validate op callback */ 8164 static int 8165 flow_drv_action_validate(struct rte_eth_dev *dev, 8166 const struct rte_flow_indir_action_conf *conf, 8167 const struct rte_flow_action *action, 8168 const struct mlx5_flow_driver_ops *fops, 8169 struct rte_flow_error *error) 8170 { 8171 static const char err_msg[] = "indirect action validation unsupported"; 8172 8173 if (!fops->action_validate) { 8174 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8175 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8176 NULL, err_msg); 8177 return -rte_errno; 8178 } 8179 return fops->action_validate(dev, conf, action, error); 8180 } 8181 8182 /** 8183 * Destroys the shared action by handle. 8184 * 8185 * @param dev 8186 * Pointer to Ethernet device structure. 8187 * @param[in] handle 8188 * Handle for the indirect action object to be destroyed. 8189 * @param[out] error 8190 * Perform verbose error reporting if not NULL. PMDs initialize this 8191 * structure in case of error only. 8192 * 8193 * @return 8194 * 0 on success, a negative errno value otherwise and rte_errno is set. 8195 * 8196 * @note: wrapper for driver action_create op callback. 8197 */ 8198 static int 8199 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8200 struct rte_flow_action_handle *handle, 8201 struct rte_flow_error *error) 8202 { 8203 static const char err_msg[] = "indirect action destruction unsupported"; 8204 struct rte_flow_attr attr = { .transfer = 0 }; 8205 const struct mlx5_flow_driver_ops *fops = 8206 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8207 8208 if (!fops->action_destroy) { 8209 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8210 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8211 NULL, err_msg); 8212 return -rte_errno; 8213 } 8214 return fops->action_destroy(dev, handle, error); 8215 } 8216 8217 /* Wrapper for driver action_destroy op callback */ 8218 static int 8219 flow_drv_action_update(struct rte_eth_dev *dev, 8220 struct rte_flow_action_handle *handle, 8221 const void *update, 8222 const struct mlx5_flow_driver_ops *fops, 8223 struct rte_flow_error *error) 8224 { 8225 static const char err_msg[] = "indirect action update unsupported"; 8226 8227 if (!fops->action_update) { 8228 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8229 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8230 NULL, err_msg); 8231 return -rte_errno; 8232 } 8233 return fops->action_update(dev, handle, update, error); 8234 } 8235 8236 /* Wrapper for driver action_destroy op callback */ 8237 static int 8238 flow_drv_action_query(struct rte_eth_dev *dev, 8239 const struct rte_flow_action_handle *handle, 8240 void *data, 8241 const struct mlx5_flow_driver_ops *fops, 8242 struct rte_flow_error *error) 8243 { 8244 static const char err_msg[] = "indirect action query unsupported"; 8245 8246 if (!fops->action_query) { 8247 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8248 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8249 NULL, err_msg); 8250 return -rte_errno; 8251 } 8252 return fops->action_query(dev, handle, data, error); 8253 } 8254 8255 /** 8256 * Create indirect action for reuse in multiple flow rules. 8257 * 8258 * @param dev 8259 * Pointer to Ethernet device structure. 8260 * @param conf 8261 * Pointer to indirect action object configuration. 8262 * @param[in] action 8263 * Action configuration for indirect action object creation. 8264 * @param[out] error 8265 * Perform verbose error reporting if not NULL. PMDs initialize this 8266 * structure in case of error only. 8267 * @return 8268 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8269 */ 8270 static struct rte_flow_action_handle * 8271 mlx5_action_handle_create(struct rte_eth_dev *dev, 8272 const struct rte_flow_indir_action_conf *conf, 8273 const struct rte_flow_action *action, 8274 struct rte_flow_error *error) 8275 { 8276 static const char err_msg[] = "indirect action creation unsupported"; 8277 struct rte_flow_attr attr = { .transfer = 0 }; 8278 const struct mlx5_flow_driver_ops *fops = 8279 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8280 8281 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8282 return NULL; 8283 if (!fops->action_create) { 8284 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8285 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8286 NULL, err_msg); 8287 return NULL; 8288 } 8289 return fops->action_create(dev, conf, action, error); 8290 } 8291 8292 /** 8293 * Updates inplace the indirect action configuration pointed by *handle* 8294 * with the configuration provided as *update* argument. 8295 * The update of the indirect action configuration effects all flow rules 8296 * reusing the action via handle. 8297 * 8298 * @param dev 8299 * Pointer to Ethernet device structure. 8300 * @param[in] handle 8301 * Handle for the indirect action to be updated. 8302 * @param[in] update 8303 * Action specification used to modify the action pointed by handle. 8304 * *update* could be of same type with the action pointed by the *handle* 8305 * handle argument, or some other structures like a wrapper, depending on 8306 * the indirect action type. 8307 * @param[out] error 8308 * Perform verbose error reporting if not NULL. PMDs initialize this 8309 * structure in case of error only. 8310 * 8311 * @return 8312 * 0 on success, a negative errno value otherwise and rte_errno is set. 8313 */ 8314 static int 8315 mlx5_action_handle_update(struct rte_eth_dev *dev, 8316 struct rte_flow_action_handle *handle, 8317 const void *update, 8318 struct rte_flow_error *error) 8319 { 8320 struct rte_flow_attr attr = { .transfer = 0 }; 8321 const struct mlx5_flow_driver_ops *fops = 8322 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8323 int ret; 8324 8325 ret = flow_drv_action_validate(dev, NULL, 8326 (const struct rte_flow_action *)update, fops, error); 8327 if (ret) 8328 return ret; 8329 return flow_drv_action_update(dev, handle, update, fops, 8330 error); 8331 } 8332 8333 /** 8334 * Query the indirect action by handle. 8335 * 8336 * This function allows retrieving action-specific data such as counters. 8337 * Data is gathered by special action which may be present/referenced in 8338 * more than one flow rule definition. 8339 * 8340 * see @RTE_FLOW_ACTION_TYPE_COUNT 8341 * 8342 * @param dev 8343 * Pointer to Ethernet device structure. 8344 * @param[in] handle 8345 * Handle for the indirect action to query. 8346 * @param[in, out] data 8347 * Pointer to storage for the associated query data type. 8348 * @param[out] error 8349 * Perform verbose error reporting if not NULL. PMDs initialize this 8350 * structure in case of error only. 8351 * 8352 * @return 8353 * 0 on success, a negative errno value otherwise and rte_errno is set. 8354 */ 8355 static int 8356 mlx5_action_handle_query(struct rte_eth_dev *dev, 8357 const struct rte_flow_action_handle *handle, 8358 void *data, 8359 struct rte_flow_error *error) 8360 { 8361 struct rte_flow_attr attr = { .transfer = 0 }; 8362 const struct mlx5_flow_driver_ops *fops = 8363 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8364 8365 return flow_drv_action_query(dev, handle, data, fops, error); 8366 } 8367 8368 /** 8369 * Destroy all indirect actions (shared RSS). 8370 * 8371 * @param dev 8372 * Pointer to Ethernet device. 8373 * 8374 * @return 8375 * 0 on success, a negative errno value otherwise and rte_errno is set. 8376 */ 8377 int 8378 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8379 { 8380 struct rte_flow_error error; 8381 struct mlx5_priv *priv = dev->data->dev_private; 8382 struct mlx5_shared_action_rss *shared_rss; 8383 int ret = 0; 8384 uint32_t idx; 8385 8386 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8387 priv->rss_shared_actions, idx, shared_rss, next) { 8388 ret |= mlx5_action_handle_destroy(dev, 8389 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8390 } 8391 return ret; 8392 } 8393 8394 #ifndef HAVE_MLX5DV_DR 8395 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8396 #else 8397 #define MLX5_DOMAIN_SYNC_FLOW \ 8398 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8399 #endif 8400 8401 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8402 { 8403 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8404 const struct mlx5_flow_driver_ops *fops; 8405 int ret; 8406 struct rte_flow_attr attr = { .transfer = 0 }; 8407 8408 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8409 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8410 if (ret > 0) 8411 ret = -ret; 8412 return ret; 8413 } 8414 8415 const struct mlx5_flow_tunnel * 8416 mlx5_get_tof(const struct rte_flow_item *item, 8417 const struct rte_flow_action *action, 8418 enum mlx5_tof_rule_type *rule_type) 8419 { 8420 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 8421 if (item->type == (typeof(item->type)) 8422 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 8423 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 8424 return flow_items_to_tunnel(item); 8425 } 8426 } 8427 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 8428 if (action->type == (typeof(action->type)) 8429 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 8430 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 8431 return flow_actions_to_tunnel(action); 8432 } 8433 } 8434 return NULL; 8435 } 8436 8437 /** 8438 * tunnel offload functionalilty is defined for DV environment only 8439 */ 8440 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8441 __extension__ 8442 union tunnel_offload_mark { 8443 uint32_t val; 8444 struct { 8445 uint32_t app_reserve:8; 8446 uint32_t table_id:15; 8447 uint32_t transfer:1; 8448 uint32_t _unused_:8; 8449 }; 8450 }; 8451 8452 static bool 8453 mlx5_access_tunnel_offload_db 8454 (struct rte_eth_dev *dev, 8455 bool (*match)(struct rte_eth_dev *, 8456 struct mlx5_flow_tunnel *, const void *), 8457 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8458 void (*miss)(struct rte_eth_dev *, void *), 8459 void *ctx, bool lock_op); 8460 8461 static int 8462 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 8463 struct rte_flow *flow, 8464 const struct rte_flow_attr *attr, 8465 const struct rte_flow_action *app_actions, 8466 uint32_t flow_idx, 8467 const struct mlx5_flow_tunnel *tunnel, 8468 struct tunnel_default_miss_ctx *ctx, 8469 struct rte_flow_error *error) 8470 { 8471 struct mlx5_priv *priv = dev->data->dev_private; 8472 struct mlx5_flow *dev_flow; 8473 struct rte_flow_attr miss_attr = *attr; 8474 const struct rte_flow_item miss_items[2] = { 8475 { 8476 .type = RTE_FLOW_ITEM_TYPE_ETH, 8477 .spec = NULL, 8478 .last = NULL, 8479 .mask = NULL 8480 }, 8481 { 8482 .type = RTE_FLOW_ITEM_TYPE_END, 8483 .spec = NULL, 8484 .last = NULL, 8485 .mask = NULL 8486 } 8487 }; 8488 union tunnel_offload_mark mark_id; 8489 struct rte_flow_action_mark miss_mark; 8490 struct rte_flow_action miss_actions[3] = { 8491 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 8492 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 8493 }; 8494 const struct rte_flow_action_jump *jump_data; 8495 uint32_t i, flow_table = 0; /* prevent compilation warning */ 8496 struct flow_grp_info grp_info = { 8497 .external = 1, 8498 .transfer = attr->transfer, 8499 .fdb_def_rule = !!priv->fdb_def_rule, 8500 .std_tbl_fix = 0, 8501 }; 8502 int ret; 8503 8504 if (!attr->transfer) { 8505 uint32_t q_size; 8506 8507 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 8508 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 8509 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 8510 0, SOCKET_ID_ANY); 8511 if (!ctx->queue) 8512 return rte_flow_error_set 8513 (error, ENOMEM, 8514 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8515 NULL, "invalid default miss RSS"); 8516 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8517 ctx->action_rss.level = 0, 8518 ctx->action_rss.types = priv->rss_conf.rss_hf, 8519 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 8520 ctx->action_rss.queue_num = priv->reta_idx_n, 8521 ctx->action_rss.key = priv->rss_conf.rss_key, 8522 ctx->action_rss.queue = ctx->queue; 8523 if (!priv->reta_idx_n || !priv->rxqs_n) 8524 return rte_flow_error_set 8525 (error, EINVAL, 8526 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8527 NULL, "invalid port configuration"); 8528 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 8529 ctx->action_rss.types = 0; 8530 for (i = 0; i != priv->reta_idx_n; ++i) 8531 ctx->queue[i] = (*priv->reta_idx)[i]; 8532 } else { 8533 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 8534 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 8535 } 8536 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 8537 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 8538 jump_data = app_actions->conf; 8539 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 8540 miss_attr.group = jump_data->group; 8541 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 8542 &flow_table, &grp_info, error); 8543 if (ret) 8544 return rte_flow_error_set(error, EINVAL, 8545 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8546 NULL, "invalid tunnel id"); 8547 mark_id.app_reserve = 0; 8548 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 8549 mark_id.transfer = !!attr->transfer; 8550 mark_id._unused_ = 0; 8551 miss_mark.id = mark_id.val; 8552 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 8553 miss_items, miss_actions, flow_idx, error); 8554 if (!dev_flow) 8555 return -rte_errno; 8556 dev_flow->flow = flow; 8557 dev_flow->external = true; 8558 dev_flow->tunnel = tunnel; 8559 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 8560 /* Subflow object was created, we must include one in the list. */ 8561 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 8562 dev_flow->handle, next); 8563 DRV_LOG(DEBUG, 8564 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 8565 dev->data->port_id, tunnel->app_tunnel.type, 8566 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 8567 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 8568 miss_actions, error); 8569 if (!ret) 8570 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 8571 error); 8572 8573 return ret; 8574 } 8575 8576 static const struct mlx5_flow_tbl_data_entry * 8577 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 8578 { 8579 struct mlx5_priv *priv = dev->data->dev_private; 8580 struct mlx5_dev_ctx_shared *sh = priv->sh; 8581 struct mlx5_hlist_entry *he; 8582 union tunnel_offload_mark mbits = { .val = mark }; 8583 union mlx5_flow_tbl_key table_key = { 8584 { 8585 .level = tunnel_id_to_flow_tbl(mbits.table_id), 8586 .id = 0, 8587 .reserved = 0, 8588 .dummy = 0, 8589 .is_fdb = !!mbits.transfer, 8590 .is_egress = 0, 8591 } 8592 }; 8593 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 8594 return he ? 8595 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 8596 } 8597 8598 static void 8599 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 8600 struct mlx5_hlist_entry *entry) 8601 { 8602 struct mlx5_dev_ctx_shared *sh = list->ctx; 8603 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8604 8605 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8606 tunnel_flow_tbl_to_id(tte->flow_table)); 8607 mlx5_free(tte); 8608 } 8609 8610 static int 8611 mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused, 8612 struct mlx5_hlist_entry *entry, 8613 uint64_t key, void *cb_ctx __rte_unused) 8614 { 8615 union tunnel_tbl_key tbl = { 8616 .val = key, 8617 }; 8618 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8619 8620 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 8621 } 8622 8623 static struct mlx5_hlist_entry * 8624 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key, 8625 void *ctx __rte_unused) 8626 { 8627 struct mlx5_dev_ctx_shared *sh = list->ctx; 8628 struct tunnel_tbl_entry *tte; 8629 union tunnel_tbl_key tbl = { 8630 .val = key, 8631 }; 8632 8633 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 8634 sizeof(*tte), 0, 8635 SOCKET_ID_ANY); 8636 if (!tte) 8637 goto err; 8638 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8639 &tte->flow_table); 8640 if (tte->flow_table >= MLX5_MAX_TABLES) { 8641 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 8642 tte->flow_table); 8643 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8644 tte->flow_table); 8645 goto err; 8646 } else if (!tte->flow_table) { 8647 goto err; 8648 } 8649 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 8650 tte->tunnel_id = tbl.tunnel_id; 8651 tte->group = tbl.group; 8652 return &tte->hash; 8653 err: 8654 if (tte) 8655 mlx5_free(tte); 8656 return NULL; 8657 } 8658 8659 static uint32_t 8660 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 8661 const struct mlx5_flow_tunnel *tunnel, 8662 uint32_t group, uint32_t *table, 8663 struct rte_flow_error *error) 8664 { 8665 struct mlx5_hlist_entry *he; 8666 struct tunnel_tbl_entry *tte; 8667 union tunnel_tbl_key key = { 8668 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 8669 .group = group 8670 }; 8671 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8672 struct mlx5_hlist *group_hash; 8673 8674 group_hash = tunnel ? tunnel->groups : thub->groups; 8675 he = mlx5_hlist_register(group_hash, key.val, NULL); 8676 if (!he) 8677 return rte_flow_error_set(error, EINVAL, 8678 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8679 NULL, 8680 "tunnel group index not supported"); 8681 tte = container_of(he, typeof(*tte), hash); 8682 *table = tte->flow_table; 8683 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 8684 dev->data->port_id, key.tunnel_id, group, *table); 8685 return 0; 8686 } 8687 8688 static void 8689 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8690 struct mlx5_flow_tunnel *tunnel) 8691 { 8692 struct mlx5_priv *priv = dev->data->dev_private; 8693 struct mlx5_indexed_pool *ipool; 8694 8695 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8696 dev->data->port_id, tunnel->tunnel_id); 8697 LIST_REMOVE(tunnel, chain); 8698 mlx5_hlist_destroy(tunnel->groups); 8699 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8700 mlx5_ipool_free(ipool, tunnel->tunnel_id); 8701 } 8702 8703 static bool 8704 mlx5_access_tunnel_offload_db 8705 (struct rte_eth_dev *dev, 8706 bool (*match)(struct rte_eth_dev *, 8707 struct mlx5_flow_tunnel *, const void *), 8708 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8709 void (*miss)(struct rte_eth_dev *, void *), 8710 void *ctx, bool lock_op) 8711 { 8712 bool verdict = false; 8713 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8714 struct mlx5_flow_tunnel *tunnel; 8715 8716 rte_spinlock_lock(&thub->sl); 8717 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 8718 verdict = match(dev, tunnel, (const void *)ctx); 8719 if (verdict) 8720 break; 8721 } 8722 if (!lock_op) 8723 rte_spinlock_unlock(&thub->sl); 8724 if (verdict && hit) 8725 hit(dev, tunnel, ctx); 8726 if (!verdict && miss) 8727 miss(dev, ctx); 8728 if (lock_op) 8729 rte_spinlock_unlock(&thub->sl); 8730 8731 return verdict; 8732 } 8733 8734 struct tunnel_db_find_tunnel_id_ctx { 8735 uint32_t tunnel_id; 8736 struct mlx5_flow_tunnel *tunnel; 8737 }; 8738 8739 static bool 8740 find_tunnel_id_match(struct rte_eth_dev *dev, 8741 struct mlx5_flow_tunnel *tunnel, const void *x) 8742 { 8743 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8744 8745 RTE_SET_USED(dev); 8746 return tunnel->tunnel_id == ctx->tunnel_id; 8747 } 8748 8749 static void 8750 find_tunnel_id_hit(struct rte_eth_dev *dev, 8751 struct mlx5_flow_tunnel *tunnel, void *x) 8752 { 8753 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8754 RTE_SET_USED(dev); 8755 ctx->tunnel = tunnel; 8756 } 8757 8758 static struct mlx5_flow_tunnel * 8759 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 8760 { 8761 struct tunnel_db_find_tunnel_id_ctx ctx = { 8762 .tunnel_id = id, 8763 }; 8764 8765 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 8766 find_tunnel_id_hit, NULL, &ctx, true); 8767 8768 return ctx.tunnel; 8769 } 8770 8771 static struct mlx5_flow_tunnel * 8772 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 8773 const struct rte_flow_tunnel *app_tunnel) 8774 { 8775 struct mlx5_priv *priv = dev->data->dev_private; 8776 struct mlx5_indexed_pool *ipool; 8777 struct mlx5_flow_tunnel *tunnel; 8778 uint32_t id; 8779 8780 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8781 tunnel = mlx5_ipool_zmalloc(ipool, &id); 8782 if (!tunnel) 8783 return NULL; 8784 if (id >= MLX5_MAX_TUNNELS) { 8785 mlx5_ipool_free(ipool, id); 8786 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 8787 return NULL; 8788 } 8789 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 8790 mlx5_flow_tunnel_grp2tbl_create_cb, 8791 mlx5_flow_tunnel_grp2tbl_match_cb, 8792 mlx5_flow_tunnel_grp2tbl_remove_cb); 8793 if (!tunnel->groups) { 8794 mlx5_ipool_free(ipool, id); 8795 return NULL; 8796 } 8797 tunnel->groups->ctx = priv->sh; 8798 /* initiate new PMD tunnel */ 8799 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8800 tunnel->tunnel_id = id; 8801 tunnel->action.type = (typeof(tunnel->action.type)) 8802 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8803 tunnel->action.conf = tunnel; 8804 tunnel->item.type = (typeof(tunnel->item.type)) 8805 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8806 tunnel->item.spec = tunnel; 8807 tunnel->item.last = NULL; 8808 tunnel->item.mask = NULL; 8809 8810 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8811 dev->data->port_id, tunnel->tunnel_id); 8812 8813 return tunnel; 8814 } 8815 8816 struct tunnel_db_get_tunnel_ctx { 8817 const struct rte_flow_tunnel *app_tunnel; 8818 struct mlx5_flow_tunnel *tunnel; 8819 }; 8820 8821 static bool get_tunnel_match(struct rte_eth_dev *dev, 8822 struct mlx5_flow_tunnel *tunnel, const void *x) 8823 { 8824 const struct tunnel_db_get_tunnel_ctx *ctx = x; 8825 8826 RTE_SET_USED(dev); 8827 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 8828 sizeof(*ctx->app_tunnel)); 8829 } 8830 8831 static void get_tunnel_hit(struct rte_eth_dev *dev, 8832 struct mlx5_flow_tunnel *tunnel, void *x) 8833 { 8834 /* called under tunnel spinlock protection */ 8835 struct tunnel_db_get_tunnel_ctx *ctx = x; 8836 8837 RTE_SET_USED(dev); 8838 tunnel->refctn++; 8839 ctx->tunnel = tunnel; 8840 } 8841 8842 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 8843 { 8844 /* called under tunnel spinlock protection */ 8845 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8846 struct tunnel_db_get_tunnel_ctx *ctx = x; 8847 8848 rte_spinlock_unlock(&thub->sl); 8849 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 8850 rte_spinlock_lock(&thub->sl); 8851 if (ctx->tunnel) { 8852 ctx->tunnel->refctn = 1; 8853 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 8854 } 8855 } 8856 8857 8858 static int 8859 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 8860 const struct rte_flow_tunnel *app_tunnel, 8861 struct mlx5_flow_tunnel **tunnel) 8862 { 8863 struct tunnel_db_get_tunnel_ctx ctx = { 8864 .app_tunnel = app_tunnel, 8865 }; 8866 8867 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 8868 get_tunnel_miss, &ctx, true); 8869 *tunnel = ctx.tunnel; 8870 return ctx.tunnel ? 0 : -ENOMEM; 8871 } 8872 8873 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 8874 { 8875 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 8876 8877 if (!thub) 8878 return; 8879 if (!LIST_EMPTY(&thub->tunnels)) 8880 DRV_LOG(WARNING, "port %u tunnels present", port_id); 8881 mlx5_hlist_destroy(thub->groups); 8882 mlx5_free(thub); 8883 } 8884 8885 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 8886 { 8887 int err; 8888 struct mlx5_flow_tunnel_hub *thub; 8889 8890 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 8891 0, SOCKET_ID_ANY); 8892 if (!thub) 8893 return -ENOMEM; 8894 LIST_INIT(&thub->tunnels); 8895 rte_spinlock_init(&thub->sl); 8896 thub->groups = mlx5_hlist_create("flow groups", 8897 rte_align32pow2(MLX5_MAX_TABLES), 0, 8898 0, mlx5_flow_tunnel_grp2tbl_create_cb, 8899 mlx5_flow_tunnel_grp2tbl_match_cb, 8900 mlx5_flow_tunnel_grp2tbl_remove_cb); 8901 if (!thub->groups) { 8902 err = -rte_errno; 8903 goto err; 8904 } 8905 thub->groups->ctx = sh; 8906 sh->tunnel_hub = thub; 8907 8908 return 0; 8909 8910 err: 8911 if (thub->groups) 8912 mlx5_hlist_destroy(thub->groups); 8913 if (thub) 8914 mlx5_free(thub); 8915 return err; 8916 } 8917 8918 static inline bool 8919 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 8920 struct rte_flow_tunnel *tunnel, 8921 const char *err_msg) 8922 { 8923 err_msg = NULL; 8924 if (!is_tunnel_offload_active(dev)) { 8925 err_msg = "tunnel offload was not activated"; 8926 goto out; 8927 } else if (!tunnel) { 8928 err_msg = "no application tunnel"; 8929 goto out; 8930 } 8931 8932 switch (tunnel->type) { 8933 default: 8934 err_msg = "unsupported tunnel type"; 8935 goto out; 8936 case RTE_FLOW_ITEM_TYPE_VXLAN: 8937 break; 8938 } 8939 8940 out: 8941 return !err_msg; 8942 } 8943 8944 static int 8945 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 8946 struct rte_flow_tunnel *app_tunnel, 8947 struct rte_flow_action **actions, 8948 uint32_t *num_of_actions, 8949 struct rte_flow_error *error) 8950 { 8951 int ret; 8952 struct mlx5_flow_tunnel *tunnel; 8953 const char *err_msg = NULL; 8954 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8955 8956 if (!verdict) 8957 return rte_flow_error_set(error, EINVAL, 8958 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8959 err_msg); 8960 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8961 if (ret < 0) { 8962 return rte_flow_error_set(error, ret, 8963 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 8964 "failed to initialize pmd tunnel"); 8965 } 8966 *actions = &tunnel->action; 8967 *num_of_actions = 1; 8968 return 0; 8969 } 8970 8971 static int 8972 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 8973 struct rte_flow_tunnel *app_tunnel, 8974 struct rte_flow_item **items, 8975 uint32_t *num_of_items, 8976 struct rte_flow_error *error) 8977 { 8978 int ret; 8979 struct mlx5_flow_tunnel *tunnel; 8980 const char *err_msg = NULL; 8981 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 8982 8983 if (!verdict) 8984 return rte_flow_error_set(error, EINVAL, 8985 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8986 err_msg); 8987 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 8988 if (ret < 0) { 8989 return rte_flow_error_set(error, ret, 8990 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 8991 "failed to initialize pmd tunnel"); 8992 } 8993 *items = &tunnel->item; 8994 *num_of_items = 1; 8995 return 0; 8996 } 8997 8998 struct tunnel_db_element_release_ctx { 8999 struct rte_flow_item *items; 9000 struct rte_flow_action *actions; 9001 uint32_t num_elements; 9002 struct rte_flow_error *error; 9003 int ret; 9004 }; 9005 9006 static bool 9007 tunnel_element_release_match(struct rte_eth_dev *dev, 9008 struct mlx5_flow_tunnel *tunnel, const void *x) 9009 { 9010 const struct tunnel_db_element_release_ctx *ctx = x; 9011 9012 RTE_SET_USED(dev); 9013 if (ctx->num_elements != 1) 9014 return false; 9015 else if (ctx->items) 9016 return ctx->items == &tunnel->item; 9017 else if (ctx->actions) 9018 return ctx->actions == &tunnel->action; 9019 9020 return false; 9021 } 9022 9023 static void 9024 tunnel_element_release_hit(struct rte_eth_dev *dev, 9025 struct mlx5_flow_tunnel *tunnel, void *x) 9026 { 9027 struct tunnel_db_element_release_ctx *ctx = x; 9028 ctx->ret = 0; 9029 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9030 mlx5_flow_tunnel_free(dev, tunnel); 9031 } 9032 9033 static void 9034 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9035 { 9036 struct tunnel_db_element_release_ctx *ctx = x; 9037 RTE_SET_USED(dev); 9038 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9039 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9040 "invalid argument"); 9041 } 9042 9043 static int 9044 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9045 struct rte_flow_item *pmd_items, 9046 uint32_t num_items, struct rte_flow_error *err) 9047 { 9048 struct tunnel_db_element_release_ctx ctx = { 9049 .items = pmd_items, 9050 .actions = NULL, 9051 .num_elements = num_items, 9052 .error = err, 9053 }; 9054 9055 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9056 tunnel_element_release_hit, 9057 tunnel_element_release_miss, &ctx, false); 9058 9059 return ctx.ret; 9060 } 9061 9062 static int 9063 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9064 struct rte_flow_action *pmd_actions, 9065 uint32_t num_actions, struct rte_flow_error *err) 9066 { 9067 struct tunnel_db_element_release_ctx ctx = { 9068 .items = NULL, 9069 .actions = pmd_actions, 9070 .num_elements = num_actions, 9071 .error = err, 9072 }; 9073 9074 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9075 tunnel_element_release_hit, 9076 tunnel_element_release_miss, &ctx, false); 9077 9078 return ctx.ret; 9079 } 9080 9081 static int 9082 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9083 struct rte_mbuf *m, 9084 struct rte_flow_restore_info *info, 9085 struct rte_flow_error *err) 9086 { 9087 uint64_t ol_flags = m->ol_flags; 9088 const struct mlx5_flow_tbl_data_entry *tble; 9089 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 9090 9091 if (!is_tunnel_offload_active(dev)) { 9092 info->flags = 0; 9093 return 0; 9094 } 9095 9096 if ((ol_flags & mask) != mask) 9097 goto err; 9098 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9099 if (!tble) { 9100 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9101 dev->data->port_id, m->hash.fdir.hi); 9102 goto err; 9103 } 9104 MLX5_ASSERT(tble->tunnel); 9105 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9106 info->group_id = tble->group_id; 9107 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9108 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9109 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9110 9111 return 0; 9112 9113 err: 9114 return rte_flow_error_set(err, EINVAL, 9115 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9116 "failed to get restore info"); 9117 } 9118 9119 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9120 static int 9121 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9122 __rte_unused struct rte_flow_tunnel *app_tunnel, 9123 __rte_unused struct rte_flow_action **actions, 9124 __rte_unused uint32_t *num_of_actions, 9125 __rte_unused struct rte_flow_error *error) 9126 { 9127 return -ENOTSUP; 9128 } 9129 9130 static int 9131 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9132 __rte_unused struct rte_flow_tunnel *app_tunnel, 9133 __rte_unused struct rte_flow_item **items, 9134 __rte_unused uint32_t *num_of_items, 9135 __rte_unused struct rte_flow_error *error) 9136 { 9137 return -ENOTSUP; 9138 } 9139 9140 static int 9141 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9142 __rte_unused struct rte_flow_item *pmd_items, 9143 __rte_unused uint32_t num_items, 9144 __rte_unused struct rte_flow_error *err) 9145 { 9146 return -ENOTSUP; 9147 } 9148 9149 static int 9150 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9151 __rte_unused struct rte_flow_action *pmd_action, 9152 __rte_unused uint32_t num_actions, 9153 __rte_unused struct rte_flow_error *err) 9154 { 9155 return -ENOTSUP; 9156 } 9157 9158 static int 9159 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9160 __rte_unused struct rte_mbuf *m, 9161 __rte_unused struct rte_flow_restore_info *i, 9162 __rte_unused struct rte_flow_error *err) 9163 { 9164 return -ENOTSUP; 9165 } 9166 9167 static int 9168 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9169 __rte_unused struct rte_flow *flow, 9170 __rte_unused const struct rte_flow_attr *attr, 9171 __rte_unused const struct rte_flow_action *actions, 9172 __rte_unused uint32_t flow_idx, 9173 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9174 __rte_unused struct tunnel_default_miss_ctx *ctx, 9175 __rte_unused struct rte_flow_error *error) 9176 { 9177 return -ENOTSUP; 9178 } 9179 9180 static struct mlx5_flow_tunnel * 9181 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9182 __rte_unused uint32_t id) 9183 { 9184 return NULL; 9185 } 9186 9187 static void 9188 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9189 __rte_unused struct mlx5_flow_tunnel *tunnel) 9190 { 9191 } 9192 9193 static uint32_t 9194 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9195 __rte_unused const struct mlx5_flow_tunnel *t, 9196 __rte_unused uint32_t group, 9197 __rte_unused uint32_t *table, 9198 struct rte_flow_error *error) 9199 { 9200 return rte_flow_error_set(error, ENOTSUP, 9201 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9202 "tunnel offload requires DV support"); 9203 } 9204 9205 void 9206 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9207 __rte_unused uint16_t port_id) 9208 { 9209 } 9210 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9211 9212 static void 9213 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9214 { 9215 int ret; 9216 struct rte_flow_error error; 9217 9218 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9219 char *item_name; 9220 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9221 sizeof(item_name), 9222 (void *)(uintptr_t)item->type, &error); 9223 if (ret > 0) 9224 printf("%s ", item_name); 9225 else 9226 printf("%d\n", (int)item->type); 9227 } 9228 printf("END\n"); 9229 } 9230