1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see ETH_RSS_* definitions). 102 */ 103 uint8_t optional; 104 /**< optional expand field. Default 0 to expand, 1 not go deeper. */ 105 }; 106 107 /** Object returned by mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_rss { 109 uint32_t entries; 110 /**< Number of entries @p patterns and @p priorities. */ 111 struct { 112 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 113 uint32_t priority; /**< Priority offset for each expansion. */ 114 } entry[]; 115 }; 116 117 static void 118 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 119 120 static bool 121 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 122 { 123 switch (item->type) { 124 case RTE_FLOW_ITEM_TYPE_ETH: 125 case RTE_FLOW_ITEM_TYPE_VLAN: 126 case RTE_FLOW_ITEM_TYPE_IPV4: 127 case RTE_FLOW_ITEM_TYPE_IPV6: 128 case RTE_FLOW_ITEM_TYPE_UDP: 129 case RTE_FLOW_ITEM_TYPE_TCP: 130 case RTE_FLOW_ITEM_TYPE_VXLAN: 131 case RTE_FLOW_ITEM_TYPE_NVGRE: 132 case RTE_FLOW_ITEM_TYPE_GRE: 133 case RTE_FLOW_ITEM_TYPE_GENEVE: 134 case RTE_FLOW_ITEM_TYPE_MPLS: 135 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 136 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 137 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 138 case RTE_FLOW_ITEM_TYPE_GTP: 139 return true; 140 default: 141 break; 142 } 143 return false; 144 } 145 146 static enum rte_flow_item_type 147 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 148 { 149 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 150 uint16_t ether_type = 0; 151 uint16_t ether_type_m; 152 uint8_t ip_next_proto = 0; 153 uint8_t ip_next_proto_m; 154 155 if (item == NULL || item->spec == NULL) 156 return ret; 157 switch (item->type) { 158 case RTE_FLOW_ITEM_TYPE_ETH: 159 if (item->mask) 160 ether_type_m = ((const struct rte_flow_item_eth *) 161 (item->mask))->type; 162 else 163 ether_type_m = rte_flow_item_eth_mask.type; 164 if (ether_type_m != RTE_BE16(0xFFFF)) 165 break; 166 ether_type = ((const struct rte_flow_item_eth *) 167 (item->spec))->type; 168 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 169 ret = RTE_FLOW_ITEM_TYPE_IPV4; 170 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 171 ret = RTE_FLOW_ITEM_TYPE_IPV6; 172 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 173 ret = RTE_FLOW_ITEM_TYPE_VLAN; 174 else 175 ret = RTE_FLOW_ITEM_TYPE_END; 176 break; 177 case RTE_FLOW_ITEM_TYPE_VLAN: 178 if (item->mask) 179 ether_type_m = ((const struct rte_flow_item_vlan *) 180 (item->mask))->inner_type; 181 else 182 ether_type_m = rte_flow_item_vlan_mask.inner_type; 183 if (ether_type_m != RTE_BE16(0xFFFF)) 184 break; 185 ether_type = ((const struct rte_flow_item_vlan *) 186 (item->spec))->inner_type; 187 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 188 ret = RTE_FLOW_ITEM_TYPE_IPV4; 189 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 190 ret = RTE_FLOW_ITEM_TYPE_IPV6; 191 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 192 ret = RTE_FLOW_ITEM_TYPE_VLAN; 193 else 194 ret = RTE_FLOW_ITEM_TYPE_END; 195 break; 196 case RTE_FLOW_ITEM_TYPE_IPV4: 197 if (item->mask) 198 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 199 (item->mask))->hdr.next_proto_id; 200 else 201 ip_next_proto_m = 202 rte_flow_item_ipv4_mask.hdr.next_proto_id; 203 if (ip_next_proto_m != 0xFF) 204 break; 205 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 206 (item->spec))->hdr.next_proto_id; 207 if (ip_next_proto == IPPROTO_UDP) 208 ret = RTE_FLOW_ITEM_TYPE_UDP; 209 else if (ip_next_proto == IPPROTO_TCP) 210 ret = RTE_FLOW_ITEM_TYPE_TCP; 211 else if (ip_next_proto == IPPROTO_IP) 212 ret = RTE_FLOW_ITEM_TYPE_IPV4; 213 else if (ip_next_proto == IPPROTO_IPV6) 214 ret = RTE_FLOW_ITEM_TYPE_IPV6; 215 else 216 ret = RTE_FLOW_ITEM_TYPE_END; 217 break; 218 case RTE_FLOW_ITEM_TYPE_IPV6: 219 if (item->mask) 220 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 221 (item->mask))->hdr.proto; 222 else 223 ip_next_proto_m = 224 rte_flow_item_ipv6_mask.hdr.proto; 225 if (ip_next_proto_m != 0xFF) 226 break; 227 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 228 (item->spec))->hdr.proto; 229 if (ip_next_proto == IPPROTO_UDP) 230 ret = RTE_FLOW_ITEM_TYPE_UDP; 231 else if (ip_next_proto == IPPROTO_TCP) 232 ret = RTE_FLOW_ITEM_TYPE_TCP; 233 else if (ip_next_proto == IPPROTO_IP) 234 ret = RTE_FLOW_ITEM_TYPE_IPV4; 235 else if (ip_next_proto == IPPROTO_IPV6) 236 ret = RTE_FLOW_ITEM_TYPE_IPV6; 237 else 238 ret = RTE_FLOW_ITEM_TYPE_END; 239 break; 240 default: 241 ret = RTE_FLOW_ITEM_TYPE_VOID; 242 break; 243 } 244 return ret; 245 } 246 247 #define MLX5_RSS_EXP_ELT_N 16 248 249 /** 250 * Expand RSS flows into several possible flows according to the RSS hash 251 * fields requested and the driver capabilities. 252 * 253 * @param[out] buf 254 * Buffer to store the result expansion. 255 * @param[in] size 256 * Buffer size in bytes. If 0, @p buf can be NULL. 257 * @param[in] pattern 258 * User flow pattern. 259 * @param[in] types 260 * RSS types to expand (see ETH_RSS_* definitions). 261 * @param[in] graph 262 * Input graph to expand @p pattern according to @p types. 263 * @param[in] graph_root_index 264 * Index of root node in @p graph, typically 0. 265 * 266 * @return 267 * A positive value representing the size of @p buf in bytes regardless of 268 * @p size on success, a negative errno value otherwise and rte_errno is 269 * set, the following errors are defined: 270 * 271 * -E2BIG: graph-depth @p graph is too deep. 272 * -EINVAL: @p size has not enough space for expanded pattern. 273 */ 274 static int 275 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 276 const struct rte_flow_item *pattern, uint64_t types, 277 const struct mlx5_flow_expand_node graph[], 278 int graph_root_index) 279 { 280 const struct rte_flow_item *item; 281 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 282 const int *next_node; 283 const int *stack[MLX5_RSS_EXP_ELT_N]; 284 int stack_pos = 0; 285 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 286 unsigned int i; 287 size_t lsize; 288 size_t user_pattern_size = 0; 289 void *addr = NULL; 290 const struct mlx5_flow_expand_node *next = NULL; 291 struct rte_flow_item missed_item; 292 int missed = 0; 293 int elt = 0; 294 const struct rte_flow_item *last_item = NULL; 295 296 memset(&missed_item, 0, sizeof(missed_item)); 297 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 298 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 299 if (lsize > size) 300 return -EINVAL; 301 buf->entry[0].priority = 0; 302 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 303 buf->entries = 0; 304 addr = buf->entry[0].pattern; 305 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 306 if (!mlx5_flow_is_rss_expandable_item(item)) { 307 user_pattern_size += sizeof(*item); 308 continue; 309 } 310 last_item = item; 311 for (i = 0; node->next && node->next[i]; ++i) { 312 next = &graph[node->next[i]]; 313 if (next->type == item->type) 314 break; 315 } 316 if (next) 317 node = next; 318 user_pattern_size += sizeof(*item); 319 } 320 user_pattern_size += sizeof(*item); /* Handle END item. */ 321 lsize += user_pattern_size; 322 if (lsize > size) 323 return -EINVAL; 324 /* Copy the user pattern in the first entry of the buffer. */ 325 rte_memcpy(addr, pattern, user_pattern_size); 326 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 327 buf->entries = 1; 328 /* Start expanding. */ 329 memset(flow_items, 0, sizeof(flow_items)); 330 user_pattern_size -= sizeof(*item); 331 /* 332 * Check if the last valid item has spec set, need complete pattern, 333 * and the pattern can be used for expansion. 334 */ 335 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 336 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 337 /* Item type END indicates expansion is not required. */ 338 return lsize; 339 } 340 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 341 next = NULL; 342 missed = 1; 343 for (i = 0; node->next && node->next[i]; ++i) { 344 next = &graph[node->next[i]]; 345 if (next->type == missed_item.type) { 346 flow_items[0].type = missed_item.type; 347 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 348 break; 349 } 350 next = NULL; 351 } 352 } 353 if (next && missed) { 354 elt = 2; /* missed item + item end. */ 355 node = next; 356 lsize += elt * sizeof(*item) + user_pattern_size; 357 if (lsize > size) 358 return -EINVAL; 359 if (node->rss_types & types) { 360 buf->entry[buf->entries].priority = 1; 361 buf->entry[buf->entries].pattern = addr; 362 buf->entries++; 363 rte_memcpy(addr, buf->entry[0].pattern, 364 user_pattern_size); 365 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 366 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 367 addr = (void *)(((uintptr_t)addr) + 368 elt * sizeof(*item)); 369 } 370 } 371 memset(flow_items, 0, sizeof(flow_items)); 372 next_node = node->next; 373 stack[stack_pos] = next_node; 374 node = next_node ? &graph[*next_node] : NULL; 375 while (node) { 376 flow_items[stack_pos].type = node->type; 377 if (node->rss_types & types) { 378 size_t n; 379 /* 380 * compute the number of items to copy from the 381 * expansion and copy it. 382 * When the stack_pos is 0, there are 1 element in it, 383 * plus the addition END item. 384 */ 385 elt = stack_pos + 2; 386 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 387 lsize += elt * sizeof(*item) + user_pattern_size; 388 if (lsize > size) 389 return -EINVAL; 390 n = elt * sizeof(*item); 391 buf->entry[buf->entries].priority = 392 stack_pos + 1 + missed; 393 buf->entry[buf->entries].pattern = addr; 394 buf->entries++; 395 rte_memcpy(addr, buf->entry[0].pattern, 396 user_pattern_size); 397 addr = (void *)(((uintptr_t)addr) + 398 user_pattern_size); 399 rte_memcpy(addr, &missed_item, 400 missed * sizeof(*item)); 401 addr = (void *)(((uintptr_t)addr) + 402 missed * sizeof(*item)); 403 rte_memcpy(addr, flow_items, n); 404 addr = (void *)(((uintptr_t)addr) + n); 405 } 406 /* Go deeper. */ 407 if (!node->optional && node->next) { 408 next_node = node->next; 409 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 410 rte_errno = E2BIG; 411 return -rte_errno; 412 } 413 stack[stack_pos] = next_node; 414 } else if (*(next_node + 1)) { 415 /* Follow up with the next possibility. */ 416 ++next_node; 417 } else { 418 /* Move to the next path. */ 419 if (stack_pos) 420 next_node = stack[--stack_pos]; 421 next_node++; 422 stack[stack_pos] = next_node; 423 } 424 node = *next_node ? &graph[*next_node] : NULL; 425 }; 426 return lsize; 427 } 428 429 enum mlx5_expansion { 430 MLX5_EXPANSION_ROOT, 431 MLX5_EXPANSION_ROOT_OUTER, 432 MLX5_EXPANSION_ROOT_ETH_VLAN, 433 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 434 MLX5_EXPANSION_OUTER_ETH, 435 MLX5_EXPANSION_OUTER_ETH_VLAN, 436 MLX5_EXPANSION_OUTER_VLAN, 437 MLX5_EXPANSION_OUTER_IPV4, 438 MLX5_EXPANSION_OUTER_IPV4_UDP, 439 MLX5_EXPANSION_OUTER_IPV4_TCP, 440 MLX5_EXPANSION_OUTER_IPV6, 441 MLX5_EXPANSION_OUTER_IPV6_UDP, 442 MLX5_EXPANSION_OUTER_IPV6_TCP, 443 MLX5_EXPANSION_VXLAN, 444 MLX5_EXPANSION_VXLAN_GPE, 445 MLX5_EXPANSION_GRE, 446 MLX5_EXPANSION_NVGRE, 447 MLX5_EXPANSION_GRE_KEY, 448 MLX5_EXPANSION_MPLS, 449 MLX5_EXPANSION_ETH, 450 MLX5_EXPANSION_ETH_VLAN, 451 MLX5_EXPANSION_VLAN, 452 MLX5_EXPANSION_IPV4, 453 MLX5_EXPANSION_IPV4_UDP, 454 MLX5_EXPANSION_IPV4_TCP, 455 MLX5_EXPANSION_IPV6, 456 MLX5_EXPANSION_IPV6_UDP, 457 MLX5_EXPANSION_IPV6_TCP, 458 MLX5_EXPANSION_IPV6_FRAG_EXT, 459 MLX5_EXPANSION_GTP 460 }; 461 462 /** Supported expansion of items. */ 463 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 464 [MLX5_EXPANSION_ROOT] = { 465 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 466 MLX5_EXPANSION_IPV4, 467 MLX5_EXPANSION_IPV6), 468 .type = RTE_FLOW_ITEM_TYPE_END, 469 }, 470 [MLX5_EXPANSION_ROOT_OUTER] = { 471 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 472 MLX5_EXPANSION_OUTER_IPV4, 473 MLX5_EXPANSION_OUTER_IPV6), 474 .type = RTE_FLOW_ITEM_TYPE_END, 475 }, 476 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 477 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 478 .type = RTE_FLOW_ITEM_TYPE_END, 479 }, 480 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 481 .next = MLX5_FLOW_EXPAND_RSS_NEXT 482 (MLX5_EXPANSION_OUTER_ETH_VLAN), 483 .type = RTE_FLOW_ITEM_TYPE_END, 484 }, 485 [MLX5_EXPANSION_OUTER_ETH] = { 486 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 487 MLX5_EXPANSION_OUTER_IPV6), 488 .type = RTE_FLOW_ITEM_TYPE_ETH, 489 .rss_types = 0, 490 }, 491 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 492 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 493 .type = RTE_FLOW_ITEM_TYPE_ETH, 494 .rss_types = 0, 495 }, 496 [MLX5_EXPANSION_OUTER_VLAN] = { 497 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 498 MLX5_EXPANSION_OUTER_IPV6), 499 .type = RTE_FLOW_ITEM_TYPE_VLAN, 500 }, 501 [MLX5_EXPANSION_OUTER_IPV4] = { 502 .next = MLX5_FLOW_EXPAND_RSS_NEXT 503 (MLX5_EXPANSION_OUTER_IPV4_UDP, 504 MLX5_EXPANSION_OUTER_IPV4_TCP, 505 MLX5_EXPANSION_GRE, 506 MLX5_EXPANSION_NVGRE, 507 MLX5_EXPANSION_IPV4, 508 MLX5_EXPANSION_IPV6), 509 .type = RTE_FLOW_ITEM_TYPE_IPV4, 510 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 511 ETH_RSS_NONFRAG_IPV4_OTHER, 512 }, 513 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 514 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 515 MLX5_EXPANSION_VXLAN_GPE, 516 MLX5_EXPANSION_MPLS, 517 MLX5_EXPANSION_GTP), 518 .type = RTE_FLOW_ITEM_TYPE_UDP, 519 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 520 }, 521 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 522 .type = RTE_FLOW_ITEM_TYPE_TCP, 523 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 524 }, 525 [MLX5_EXPANSION_OUTER_IPV6] = { 526 .next = MLX5_FLOW_EXPAND_RSS_NEXT 527 (MLX5_EXPANSION_OUTER_IPV6_UDP, 528 MLX5_EXPANSION_OUTER_IPV6_TCP, 529 MLX5_EXPANSION_IPV4, 530 MLX5_EXPANSION_IPV6, 531 MLX5_EXPANSION_GRE, 532 MLX5_EXPANSION_NVGRE), 533 .type = RTE_FLOW_ITEM_TYPE_IPV6, 534 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 535 ETH_RSS_NONFRAG_IPV6_OTHER, 536 }, 537 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 538 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 539 MLX5_EXPANSION_VXLAN_GPE, 540 MLX5_EXPANSION_MPLS, 541 MLX5_EXPANSION_GTP), 542 .type = RTE_FLOW_ITEM_TYPE_UDP, 543 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 544 }, 545 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 546 .type = RTE_FLOW_ITEM_TYPE_TCP, 547 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 548 }, 549 [MLX5_EXPANSION_VXLAN] = { 550 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 551 MLX5_EXPANSION_IPV4, 552 MLX5_EXPANSION_IPV6), 553 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 554 }, 555 [MLX5_EXPANSION_VXLAN_GPE] = { 556 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 557 MLX5_EXPANSION_IPV4, 558 MLX5_EXPANSION_IPV6), 559 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 560 }, 561 [MLX5_EXPANSION_GRE] = { 562 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 563 MLX5_EXPANSION_IPV6, 564 MLX5_EXPANSION_GRE_KEY, 565 MLX5_EXPANSION_MPLS), 566 .type = RTE_FLOW_ITEM_TYPE_GRE, 567 }, 568 [MLX5_EXPANSION_GRE_KEY] = { 569 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 570 MLX5_EXPANSION_IPV6, 571 MLX5_EXPANSION_MPLS), 572 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 573 .optional = 1, 574 }, 575 [MLX5_EXPANSION_NVGRE] = { 576 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 577 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 578 }, 579 [MLX5_EXPANSION_MPLS] = { 580 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6, 582 MLX5_EXPANSION_ETH), 583 .type = RTE_FLOW_ITEM_TYPE_MPLS, 584 }, 585 [MLX5_EXPANSION_ETH] = { 586 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 587 MLX5_EXPANSION_IPV6), 588 .type = RTE_FLOW_ITEM_TYPE_ETH, 589 }, 590 [MLX5_EXPANSION_ETH_VLAN] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 592 .type = RTE_FLOW_ITEM_TYPE_ETH, 593 }, 594 [MLX5_EXPANSION_VLAN] = { 595 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 596 MLX5_EXPANSION_IPV6), 597 .type = RTE_FLOW_ITEM_TYPE_VLAN, 598 }, 599 [MLX5_EXPANSION_IPV4] = { 600 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 601 MLX5_EXPANSION_IPV4_TCP), 602 .type = RTE_FLOW_ITEM_TYPE_IPV4, 603 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 604 ETH_RSS_NONFRAG_IPV4_OTHER, 605 }, 606 [MLX5_EXPANSION_IPV4_UDP] = { 607 .type = RTE_FLOW_ITEM_TYPE_UDP, 608 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 609 }, 610 [MLX5_EXPANSION_IPV4_TCP] = { 611 .type = RTE_FLOW_ITEM_TYPE_TCP, 612 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 613 }, 614 [MLX5_EXPANSION_IPV6] = { 615 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 616 MLX5_EXPANSION_IPV6_TCP, 617 MLX5_EXPANSION_IPV6_FRAG_EXT), 618 .type = RTE_FLOW_ITEM_TYPE_IPV6, 619 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 620 ETH_RSS_NONFRAG_IPV6_OTHER, 621 }, 622 [MLX5_EXPANSION_IPV6_UDP] = { 623 .type = RTE_FLOW_ITEM_TYPE_UDP, 624 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 625 }, 626 [MLX5_EXPANSION_IPV6_TCP] = { 627 .type = RTE_FLOW_ITEM_TYPE_TCP, 628 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 629 }, 630 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 631 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 632 }, 633 [MLX5_EXPANSION_GTP] = { 634 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 635 MLX5_EXPANSION_IPV6), 636 .type = RTE_FLOW_ITEM_TYPE_GTP 637 } 638 }; 639 640 static struct rte_flow_action_handle * 641 mlx5_action_handle_create(struct rte_eth_dev *dev, 642 const struct rte_flow_indir_action_conf *conf, 643 const struct rte_flow_action *action, 644 struct rte_flow_error *error); 645 static int mlx5_action_handle_destroy 646 (struct rte_eth_dev *dev, 647 struct rte_flow_action_handle *handle, 648 struct rte_flow_error *error); 649 static int mlx5_action_handle_update 650 (struct rte_eth_dev *dev, 651 struct rte_flow_action_handle *handle, 652 const void *update, 653 struct rte_flow_error *error); 654 static int mlx5_action_handle_query 655 (struct rte_eth_dev *dev, 656 const struct rte_flow_action_handle *handle, 657 void *data, 658 struct rte_flow_error *error); 659 static int 660 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 661 struct rte_flow_tunnel *app_tunnel, 662 struct rte_flow_action **actions, 663 uint32_t *num_of_actions, 664 struct rte_flow_error *error); 665 static int 666 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 667 struct rte_flow_tunnel *app_tunnel, 668 struct rte_flow_item **items, 669 uint32_t *num_of_items, 670 struct rte_flow_error *error); 671 static int 672 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 673 struct rte_flow_item *pmd_items, 674 uint32_t num_items, struct rte_flow_error *err); 675 static int 676 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 677 struct rte_flow_action *pmd_actions, 678 uint32_t num_actions, 679 struct rte_flow_error *err); 680 static int 681 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 682 struct rte_mbuf *m, 683 struct rte_flow_restore_info *info, 684 struct rte_flow_error *err); 685 686 static const struct rte_flow_ops mlx5_flow_ops = { 687 .validate = mlx5_flow_validate, 688 .create = mlx5_flow_create, 689 .destroy = mlx5_flow_destroy, 690 .flush = mlx5_flow_flush, 691 .isolate = mlx5_flow_isolate, 692 .query = mlx5_flow_query, 693 .dev_dump = mlx5_flow_dev_dump, 694 .get_aged_flows = mlx5_flow_get_aged_flows, 695 .action_handle_create = mlx5_action_handle_create, 696 .action_handle_destroy = mlx5_action_handle_destroy, 697 .action_handle_update = mlx5_action_handle_update, 698 .action_handle_query = mlx5_action_handle_query, 699 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 700 .tunnel_match = mlx5_flow_tunnel_match, 701 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 702 .tunnel_item_release = mlx5_flow_tunnel_item_release, 703 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 704 }; 705 706 /* Tunnel information. */ 707 struct mlx5_flow_tunnel_info { 708 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 709 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 710 }; 711 712 static struct mlx5_flow_tunnel_info tunnels_info[] = { 713 { 714 .tunnel = MLX5_FLOW_LAYER_VXLAN, 715 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 716 }, 717 { 718 .tunnel = MLX5_FLOW_LAYER_GENEVE, 719 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 720 }, 721 { 722 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 723 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 724 }, 725 { 726 .tunnel = MLX5_FLOW_LAYER_GRE, 727 .ptype = RTE_PTYPE_TUNNEL_GRE, 728 }, 729 { 730 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 731 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 732 }, 733 { 734 .tunnel = MLX5_FLOW_LAYER_MPLS, 735 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 736 }, 737 { 738 .tunnel = MLX5_FLOW_LAYER_NVGRE, 739 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 740 }, 741 { 742 .tunnel = MLX5_FLOW_LAYER_IPIP, 743 .ptype = RTE_PTYPE_TUNNEL_IP, 744 }, 745 { 746 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 747 .ptype = RTE_PTYPE_TUNNEL_IP, 748 }, 749 { 750 .tunnel = MLX5_FLOW_LAYER_GTP, 751 .ptype = RTE_PTYPE_TUNNEL_GTPU, 752 }, 753 }; 754 755 756 757 /** 758 * Translate tag ID to register. 759 * 760 * @param[in] dev 761 * Pointer to the Ethernet device structure. 762 * @param[in] feature 763 * The feature that request the register. 764 * @param[in] id 765 * The request register ID. 766 * @param[out] error 767 * Error description in case of any. 768 * 769 * @return 770 * The request register on success, a negative errno 771 * value otherwise and rte_errno is set. 772 */ 773 int 774 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 775 enum mlx5_feature_name feature, 776 uint32_t id, 777 struct rte_flow_error *error) 778 { 779 struct mlx5_priv *priv = dev->data->dev_private; 780 struct mlx5_dev_config *config = &priv->config; 781 enum modify_reg start_reg; 782 bool skip_mtr_reg = false; 783 784 switch (feature) { 785 case MLX5_HAIRPIN_RX: 786 return REG_B; 787 case MLX5_HAIRPIN_TX: 788 return REG_A; 789 case MLX5_METADATA_RX: 790 switch (config->dv_xmeta_en) { 791 case MLX5_XMETA_MODE_LEGACY: 792 return REG_B; 793 case MLX5_XMETA_MODE_META16: 794 return REG_C_0; 795 case MLX5_XMETA_MODE_META32: 796 return REG_C_1; 797 } 798 break; 799 case MLX5_METADATA_TX: 800 return REG_A; 801 case MLX5_METADATA_FDB: 802 switch (config->dv_xmeta_en) { 803 case MLX5_XMETA_MODE_LEGACY: 804 return REG_NON; 805 case MLX5_XMETA_MODE_META16: 806 return REG_C_0; 807 case MLX5_XMETA_MODE_META32: 808 return REG_C_1; 809 } 810 break; 811 case MLX5_FLOW_MARK: 812 switch (config->dv_xmeta_en) { 813 case MLX5_XMETA_MODE_LEGACY: 814 return REG_NON; 815 case MLX5_XMETA_MODE_META16: 816 return REG_C_1; 817 case MLX5_XMETA_MODE_META32: 818 return REG_C_0; 819 } 820 break; 821 case MLX5_MTR_ID: 822 /* 823 * If meter color and meter id share one register, flow match 824 * should use the meter color register for match. 825 */ 826 if (priv->mtr_reg_share) 827 return priv->mtr_color_reg; 828 else 829 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 830 REG_C_3; 831 case MLX5_MTR_COLOR: 832 case MLX5_ASO_FLOW_HIT: 833 case MLX5_ASO_CONNTRACK: 834 /* All features use the same REG_C. */ 835 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 836 return priv->mtr_color_reg; 837 case MLX5_COPY_MARK: 838 /* 839 * Metadata COPY_MARK register using is in meter suffix sub 840 * flow while with meter. It's safe to share the same register. 841 */ 842 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 843 case MLX5_APP_TAG: 844 /* 845 * If meter is enable, it will engage the register for color 846 * match and flow match. If meter color match is not using the 847 * REG_C_2, need to skip the REG_C_x be used by meter color 848 * match. 849 * If meter is disable, free to use all available registers. 850 */ 851 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 852 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 853 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 854 if (id > (uint32_t)(REG_C_7 - start_reg)) 855 return rte_flow_error_set(error, EINVAL, 856 RTE_FLOW_ERROR_TYPE_ITEM, 857 NULL, "invalid tag id"); 858 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 859 return rte_flow_error_set(error, ENOTSUP, 860 RTE_FLOW_ERROR_TYPE_ITEM, 861 NULL, "unsupported tag id"); 862 /* 863 * This case means meter is using the REG_C_x great than 2. 864 * Take care not to conflict with meter color REG_C_x. 865 * If the available index REG_C_y >= REG_C_x, skip the 866 * color register. 867 */ 868 if (skip_mtr_reg && config->flow_mreg_c 869 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 870 if (id >= (uint32_t)(REG_C_7 - start_reg)) 871 return rte_flow_error_set(error, EINVAL, 872 RTE_FLOW_ERROR_TYPE_ITEM, 873 NULL, "invalid tag id"); 874 if (config->flow_mreg_c 875 [id + 1 + start_reg - REG_C_0] != REG_NON) 876 return config->flow_mreg_c 877 [id + 1 + start_reg - REG_C_0]; 878 return rte_flow_error_set(error, ENOTSUP, 879 RTE_FLOW_ERROR_TYPE_ITEM, 880 NULL, "unsupported tag id"); 881 } 882 return config->flow_mreg_c[id + start_reg - REG_C_0]; 883 } 884 MLX5_ASSERT(false); 885 return rte_flow_error_set(error, EINVAL, 886 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 887 NULL, "invalid feature name"); 888 } 889 890 /** 891 * Check extensive flow metadata register support. 892 * 893 * @param dev 894 * Pointer to rte_eth_dev structure. 895 * 896 * @return 897 * True if device supports extensive flow metadata register, otherwise false. 898 */ 899 bool 900 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 901 { 902 struct mlx5_priv *priv = dev->data->dev_private; 903 struct mlx5_dev_config *config = &priv->config; 904 905 /* 906 * Having available reg_c can be regarded inclusively as supporting 907 * extensive flow metadata register, which could mean, 908 * - metadata register copy action by modify header. 909 * - 16 modify header actions is supported. 910 * - reg_c's are preserved across different domain (FDB and NIC) on 911 * packet loopback by flow lookup miss. 912 */ 913 return config->flow_mreg_c[2] != REG_NON; 914 } 915 916 /** 917 * Get the lowest priority. 918 * 919 * @param[in] dev 920 * Pointer to the Ethernet device structure. 921 * @param[in] attributes 922 * Pointer to device flow rule attributes. 923 * 924 * @return 925 * The value of lowest priority of flow. 926 */ 927 uint32_t 928 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 929 const struct rte_flow_attr *attr) 930 { 931 struct mlx5_priv *priv = dev->data->dev_private; 932 933 if (!attr->group && !attr->transfer) 934 return priv->config.flow_prio - 2; 935 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 936 } 937 938 /** 939 * Calculate matcher priority of the flow. 940 * 941 * @param[in] dev 942 * Pointer to the Ethernet device structure. 943 * @param[in] attr 944 * Pointer to device flow rule attributes. 945 * @param[in] subpriority 946 * The priority based on the items. 947 * @return 948 * The matcher priority of the flow. 949 */ 950 uint16_t 951 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 952 const struct rte_flow_attr *attr, 953 uint32_t subpriority) 954 { 955 uint16_t priority = (uint16_t)attr->priority; 956 struct mlx5_priv *priv = dev->data->dev_private; 957 958 if (!attr->group && !attr->transfer) { 959 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 960 priority = priv->config.flow_prio - 1; 961 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 962 } 963 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 964 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 965 return priority * 3 + subpriority; 966 } 967 968 /** 969 * Verify the @p item specifications (spec, last, mask) are compatible with the 970 * NIC capabilities. 971 * 972 * @param[in] item 973 * Item specification. 974 * @param[in] mask 975 * @p item->mask or flow default bit-masks. 976 * @param[in] nic_mask 977 * Bit-masks covering supported fields by the NIC to compare with user mask. 978 * @param[in] size 979 * Bit-masks size in bytes. 980 * @param[in] range_accepted 981 * True if range of values is accepted for specific fields, false otherwise. 982 * @param[out] error 983 * Pointer to error structure. 984 * 985 * @return 986 * 0 on success, a negative errno value otherwise and rte_errno is set. 987 */ 988 int 989 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 990 const uint8_t *mask, 991 const uint8_t *nic_mask, 992 unsigned int size, 993 bool range_accepted, 994 struct rte_flow_error *error) 995 { 996 unsigned int i; 997 998 MLX5_ASSERT(nic_mask); 999 for (i = 0; i < size; ++i) 1000 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1001 return rte_flow_error_set(error, ENOTSUP, 1002 RTE_FLOW_ERROR_TYPE_ITEM, 1003 item, 1004 "mask enables non supported" 1005 " bits"); 1006 if (!item->spec && (item->mask || item->last)) 1007 return rte_flow_error_set(error, EINVAL, 1008 RTE_FLOW_ERROR_TYPE_ITEM, item, 1009 "mask/last without a spec is not" 1010 " supported"); 1011 if (item->spec && item->last && !range_accepted) { 1012 uint8_t spec[size]; 1013 uint8_t last[size]; 1014 unsigned int i; 1015 int ret; 1016 1017 for (i = 0; i < size; ++i) { 1018 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1019 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1020 } 1021 ret = memcmp(spec, last, size); 1022 if (ret != 0) 1023 return rte_flow_error_set(error, EINVAL, 1024 RTE_FLOW_ERROR_TYPE_ITEM, 1025 item, 1026 "range is not valid"); 1027 } 1028 return 0; 1029 } 1030 1031 /** 1032 * Adjust the hash fields according to the @p flow information. 1033 * 1034 * @param[in] dev_flow. 1035 * Pointer to the mlx5_flow. 1036 * @param[in] tunnel 1037 * 1 when the hash field is for a tunnel item. 1038 * @param[in] layer_types 1039 * ETH_RSS_* types. 1040 * @param[in] hash_fields 1041 * Item hash fields. 1042 * 1043 * @return 1044 * The hash fields that should be used. 1045 */ 1046 uint64_t 1047 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1048 int tunnel __rte_unused, uint64_t layer_types, 1049 uint64_t hash_fields) 1050 { 1051 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1052 int rss_request_inner = rss_desc->level >= 2; 1053 1054 /* Check RSS hash level for tunnel. */ 1055 if (tunnel && rss_request_inner) 1056 hash_fields |= IBV_RX_HASH_INNER; 1057 else if (tunnel || rss_request_inner) 1058 return 0; 1059 #endif 1060 /* Check if requested layer matches RSS hash fields. */ 1061 if (!(rss_desc->types & layer_types)) 1062 return 0; 1063 return hash_fields; 1064 } 1065 1066 /** 1067 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1068 * if several tunnel rules are used on this queue, the tunnel ptype will be 1069 * cleared. 1070 * 1071 * @param rxq_ctrl 1072 * Rx queue to update. 1073 */ 1074 static void 1075 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1076 { 1077 unsigned int i; 1078 uint32_t tunnel_ptype = 0; 1079 1080 /* Look up for the ptype to use. */ 1081 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1082 if (!rxq_ctrl->flow_tunnels_n[i]) 1083 continue; 1084 if (!tunnel_ptype) { 1085 tunnel_ptype = tunnels_info[i].ptype; 1086 } else { 1087 tunnel_ptype = 0; 1088 break; 1089 } 1090 } 1091 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1092 } 1093 1094 /** 1095 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1096 * flow. 1097 * 1098 * @param[in] dev 1099 * Pointer to the Ethernet device structure. 1100 * @param[in] dev_handle 1101 * Pointer to device flow handle structure. 1102 */ 1103 void 1104 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1105 struct mlx5_flow_handle *dev_handle) 1106 { 1107 struct mlx5_priv *priv = dev->data->dev_private; 1108 const int mark = dev_handle->mark; 1109 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1110 struct mlx5_ind_table_obj *ind_tbl = NULL; 1111 unsigned int i; 1112 1113 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1114 struct mlx5_hrxq *hrxq; 1115 1116 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1117 dev_handle->rix_hrxq); 1118 if (hrxq) 1119 ind_tbl = hrxq->ind_table; 1120 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1121 struct mlx5_shared_action_rss *shared_rss; 1122 1123 shared_rss = mlx5_ipool_get 1124 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1125 dev_handle->rix_srss); 1126 if (shared_rss) 1127 ind_tbl = shared_rss->ind_tbl; 1128 } 1129 if (!ind_tbl) 1130 return; 1131 for (i = 0; i != ind_tbl->queues_n; ++i) { 1132 int idx = ind_tbl->queues[i]; 1133 struct mlx5_rxq_ctrl *rxq_ctrl = 1134 container_of((*priv->rxqs)[idx], 1135 struct mlx5_rxq_ctrl, rxq); 1136 1137 /* 1138 * To support metadata register copy on Tx loopback, 1139 * this must be always enabled (metadata may arive 1140 * from other port - not from local flows only. 1141 */ 1142 if (priv->config.dv_flow_en && 1143 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1144 mlx5_flow_ext_mreg_supported(dev)) { 1145 rxq_ctrl->rxq.mark = 1; 1146 rxq_ctrl->flow_mark_n = 1; 1147 } else if (mark) { 1148 rxq_ctrl->rxq.mark = 1; 1149 rxq_ctrl->flow_mark_n++; 1150 } 1151 if (tunnel) { 1152 unsigned int j; 1153 1154 /* Increase the counter matching the flow. */ 1155 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1156 if ((tunnels_info[j].tunnel & 1157 dev_handle->layers) == 1158 tunnels_info[j].tunnel) { 1159 rxq_ctrl->flow_tunnels_n[j]++; 1160 break; 1161 } 1162 } 1163 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1164 } 1165 } 1166 } 1167 1168 /** 1169 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1170 * 1171 * @param[in] dev 1172 * Pointer to the Ethernet device structure. 1173 * @param[in] flow 1174 * Pointer to flow structure. 1175 */ 1176 static void 1177 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1178 { 1179 struct mlx5_priv *priv = dev->data->dev_private; 1180 uint32_t handle_idx; 1181 struct mlx5_flow_handle *dev_handle; 1182 1183 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1184 handle_idx, dev_handle, next) 1185 flow_drv_rxq_flags_set(dev, dev_handle); 1186 } 1187 1188 /** 1189 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1190 * device flow if no other flow uses it with the same kind of request. 1191 * 1192 * @param dev 1193 * Pointer to Ethernet device. 1194 * @param[in] dev_handle 1195 * Pointer to the device flow handle structure. 1196 */ 1197 static void 1198 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1199 struct mlx5_flow_handle *dev_handle) 1200 { 1201 struct mlx5_priv *priv = dev->data->dev_private; 1202 const int mark = dev_handle->mark; 1203 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1204 struct mlx5_ind_table_obj *ind_tbl = NULL; 1205 unsigned int i; 1206 1207 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1208 struct mlx5_hrxq *hrxq; 1209 1210 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1211 dev_handle->rix_hrxq); 1212 if (hrxq) 1213 ind_tbl = hrxq->ind_table; 1214 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1215 struct mlx5_shared_action_rss *shared_rss; 1216 1217 shared_rss = mlx5_ipool_get 1218 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1219 dev_handle->rix_srss); 1220 if (shared_rss) 1221 ind_tbl = shared_rss->ind_tbl; 1222 } 1223 if (!ind_tbl) 1224 return; 1225 MLX5_ASSERT(dev->data->dev_started); 1226 for (i = 0; i != ind_tbl->queues_n; ++i) { 1227 int idx = ind_tbl->queues[i]; 1228 struct mlx5_rxq_ctrl *rxq_ctrl = 1229 container_of((*priv->rxqs)[idx], 1230 struct mlx5_rxq_ctrl, rxq); 1231 1232 if (priv->config.dv_flow_en && 1233 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1234 mlx5_flow_ext_mreg_supported(dev)) { 1235 rxq_ctrl->rxq.mark = 1; 1236 rxq_ctrl->flow_mark_n = 1; 1237 } else if (mark) { 1238 rxq_ctrl->flow_mark_n--; 1239 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1240 } 1241 if (tunnel) { 1242 unsigned int j; 1243 1244 /* Decrease the counter matching the flow. */ 1245 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1246 if ((tunnels_info[j].tunnel & 1247 dev_handle->layers) == 1248 tunnels_info[j].tunnel) { 1249 rxq_ctrl->flow_tunnels_n[j]--; 1250 break; 1251 } 1252 } 1253 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1254 } 1255 } 1256 } 1257 1258 /** 1259 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1260 * @p flow if no other flow uses it with the same kind of request. 1261 * 1262 * @param dev 1263 * Pointer to Ethernet device. 1264 * @param[in] flow 1265 * Pointer to the flow. 1266 */ 1267 static void 1268 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1269 { 1270 struct mlx5_priv *priv = dev->data->dev_private; 1271 uint32_t handle_idx; 1272 struct mlx5_flow_handle *dev_handle; 1273 1274 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1275 handle_idx, dev_handle, next) 1276 flow_drv_rxq_flags_trim(dev, dev_handle); 1277 } 1278 1279 /** 1280 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1281 * 1282 * @param dev 1283 * Pointer to Ethernet device. 1284 */ 1285 static void 1286 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1287 { 1288 struct mlx5_priv *priv = dev->data->dev_private; 1289 unsigned int i; 1290 1291 for (i = 0; i != priv->rxqs_n; ++i) { 1292 struct mlx5_rxq_ctrl *rxq_ctrl; 1293 unsigned int j; 1294 1295 if (!(*priv->rxqs)[i]) 1296 continue; 1297 rxq_ctrl = container_of((*priv->rxqs)[i], 1298 struct mlx5_rxq_ctrl, rxq); 1299 rxq_ctrl->flow_mark_n = 0; 1300 rxq_ctrl->rxq.mark = 0; 1301 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1302 rxq_ctrl->flow_tunnels_n[j] = 0; 1303 rxq_ctrl->rxq.tunnel = 0; 1304 } 1305 } 1306 1307 /** 1308 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1309 * 1310 * @param[in] dev 1311 * Pointer to the Ethernet device structure. 1312 */ 1313 void 1314 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1315 { 1316 struct mlx5_priv *priv = dev->data->dev_private; 1317 struct mlx5_rxq_data *data; 1318 unsigned int i; 1319 1320 for (i = 0; i != priv->rxqs_n; ++i) { 1321 if (!(*priv->rxqs)[i]) 1322 continue; 1323 data = (*priv->rxqs)[i]; 1324 if (!rte_flow_dynf_metadata_avail()) { 1325 data->dynf_meta = 0; 1326 data->flow_meta_mask = 0; 1327 data->flow_meta_offset = -1; 1328 data->flow_meta_port_mask = 0; 1329 } else { 1330 data->dynf_meta = 1; 1331 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1332 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1333 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1334 } 1335 } 1336 } 1337 1338 /* 1339 * return a pointer to the desired action in the list of actions. 1340 * 1341 * @param[in] actions 1342 * The list of actions to search the action in. 1343 * @param[in] action 1344 * The action to find. 1345 * 1346 * @return 1347 * Pointer to the action in the list, if found. NULL otherwise. 1348 */ 1349 const struct rte_flow_action * 1350 mlx5_flow_find_action(const struct rte_flow_action *actions, 1351 enum rte_flow_action_type action) 1352 { 1353 if (actions == NULL) 1354 return NULL; 1355 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1356 if (actions->type == action) 1357 return actions; 1358 return NULL; 1359 } 1360 1361 /* 1362 * Validate the flag action. 1363 * 1364 * @param[in] action_flags 1365 * Bit-fields that holds the actions detected until now. 1366 * @param[in] attr 1367 * Attributes of flow that includes this action. 1368 * @param[out] error 1369 * Pointer to error structure. 1370 * 1371 * @return 1372 * 0 on success, a negative errno value otherwise and rte_errno is set. 1373 */ 1374 int 1375 mlx5_flow_validate_action_flag(uint64_t action_flags, 1376 const struct rte_flow_attr *attr, 1377 struct rte_flow_error *error) 1378 { 1379 if (action_flags & MLX5_FLOW_ACTION_MARK) 1380 return rte_flow_error_set(error, EINVAL, 1381 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1382 "can't mark and flag in same flow"); 1383 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1384 return rte_flow_error_set(error, EINVAL, 1385 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1386 "can't have 2 flag" 1387 " actions in same flow"); 1388 if (attr->egress) 1389 return rte_flow_error_set(error, ENOTSUP, 1390 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1391 "flag action not supported for " 1392 "egress"); 1393 return 0; 1394 } 1395 1396 /* 1397 * Validate the mark action. 1398 * 1399 * @param[in] action 1400 * Pointer to the queue action. 1401 * @param[in] action_flags 1402 * Bit-fields that holds the actions detected until now. 1403 * @param[in] attr 1404 * Attributes of flow that includes this action. 1405 * @param[out] error 1406 * Pointer to error structure. 1407 * 1408 * @return 1409 * 0 on success, a negative errno value otherwise and rte_errno is set. 1410 */ 1411 int 1412 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1413 uint64_t action_flags, 1414 const struct rte_flow_attr *attr, 1415 struct rte_flow_error *error) 1416 { 1417 const struct rte_flow_action_mark *mark = action->conf; 1418 1419 if (!mark) 1420 return rte_flow_error_set(error, EINVAL, 1421 RTE_FLOW_ERROR_TYPE_ACTION, 1422 action, 1423 "configuration cannot be null"); 1424 if (mark->id >= MLX5_FLOW_MARK_MAX) 1425 return rte_flow_error_set(error, EINVAL, 1426 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1427 &mark->id, 1428 "mark id must in 0 <= id < " 1429 RTE_STR(MLX5_FLOW_MARK_MAX)); 1430 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1431 return rte_flow_error_set(error, EINVAL, 1432 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1433 "can't flag and mark in same flow"); 1434 if (action_flags & MLX5_FLOW_ACTION_MARK) 1435 return rte_flow_error_set(error, EINVAL, 1436 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1437 "can't have 2 mark actions in same" 1438 " flow"); 1439 if (attr->egress) 1440 return rte_flow_error_set(error, ENOTSUP, 1441 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1442 "mark action not supported for " 1443 "egress"); 1444 return 0; 1445 } 1446 1447 /* 1448 * Validate the drop action. 1449 * 1450 * @param[in] action_flags 1451 * Bit-fields that holds the actions detected until now. 1452 * @param[in] attr 1453 * Attributes of flow that includes this action. 1454 * @param[out] error 1455 * Pointer to error structure. 1456 * 1457 * @return 1458 * 0 on success, a negative errno value otherwise and rte_errno is set. 1459 */ 1460 int 1461 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1462 const struct rte_flow_attr *attr, 1463 struct rte_flow_error *error) 1464 { 1465 if (attr->egress) 1466 return rte_flow_error_set(error, ENOTSUP, 1467 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1468 "drop action not supported for " 1469 "egress"); 1470 return 0; 1471 } 1472 1473 /* 1474 * Validate the queue action. 1475 * 1476 * @param[in] action 1477 * Pointer to the queue action. 1478 * @param[in] action_flags 1479 * Bit-fields that holds the actions detected until now. 1480 * @param[in] dev 1481 * Pointer to the Ethernet device structure. 1482 * @param[in] attr 1483 * Attributes of flow that includes this action. 1484 * @param[out] error 1485 * Pointer to error structure. 1486 * 1487 * @return 1488 * 0 on success, a negative errno value otherwise and rte_errno is set. 1489 */ 1490 int 1491 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1492 uint64_t action_flags, 1493 struct rte_eth_dev *dev, 1494 const struct rte_flow_attr *attr, 1495 struct rte_flow_error *error) 1496 { 1497 struct mlx5_priv *priv = dev->data->dev_private; 1498 const struct rte_flow_action_queue *queue = action->conf; 1499 1500 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1501 return rte_flow_error_set(error, EINVAL, 1502 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1503 "can't have 2 fate actions in" 1504 " same flow"); 1505 if (!priv->rxqs_n) 1506 return rte_flow_error_set(error, EINVAL, 1507 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1508 NULL, "No Rx queues configured"); 1509 if (queue->index >= priv->rxqs_n) 1510 return rte_flow_error_set(error, EINVAL, 1511 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1512 &queue->index, 1513 "queue index out of range"); 1514 if (!(*priv->rxqs)[queue->index]) 1515 return rte_flow_error_set(error, EINVAL, 1516 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1517 &queue->index, 1518 "queue is not configured"); 1519 if (attr->egress) 1520 return rte_flow_error_set(error, ENOTSUP, 1521 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1522 "queue action not supported for " 1523 "egress"); 1524 return 0; 1525 } 1526 1527 /* 1528 * Validate the rss action. 1529 * 1530 * @param[in] dev 1531 * Pointer to the Ethernet device structure. 1532 * @param[in] action 1533 * Pointer to the queue action. 1534 * @param[out] error 1535 * Pointer to error structure. 1536 * 1537 * @return 1538 * 0 on success, a negative errno value otherwise and rte_errno is set. 1539 */ 1540 int 1541 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1542 const struct rte_flow_action *action, 1543 struct rte_flow_error *error) 1544 { 1545 struct mlx5_priv *priv = dev->data->dev_private; 1546 const struct rte_flow_action_rss *rss = action->conf; 1547 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1548 unsigned int i; 1549 1550 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1551 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1552 return rte_flow_error_set(error, ENOTSUP, 1553 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1554 &rss->func, 1555 "RSS hash function not supported"); 1556 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1557 if (rss->level > 2) 1558 #else 1559 if (rss->level > 1) 1560 #endif 1561 return rte_flow_error_set(error, ENOTSUP, 1562 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1563 &rss->level, 1564 "tunnel RSS is not supported"); 1565 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1566 if (rss->key_len == 0 && rss->key != NULL) 1567 return rte_flow_error_set(error, ENOTSUP, 1568 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1569 &rss->key_len, 1570 "RSS hash key length 0"); 1571 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1572 return rte_flow_error_set(error, ENOTSUP, 1573 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1574 &rss->key_len, 1575 "RSS hash key too small"); 1576 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1577 return rte_flow_error_set(error, ENOTSUP, 1578 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1579 &rss->key_len, 1580 "RSS hash key too large"); 1581 if (rss->queue_num > priv->config.ind_table_max_size) 1582 return rte_flow_error_set(error, ENOTSUP, 1583 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1584 &rss->queue_num, 1585 "number of queues too large"); 1586 if (rss->types & MLX5_RSS_HF_MASK) 1587 return rte_flow_error_set(error, ENOTSUP, 1588 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1589 &rss->types, 1590 "some RSS protocols are not" 1591 " supported"); 1592 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1593 !(rss->types & ETH_RSS_IP)) 1594 return rte_flow_error_set(error, EINVAL, 1595 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1596 "L3 partial RSS requested but L3 RSS" 1597 " type not specified"); 1598 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1599 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1600 return rte_flow_error_set(error, EINVAL, 1601 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1602 "L4 partial RSS requested but L4 RSS" 1603 " type not specified"); 1604 if (!priv->rxqs_n) 1605 return rte_flow_error_set(error, EINVAL, 1606 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1607 NULL, "No Rx queues configured"); 1608 if (!rss->queue_num) 1609 return rte_flow_error_set(error, EINVAL, 1610 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1611 NULL, "No queues configured"); 1612 for (i = 0; i != rss->queue_num; ++i) { 1613 struct mlx5_rxq_ctrl *rxq_ctrl; 1614 1615 if (rss->queue[i] >= priv->rxqs_n) 1616 return rte_flow_error_set 1617 (error, EINVAL, 1618 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1619 &rss->queue[i], "queue index out of range"); 1620 if (!(*priv->rxqs)[rss->queue[i]]) 1621 return rte_flow_error_set 1622 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1623 &rss->queue[i], "queue is not configured"); 1624 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1625 struct mlx5_rxq_ctrl, rxq); 1626 if (i == 0) 1627 rxq_type = rxq_ctrl->type; 1628 if (rxq_type != rxq_ctrl->type) 1629 return rte_flow_error_set 1630 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1631 &rss->queue[i], 1632 "combining hairpin and regular RSS queues is not supported"); 1633 } 1634 return 0; 1635 } 1636 1637 /* 1638 * Validate the rss action. 1639 * 1640 * @param[in] action 1641 * Pointer to the queue action. 1642 * @param[in] action_flags 1643 * Bit-fields that holds the actions detected until now. 1644 * @param[in] dev 1645 * Pointer to the Ethernet device structure. 1646 * @param[in] attr 1647 * Attributes of flow that includes this action. 1648 * @param[in] item_flags 1649 * Items that were detected. 1650 * @param[out] error 1651 * Pointer to error structure. 1652 * 1653 * @return 1654 * 0 on success, a negative errno value otherwise and rte_errno is set. 1655 */ 1656 int 1657 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1658 uint64_t action_flags, 1659 struct rte_eth_dev *dev, 1660 const struct rte_flow_attr *attr, 1661 uint64_t item_flags, 1662 struct rte_flow_error *error) 1663 { 1664 const struct rte_flow_action_rss *rss = action->conf; 1665 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1666 int ret; 1667 1668 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1669 return rte_flow_error_set(error, EINVAL, 1670 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1671 "can't have 2 fate actions" 1672 " in same flow"); 1673 ret = mlx5_validate_action_rss(dev, action, error); 1674 if (ret) 1675 return ret; 1676 if (attr->egress) 1677 return rte_flow_error_set(error, ENOTSUP, 1678 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1679 "rss action not supported for " 1680 "egress"); 1681 if (rss->level > 1 && !tunnel) 1682 return rte_flow_error_set(error, EINVAL, 1683 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1684 "inner RSS is not supported for " 1685 "non-tunnel flows"); 1686 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1687 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1688 return rte_flow_error_set(error, EINVAL, 1689 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1690 "RSS on eCPRI is not supported now"); 1691 } 1692 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1693 !(item_flags & 1694 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1695 rss->level > 1) 1696 return rte_flow_error_set(error, EINVAL, 1697 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1698 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1699 return 0; 1700 } 1701 1702 /* 1703 * Validate the default miss action. 1704 * 1705 * @param[in] action_flags 1706 * Bit-fields that holds the actions detected until now. 1707 * @param[out] error 1708 * Pointer to error structure. 1709 * 1710 * @return 1711 * 0 on success, a negative errno value otherwise and rte_errno is set. 1712 */ 1713 int 1714 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1715 const struct rte_flow_attr *attr, 1716 struct rte_flow_error *error) 1717 { 1718 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1719 return rte_flow_error_set(error, EINVAL, 1720 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1721 "can't have 2 fate actions in" 1722 " same flow"); 1723 if (attr->egress) 1724 return rte_flow_error_set(error, ENOTSUP, 1725 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1726 "default miss action not supported " 1727 "for egress"); 1728 if (attr->group) 1729 return rte_flow_error_set(error, ENOTSUP, 1730 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1731 "only group 0 is supported"); 1732 if (attr->transfer) 1733 return rte_flow_error_set(error, ENOTSUP, 1734 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1735 NULL, "transfer is not supported"); 1736 return 0; 1737 } 1738 1739 /* 1740 * Validate the count action. 1741 * 1742 * @param[in] dev 1743 * Pointer to the Ethernet device structure. 1744 * @param[in] attr 1745 * Attributes of flow that includes this action. 1746 * @param[out] error 1747 * Pointer to error structure. 1748 * 1749 * @return 1750 * 0 on success, a negative errno value otherwise and rte_errno is set. 1751 */ 1752 int 1753 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1754 const struct rte_flow_attr *attr, 1755 struct rte_flow_error *error) 1756 { 1757 if (attr->egress) 1758 return rte_flow_error_set(error, ENOTSUP, 1759 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1760 "count action not supported for " 1761 "egress"); 1762 return 0; 1763 } 1764 1765 /* 1766 * Validate the ASO CT action. 1767 * 1768 * @param[in] dev 1769 * Pointer to the Ethernet device structure. 1770 * @param[in] conntrack 1771 * Pointer to the CT action profile. 1772 * @param[out] error 1773 * Pointer to error structure. 1774 * 1775 * @return 1776 * 0 on success, a negative errno value otherwise and rte_errno is set. 1777 */ 1778 int 1779 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1780 const struct rte_flow_action_conntrack *conntrack, 1781 struct rte_flow_error *error) 1782 { 1783 RTE_SET_USED(dev); 1784 1785 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1786 return rte_flow_error_set(error, EINVAL, 1787 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1788 "Invalid CT state"); 1789 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1790 return rte_flow_error_set(error, EINVAL, 1791 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1792 "Invalid last TCP packet flag"); 1793 return 0; 1794 } 1795 1796 /** 1797 * Verify the @p attributes will be correctly understood by the NIC and store 1798 * them in the @p flow if everything is correct. 1799 * 1800 * @param[in] dev 1801 * Pointer to the Ethernet device structure. 1802 * @param[in] attributes 1803 * Pointer to flow attributes 1804 * @param[out] error 1805 * Pointer to error structure. 1806 * 1807 * @return 1808 * 0 on success, a negative errno value otherwise and rte_errno is set. 1809 */ 1810 int 1811 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1812 const struct rte_flow_attr *attributes, 1813 struct rte_flow_error *error) 1814 { 1815 struct mlx5_priv *priv = dev->data->dev_private; 1816 uint32_t priority_max = priv->config.flow_prio - 1; 1817 1818 if (attributes->group) 1819 return rte_flow_error_set(error, ENOTSUP, 1820 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1821 NULL, "groups is not supported"); 1822 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1823 attributes->priority >= priority_max) 1824 return rte_flow_error_set(error, ENOTSUP, 1825 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1826 NULL, "priority out of range"); 1827 if (attributes->egress) 1828 return rte_flow_error_set(error, ENOTSUP, 1829 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1830 "egress is not supported"); 1831 if (attributes->transfer && !priv->config.dv_esw_en) 1832 return rte_flow_error_set(error, ENOTSUP, 1833 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1834 NULL, "transfer is not supported"); 1835 if (!attributes->ingress) 1836 return rte_flow_error_set(error, EINVAL, 1837 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1838 NULL, 1839 "ingress attribute is mandatory"); 1840 return 0; 1841 } 1842 1843 /** 1844 * Validate ICMP6 item. 1845 * 1846 * @param[in] item 1847 * Item specification. 1848 * @param[in] item_flags 1849 * Bit-fields that holds the items detected until now. 1850 * @param[in] ext_vlan_sup 1851 * Whether extended VLAN features are supported or not. 1852 * @param[out] error 1853 * Pointer to error structure. 1854 * 1855 * @return 1856 * 0 on success, a negative errno value otherwise and rte_errno is set. 1857 */ 1858 int 1859 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1860 uint64_t item_flags, 1861 uint8_t target_protocol, 1862 struct rte_flow_error *error) 1863 { 1864 const struct rte_flow_item_icmp6 *mask = item->mask; 1865 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1866 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1867 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1868 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1869 MLX5_FLOW_LAYER_OUTER_L4; 1870 int ret; 1871 1872 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1873 return rte_flow_error_set(error, EINVAL, 1874 RTE_FLOW_ERROR_TYPE_ITEM, item, 1875 "protocol filtering not compatible" 1876 " with ICMP6 layer"); 1877 if (!(item_flags & l3m)) 1878 return rte_flow_error_set(error, EINVAL, 1879 RTE_FLOW_ERROR_TYPE_ITEM, item, 1880 "IPv6 is mandatory to filter on" 1881 " ICMP6"); 1882 if (item_flags & l4m) 1883 return rte_flow_error_set(error, EINVAL, 1884 RTE_FLOW_ERROR_TYPE_ITEM, item, 1885 "multiple L4 layers not supported"); 1886 if (!mask) 1887 mask = &rte_flow_item_icmp6_mask; 1888 ret = mlx5_flow_item_acceptable 1889 (item, (const uint8_t *)mask, 1890 (const uint8_t *)&rte_flow_item_icmp6_mask, 1891 sizeof(struct rte_flow_item_icmp6), 1892 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1893 if (ret < 0) 1894 return ret; 1895 return 0; 1896 } 1897 1898 /** 1899 * Validate ICMP item. 1900 * 1901 * @param[in] item 1902 * Item specification. 1903 * @param[in] item_flags 1904 * Bit-fields that holds the items detected until now. 1905 * @param[out] error 1906 * Pointer to error structure. 1907 * 1908 * @return 1909 * 0 on success, a negative errno value otherwise and rte_errno is set. 1910 */ 1911 int 1912 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1913 uint64_t item_flags, 1914 uint8_t target_protocol, 1915 struct rte_flow_error *error) 1916 { 1917 const struct rte_flow_item_icmp *mask = item->mask; 1918 const struct rte_flow_item_icmp nic_mask = { 1919 .hdr.icmp_type = 0xff, 1920 .hdr.icmp_code = 0xff, 1921 .hdr.icmp_ident = RTE_BE16(0xffff), 1922 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1923 }; 1924 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1925 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1926 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1927 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1928 MLX5_FLOW_LAYER_OUTER_L4; 1929 int ret; 1930 1931 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1932 return rte_flow_error_set(error, EINVAL, 1933 RTE_FLOW_ERROR_TYPE_ITEM, item, 1934 "protocol filtering not compatible" 1935 " with ICMP layer"); 1936 if (!(item_flags & l3m)) 1937 return rte_flow_error_set(error, EINVAL, 1938 RTE_FLOW_ERROR_TYPE_ITEM, item, 1939 "IPv4 is mandatory to filter" 1940 " on ICMP"); 1941 if (item_flags & l4m) 1942 return rte_flow_error_set(error, EINVAL, 1943 RTE_FLOW_ERROR_TYPE_ITEM, item, 1944 "multiple L4 layers not supported"); 1945 if (!mask) 1946 mask = &nic_mask; 1947 ret = mlx5_flow_item_acceptable 1948 (item, (const uint8_t *)mask, 1949 (const uint8_t *)&nic_mask, 1950 sizeof(struct rte_flow_item_icmp), 1951 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1952 if (ret < 0) 1953 return ret; 1954 return 0; 1955 } 1956 1957 /** 1958 * Validate Ethernet item. 1959 * 1960 * @param[in] item 1961 * Item specification. 1962 * @param[in] item_flags 1963 * Bit-fields that holds the items detected until now. 1964 * @param[out] error 1965 * Pointer to error structure. 1966 * 1967 * @return 1968 * 0 on success, a negative errno value otherwise and rte_errno is set. 1969 */ 1970 int 1971 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1972 uint64_t item_flags, bool ext_vlan_sup, 1973 struct rte_flow_error *error) 1974 { 1975 const struct rte_flow_item_eth *mask = item->mask; 1976 const struct rte_flow_item_eth nic_mask = { 1977 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1978 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1979 .type = RTE_BE16(0xffff), 1980 .has_vlan = ext_vlan_sup ? 1 : 0, 1981 }; 1982 int ret; 1983 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1984 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1985 MLX5_FLOW_LAYER_OUTER_L2; 1986 1987 if (item_flags & ethm) 1988 return rte_flow_error_set(error, ENOTSUP, 1989 RTE_FLOW_ERROR_TYPE_ITEM, item, 1990 "multiple L2 layers not supported"); 1991 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1992 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1993 return rte_flow_error_set(error, EINVAL, 1994 RTE_FLOW_ERROR_TYPE_ITEM, item, 1995 "L2 layer should not follow " 1996 "L3 layers"); 1997 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1998 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1999 return rte_flow_error_set(error, EINVAL, 2000 RTE_FLOW_ERROR_TYPE_ITEM, item, 2001 "L2 layer should not follow VLAN"); 2002 if (!mask) 2003 mask = &rte_flow_item_eth_mask; 2004 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2005 (const uint8_t *)&nic_mask, 2006 sizeof(struct rte_flow_item_eth), 2007 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2008 return ret; 2009 } 2010 2011 /** 2012 * Validate VLAN item. 2013 * 2014 * @param[in] item 2015 * Item specification. 2016 * @param[in] item_flags 2017 * Bit-fields that holds the items detected until now. 2018 * @param[in] dev 2019 * Ethernet device flow is being created on. 2020 * @param[out] error 2021 * Pointer to error structure. 2022 * 2023 * @return 2024 * 0 on success, a negative errno value otherwise and rte_errno is set. 2025 */ 2026 int 2027 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2028 uint64_t item_flags, 2029 struct rte_eth_dev *dev, 2030 struct rte_flow_error *error) 2031 { 2032 const struct rte_flow_item_vlan *spec = item->spec; 2033 const struct rte_flow_item_vlan *mask = item->mask; 2034 const struct rte_flow_item_vlan nic_mask = { 2035 .tci = RTE_BE16(UINT16_MAX), 2036 .inner_type = RTE_BE16(UINT16_MAX), 2037 }; 2038 uint16_t vlan_tag = 0; 2039 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2040 int ret; 2041 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2042 MLX5_FLOW_LAYER_INNER_L4) : 2043 (MLX5_FLOW_LAYER_OUTER_L3 | 2044 MLX5_FLOW_LAYER_OUTER_L4); 2045 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2046 MLX5_FLOW_LAYER_OUTER_VLAN; 2047 2048 if (item_flags & vlanm) 2049 return rte_flow_error_set(error, EINVAL, 2050 RTE_FLOW_ERROR_TYPE_ITEM, item, 2051 "multiple VLAN layers not supported"); 2052 else if ((item_flags & l34m) != 0) 2053 return rte_flow_error_set(error, EINVAL, 2054 RTE_FLOW_ERROR_TYPE_ITEM, item, 2055 "VLAN cannot follow L3/L4 layer"); 2056 if (!mask) 2057 mask = &rte_flow_item_vlan_mask; 2058 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2059 (const uint8_t *)&nic_mask, 2060 sizeof(struct rte_flow_item_vlan), 2061 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2062 if (ret) 2063 return ret; 2064 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2065 struct mlx5_priv *priv = dev->data->dev_private; 2066 2067 if (priv->vmwa_context) { 2068 /* 2069 * Non-NULL context means we have a virtual machine 2070 * and SR-IOV enabled, we have to create VLAN interface 2071 * to make hypervisor to setup E-Switch vport 2072 * context correctly. We avoid creating the multiple 2073 * VLAN interfaces, so we cannot support VLAN tag mask. 2074 */ 2075 return rte_flow_error_set(error, EINVAL, 2076 RTE_FLOW_ERROR_TYPE_ITEM, 2077 item, 2078 "VLAN tag mask is not" 2079 " supported in virtual" 2080 " environment"); 2081 } 2082 } 2083 if (spec) { 2084 vlan_tag = spec->tci; 2085 vlan_tag &= mask->tci; 2086 } 2087 /* 2088 * From verbs perspective an empty VLAN is equivalent 2089 * to a packet without VLAN layer. 2090 */ 2091 if (!vlan_tag) 2092 return rte_flow_error_set(error, EINVAL, 2093 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2094 item->spec, 2095 "VLAN cannot be empty"); 2096 return 0; 2097 } 2098 2099 /** 2100 * Validate IPV4 item. 2101 * 2102 * @param[in] item 2103 * Item specification. 2104 * @param[in] item_flags 2105 * Bit-fields that holds the items detected until now. 2106 * @param[in] last_item 2107 * Previous validated item in the pattern items. 2108 * @param[in] ether_type 2109 * Type in the ethernet layer header (including dot1q). 2110 * @param[in] acc_mask 2111 * Acceptable mask, if NULL default internal default mask 2112 * will be used to check whether item fields are supported. 2113 * @param[in] range_accepted 2114 * True if range of values is accepted for specific fields, false otherwise. 2115 * @param[out] error 2116 * Pointer to error structure. 2117 * 2118 * @return 2119 * 0 on success, a negative errno value otherwise and rte_errno is set. 2120 */ 2121 int 2122 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2123 uint64_t item_flags, 2124 uint64_t last_item, 2125 uint16_t ether_type, 2126 const struct rte_flow_item_ipv4 *acc_mask, 2127 bool range_accepted, 2128 struct rte_flow_error *error) 2129 { 2130 const struct rte_flow_item_ipv4 *mask = item->mask; 2131 const struct rte_flow_item_ipv4 *spec = item->spec; 2132 const struct rte_flow_item_ipv4 nic_mask = { 2133 .hdr = { 2134 .src_addr = RTE_BE32(0xffffffff), 2135 .dst_addr = RTE_BE32(0xffffffff), 2136 .type_of_service = 0xff, 2137 .next_proto_id = 0xff, 2138 }, 2139 }; 2140 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2141 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2142 MLX5_FLOW_LAYER_OUTER_L3; 2143 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2144 MLX5_FLOW_LAYER_OUTER_L4; 2145 int ret; 2146 uint8_t next_proto = 0xFF; 2147 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2148 MLX5_FLOW_LAYER_OUTER_VLAN | 2149 MLX5_FLOW_LAYER_INNER_VLAN); 2150 2151 if ((last_item & l2_vlan) && ether_type && 2152 ether_type != RTE_ETHER_TYPE_IPV4) 2153 return rte_flow_error_set(error, EINVAL, 2154 RTE_FLOW_ERROR_TYPE_ITEM, item, 2155 "IPv4 cannot follow L2/VLAN layer " 2156 "which ether type is not IPv4"); 2157 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2158 if (mask && spec) 2159 next_proto = mask->hdr.next_proto_id & 2160 spec->hdr.next_proto_id; 2161 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2162 return rte_flow_error_set(error, EINVAL, 2163 RTE_FLOW_ERROR_TYPE_ITEM, 2164 item, 2165 "multiple tunnel " 2166 "not supported"); 2167 } 2168 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2169 return rte_flow_error_set(error, EINVAL, 2170 RTE_FLOW_ERROR_TYPE_ITEM, item, 2171 "wrong tunnel type - IPv6 specified " 2172 "but IPv4 item provided"); 2173 if (item_flags & l3m) 2174 return rte_flow_error_set(error, ENOTSUP, 2175 RTE_FLOW_ERROR_TYPE_ITEM, item, 2176 "multiple L3 layers not supported"); 2177 else if (item_flags & l4m) 2178 return rte_flow_error_set(error, EINVAL, 2179 RTE_FLOW_ERROR_TYPE_ITEM, item, 2180 "L3 cannot follow an L4 layer."); 2181 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2182 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2183 return rte_flow_error_set(error, EINVAL, 2184 RTE_FLOW_ERROR_TYPE_ITEM, item, 2185 "L3 cannot follow an NVGRE layer."); 2186 if (!mask) 2187 mask = &rte_flow_item_ipv4_mask; 2188 else if (mask->hdr.next_proto_id != 0 && 2189 mask->hdr.next_proto_id != 0xff) 2190 return rte_flow_error_set(error, EINVAL, 2191 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2192 "partial mask is not supported" 2193 " for protocol"); 2194 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2195 acc_mask ? (const uint8_t *)acc_mask 2196 : (const uint8_t *)&nic_mask, 2197 sizeof(struct rte_flow_item_ipv4), 2198 range_accepted, error); 2199 if (ret < 0) 2200 return ret; 2201 return 0; 2202 } 2203 2204 /** 2205 * Validate IPV6 item. 2206 * 2207 * @param[in] item 2208 * Item specification. 2209 * @param[in] item_flags 2210 * Bit-fields that holds the items detected until now. 2211 * @param[in] last_item 2212 * Previous validated item in the pattern items. 2213 * @param[in] ether_type 2214 * Type in the ethernet layer header (including dot1q). 2215 * @param[in] acc_mask 2216 * Acceptable mask, if NULL default internal default mask 2217 * will be used to check whether item fields are supported. 2218 * @param[out] error 2219 * Pointer to error structure. 2220 * 2221 * @return 2222 * 0 on success, a negative errno value otherwise and rte_errno is set. 2223 */ 2224 int 2225 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2226 uint64_t item_flags, 2227 uint64_t last_item, 2228 uint16_t ether_type, 2229 const struct rte_flow_item_ipv6 *acc_mask, 2230 struct rte_flow_error *error) 2231 { 2232 const struct rte_flow_item_ipv6 *mask = item->mask; 2233 const struct rte_flow_item_ipv6 *spec = item->spec; 2234 const struct rte_flow_item_ipv6 nic_mask = { 2235 .hdr = { 2236 .src_addr = 2237 "\xff\xff\xff\xff\xff\xff\xff\xff" 2238 "\xff\xff\xff\xff\xff\xff\xff\xff", 2239 .dst_addr = 2240 "\xff\xff\xff\xff\xff\xff\xff\xff" 2241 "\xff\xff\xff\xff\xff\xff\xff\xff", 2242 .vtc_flow = RTE_BE32(0xffffffff), 2243 .proto = 0xff, 2244 }, 2245 }; 2246 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2247 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2248 MLX5_FLOW_LAYER_OUTER_L3; 2249 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2250 MLX5_FLOW_LAYER_OUTER_L4; 2251 int ret; 2252 uint8_t next_proto = 0xFF; 2253 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2254 MLX5_FLOW_LAYER_OUTER_VLAN | 2255 MLX5_FLOW_LAYER_INNER_VLAN); 2256 2257 if ((last_item & l2_vlan) && ether_type && 2258 ether_type != RTE_ETHER_TYPE_IPV6) 2259 return rte_flow_error_set(error, EINVAL, 2260 RTE_FLOW_ERROR_TYPE_ITEM, item, 2261 "IPv6 cannot follow L2/VLAN layer " 2262 "which ether type is not IPv6"); 2263 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2264 next_proto = spec->hdr.proto; 2265 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2266 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2267 return rte_flow_error_set(error, EINVAL, 2268 RTE_FLOW_ERROR_TYPE_ITEM, 2269 item, 2270 "multiple tunnel " 2271 "not supported"); 2272 } 2273 if (next_proto == IPPROTO_HOPOPTS || 2274 next_proto == IPPROTO_ROUTING || 2275 next_proto == IPPROTO_FRAGMENT || 2276 next_proto == IPPROTO_ESP || 2277 next_proto == IPPROTO_AH || 2278 next_proto == IPPROTO_DSTOPTS) 2279 return rte_flow_error_set(error, EINVAL, 2280 RTE_FLOW_ERROR_TYPE_ITEM, item, 2281 "IPv6 proto (next header) should " 2282 "not be set as extension header"); 2283 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2284 return rte_flow_error_set(error, EINVAL, 2285 RTE_FLOW_ERROR_TYPE_ITEM, item, 2286 "wrong tunnel type - IPv4 specified " 2287 "but IPv6 item provided"); 2288 if (item_flags & l3m) 2289 return rte_flow_error_set(error, ENOTSUP, 2290 RTE_FLOW_ERROR_TYPE_ITEM, item, 2291 "multiple L3 layers not supported"); 2292 else if (item_flags & l4m) 2293 return rte_flow_error_set(error, EINVAL, 2294 RTE_FLOW_ERROR_TYPE_ITEM, item, 2295 "L3 cannot follow an L4 layer."); 2296 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2297 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2298 return rte_flow_error_set(error, EINVAL, 2299 RTE_FLOW_ERROR_TYPE_ITEM, item, 2300 "L3 cannot follow an NVGRE layer."); 2301 if (!mask) 2302 mask = &rte_flow_item_ipv6_mask; 2303 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2304 acc_mask ? (const uint8_t *)acc_mask 2305 : (const uint8_t *)&nic_mask, 2306 sizeof(struct rte_flow_item_ipv6), 2307 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2308 if (ret < 0) 2309 return ret; 2310 return 0; 2311 } 2312 2313 /** 2314 * Validate UDP item. 2315 * 2316 * @param[in] item 2317 * Item specification. 2318 * @param[in] item_flags 2319 * Bit-fields that holds the items detected until now. 2320 * @param[in] target_protocol 2321 * The next protocol in the previous item. 2322 * @param[in] flow_mask 2323 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2324 * @param[out] error 2325 * Pointer to error structure. 2326 * 2327 * @return 2328 * 0 on success, a negative errno value otherwise and rte_errno is set. 2329 */ 2330 int 2331 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2332 uint64_t item_flags, 2333 uint8_t target_protocol, 2334 struct rte_flow_error *error) 2335 { 2336 const struct rte_flow_item_udp *mask = item->mask; 2337 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2338 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2339 MLX5_FLOW_LAYER_OUTER_L3; 2340 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2341 MLX5_FLOW_LAYER_OUTER_L4; 2342 int ret; 2343 2344 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2345 return rte_flow_error_set(error, EINVAL, 2346 RTE_FLOW_ERROR_TYPE_ITEM, item, 2347 "protocol filtering not compatible" 2348 " with UDP layer"); 2349 if (!(item_flags & l3m)) 2350 return rte_flow_error_set(error, EINVAL, 2351 RTE_FLOW_ERROR_TYPE_ITEM, item, 2352 "L3 is mandatory to filter on L4"); 2353 if (item_flags & l4m) 2354 return rte_flow_error_set(error, EINVAL, 2355 RTE_FLOW_ERROR_TYPE_ITEM, item, 2356 "multiple L4 layers not supported"); 2357 if (!mask) 2358 mask = &rte_flow_item_udp_mask; 2359 ret = mlx5_flow_item_acceptable 2360 (item, (const uint8_t *)mask, 2361 (const uint8_t *)&rte_flow_item_udp_mask, 2362 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2363 error); 2364 if (ret < 0) 2365 return ret; 2366 return 0; 2367 } 2368 2369 /** 2370 * Validate TCP item. 2371 * 2372 * @param[in] item 2373 * Item specification. 2374 * @param[in] item_flags 2375 * Bit-fields that holds the items detected until now. 2376 * @param[in] target_protocol 2377 * The next protocol in the previous item. 2378 * @param[out] error 2379 * Pointer to error structure. 2380 * 2381 * @return 2382 * 0 on success, a negative errno value otherwise and rte_errno is set. 2383 */ 2384 int 2385 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2386 uint64_t item_flags, 2387 uint8_t target_protocol, 2388 const struct rte_flow_item_tcp *flow_mask, 2389 struct rte_flow_error *error) 2390 { 2391 const struct rte_flow_item_tcp *mask = item->mask; 2392 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2393 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2394 MLX5_FLOW_LAYER_OUTER_L3; 2395 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2396 MLX5_FLOW_LAYER_OUTER_L4; 2397 int ret; 2398 2399 MLX5_ASSERT(flow_mask); 2400 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2401 return rte_flow_error_set(error, EINVAL, 2402 RTE_FLOW_ERROR_TYPE_ITEM, item, 2403 "protocol filtering not compatible" 2404 " with TCP layer"); 2405 if (!(item_flags & l3m)) 2406 return rte_flow_error_set(error, EINVAL, 2407 RTE_FLOW_ERROR_TYPE_ITEM, item, 2408 "L3 is mandatory to filter on L4"); 2409 if (item_flags & l4m) 2410 return rte_flow_error_set(error, EINVAL, 2411 RTE_FLOW_ERROR_TYPE_ITEM, item, 2412 "multiple L4 layers not supported"); 2413 if (!mask) 2414 mask = &rte_flow_item_tcp_mask; 2415 ret = mlx5_flow_item_acceptable 2416 (item, (const uint8_t *)mask, 2417 (const uint8_t *)flow_mask, 2418 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2419 error); 2420 if (ret < 0) 2421 return ret; 2422 return 0; 2423 } 2424 2425 /** 2426 * Validate VXLAN item. 2427 * 2428 * @param[in] dev 2429 * Pointer to the Ethernet device structure. 2430 * @param[in] item 2431 * Item specification. 2432 * @param[in] item_flags 2433 * Bit-fields that holds the items detected until now. 2434 * @param[in] attr 2435 * Flow rule attributes. 2436 * @param[out] error 2437 * Pointer to error structure. 2438 * 2439 * @return 2440 * 0 on success, a negative errno value otherwise and rte_errno is set. 2441 */ 2442 int 2443 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2444 const struct rte_flow_item *item, 2445 uint64_t item_flags, 2446 const struct rte_flow_attr *attr, 2447 struct rte_flow_error *error) 2448 { 2449 const struct rte_flow_item_vxlan *spec = item->spec; 2450 const struct rte_flow_item_vxlan *mask = item->mask; 2451 int ret; 2452 struct mlx5_priv *priv = dev->data->dev_private; 2453 union vni { 2454 uint32_t vlan_id; 2455 uint8_t vni[4]; 2456 } id = { .vlan_id = 0, }; 2457 const struct rte_flow_item_vxlan nic_mask = { 2458 .vni = "\xff\xff\xff", 2459 .rsvd1 = 0xff, 2460 }; 2461 const struct rte_flow_item_vxlan *valid_mask; 2462 2463 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2464 return rte_flow_error_set(error, ENOTSUP, 2465 RTE_FLOW_ERROR_TYPE_ITEM, item, 2466 "multiple tunnel layers not" 2467 " supported"); 2468 valid_mask = &rte_flow_item_vxlan_mask; 2469 /* 2470 * Verify only UDPv4 is present as defined in 2471 * https://tools.ietf.org/html/rfc7348 2472 */ 2473 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2474 return rte_flow_error_set(error, EINVAL, 2475 RTE_FLOW_ERROR_TYPE_ITEM, item, 2476 "no outer UDP layer found"); 2477 if (!mask) 2478 mask = &rte_flow_item_vxlan_mask; 2479 /* FDB domain & NIC domain non-zero group */ 2480 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2481 valid_mask = &nic_mask; 2482 /* Group zero in NIC domain */ 2483 if (!attr->group && !attr->transfer && priv->sh->tunnel_header_0_1) 2484 valid_mask = &nic_mask; 2485 ret = mlx5_flow_item_acceptable 2486 (item, (const uint8_t *)mask, 2487 (const uint8_t *)valid_mask, 2488 sizeof(struct rte_flow_item_vxlan), 2489 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2490 if (ret < 0) 2491 return ret; 2492 if (spec) { 2493 memcpy(&id.vni[1], spec->vni, 3); 2494 memcpy(&id.vni[1], mask->vni, 3); 2495 } 2496 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2497 return rte_flow_error_set(error, ENOTSUP, 2498 RTE_FLOW_ERROR_TYPE_ITEM, item, 2499 "VXLAN tunnel must be fully defined"); 2500 return 0; 2501 } 2502 2503 /** 2504 * Validate VXLAN_GPE item. 2505 * 2506 * @param[in] item 2507 * Item specification. 2508 * @param[in] item_flags 2509 * Bit-fields that holds the items detected until now. 2510 * @param[in] priv 2511 * Pointer to the private data structure. 2512 * @param[in] target_protocol 2513 * The next protocol in the previous item. 2514 * @param[out] error 2515 * Pointer to error structure. 2516 * 2517 * @return 2518 * 0 on success, a negative errno value otherwise and rte_errno is set. 2519 */ 2520 int 2521 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2522 uint64_t item_flags, 2523 struct rte_eth_dev *dev, 2524 struct rte_flow_error *error) 2525 { 2526 struct mlx5_priv *priv = dev->data->dev_private; 2527 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2528 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2529 int ret; 2530 union vni { 2531 uint32_t vlan_id; 2532 uint8_t vni[4]; 2533 } id = { .vlan_id = 0, }; 2534 2535 if (!priv->config.l3_vxlan_en) 2536 return rte_flow_error_set(error, ENOTSUP, 2537 RTE_FLOW_ERROR_TYPE_ITEM, item, 2538 "L3 VXLAN is not enabled by device" 2539 " parameter and/or not configured in" 2540 " firmware"); 2541 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2542 return rte_flow_error_set(error, ENOTSUP, 2543 RTE_FLOW_ERROR_TYPE_ITEM, item, 2544 "multiple tunnel layers not" 2545 " supported"); 2546 /* 2547 * Verify only UDPv4 is present as defined in 2548 * https://tools.ietf.org/html/rfc7348 2549 */ 2550 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2551 return rte_flow_error_set(error, EINVAL, 2552 RTE_FLOW_ERROR_TYPE_ITEM, item, 2553 "no outer UDP layer found"); 2554 if (!mask) 2555 mask = &rte_flow_item_vxlan_gpe_mask; 2556 ret = mlx5_flow_item_acceptable 2557 (item, (const uint8_t *)mask, 2558 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2559 sizeof(struct rte_flow_item_vxlan_gpe), 2560 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2561 if (ret < 0) 2562 return ret; 2563 if (spec) { 2564 if (spec->protocol) 2565 return rte_flow_error_set(error, ENOTSUP, 2566 RTE_FLOW_ERROR_TYPE_ITEM, 2567 item, 2568 "VxLAN-GPE protocol" 2569 " not supported"); 2570 memcpy(&id.vni[1], spec->vni, 3); 2571 memcpy(&id.vni[1], mask->vni, 3); 2572 } 2573 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2574 return rte_flow_error_set(error, ENOTSUP, 2575 RTE_FLOW_ERROR_TYPE_ITEM, item, 2576 "VXLAN-GPE tunnel must be fully" 2577 " defined"); 2578 return 0; 2579 } 2580 /** 2581 * Validate GRE Key item. 2582 * 2583 * @param[in] item 2584 * Item specification. 2585 * @param[in] item_flags 2586 * Bit flags to mark detected items. 2587 * @param[in] gre_item 2588 * Pointer to gre_item 2589 * @param[out] error 2590 * Pointer to error structure. 2591 * 2592 * @return 2593 * 0 on success, a negative errno value otherwise and rte_errno is set. 2594 */ 2595 int 2596 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2597 uint64_t item_flags, 2598 const struct rte_flow_item *gre_item, 2599 struct rte_flow_error *error) 2600 { 2601 const rte_be32_t *mask = item->mask; 2602 int ret = 0; 2603 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2604 const struct rte_flow_item_gre *gre_spec; 2605 const struct rte_flow_item_gre *gre_mask; 2606 2607 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2608 return rte_flow_error_set(error, ENOTSUP, 2609 RTE_FLOW_ERROR_TYPE_ITEM, item, 2610 "Multiple GRE key not support"); 2611 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2612 return rte_flow_error_set(error, ENOTSUP, 2613 RTE_FLOW_ERROR_TYPE_ITEM, item, 2614 "No preceding GRE header"); 2615 if (item_flags & MLX5_FLOW_LAYER_INNER) 2616 return rte_flow_error_set(error, ENOTSUP, 2617 RTE_FLOW_ERROR_TYPE_ITEM, item, 2618 "GRE key following a wrong item"); 2619 gre_mask = gre_item->mask; 2620 if (!gre_mask) 2621 gre_mask = &rte_flow_item_gre_mask; 2622 gre_spec = gre_item->spec; 2623 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2624 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2625 return rte_flow_error_set(error, EINVAL, 2626 RTE_FLOW_ERROR_TYPE_ITEM, item, 2627 "Key bit must be on"); 2628 2629 if (!mask) 2630 mask = &gre_key_default_mask; 2631 ret = mlx5_flow_item_acceptable 2632 (item, (const uint8_t *)mask, 2633 (const uint8_t *)&gre_key_default_mask, 2634 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2635 return ret; 2636 } 2637 2638 /** 2639 * Validate GRE item. 2640 * 2641 * @param[in] item 2642 * Item specification. 2643 * @param[in] item_flags 2644 * Bit flags to mark detected items. 2645 * @param[in] target_protocol 2646 * The next protocol in the previous item. 2647 * @param[out] error 2648 * Pointer to error structure. 2649 * 2650 * @return 2651 * 0 on success, a negative errno value otherwise and rte_errno is set. 2652 */ 2653 int 2654 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2655 uint64_t item_flags, 2656 uint8_t target_protocol, 2657 struct rte_flow_error *error) 2658 { 2659 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2660 const struct rte_flow_item_gre *mask = item->mask; 2661 int ret; 2662 const struct rte_flow_item_gre nic_mask = { 2663 .c_rsvd0_ver = RTE_BE16(0xB000), 2664 .protocol = RTE_BE16(UINT16_MAX), 2665 }; 2666 2667 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2668 return rte_flow_error_set(error, EINVAL, 2669 RTE_FLOW_ERROR_TYPE_ITEM, item, 2670 "protocol filtering not compatible" 2671 " with this GRE layer"); 2672 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2673 return rte_flow_error_set(error, ENOTSUP, 2674 RTE_FLOW_ERROR_TYPE_ITEM, item, 2675 "multiple tunnel layers not" 2676 " supported"); 2677 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2678 return rte_flow_error_set(error, ENOTSUP, 2679 RTE_FLOW_ERROR_TYPE_ITEM, item, 2680 "L3 Layer is missing"); 2681 if (!mask) 2682 mask = &rte_flow_item_gre_mask; 2683 ret = mlx5_flow_item_acceptable 2684 (item, (const uint8_t *)mask, 2685 (const uint8_t *)&nic_mask, 2686 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2687 error); 2688 if (ret < 0) 2689 return ret; 2690 #ifndef HAVE_MLX5DV_DR 2691 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2692 if (spec && (spec->protocol & mask->protocol)) 2693 return rte_flow_error_set(error, ENOTSUP, 2694 RTE_FLOW_ERROR_TYPE_ITEM, item, 2695 "without MPLS support the" 2696 " specification cannot be used for" 2697 " filtering"); 2698 #endif 2699 #endif 2700 return 0; 2701 } 2702 2703 /** 2704 * Validate Geneve item. 2705 * 2706 * @param[in] item 2707 * Item specification. 2708 * @param[in] itemFlags 2709 * Bit-fields that holds the items detected until now. 2710 * @param[in] enPriv 2711 * Pointer to the private data structure. 2712 * @param[out] error 2713 * Pointer to error structure. 2714 * 2715 * @return 2716 * 0 on success, a negative errno value otherwise and rte_errno is set. 2717 */ 2718 2719 int 2720 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2721 uint64_t item_flags, 2722 struct rte_eth_dev *dev, 2723 struct rte_flow_error *error) 2724 { 2725 struct mlx5_priv *priv = dev->data->dev_private; 2726 const struct rte_flow_item_geneve *spec = item->spec; 2727 const struct rte_flow_item_geneve *mask = item->mask; 2728 int ret; 2729 uint16_t gbhdr; 2730 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2731 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2732 const struct rte_flow_item_geneve nic_mask = { 2733 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2734 .vni = "\xff\xff\xff", 2735 .protocol = RTE_BE16(UINT16_MAX), 2736 }; 2737 2738 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2739 return rte_flow_error_set(error, ENOTSUP, 2740 RTE_FLOW_ERROR_TYPE_ITEM, item, 2741 "L3 Geneve is not enabled by device" 2742 " parameter and/or not configured in" 2743 " firmware"); 2744 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2745 return rte_flow_error_set(error, ENOTSUP, 2746 RTE_FLOW_ERROR_TYPE_ITEM, item, 2747 "multiple tunnel layers not" 2748 " supported"); 2749 /* 2750 * Verify only UDPv4 is present as defined in 2751 * https://tools.ietf.org/html/rfc7348 2752 */ 2753 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2754 return rte_flow_error_set(error, EINVAL, 2755 RTE_FLOW_ERROR_TYPE_ITEM, item, 2756 "no outer UDP layer found"); 2757 if (!mask) 2758 mask = &rte_flow_item_geneve_mask; 2759 ret = mlx5_flow_item_acceptable 2760 (item, (const uint8_t *)mask, 2761 (const uint8_t *)&nic_mask, 2762 sizeof(struct rte_flow_item_geneve), 2763 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2764 if (ret) 2765 return ret; 2766 if (spec) { 2767 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2768 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2769 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2770 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2771 return rte_flow_error_set(error, ENOTSUP, 2772 RTE_FLOW_ERROR_TYPE_ITEM, 2773 item, 2774 "Geneve protocol unsupported" 2775 " fields are being used"); 2776 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2777 return rte_flow_error_set 2778 (error, ENOTSUP, 2779 RTE_FLOW_ERROR_TYPE_ITEM, 2780 item, 2781 "Unsupported Geneve options length"); 2782 } 2783 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2784 return rte_flow_error_set 2785 (error, ENOTSUP, 2786 RTE_FLOW_ERROR_TYPE_ITEM, item, 2787 "Geneve tunnel must be fully defined"); 2788 return 0; 2789 } 2790 2791 /** 2792 * Validate Geneve TLV option item. 2793 * 2794 * @param[in] item 2795 * Item specification. 2796 * @param[in] last_item 2797 * Previous validated item in the pattern items. 2798 * @param[in] geneve_item 2799 * Previous GENEVE item specification. 2800 * @param[in] dev 2801 * Pointer to the rte_eth_dev structure. 2802 * @param[out] error 2803 * Pointer to error structure. 2804 * 2805 * @return 2806 * 0 on success, a negative errno value otherwise and rte_errno is set. 2807 */ 2808 int 2809 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2810 uint64_t last_item, 2811 const struct rte_flow_item *geneve_item, 2812 struct rte_eth_dev *dev, 2813 struct rte_flow_error *error) 2814 { 2815 struct mlx5_priv *priv = dev->data->dev_private; 2816 struct mlx5_dev_ctx_shared *sh = priv->sh; 2817 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2818 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2819 uint8_t data_max_supported = 2820 hca_attr->max_geneve_tlv_option_data_len * 4; 2821 struct mlx5_dev_config *config = &priv->config; 2822 const struct rte_flow_item_geneve *geneve_spec; 2823 const struct rte_flow_item_geneve *geneve_mask; 2824 const struct rte_flow_item_geneve_opt *spec = item->spec; 2825 const struct rte_flow_item_geneve_opt *mask = item->mask; 2826 unsigned int i; 2827 unsigned int data_len; 2828 uint8_t tlv_option_len; 2829 uint16_t optlen_m, optlen_v; 2830 const struct rte_flow_item_geneve_opt full_mask = { 2831 .option_class = RTE_BE16(0xffff), 2832 .option_type = 0xff, 2833 .option_len = 0x1f, 2834 }; 2835 2836 if (!mask) 2837 mask = &rte_flow_item_geneve_opt_mask; 2838 if (!spec) 2839 return rte_flow_error_set 2840 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2841 "Geneve TLV opt class/type/length must be specified"); 2842 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2843 return rte_flow_error_set 2844 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2845 "Geneve TLV opt length exceeeds the limit (31)"); 2846 /* Check if class type and length masks are full. */ 2847 if (full_mask.option_class != mask->option_class || 2848 full_mask.option_type != mask->option_type || 2849 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2850 return rte_flow_error_set 2851 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2852 "Geneve TLV opt class/type/length masks must be full"); 2853 /* Check if length is supported */ 2854 if ((uint32_t)spec->option_len > 2855 config->hca_attr.max_geneve_tlv_option_data_len) 2856 return rte_flow_error_set 2857 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2858 "Geneve TLV opt length not supported"); 2859 if (config->hca_attr.max_geneve_tlv_options > 1) 2860 DRV_LOG(DEBUG, 2861 "max_geneve_tlv_options supports more than 1 option"); 2862 /* Check GENEVE item preceding. */ 2863 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2864 return rte_flow_error_set 2865 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2866 "Geneve opt item must be preceded with Geneve item"); 2867 geneve_spec = geneve_item->spec; 2868 geneve_mask = geneve_item->mask ? geneve_item->mask : 2869 &rte_flow_item_geneve_mask; 2870 /* Check if GENEVE TLV option size doesn't exceed option length */ 2871 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2872 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2873 tlv_option_len = spec->option_len & mask->option_len; 2874 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2875 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2876 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2877 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2878 if ((optlen_v & optlen_m) <= tlv_option_len) 2879 return rte_flow_error_set 2880 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2881 "GENEVE TLV option length exceeds optlen"); 2882 } 2883 /* Check if length is 0 or data is 0. */ 2884 if (spec->data == NULL || spec->option_len == 0) 2885 return rte_flow_error_set 2886 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2887 "Geneve TLV opt with zero data/length not supported"); 2888 /* Check not all data & mask are 0. */ 2889 data_len = spec->option_len * 4; 2890 if (mask->data == NULL) { 2891 for (i = 0; i < data_len; i++) 2892 if (spec->data[i]) 2893 break; 2894 if (i == data_len) 2895 return rte_flow_error_set(error, ENOTSUP, 2896 RTE_FLOW_ERROR_TYPE_ITEM, item, 2897 "Can't match on Geneve option data 0"); 2898 } else { 2899 for (i = 0; i < data_len; i++) 2900 if (spec->data[i] & mask->data[i]) 2901 break; 2902 if (i == data_len) 2903 return rte_flow_error_set(error, ENOTSUP, 2904 RTE_FLOW_ERROR_TYPE_ITEM, item, 2905 "Can't match on Geneve option data and mask 0"); 2906 /* Check data mask supported. */ 2907 for (i = data_max_supported; i < data_len ; i++) 2908 if (mask->data[i]) 2909 return rte_flow_error_set(error, ENOTSUP, 2910 RTE_FLOW_ERROR_TYPE_ITEM, item, 2911 "Data mask is of unsupported size"); 2912 } 2913 /* Check GENEVE option is supported in NIC. */ 2914 if (!config->hca_attr.geneve_tlv_opt) 2915 return rte_flow_error_set 2916 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2917 "Geneve TLV opt not supported"); 2918 /* Check if we already have geneve option with different type/class. */ 2919 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2920 geneve_opt_resource = sh->geneve_tlv_option_resource; 2921 if (geneve_opt_resource != NULL) 2922 if (geneve_opt_resource->option_class != spec->option_class || 2923 geneve_opt_resource->option_type != spec->option_type || 2924 geneve_opt_resource->length != spec->option_len) { 2925 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2926 return rte_flow_error_set(error, ENOTSUP, 2927 RTE_FLOW_ERROR_TYPE_ITEM, item, 2928 "Only one Geneve TLV option supported"); 2929 } 2930 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2931 return 0; 2932 } 2933 2934 /** 2935 * Validate MPLS item. 2936 * 2937 * @param[in] dev 2938 * Pointer to the rte_eth_dev structure. 2939 * @param[in] item 2940 * Item specification. 2941 * @param[in] item_flags 2942 * Bit-fields that holds the items detected until now. 2943 * @param[in] prev_layer 2944 * The protocol layer indicated in previous item. 2945 * @param[out] error 2946 * Pointer to error structure. 2947 * 2948 * @return 2949 * 0 on success, a negative errno value otherwise and rte_errno is set. 2950 */ 2951 int 2952 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2953 const struct rte_flow_item *item __rte_unused, 2954 uint64_t item_flags __rte_unused, 2955 uint64_t prev_layer __rte_unused, 2956 struct rte_flow_error *error) 2957 { 2958 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2959 const struct rte_flow_item_mpls *mask = item->mask; 2960 struct mlx5_priv *priv = dev->data->dev_private; 2961 int ret; 2962 2963 if (!priv->config.mpls_en) 2964 return rte_flow_error_set(error, ENOTSUP, 2965 RTE_FLOW_ERROR_TYPE_ITEM, item, 2966 "MPLS not supported or" 2967 " disabled in firmware" 2968 " configuration."); 2969 /* MPLS over UDP, GRE is allowed */ 2970 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 2971 MLX5_FLOW_LAYER_GRE | 2972 MLX5_FLOW_LAYER_GRE_KEY))) 2973 return rte_flow_error_set(error, EINVAL, 2974 RTE_FLOW_ERROR_TYPE_ITEM, item, 2975 "protocol filtering not compatible" 2976 " with MPLS layer"); 2977 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2978 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2979 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2980 return rte_flow_error_set(error, ENOTSUP, 2981 RTE_FLOW_ERROR_TYPE_ITEM, item, 2982 "multiple tunnel layers not" 2983 " supported"); 2984 if (!mask) 2985 mask = &rte_flow_item_mpls_mask; 2986 ret = mlx5_flow_item_acceptable 2987 (item, (const uint8_t *)mask, 2988 (const uint8_t *)&rte_flow_item_mpls_mask, 2989 sizeof(struct rte_flow_item_mpls), 2990 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2991 if (ret < 0) 2992 return ret; 2993 return 0; 2994 #else 2995 return rte_flow_error_set(error, ENOTSUP, 2996 RTE_FLOW_ERROR_TYPE_ITEM, item, 2997 "MPLS is not supported by Verbs, please" 2998 " update."); 2999 #endif 3000 } 3001 3002 /** 3003 * Validate NVGRE item. 3004 * 3005 * @param[in] item 3006 * Item specification. 3007 * @param[in] item_flags 3008 * Bit flags to mark detected items. 3009 * @param[in] target_protocol 3010 * The next protocol in the previous item. 3011 * @param[out] error 3012 * Pointer to error structure. 3013 * 3014 * @return 3015 * 0 on success, a negative errno value otherwise and rte_errno is set. 3016 */ 3017 int 3018 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3019 uint64_t item_flags, 3020 uint8_t target_protocol, 3021 struct rte_flow_error *error) 3022 { 3023 const struct rte_flow_item_nvgre *mask = item->mask; 3024 int ret; 3025 3026 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3027 return rte_flow_error_set(error, EINVAL, 3028 RTE_FLOW_ERROR_TYPE_ITEM, item, 3029 "protocol filtering not compatible" 3030 " with this GRE layer"); 3031 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3032 return rte_flow_error_set(error, ENOTSUP, 3033 RTE_FLOW_ERROR_TYPE_ITEM, item, 3034 "multiple tunnel layers not" 3035 " supported"); 3036 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3037 return rte_flow_error_set(error, ENOTSUP, 3038 RTE_FLOW_ERROR_TYPE_ITEM, item, 3039 "L3 Layer is missing"); 3040 if (!mask) 3041 mask = &rte_flow_item_nvgre_mask; 3042 ret = mlx5_flow_item_acceptable 3043 (item, (const uint8_t *)mask, 3044 (const uint8_t *)&rte_flow_item_nvgre_mask, 3045 sizeof(struct rte_flow_item_nvgre), 3046 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3047 if (ret < 0) 3048 return ret; 3049 return 0; 3050 } 3051 3052 /** 3053 * Validate eCPRI item. 3054 * 3055 * @param[in] item 3056 * Item specification. 3057 * @param[in] item_flags 3058 * Bit-fields that holds the items detected until now. 3059 * @param[in] last_item 3060 * Previous validated item in the pattern items. 3061 * @param[in] ether_type 3062 * Type in the ethernet layer header (including dot1q). 3063 * @param[in] acc_mask 3064 * Acceptable mask, if NULL default internal default mask 3065 * will be used to check whether item fields are supported. 3066 * @param[out] error 3067 * Pointer to error structure. 3068 * 3069 * @return 3070 * 0 on success, a negative errno value otherwise and rte_errno is set. 3071 */ 3072 int 3073 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3074 uint64_t item_flags, 3075 uint64_t last_item, 3076 uint16_t ether_type, 3077 const struct rte_flow_item_ecpri *acc_mask, 3078 struct rte_flow_error *error) 3079 { 3080 const struct rte_flow_item_ecpri *mask = item->mask; 3081 const struct rte_flow_item_ecpri nic_mask = { 3082 .hdr = { 3083 .common = { 3084 .u32 = 3085 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3086 .type = 0xFF, 3087 }).u32), 3088 }, 3089 .dummy[0] = 0xFFFFFFFF, 3090 }, 3091 }; 3092 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3093 MLX5_FLOW_LAYER_OUTER_VLAN); 3094 struct rte_flow_item_ecpri mask_lo; 3095 3096 if (!(last_item & outer_l2_vlan) && 3097 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3098 return rte_flow_error_set(error, EINVAL, 3099 RTE_FLOW_ERROR_TYPE_ITEM, item, 3100 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3101 if ((last_item & outer_l2_vlan) && ether_type && 3102 ether_type != RTE_ETHER_TYPE_ECPRI) 3103 return rte_flow_error_set(error, EINVAL, 3104 RTE_FLOW_ERROR_TYPE_ITEM, item, 3105 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3106 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3107 return rte_flow_error_set(error, EINVAL, 3108 RTE_FLOW_ERROR_TYPE_ITEM, item, 3109 "eCPRI with tunnel is not supported right now"); 3110 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3111 return rte_flow_error_set(error, ENOTSUP, 3112 RTE_FLOW_ERROR_TYPE_ITEM, item, 3113 "multiple L3 layers not supported"); 3114 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3115 return rte_flow_error_set(error, EINVAL, 3116 RTE_FLOW_ERROR_TYPE_ITEM, item, 3117 "eCPRI cannot coexist with a TCP layer"); 3118 /* In specification, eCPRI could be over UDP layer. */ 3119 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3120 return rte_flow_error_set(error, EINVAL, 3121 RTE_FLOW_ERROR_TYPE_ITEM, item, 3122 "eCPRI over UDP layer is not yet supported right now"); 3123 /* Mask for type field in common header could be zero. */ 3124 if (!mask) 3125 mask = &rte_flow_item_ecpri_mask; 3126 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3127 /* Input mask is in big-endian format. */ 3128 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3129 return rte_flow_error_set(error, EINVAL, 3130 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3131 "partial mask is not supported for protocol"); 3132 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3133 return rte_flow_error_set(error, EINVAL, 3134 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3135 "message header mask must be after a type mask"); 3136 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3137 acc_mask ? (const uint8_t *)acc_mask 3138 : (const uint8_t *)&nic_mask, 3139 sizeof(struct rte_flow_item_ecpri), 3140 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3141 } 3142 3143 static int 3144 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3145 const struct rte_flow_attr *attr __rte_unused, 3146 const struct rte_flow_item items[] __rte_unused, 3147 const struct rte_flow_action actions[] __rte_unused, 3148 bool external __rte_unused, 3149 int hairpin __rte_unused, 3150 struct rte_flow_error *error) 3151 { 3152 return rte_flow_error_set(error, ENOTSUP, 3153 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3154 } 3155 3156 static struct mlx5_flow * 3157 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3158 const struct rte_flow_attr *attr __rte_unused, 3159 const struct rte_flow_item items[] __rte_unused, 3160 const struct rte_flow_action actions[] __rte_unused, 3161 struct rte_flow_error *error) 3162 { 3163 rte_flow_error_set(error, ENOTSUP, 3164 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3165 return NULL; 3166 } 3167 3168 static int 3169 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3170 struct mlx5_flow *dev_flow __rte_unused, 3171 const struct rte_flow_attr *attr __rte_unused, 3172 const struct rte_flow_item items[] __rte_unused, 3173 const struct rte_flow_action actions[] __rte_unused, 3174 struct rte_flow_error *error) 3175 { 3176 return rte_flow_error_set(error, ENOTSUP, 3177 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3178 } 3179 3180 static int 3181 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3182 struct rte_flow *flow __rte_unused, 3183 struct rte_flow_error *error) 3184 { 3185 return rte_flow_error_set(error, ENOTSUP, 3186 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3187 } 3188 3189 static void 3190 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3191 struct rte_flow *flow __rte_unused) 3192 { 3193 } 3194 3195 static void 3196 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3197 struct rte_flow *flow __rte_unused) 3198 { 3199 } 3200 3201 static int 3202 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3203 struct rte_flow *flow __rte_unused, 3204 const struct rte_flow_action *actions __rte_unused, 3205 void *data __rte_unused, 3206 struct rte_flow_error *error) 3207 { 3208 return rte_flow_error_set(error, ENOTSUP, 3209 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3210 } 3211 3212 static int 3213 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3214 uint32_t domains __rte_unused, 3215 uint32_t flags __rte_unused) 3216 { 3217 return 0; 3218 } 3219 3220 /* Void driver to protect from null pointer reference. */ 3221 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3222 .validate = flow_null_validate, 3223 .prepare = flow_null_prepare, 3224 .translate = flow_null_translate, 3225 .apply = flow_null_apply, 3226 .remove = flow_null_remove, 3227 .destroy = flow_null_destroy, 3228 .query = flow_null_query, 3229 .sync_domain = flow_null_sync_domain, 3230 }; 3231 3232 /** 3233 * Select flow driver type according to flow attributes and device 3234 * configuration. 3235 * 3236 * @param[in] dev 3237 * Pointer to the dev structure. 3238 * @param[in] attr 3239 * Pointer to the flow attributes. 3240 * 3241 * @return 3242 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3243 */ 3244 static enum mlx5_flow_drv_type 3245 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3246 { 3247 struct mlx5_priv *priv = dev->data->dev_private; 3248 /* The OS can determine first a specific flow type (DV, VERBS) */ 3249 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3250 3251 if (type != MLX5_FLOW_TYPE_MAX) 3252 return type; 3253 /* If no OS specific type - continue with DV/VERBS selection */ 3254 if (attr->transfer && priv->config.dv_esw_en) 3255 type = MLX5_FLOW_TYPE_DV; 3256 if (!attr->transfer) 3257 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3258 MLX5_FLOW_TYPE_VERBS; 3259 return type; 3260 } 3261 3262 #define flow_get_drv_ops(type) flow_drv_ops[type] 3263 3264 /** 3265 * Flow driver validation API. This abstracts calling driver specific functions. 3266 * The type of flow driver is determined according to flow attributes. 3267 * 3268 * @param[in] dev 3269 * Pointer to the dev structure. 3270 * @param[in] attr 3271 * Pointer to the flow attributes. 3272 * @param[in] items 3273 * Pointer to the list of items. 3274 * @param[in] actions 3275 * Pointer to the list of actions. 3276 * @param[in] external 3277 * This flow rule is created by request external to PMD. 3278 * @param[in] hairpin 3279 * Number of hairpin TX actions, 0 means classic flow. 3280 * @param[out] error 3281 * Pointer to the error structure. 3282 * 3283 * @return 3284 * 0 on success, a negative errno value otherwise and rte_errno is set. 3285 */ 3286 static inline int 3287 flow_drv_validate(struct rte_eth_dev *dev, 3288 const struct rte_flow_attr *attr, 3289 const struct rte_flow_item items[], 3290 const struct rte_flow_action actions[], 3291 bool external, int hairpin, struct rte_flow_error *error) 3292 { 3293 const struct mlx5_flow_driver_ops *fops; 3294 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3295 3296 fops = flow_get_drv_ops(type); 3297 return fops->validate(dev, attr, items, actions, external, 3298 hairpin, error); 3299 } 3300 3301 /** 3302 * Flow driver preparation API. This abstracts calling driver specific 3303 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3304 * calculates the size of memory required for device flow, allocates the memory, 3305 * initializes the device flow and returns the pointer. 3306 * 3307 * @note 3308 * This function initializes device flow structure such as dv or verbs in 3309 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3310 * rest. For example, adding returning device flow to flow->dev_flow list and 3311 * setting backward reference to the flow should be done out of this function. 3312 * layers field is not filled either. 3313 * 3314 * @param[in] dev 3315 * Pointer to the dev structure. 3316 * @param[in] attr 3317 * Pointer to the flow attributes. 3318 * @param[in] items 3319 * Pointer to the list of items. 3320 * @param[in] actions 3321 * Pointer to the list of actions. 3322 * @param[in] flow_idx 3323 * This memory pool index to the flow. 3324 * @param[out] error 3325 * Pointer to the error structure. 3326 * 3327 * @return 3328 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3329 */ 3330 static inline struct mlx5_flow * 3331 flow_drv_prepare(struct rte_eth_dev *dev, 3332 const struct rte_flow *flow, 3333 const struct rte_flow_attr *attr, 3334 const struct rte_flow_item items[], 3335 const struct rte_flow_action actions[], 3336 uint32_t flow_idx, 3337 struct rte_flow_error *error) 3338 { 3339 const struct mlx5_flow_driver_ops *fops; 3340 enum mlx5_flow_drv_type type = flow->drv_type; 3341 struct mlx5_flow *mlx5_flow = NULL; 3342 3343 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3344 fops = flow_get_drv_ops(type); 3345 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3346 if (mlx5_flow) 3347 mlx5_flow->flow_idx = flow_idx; 3348 return mlx5_flow; 3349 } 3350 3351 /** 3352 * Flow driver translation API. This abstracts calling driver specific 3353 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3354 * translates a generic flow into a driver flow. flow_drv_prepare() must 3355 * precede. 3356 * 3357 * @note 3358 * dev_flow->layers could be filled as a result of parsing during translation 3359 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3360 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3361 * flow->actions could be overwritten even though all the expanded dev_flows 3362 * have the same actions. 3363 * 3364 * @param[in] dev 3365 * Pointer to the rte dev structure. 3366 * @param[in, out] dev_flow 3367 * Pointer to the mlx5 flow. 3368 * @param[in] attr 3369 * Pointer to the flow attributes. 3370 * @param[in] items 3371 * Pointer to the list of items. 3372 * @param[in] actions 3373 * Pointer to the list of actions. 3374 * @param[out] error 3375 * Pointer to the error structure. 3376 * 3377 * @return 3378 * 0 on success, a negative errno value otherwise and rte_errno is set. 3379 */ 3380 static inline int 3381 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3382 const struct rte_flow_attr *attr, 3383 const struct rte_flow_item items[], 3384 const struct rte_flow_action actions[], 3385 struct rte_flow_error *error) 3386 { 3387 const struct mlx5_flow_driver_ops *fops; 3388 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3389 3390 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3391 fops = flow_get_drv_ops(type); 3392 return fops->translate(dev, dev_flow, attr, items, actions, error); 3393 } 3394 3395 /** 3396 * Flow driver apply API. This abstracts calling driver specific functions. 3397 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3398 * translated driver flows on to device. flow_drv_translate() must precede. 3399 * 3400 * @param[in] dev 3401 * Pointer to Ethernet device structure. 3402 * @param[in, out] flow 3403 * Pointer to flow structure. 3404 * @param[out] error 3405 * Pointer to error structure. 3406 * 3407 * @return 3408 * 0 on success, a negative errno value otherwise and rte_errno is set. 3409 */ 3410 static inline int 3411 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3412 struct rte_flow_error *error) 3413 { 3414 const struct mlx5_flow_driver_ops *fops; 3415 enum mlx5_flow_drv_type type = flow->drv_type; 3416 3417 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3418 fops = flow_get_drv_ops(type); 3419 return fops->apply(dev, flow, error); 3420 } 3421 3422 /** 3423 * Flow driver destroy API. This abstracts calling driver specific functions. 3424 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3425 * on device and releases resources of the flow. 3426 * 3427 * @param[in] dev 3428 * Pointer to Ethernet device. 3429 * @param[in, out] flow 3430 * Pointer to flow structure. 3431 */ 3432 static inline void 3433 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3434 { 3435 const struct mlx5_flow_driver_ops *fops; 3436 enum mlx5_flow_drv_type type = flow->drv_type; 3437 3438 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3439 fops = flow_get_drv_ops(type); 3440 fops->destroy(dev, flow); 3441 } 3442 3443 /** 3444 * Flow driver find RSS policy tbl API. This abstracts calling driver 3445 * specific functions. Parent flow (rte_flow) should have driver 3446 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3447 * 3448 * @param[in] dev 3449 * Pointer to Ethernet device. 3450 * @param[in, out] flow 3451 * Pointer to flow structure. 3452 * @param[in] policy 3453 * Pointer to meter policy table. 3454 * @param[in] rss_desc 3455 * Pointer to rss_desc 3456 */ 3457 static struct mlx5_flow_meter_sub_policy * 3458 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3459 struct rte_flow *flow, 3460 struct mlx5_flow_meter_policy *policy, 3461 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3462 { 3463 const struct mlx5_flow_driver_ops *fops; 3464 enum mlx5_flow_drv_type type = flow->drv_type; 3465 3466 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3467 fops = flow_get_drv_ops(type); 3468 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3469 } 3470 3471 /** 3472 * Flow driver color tag rule API. This abstracts calling driver 3473 * specific functions. Parent flow (rte_flow) should have driver 3474 * type (drv_type). It will create the color tag rules in hierarchy meter. 3475 * 3476 * @param[in] dev 3477 * Pointer to Ethernet device. 3478 * @param[in, out] flow 3479 * Pointer to flow structure. 3480 * @param[in] fm 3481 * Pointer to flow meter structure. 3482 * @param[in] src_port 3483 * The src port this extra rule should use. 3484 * @param[in] item 3485 * The src port id match item. 3486 * @param[out] error 3487 * Pointer to error structure. 3488 */ 3489 static int 3490 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3491 struct rte_flow *flow, 3492 struct mlx5_flow_meter_info *fm, 3493 int32_t src_port, 3494 const struct rte_flow_item *item, 3495 struct rte_flow_error *error) 3496 { 3497 const struct mlx5_flow_driver_ops *fops; 3498 enum mlx5_flow_drv_type type = flow->drv_type; 3499 3500 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3501 fops = flow_get_drv_ops(type); 3502 return fops->meter_hierarchy_rule_create(dev, fm, 3503 src_port, item, error); 3504 } 3505 3506 /** 3507 * Get RSS action from the action list. 3508 * 3509 * @param[in] dev 3510 * Pointer to Ethernet device. 3511 * @param[in] actions 3512 * Pointer to the list of actions. 3513 * @param[in] flow 3514 * Parent flow structure pointer. 3515 * 3516 * @return 3517 * Pointer to the RSS action if exist, else return NULL. 3518 */ 3519 static const struct rte_flow_action_rss* 3520 flow_get_rss_action(struct rte_eth_dev *dev, 3521 const struct rte_flow_action actions[]) 3522 { 3523 struct mlx5_priv *priv = dev->data->dev_private; 3524 const struct rte_flow_action_rss *rss = NULL; 3525 3526 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3527 switch (actions->type) { 3528 case RTE_FLOW_ACTION_TYPE_RSS: 3529 rss = actions->conf; 3530 break; 3531 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3532 { 3533 const struct rte_flow_action_sample *sample = 3534 actions->conf; 3535 const struct rte_flow_action *act = sample->actions; 3536 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3537 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3538 rss = act->conf; 3539 break; 3540 } 3541 case RTE_FLOW_ACTION_TYPE_METER: 3542 { 3543 uint32_t mtr_idx; 3544 struct mlx5_flow_meter_info *fm; 3545 struct mlx5_flow_meter_policy *policy; 3546 const struct rte_flow_action_meter *mtr = actions->conf; 3547 3548 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3549 if (fm && !fm->def_policy) { 3550 policy = mlx5_flow_meter_policy_find(dev, 3551 fm->policy_id, NULL); 3552 MLX5_ASSERT(policy); 3553 if (policy->is_hierarchy) { 3554 policy = 3555 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3556 policy); 3557 if (!policy) 3558 return NULL; 3559 } 3560 if (policy->is_rss) 3561 rss = 3562 policy->act_cnt[RTE_COLOR_GREEN].rss->conf; 3563 } 3564 break; 3565 } 3566 default: 3567 break; 3568 } 3569 } 3570 return rss; 3571 } 3572 3573 /** 3574 * Get ASO age action by index. 3575 * 3576 * @param[in] dev 3577 * Pointer to the Ethernet device structure. 3578 * @param[in] age_idx 3579 * Index to the ASO age action. 3580 * 3581 * @return 3582 * The specified ASO age action. 3583 */ 3584 struct mlx5_aso_age_action* 3585 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3586 { 3587 uint16_t pool_idx = age_idx & UINT16_MAX; 3588 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3589 struct mlx5_priv *priv = dev->data->dev_private; 3590 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3591 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3592 3593 return &pool->actions[offset - 1]; 3594 } 3595 3596 /* maps indirect action to translated direct in some actions array */ 3597 struct mlx5_translated_action_handle { 3598 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3599 int index; /**< Index in related array of rte_flow_action. */ 3600 }; 3601 3602 /** 3603 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3604 * direct action if translation possible. 3605 * This functionality used to run same execution path for both direct and 3606 * indirect actions on flow create. All necessary preparations for indirect 3607 * action handling should be performed on *handle* actions list returned 3608 * from this call. 3609 * 3610 * @param[in] dev 3611 * Pointer to Ethernet device. 3612 * @param[in] actions 3613 * List of actions to translate. 3614 * @param[out] handle 3615 * List to store translated indirect action object handles. 3616 * @param[in, out] indir_n 3617 * Size of *handle* array. On return should be updated with number of 3618 * indirect actions retrieved from the *actions* list. 3619 * @param[out] translated_actions 3620 * List of actions where all indirect actions were translated to direct 3621 * if possible. NULL if no translation took place. 3622 * @param[out] error 3623 * Pointer to the error structure. 3624 * 3625 * @return 3626 * 0 on success, a negative errno value otherwise and rte_errno is set. 3627 */ 3628 static int 3629 flow_action_handles_translate(struct rte_eth_dev *dev, 3630 const struct rte_flow_action actions[], 3631 struct mlx5_translated_action_handle *handle, 3632 int *indir_n, 3633 struct rte_flow_action **translated_actions, 3634 struct rte_flow_error *error) 3635 { 3636 struct mlx5_priv *priv = dev->data->dev_private; 3637 struct rte_flow_action *translated = NULL; 3638 size_t actions_size; 3639 int n; 3640 int copied_n = 0; 3641 struct mlx5_translated_action_handle *handle_end = NULL; 3642 3643 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3644 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3645 continue; 3646 if (copied_n == *indir_n) { 3647 return rte_flow_error_set 3648 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3649 NULL, "too many shared actions"); 3650 } 3651 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3652 sizeof(actions[n].conf)); 3653 handle[copied_n].index = n; 3654 copied_n++; 3655 } 3656 n++; 3657 *indir_n = copied_n; 3658 if (!copied_n) 3659 return 0; 3660 actions_size = sizeof(struct rte_flow_action) * n; 3661 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3662 if (!translated) { 3663 rte_errno = ENOMEM; 3664 return -ENOMEM; 3665 } 3666 memcpy(translated, actions, actions_size); 3667 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3668 struct mlx5_shared_action_rss *shared_rss; 3669 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3670 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3671 uint32_t idx = act_idx & 3672 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3673 3674 switch (type) { 3675 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3676 shared_rss = mlx5_ipool_get 3677 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3678 translated[handle->index].type = 3679 RTE_FLOW_ACTION_TYPE_RSS; 3680 translated[handle->index].conf = 3681 &shared_rss->origin; 3682 break; 3683 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3684 translated[handle->index].type = 3685 (enum rte_flow_action_type) 3686 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3687 translated[handle->index].conf = (void *)(uintptr_t)idx; 3688 break; 3689 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3690 if (priv->sh->flow_hit_aso_en) { 3691 translated[handle->index].type = 3692 (enum rte_flow_action_type) 3693 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3694 translated[handle->index].conf = 3695 (void *)(uintptr_t)idx; 3696 break; 3697 } 3698 /* Fall-through */ 3699 case MLX5_INDIRECT_ACTION_TYPE_CT: 3700 if (priv->sh->ct_aso_en) { 3701 translated[handle->index].type = 3702 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3703 translated[handle->index].conf = 3704 (void *)(uintptr_t)idx; 3705 break; 3706 } 3707 /* Fall-through */ 3708 default: 3709 mlx5_free(translated); 3710 return rte_flow_error_set 3711 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3712 NULL, "invalid indirect action type"); 3713 } 3714 } 3715 *translated_actions = translated; 3716 return 0; 3717 } 3718 3719 /** 3720 * Get Shared RSS action from the action list. 3721 * 3722 * @param[in] dev 3723 * Pointer to Ethernet device. 3724 * @param[in] shared 3725 * Pointer to the list of actions. 3726 * @param[in] shared_n 3727 * Actions list length. 3728 * 3729 * @return 3730 * The MLX5 RSS action ID if exists, otherwise return 0. 3731 */ 3732 static uint32_t 3733 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3734 struct mlx5_translated_action_handle *handle, 3735 int shared_n) 3736 { 3737 struct mlx5_translated_action_handle *handle_end; 3738 struct mlx5_priv *priv = dev->data->dev_private; 3739 struct mlx5_shared_action_rss *shared_rss; 3740 3741 3742 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3743 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3744 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3745 uint32_t idx = act_idx & 3746 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3747 switch (type) { 3748 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3749 shared_rss = mlx5_ipool_get 3750 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3751 idx); 3752 __atomic_add_fetch(&shared_rss->refcnt, 1, 3753 __ATOMIC_RELAXED); 3754 return idx; 3755 default: 3756 break; 3757 } 3758 } 3759 return 0; 3760 } 3761 3762 static unsigned int 3763 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3764 { 3765 const struct rte_flow_item *item; 3766 unsigned int has_vlan = 0; 3767 3768 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3769 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3770 has_vlan = 1; 3771 break; 3772 } 3773 } 3774 if (has_vlan) 3775 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3776 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3777 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3778 MLX5_EXPANSION_ROOT_OUTER; 3779 } 3780 3781 /** 3782 * Get layer flags from the prefix flow. 3783 * 3784 * Some flows may be split to several subflows, the prefix subflow gets the 3785 * match items and the suffix sub flow gets the actions. 3786 * Some actions need the user defined match item flags to get the detail for 3787 * the action. 3788 * This function helps the suffix flow to get the item layer flags from prefix 3789 * subflow. 3790 * 3791 * @param[in] dev_flow 3792 * Pointer the created preifx subflow. 3793 * 3794 * @return 3795 * The layers get from prefix subflow. 3796 */ 3797 static inline uint64_t 3798 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3799 { 3800 uint64_t layers = 0; 3801 3802 /* 3803 * Layers bits could be localization, but usually the compiler will 3804 * help to do the optimization work for source code. 3805 * If no decap actions, use the layers directly. 3806 */ 3807 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3808 return dev_flow->handle->layers; 3809 /* Convert L3 layers with decap action. */ 3810 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3811 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3812 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3813 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3814 /* Convert L4 layers with decap action. */ 3815 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3816 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3817 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3818 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3819 return layers; 3820 } 3821 3822 /** 3823 * Get metadata split action information. 3824 * 3825 * @param[in] actions 3826 * Pointer to the list of actions. 3827 * @param[out] qrss 3828 * Pointer to the return pointer. 3829 * @param[out] qrss_type 3830 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3831 * if no QUEUE/RSS is found. 3832 * @param[out] encap_idx 3833 * Pointer to the index of the encap action if exists, otherwise the last 3834 * action index. 3835 * 3836 * @return 3837 * Total number of actions. 3838 */ 3839 static int 3840 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3841 const struct rte_flow_action **qrss, 3842 int *encap_idx) 3843 { 3844 const struct rte_flow_action_raw_encap *raw_encap; 3845 int actions_n = 0; 3846 int raw_decap_idx = -1; 3847 3848 *encap_idx = -1; 3849 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3850 switch (actions->type) { 3851 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3852 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3853 *encap_idx = actions_n; 3854 break; 3855 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3856 raw_decap_idx = actions_n; 3857 break; 3858 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3859 raw_encap = actions->conf; 3860 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3861 *encap_idx = raw_decap_idx != -1 ? 3862 raw_decap_idx : actions_n; 3863 break; 3864 case RTE_FLOW_ACTION_TYPE_QUEUE: 3865 case RTE_FLOW_ACTION_TYPE_RSS: 3866 *qrss = actions; 3867 break; 3868 default: 3869 break; 3870 } 3871 actions_n++; 3872 } 3873 if (*encap_idx == -1) 3874 *encap_idx = actions_n; 3875 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3876 return actions_n + 1; 3877 } 3878 3879 /** 3880 * Check if the action will change packet. 3881 * 3882 * @param dev 3883 * Pointer to Ethernet device. 3884 * @param[in] type 3885 * action type. 3886 * 3887 * @return 3888 * true if action will change packet, false otherwise. 3889 */ 3890 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 3891 enum rte_flow_action_type type) 3892 { 3893 struct mlx5_priv *priv = dev->data->dev_private; 3894 3895 switch (type) { 3896 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3897 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3898 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3899 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3900 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3901 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3902 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3903 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3904 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3905 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3906 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3907 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3908 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3909 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3910 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 3911 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 3912 case RTE_FLOW_ACTION_TYPE_SET_META: 3913 case RTE_FLOW_ACTION_TYPE_SET_TAG: 3914 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 3915 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3916 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3917 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3918 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3919 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3920 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3921 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3922 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3923 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3924 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 3925 return true; 3926 case RTE_FLOW_ACTION_TYPE_FLAG: 3927 case RTE_FLOW_ACTION_TYPE_MARK: 3928 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 3929 return true; 3930 else 3931 return false; 3932 default: 3933 return false; 3934 } 3935 } 3936 3937 /** 3938 * Check meter action from the action list. 3939 * 3940 * @param dev 3941 * Pointer to Ethernet device. 3942 * @param[in] actions 3943 * Pointer to the list of actions. 3944 * @param[out] has_mtr 3945 * Pointer to the meter exist flag. 3946 * @param[out] has_modify 3947 * Pointer to the flag showing there's packet change action. 3948 * @param[out] meter_id 3949 * Pointer to the meter id. 3950 * 3951 * @return 3952 * Total number of actions. 3953 */ 3954 static int 3955 flow_check_meter_action(struct rte_eth_dev *dev, 3956 const struct rte_flow_action actions[], 3957 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 3958 { 3959 const struct rte_flow_action_meter *mtr = NULL; 3960 int actions_n = 0; 3961 3962 MLX5_ASSERT(has_mtr); 3963 *has_mtr = false; 3964 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3965 switch (actions->type) { 3966 case RTE_FLOW_ACTION_TYPE_METER: 3967 mtr = actions->conf; 3968 *meter_id = mtr->mtr_id; 3969 *has_mtr = true; 3970 break; 3971 default: 3972 break; 3973 } 3974 if (!*has_mtr) 3975 *has_modify |= flow_check_modify_action_type(dev, 3976 actions->type); 3977 actions_n++; 3978 } 3979 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3980 return actions_n + 1; 3981 } 3982 3983 /** 3984 * Check if the flow should be split due to hairpin. 3985 * The reason for the split is that in current HW we can't 3986 * support encap and push-vlan on Rx, so if a flow contains 3987 * these actions we move it to Tx. 3988 * 3989 * @param dev 3990 * Pointer to Ethernet device. 3991 * @param[in] attr 3992 * Flow rule attributes. 3993 * @param[in] actions 3994 * Associated actions (list terminated by the END action). 3995 * 3996 * @return 3997 * > 0 the number of actions and the flow should be split, 3998 * 0 when no split required. 3999 */ 4000 static int 4001 flow_check_hairpin_split(struct rte_eth_dev *dev, 4002 const struct rte_flow_attr *attr, 4003 const struct rte_flow_action actions[]) 4004 { 4005 int queue_action = 0; 4006 int action_n = 0; 4007 int split = 0; 4008 const struct rte_flow_action_queue *queue; 4009 const struct rte_flow_action_rss *rss; 4010 const struct rte_flow_action_raw_encap *raw_encap; 4011 const struct rte_eth_hairpin_conf *conf; 4012 4013 if (!attr->ingress) 4014 return 0; 4015 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4016 switch (actions->type) { 4017 case RTE_FLOW_ACTION_TYPE_QUEUE: 4018 queue = actions->conf; 4019 if (queue == NULL) 4020 return 0; 4021 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4022 if (conf == NULL || conf->tx_explicit != 0) 4023 return 0; 4024 queue_action = 1; 4025 action_n++; 4026 break; 4027 case RTE_FLOW_ACTION_TYPE_RSS: 4028 rss = actions->conf; 4029 if (rss == NULL || rss->queue_num == 0) 4030 return 0; 4031 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4032 if (conf == NULL || conf->tx_explicit != 0) 4033 return 0; 4034 queue_action = 1; 4035 action_n++; 4036 break; 4037 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4038 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4039 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4040 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4041 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4042 split++; 4043 action_n++; 4044 break; 4045 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4046 raw_encap = actions->conf; 4047 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4048 split++; 4049 action_n++; 4050 break; 4051 default: 4052 action_n++; 4053 break; 4054 } 4055 } 4056 if (split && queue_action) 4057 return action_n; 4058 return 0; 4059 } 4060 4061 /* Declare flow create/destroy prototype in advance. */ 4062 static uint32_t 4063 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4064 const struct rte_flow_attr *attr, 4065 const struct rte_flow_item items[], 4066 const struct rte_flow_action actions[], 4067 bool external, struct rte_flow_error *error); 4068 4069 static void 4070 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4071 uint32_t flow_idx); 4072 4073 int 4074 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4075 struct mlx5_list_entry *entry, void *cb_ctx) 4076 { 4077 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4078 struct mlx5_flow_mreg_copy_resource *mcp_res = 4079 container_of(entry, typeof(*mcp_res), hlist_ent); 4080 4081 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4082 } 4083 4084 struct mlx5_list_entry * 4085 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4086 { 4087 struct rte_eth_dev *dev = tool_ctx; 4088 struct mlx5_priv *priv = dev->data->dev_private; 4089 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4090 struct mlx5_flow_mreg_copy_resource *mcp_res; 4091 struct rte_flow_error *error = ctx->error; 4092 uint32_t idx = 0; 4093 int ret; 4094 uint32_t mark_id = *(uint32_t *)(ctx->data); 4095 struct rte_flow_attr attr = { 4096 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4097 .ingress = 1, 4098 }; 4099 struct mlx5_rte_flow_item_tag tag_spec = { 4100 .data = mark_id, 4101 }; 4102 struct rte_flow_item items[] = { 4103 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4104 }; 4105 struct rte_flow_action_mark ftag = { 4106 .id = mark_id, 4107 }; 4108 struct mlx5_flow_action_copy_mreg cp_mreg = { 4109 .dst = REG_B, 4110 .src = REG_NON, 4111 }; 4112 struct rte_flow_action_jump jump = { 4113 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4114 }; 4115 struct rte_flow_action actions[] = { 4116 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4117 }; 4118 4119 /* Fill the register fileds in the flow. */ 4120 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4121 if (ret < 0) 4122 return NULL; 4123 tag_spec.id = ret; 4124 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4125 if (ret < 0) 4126 return NULL; 4127 cp_mreg.src = ret; 4128 /* Provide the full width of FLAG specific value. */ 4129 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4130 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4131 /* Build a new flow. */ 4132 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4133 items[0] = (struct rte_flow_item){ 4134 .type = (enum rte_flow_item_type) 4135 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4136 .spec = &tag_spec, 4137 }; 4138 items[1] = (struct rte_flow_item){ 4139 .type = RTE_FLOW_ITEM_TYPE_END, 4140 }; 4141 actions[0] = (struct rte_flow_action){ 4142 .type = (enum rte_flow_action_type) 4143 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4144 .conf = &ftag, 4145 }; 4146 actions[1] = (struct rte_flow_action){ 4147 .type = (enum rte_flow_action_type) 4148 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4149 .conf = &cp_mreg, 4150 }; 4151 actions[2] = (struct rte_flow_action){ 4152 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4153 .conf = &jump, 4154 }; 4155 actions[3] = (struct rte_flow_action){ 4156 .type = RTE_FLOW_ACTION_TYPE_END, 4157 }; 4158 } else { 4159 /* Default rule, wildcard match. */ 4160 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4161 items[0] = (struct rte_flow_item){ 4162 .type = RTE_FLOW_ITEM_TYPE_END, 4163 }; 4164 actions[0] = (struct rte_flow_action){ 4165 .type = (enum rte_flow_action_type) 4166 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4167 .conf = &cp_mreg, 4168 }; 4169 actions[1] = (struct rte_flow_action){ 4170 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4171 .conf = &jump, 4172 }; 4173 actions[2] = (struct rte_flow_action){ 4174 .type = RTE_FLOW_ACTION_TYPE_END, 4175 }; 4176 } 4177 /* Build a new entry. */ 4178 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4179 if (!mcp_res) { 4180 rte_errno = ENOMEM; 4181 return NULL; 4182 } 4183 mcp_res->idx = idx; 4184 mcp_res->mark_id = mark_id; 4185 /* 4186 * The copy Flows are not included in any list. There 4187 * ones are referenced from other Flows and can not 4188 * be applied, removed, deleted in ardbitrary order 4189 * by list traversing. 4190 */ 4191 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4192 &attr, items, actions, false, error); 4193 if (!mcp_res->rix_flow) { 4194 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4195 return NULL; 4196 } 4197 return &mcp_res->hlist_ent; 4198 } 4199 4200 struct mlx5_list_entry * 4201 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4202 void *cb_ctx __rte_unused) 4203 { 4204 struct rte_eth_dev *dev = tool_ctx; 4205 struct mlx5_priv *priv = dev->data->dev_private; 4206 struct mlx5_flow_mreg_copy_resource *mcp_res; 4207 uint32_t idx = 0; 4208 4209 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4210 if (!mcp_res) { 4211 rte_errno = ENOMEM; 4212 return NULL; 4213 } 4214 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4215 mcp_res->idx = idx; 4216 return &mcp_res->hlist_ent; 4217 } 4218 4219 void 4220 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4221 { 4222 struct mlx5_flow_mreg_copy_resource *mcp_res = 4223 container_of(entry, typeof(*mcp_res), hlist_ent); 4224 struct rte_eth_dev *dev = tool_ctx; 4225 struct mlx5_priv *priv = dev->data->dev_private; 4226 4227 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4228 } 4229 4230 /** 4231 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4232 * 4233 * As mark_id is unique, if there's already a registered flow for the mark_id, 4234 * return by increasing the reference counter of the resource. Otherwise, create 4235 * the resource (mcp_res) and flow. 4236 * 4237 * Flow looks like, 4238 * - If ingress port is ANY and reg_c[1] is mark_id, 4239 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4240 * 4241 * For default flow (zero mark_id), flow is like, 4242 * - If ingress port is ANY, 4243 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4244 * 4245 * @param dev 4246 * Pointer to Ethernet device. 4247 * @param mark_id 4248 * ID of MARK action, zero means default flow for META. 4249 * @param[out] error 4250 * Perform verbose error reporting if not NULL. 4251 * 4252 * @return 4253 * Associated resource on success, NULL otherwise and rte_errno is set. 4254 */ 4255 static struct mlx5_flow_mreg_copy_resource * 4256 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4257 struct rte_flow_error *error) 4258 { 4259 struct mlx5_priv *priv = dev->data->dev_private; 4260 struct mlx5_list_entry *entry; 4261 struct mlx5_flow_cb_ctx ctx = { 4262 .dev = dev, 4263 .error = error, 4264 .data = &mark_id, 4265 }; 4266 4267 /* Check if already registered. */ 4268 MLX5_ASSERT(priv->mreg_cp_tbl); 4269 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4270 if (!entry) 4271 return NULL; 4272 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4273 hlist_ent); 4274 } 4275 4276 void 4277 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4278 { 4279 struct mlx5_flow_mreg_copy_resource *mcp_res = 4280 container_of(entry, typeof(*mcp_res), hlist_ent); 4281 struct rte_eth_dev *dev = tool_ctx; 4282 struct mlx5_priv *priv = dev->data->dev_private; 4283 4284 MLX5_ASSERT(mcp_res->rix_flow); 4285 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4286 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4287 } 4288 4289 /** 4290 * Release flow in RX_CP_TBL. 4291 * 4292 * @param dev 4293 * Pointer to Ethernet device. 4294 * @flow 4295 * Parent flow for wich copying is provided. 4296 */ 4297 static void 4298 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4299 struct rte_flow *flow) 4300 { 4301 struct mlx5_flow_mreg_copy_resource *mcp_res; 4302 struct mlx5_priv *priv = dev->data->dev_private; 4303 4304 if (!flow->rix_mreg_copy) 4305 return; 4306 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4307 flow->rix_mreg_copy); 4308 if (!mcp_res || !priv->mreg_cp_tbl) 4309 return; 4310 MLX5_ASSERT(mcp_res->rix_flow); 4311 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4312 flow->rix_mreg_copy = 0; 4313 } 4314 4315 /** 4316 * Remove the default copy action from RX_CP_TBL. 4317 * 4318 * This functions is called in the mlx5_dev_start(). No thread safe 4319 * is guaranteed. 4320 * 4321 * @param dev 4322 * Pointer to Ethernet device. 4323 */ 4324 static void 4325 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4326 { 4327 struct mlx5_list_entry *entry; 4328 struct mlx5_priv *priv = dev->data->dev_private; 4329 struct mlx5_flow_cb_ctx ctx; 4330 uint32_t mark_id; 4331 4332 /* Check if default flow is registered. */ 4333 if (!priv->mreg_cp_tbl) 4334 return; 4335 mark_id = MLX5_DEFAULT_COPY_ID; 4336 ctx.data = &mark_id; 4337 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4338 if (!entry) 4339 return; 4340 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4341 } 4342 4343 /** 4344 * Add the default copy action in in RX_CP_TBL. 4345 * 4346 * This functions is called in the mlx5_dev_start(). No thread safe 4347 * is guaranteed. 4348 * 4349 * @param dev 4350 * Pointer to Ethernet device. 4351 * @param[out] error 4352 * Perform verbose error reporting if not NULL. 4353 * 4354 * @return 4355 * 0 for success, negative value otherwise and rte_errno is set. 4356 */ 4357 static int 4358 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4359 struct rte_flow_error *error) 4360 { 4361 struct mlx5_priv *priv = dev->data->dev_private; 4362 struct mlx5_flow_mreg_copy_resource *mcp_res; 4363 struct mlx5_flow_cb_ctx ctx; 4364 uint32_t mark_id; 4365 4366 /* Check whether extensive metadata feature is engaged. */ 4367 if (!priv->config.dv_flow_en || 4368 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4369 !mlx5_flow_ext_mreg_supported(dev) || 4370 !priv->sh->dv_regc0_mask) 4371 return 0; 4372 /* 4373 * Add default mreg copy flow may be called multiple time, but 4374 * only be called once in stop. Avoid register it twice. 4375 */ 4376 mark_id = MLX5_DEFAULT_COPY_ID; 4377 ctx.data = &mark_id; 4378 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4379 return 0; 4380 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4381 if (!mcp_res) 4382 return -rte_errno; 4383 return 0; 4384 } 4385 4386 /** 4387 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4388 * 4389 * All the flow having Q/RSS action should be split by 4390 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4391 * performs the following, 4392 * - CQE->flow_tag := reg_c[1] (MARK) 4393 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4394 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4395 * but there should be a flow per each MARK ID set by MARK action. 4396 * 4397 * For the aforementioned reason, if there's a MARK action in flow's action 4398 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4399 * the MARK ID to CQE's flow_tag like, 4400 * - If reg_c[1] is mark_id, 4401 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4402 * 4403 * For SET_META action which stores value in reg_c[0], as the destination is 4404 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4405 * MARK ID means the default flow. The default flow looks like, 4406 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4407 * 4408 * @param dev 4409 * Pointer to Ethernet device. 4410 * @param flow 4411 * Pointer to flow structure. 4412 * @param[in] actions 4413 * Pointer to the list of actions. 4414 * @param[out] error 4415 * Perform verbose error reporting if not NULL. 4416 * 4417 * @return 4418 * 0 on success, negative value otherwise and rte_errno is set. 4419 */ 4420 static int 4421 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4422 struct rte_flow *flow, 4423 const struct rte_flow_action *actions, 4424 struct rte_flow_error *error) 4425 { 4426 struct mlx5_priv *priv = dev->data->dev_private; 4427 struct mlx5_dev_config *config = &priv->config; 4428 struct mlx5_flow_mreg_copy_resource *mcp_res; 4429 const struct rte_flow_action_mark *mark; 4430 4431 /* Check whether extensive metadata feature is engaged. */ 4432 if (!config->dv_flow_en || 4433 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4434 !mlx5_flow_ext_mreg_supported(dev) || 4435 !priv->sh->dv_regc0_mask) 4436 return 0; 4437 /* Find MARK action. */ 4438 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4439 switch (actions->type) { 4440 case RTE_FLOW_ACTION_TYPE_FLAG: 4441 mcp_res = flow_mreg_add_copy_action 4442 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4443 if (!mcp_res) 4444 return -rte_errno; 4445 flow->rix_mreg_copy = mcp_res->idx; 4446 return 0; 4447 case RTE_FLOW_ACTION_TYPE_MARK: 4448 mark = (const struct rte_flow_action_mark *) 4449 actions->conf; 4450 mcp_res = 4451 flow_mreg_add_copy_action(dev, mark->id, error); 4452 if (!mcp_res) 4453 return -rte_errno; 4454 flow->rix_mreg_copy = mcp_res->idx; 4455 return 0; 4456 default: 4457 break; 4458 } 4459 } 4460 return 0; 4461 } 4462 4463 #define MLX5_MAX_SPLIT_ACTIONS 24 4464 #define MLX5_MAX_SPLIT_ITEMS 24 4465 4466 /** 4467 * Split the hairpin flow. 4468 * Since HW can't support encap and push-vlan on Rx, we move these 4469 * actions to Tx. 4470 * If the count action is after the encap then we also 4471 * move the count action. in this case the count will also measure 4472 * the outer bytes. 4473 * 4474 * @param dev 4475 * Pointer to Ethernet device. 4476 * @param[in] actions 4477 * Associated actions (list terminated by the END action). 4478 * @param[out] actions_rx 4479 * Rx flow actions. 4480 * @param[out] actions_tx 4481 * Tx flow actions.. 4482 * @param[out] pattern_tx 4483 * The pattern items for the Tx flow. 4484 * @param[out] flow_id 4485 * The flow ID connected to this flow. 4486 * 4487 * @return 4488 * 0 on success. 4489 */ 4490 static int 4491 flow_hairpin_split(struct rte_eth_dev *dev, 4492 const struct rte_flow_action actions[], 4493 struct rte_flow_action actions_rx[], 4494 struct rte_flow_action actions_tx[], 4495 struct rte_flow_item pattern_tx[], 4496 uint32_t flow_id) 4497 { 4498 const struct rte_flow_action_raw_encap *raw_encap; 4499 const struct rte_flow_action_raw_decap *raw_decap; 4500 struct mlx5_rte_flow_action_set_tag *set_tag; 4501 struct rte_flow_action *tag_action; 4502 struct mlx5_rte_flow_item_tag *tag_item; 4503 struct rte_flow_item *item; 4504 char *addr; 4505 int encap = 0; 4506 4507 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4508 switch (actions->type) { 4509 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4510 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4511 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4512 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4513 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4514 rte_memcpy(actions_tx, actions, 4515 sizeof(struct rte_flow_action)); 4516 actions_tx++; 4517 break; 4518 case RTE_FLOW_ACTION_TYPE_COUNT: 4519 if (encap) { 4520 rte_memcpy(actions_tx, actions, 4521 sizeof(struct rte_flow_action)); 4522 actions_tx++; 4523 } else { 4524 rte_memcpy(actions_rx, actions, 4525 sizeof(struct rte_flow_action)); 4526 actions_rx++; 4527 } 4528 break; 4529 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4530 raw_encap = actions->conf; 4531 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4532 memcpy(actions_tx, actions, 4533 sizeof(struct rte_flow_action)); 4534 actions_tx++; 4535 encap = 1; 4536 } else { 4537 rte_memcpy(actions_rx, actions, 4538 sizeof(struct rte_flow_action)); 4539 actions_rx++; 4540 } 4541 break; 4542 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4543 raw_decap = actions->conf; 4544 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4545 memcpy(actions_tx, actions, 4546 sizeof(struct rte_flow_action)); 4547 actions_tx++; 4548 } else { 4549 rte_memcpy(actions_rx, actions, 4550 sizeof(struct rte_flow_action)); 4551 actions_rx++; 4552 } 4553 break; 4554 default: 4555 rte_memcpy(actions_rx, actions, 4556 sizeof(struct rte_flow_action)); 4557 actions_rx++; 4558 break; 4559 } 4560 } 4561 /* Add set meta action and end action for the Rx flow. */ 4562 tag_action = actions_rx; 4563 tag_action->type = (enum rte_flow_action_type) 4564 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4565 actions_rx++; 4566 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4567 actions_rx++; 4568 set_tag = (void *)actions_rx; 4569 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4570 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4571 .data = flow_id, 4572 }; 4573 MLX5_ASSERT(set_tag->id > REG_NON); 4574 tag_action->conf = set_tag; 4575 /* Create Tx item list. */ 4576 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4577 addr = (void *)&pattern_tx[2]; 4578 item = pattern_tx; 4579 item->type = (enum rte_flow_item_type) 4580 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4581 tag_item = (void *)addr; 4582 tag_item->data = flow_id; 4583 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4584 MLX5_ASSERT(set_tag->id > REG_NON); 4585 item->spec = tag_item; 4586 addr += sizeof(struct mlx5_rte_flow_item_tag); 4587 tag_item = (void *)addr; 4588 tag_item->data = UINT32_MAX; 4589 tag_item->id = UINT16_MAX; 4590 item->mask = tag_item; 4591 item->last = NULL; 4592 item++; 4593 item->type = RTE_FLOW_ITEM_TYPE_END; 4594 return 0; 4595 } 4596 4597 /** 4598 * The last stage of splitting chain, just creates the subflow 4599 * without any modification. 4600 * 4601 * @param[in] dev 4602 * Pointer to Ethernet device. 4603 * @param[in] flow 4604 * Parent flow structure pointer. 4605 * @param[in, out] sub_flow 4606 * Pointer to return the created subflow, may be NULL. 4607 * @param[in] attr 4608 * Flow rule attributes. 4609 * @param[in] items 4610 * Pattern specification (list terminated by the END pattern item). 4611 * @param[in] actions 4612 * Associated actions (list terminated by the END action). 4613 * @param[in] flow_split_info 4614 * Pointer to flow split info structure. 4615 * @param[out] error 4616 * Perform verbose error reporting if not NULL. 4617 * @return 4618 * 0 on success, negative value otherwise 4619 */ 4620 static int 4621 flow_create_split_inner(struct rte_eth_dev *dev, 4622 struct rte_flow *flow, 4623 struct mlx5_flow **sub_flow, 4624 const struct rte_flow_attr *attr, 4625 const struct rte_flow_item items[], 4626 const struct rte_flow_action actions[], 4627 struct mlx5_flow_split_info *flow_split_info, 4628 struct rte_flow_error *error) 4629 { 4630 struct mlx5_flow *dev_flow; 4631 4632 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4633 flow_split_info->flow_idx, error); 4634 if (!dev_flow) 4635 return -rte_errno; 4636 dev_flow->flow = flow; 4637 dev_flow->external = flow_split_info->external; 4638 dev_flow->skip_scale = flow_split_info->skip_scale; 4639 /* Subflow object was created, we must include one in the list. */ 4640 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4641 dev_flow->handle, next); 4642 /* 4643 * If dev_flow is as one of the suffix flow, some actions in suffix 4644 * flow may need some user defined item layer flags, and pass the 4645 * Metadate rxq mark flag to suffix flow as well. 4646 */ 4647 if (flow_split_info->prefix_layers) 4648 dev_flow->handle->layers = flow_split_info->prefix_layers; 4649 if (flow_split_info->prefix_mark) 4650 dev_flow->handle->mark = 1; 4651 if (sub_flow) 4652 *sub_flow = dev_flow; 4653 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4654 dev_flow->dv.table_id = flow_split_info->table_id; 4655 #endif 4656 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4657 } 4658 4659 /** 4660 * Get the sub policy of a meter. 4661 * 4662 * @param[in] dev 4663 * Pointer to Ethernet device. 4664 * @param[in] flow 4665 * Parent flow structure pointer. 4666 * @param wks 4667 * Pointer to thread flow work space. 4668 * @param[in] attr 4669 * Flow rule attributes. 4670 * @param[in] items 4671 * Pattern specification (list terminated by the END pattern item). 4672 * @param[out] error 4673 * Perform verbose error reporting if not NULL. 4674 * 4675 * @return 4676 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4677 */ 4678 static struct mlx5_flow_meter_sub_policy * 4679 get_meter_sub_policy(struct rte_eth_dev *dev, 4680 struct rte_flow *flow, 4681 struct mlx5_flow_workspace *wks, 4682 const struct rte_flow_attr *attr, 4683 const struct rte_flow_item items[], 4684 struct rte_flow_error *error) 4685 { 4686 struct mlx5_flow_meter_policy *policy; 4687 struct mlx5_flow_meter_policy *final_policy; 4688 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4689 4690 policy = wks->policy; 4691 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4692 if (final_policy->is_rss || final_policy->is_queue) { 4693 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4694 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4695 uint32_t i; 4696 4697 /* 4698 * This is a tmp dev_flow, 4699 * no need to register any matcher for it in translate. 4700 */ 4701 wks->skip_matcher_reg = 1; 4702 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4703 struct mlx5_flow dev_flow = {0}; 4704 struct mlx5_flow_handle dev_handle = { {0} }; 4705 uint8_t fate = final_policy->act_cnt[i].fate_action; 4706 4707 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4708 const void *rss_act = 4709 final_policy->act_cnt[i].rss->conf; 4710 struct rte_flow_action rss_actions[2] = { 4711 [0] = { 4712 .type = RTE_FLOW_ACTION_TYPE_RSS, 4713 .conf = rss_act, 4714 }, 4715 [1] = { 4716 .type = RTE_FLOW_ACTION_TYPE_END, 4717 .conf = NULL, 4718 } 4719 }; 4720 4721 dev_flow.handle = &dev_handle; 4722 dev_flow.ingress = attr->ingress; 4723 dev_flow.flow = flow; 4724 dev_flow.external = 0; 4725 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4726 dev_flow.dv.transfer = attr->transfer; 4727 #endif 4728 /** 4729 * Translate RSS action to get rss hash fields. 4730 */ 4731 if (flow_drv_translate(dev, &dev_flow, attr, 4732 items, rss_actions, error)) 4733 goto exit; 4734 rss_desc_v[i] = wks->rss_desc; 4735 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4736 rss_desc_v[i].hash_fields = 4737 dev_flow.hash_fields; 4738 rss_desc_v[i].queue_num = 4739 rss_desc_v[i].hash_fields ? 4740 rss_desc_v[i].queue_num : 1; 4741 rss_desc_v[i].tunnel = 4742 !!(dev_flow.handle->layers & 4743 MLX5_FLOW_LAYER_TUNNEL); 4744 rss_desc[i] = &rss_desc_v[i]; 4745 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4746 /* This is queue action. */ 4747 rss_desc_v[i] = wks->rss_desc; 4748 rss_desc_v[i].key_len = 0; 4749 rss_desc_v[i].hash_fields = 0; 4750 rss_desc_v[i].queue = 4751 &final_policy->act_cnt[i].queue; 4752 rss_desc_v[i].queue_num = 1; 4753 rss_desc[i] = &rss_desc_v[i]; 4754 } else { 4755 rss_desc[i] = NULL; 4756 } 4757 } 4758 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4759 flow, policy, rss_desc); 4760 } else { 4761 enum mlx5_meter_domain mtr_domain = 4762 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4763 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4764 MLX5_MTR_DOMAIN_INGRESS); 4765 sub_policy = policy->sub_policys[mtr_domain][0]; 4766 } 4767 if (!sub_policy) 4768 rte_flow_error_set(error, EINVAL, 4769 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4770 "Failed to get meter sub-policy."); 4771 exit: 4772 return sub_policy; 4773 } 4774 4775 /** 4776 * Split the meter flow. 4777 * 4778 * As meter flow will split to three sub flow, other than meter 4779 * action, the other actions make sense to only meter accepts 4780 * the packet. If it need to be dropped, no other additional 4781 * actions should be take. 4782 * 4783 * One kind of special action which decapsulates the L3 tunnel 4784 * header will be in the prefix sub flow, as not to take the 4785 * L3 tunnel header into account. 4786 * 4787 * @param[in] dev 4788 * Pointer to Ethernet device. 4789 * @param[in] flow 4790 * Parent flow structure pointer. 4791 * @param wks 4792 * Pointer to thread flow work space. 4793 * @param[in] attr 4794 * Flow rule attributes. 4795 * @param[in] items 4796 * Pattern specification (list terminated by the END pattern item). 4797 * @param[out] sfx_items 4798 * Suffix flow match items (list terminated by the END pattern item). 4799 * @param[in] actions 4800 * Associated actions (list terminated by the END action). 4801 * @param[out] actions_sfx 4802 * Suffix flow actions. 4803 * @param[out] actions_pre 4804 * Prefix flow actions. 4805 * @param[out] mtr_flow_id 4806 * Pointer to meter flow id. 4807 * @param[out] error 4808 * Perform verbose error reporting if not NULL. 4809 * 4810 * @return 4811 * 0 on success, a negative errno value otherwise and rte_errno is set. 4812 */ 4813 static int 4814 flow_meter_split_prep(struct rte_eth_dev *dev, 4815 struct rte_flow *flow, 4816 struct mlx5_flow_workspace *wks, 4817 const struct rte_flow_attr *attr, 4818 const struct rte_flow_item items[], 4819 struct rte_flow_item sfx_items[], 4820 const struct rte_flow_action actions[], 4821 struct rte_flow_action actions_sfx[], 4822 struct rte_flow_action actions_pre[], 4823 uint32_t *mtr_flow_id, 4824 struct rte_flow_error *error) 4825 { 4826 struct mlx5_priv *priv = dev->data->dev_private; 4827 struct mlx5_flow_meter_info *fm = wks->fm; 4828 struct rte_flow_action *tag_action = NULL; 4829 struct rte_flow_item *tag_item; 4830 struct mlx5_rte_flow_action_set_tag *set_tag; 4831 const struct rte_flow_action_raw_encap *raw_encap; 4832 const struct rte_flow_action_raw_decap *raw_decap; 4833 struct mlx5_rte_flow_item_tag *tag_item_spec; 4834 struct mlx5_rte_flow_item_tag *tag_item_mask; 4835 uint32_t tag_id = 0; 4836 struct rte_flow_item *vlan_item_dst = NULL; 4837 const struct rte_flow_item *vlan_item_src = NULL; 4838 struct rte_flow_action *hw_mtr_action; 4839 struct rte_flow_action *action_pre_head = NULL; 4840 int32_t flow_src_port = priv->representor_id; 4841 bool mtr_first; 4842 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 4843 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 4844 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 4845 uint32_t flow_id = 0; 4846 uint32_t flow_id_reversed = 0; 4847 uint8_t flow_id_bits = 0; 4848 int shift; 4849 4850 /* Prepare the suffix subflow items. */ 4851 tag_item = sfx_items++; 4852 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4853 struct mlx5_priv *port_priv; 4854 const struct rte_flow_item_port_id *pid_v; 4855 int item_type = items->type; 4856 4857 switch (item_type) { 4858 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4859 pid_v = items->spec; 4860 MLX5_ASSERT(pid_v); 4861 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 4862 if (!port_priv) 4863 return rte_flow_error_set(error, 4864 rte_errno, 4865 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 4866 pid_v, 4867 "Failed to get port info."); 4868 flow_src_port = port_priv->representor_id; 4869 if (!fm->def_policy && wks->policy->is_hierarchy && 4870 flow_src_port != priv->representor_id) { 4871 if (flow_drv_mtr_hierarchy_rule_create(dev, 4872 flow, fm, 4873 flow_src_port, 4874 items, 4875 error)) 4876 return -rte_errno; 4877 } 4878 memcpy(sfx_items, items, sizeof(*sfx_items)); 4879 sfx_items++; 4880 break; 4881 case RTE_FLOW_ITEM_TYPE_VLAN: 4882 /* Determine if copy vlan item below. */ 4883 vlan_item_src = items; 4884 vlan_item_dst = sfx_items++; 4885 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 4886 break; 4887 default: 4888 break; 4889 } 4890 } 4891 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4892 sfx_items++; 4893 mtr_first = priv->sh->meter_aso_en && 4894 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 4895 /* For ASO meter, meter must be before tag in TX direction. */ 4896 if (mtr_first) { 4897 action_pre_head = actions_pre++; 4898 /* Leave space for tag action. */ 4899 tag_action = actions_pre++; 4900 } 4901 /* Prepare the actions for prefix and suffix flow. */ 4902 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4903 struct rte_flow_action *action_cur = NULL; 4904 4905 switch (actions->type) { 4906 case RTE_FLOW_ACTION_TYPE_METER: 4907 if (mtr_first) { 4908 action_cur = action_pre_head; 4909 } else { 4910 /* Leave space for tag action. */ 4911 tag_action = actions_pre++; 4912 action_cur = actions_pre++; 4913 } 4914 break; 4915 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4916 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4917 action_cur = actions_pre++; 4918 break; 4919 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4920 raw_encap = actions->conf; 4921 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4922 action_cur = actions_pre++; 4923 break; 4924 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4925 raw_decap = actions->conf; 4926 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4927 action_cur = actions_pre++; 4928 break; 4929 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4930 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4931 if (vlan_item_dst && vlan_item_src) { 4932 memcpy(vlan_item_dst, vlan_item_src, 4933 sizeof(*vlan_item_dst)); 4934 /* 4935 * Convert to internal match item, it is used 4936 * for vlan push and set vid. 4937 */ 4938 vlan_item_dst->type = (enum rte_flow_item_type) 4939 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4940 } 4941 break; 4942 default: 4943 break; 4944 } 4945 if (!action_cur) 4946 action_cur = (fm->def_policy) ? 4947 actions_sfx++ : actions_pre++; 4948 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 4949 } 4950 /* Add end action to the actions. */ 4951 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4952 if (priv->sh->meter_aso_en) { 4953 /** 4954 * For ASO meter, need to add an extra jump action explicitly, 4955 * to jump from meter to policer table. 4956 */ 4957 struct mlx5_flow_meter_sub_policy *sub_policy; 4958 struct mlx5_flow_tbl_data_entry *tbl_data; 4959 4960 if (!fm->def_policy) { 4961 sub_policy = get_meter_sub_policy(dev, flow, wks, 4962 attr, items, error); 4963 if (!sub_policy) 4964 return -rte_errno; 4965 } else { 4966 enum mlx5_meter_domain mtr_domain = 4967 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4968 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4969 MLX5_MTR_DOMAIN_INGRESS); 4970 4971 sub_policy = 4972 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 4973 } 4974 tbl_data = container_of(sub_policy->tbl_rsc, 4975 struct mlx5_flow_tbl_data_entry, tbl); 4976 hw_mtr_action = actions_pre++; 4977 hw_mtr_action->type = (enum rte_flow_action_type) 4978 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 4979 hw_mtr_action->conf = tbl_data->jump.action; 4980 } 4981 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4982 actions_pre++; 4983 if (!tag_action) 4984 return rte_flow_error_set(error, ENOMEM, 4985 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4986 NULL, "No tag action space."); 4987 if (!mtr_flow_id) { 4988 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 4989 goto exit; 4990 } 4991 /* Only default-policy Meter creates mtr flow id. */ 4992 if (fm->def_policy) { 4993 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 4994 if (!tag_id) 4995 return rte_flow_error_set(error, ENOMEM, 4996 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4997 "Failed to allocate meter flow id."); 4998 flow_id = tag_id - 1; 4999 flow_id_bits = (!flow_id) ? 1 : 5000 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5001 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5002 mtr_reg_bits) { 5003 mlx5_ipool_free(fm->flow_ipool, tag_id); 5004 return rte_flow_error_set(error, EINVAL, 5005 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5006 "Meter flow id exceeds max limit."); 5007 } 5008 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5009 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5010 } 5011 /* Build tag actions and items for meter_id/meter flow_id. */ 5012 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5013 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5014 tag_item_mask = tag_item_spec + 1; 5015 /* Both flow_id and meter_id share the same register. */ 5016 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5017 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5018 0, error), 5019 .offset = mtr_id_offset, 5020 .length = mtr_reg_bits, 5021 .data = flow->meter, 5022 }; 5023 /* 5024 * The color Reg bits used by flow_id are growing from 5025 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5026 */ 5027 for (shift = 0; shift < flow_id_bits; shift++) 5028 flow_id_reversed = (flow_id_reversed << 1) | 5029 ((flow_id >> shift) & 0x1); 5030 set_tag->data |= 5031 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5032 tag_item_spec->id = set_tag->id; 5033 tag_item_spec->data = set_tag->data << mtr_id_offset; 5034 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5035 tag_action->type = (enum rte_flow_action_type) 5036 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5037 tag_action->conf = set_tag; 5038 tag_item->type = (enum rte_flow_item_type) 5039 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5040 tag_item->spec = tag_item_spec; 5041 tag_item->last = NULL; 5042 tag_item->mask = tag_item_mask; 5043 exit: 5044 if (mtr_flow_id) 5045 *mtr_flow_id = tag_id; 5046 return 0; 5047 } 5048 5049 /** 5050 * Split action list having QUEUE/RSS for metadata register copy. 5051 * 5052 * Once Q/RSS action is detected in user's action list, the flow action 5053 * should be split in order to copy metadata registers, which will happen in 5054 * RX_CP_TBL like, 5055 * - CQE->flow_tag := reg_c[1] (MARK) 5056 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5057 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5058 * This is because the last action of each flow must be a terminal action 5059 * (QUEUE, RSS or DROP). 5060 * 5061 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5062 * stored and kept in the mlx5_flow structure per each sub_flow. 5063 * 5064 * The Q/RSS action is replaced with, 5065 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5066 * And the following JUMP action is added at the end, 5067 * - JUMP, to RX_CP_TBL. 5068 * 5069 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5070 * flow_create_split_metadata() routine. The flow will look like, 5071 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5072 * 5073 * @param dev 5074 * Pointer to Ethernet device. 5075 * @param[out] split_actions 5076 * Pointer to store split actions to jump to CP_TBL. 5077 * @param[in] actions 5078 * Pointer to the list of original flow actions. 5079 * @param[in] qrss 5080 * Pointer to the Q/RSS action. 5081 * @param[in] actions_n 5082 * Number of original actions. 5083 * @param[out] error 5084 * Perform verbose error reporting if not NULL. 5085 * 5086 * @return 5087 * non-zero unique flow_id on success, otherwise 0 and 5088 * error/rte_error are set. 5089 */ 5090 static uint32_t 5091 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5092 struct rte_flow_action *split_actions, 5093 const struct rte_flow_action *actions, 5094 const struct rte_flow_action *qrss, 5095 int actions_n, struct rte_flow_error *error) 5096 { 5097 struct mlx5_priv *priv = dev->data->dev_private; 5098 struct mlx5_rte_flow_action_set_tag *set_tag; 5099 struct rte_flow_action_jump *jump; 5100 const int qrss_idx = qrss - actions; 5101 uint32_t flow_id = 0; 5102 int ret = 0; 5103 5104 /* 5105 * Given actions will be split 5106 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5107 * - Add jump to mreg CP_TBL. 5108 * As a result, there will be one more action. 5109 */ 5110 ++actions_n; 5111 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5112 set_tag = (void *)(split_actions + actions_n); 5113 /* 5114 * If tag action is not set to void(it means we are not the meter 5115 * suffix flow), add the tag action. Since meter suffix flow already 5116 * has the tag added. 5117 */ 5118 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 5119 /* 5120 * Allocate the new subflow ID. This one is unique within 5121 * device and not shared with representors. Otherwise, 5122 * we would have to resolve multi-thread access synch 5123 * issue. Each flow on the shared device is appended 5124 * with source vport identifier, so the resulting 5125 * flows will be unique in the shared (by master and 5126 * representors) domain even if they have coinciding 5127 * IDs. 5128 */ 5129 mlx5_ipool_malloc(priv->sh->ipool 5130 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5131 if (!flow_id) 5132 return rte_flow_error_set(error, ENOMEM, 5133 RTE_FLOW_ERROR_TYPE_ACTION, 5134 NULL, "can't allocate id " 5135 "for split Q/RSS subflow"); 5136 /* Internal SET_TAG action to set flow ID. */ 5137 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5138 .data = flow_id, 5139 }; 5140 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5141 if (ret < 0) 5142 return ret; 5143 set_tag->id = ret; 5144 /* Construct new actions array. */ 5145 /* Replace QUEUE/RSS action. */ 5146 split_actions[qrss_idx] = (struct rte_flow_action){ 5147 .type = (enum rte_flow_action_type) 5148 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5149 .conf = set_tag, 5150 }; 5151 } 5152 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5153 jump = (void *)(set_tag + 1); 5154 *jump = (struct rte_flow_action_jump){ 5155 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5156 }; 5157 split_actions[actions_n - 2] = (struct rte_flow_action){ 5158 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5159 .conf = jump, 5160 }; 5161 split_actions[actions_n - 1] = (struct rte_flow_action){ 5162 .type = RTE_FLOW_ACTION_TYPE_END, 5163 }; 5164 return flow_id; 5165 } 5166 5167 /** 5168 * Extend the given action list for Tx metadata copy. 5169 * 5170 * Copy the given action list to the ext_actions and add flow metadata register 5171 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5172 * 5173 * @param[out] ext_actions 5174 * Pointer to the extended action list. 5175 * @param[in] actions 5176 * Pointer to the list of actions. 5177 * @param[in] actions_n 5178 * Number of actions in the list. 5179 * @param[out] error 5180 * Perform verbose error reporting if not NULL. 5181 * @param[in] encap_idx 5182 * The encap action inndex. 5183 * 5184 * @return 5185 * 0 on success, negative value otherwise 5186 */ 5187 static int 5188 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5189 struct rte_flow_action *ext_actions, 5190 const struct rte_flow_action *actions, 5191 int actions_n, struct rte_flow_error *error, 5192 int encap_idx) 5193 { 5194 struct mlx5_flow_action_copy_mreg *cp_mreg = 5195 (struct mlx5_flow_action_copy_mreg *) 5196 (ext_actions + actions_n + 1); 5197 int ret; 5198 5199 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5200 if (ret < 0) 5201 return ret; 5202 cp_mreg->dst = ret; 5203 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5204 if (ret < 0) 5205 return ret; 5206 cp_mreg->src = ret; 5207 if (encap_idx != 0) 5208 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5209 if (encap_idx == actions_n - 1) { 5210 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5211 .type = (enum rte_flow_action_type) 5212 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5213 .conf = cp_mreg, 5214 }; 5215 ext_actions[actions_n] = (struct rte_flow_action){ 5216 .type = RTE_FLOW_ACTION_TYPE_END, 5217 }; 5218 } else { 5219 ext_actions[encap_idx] = (struct rte_flow_action){ 5220 .type = (enum rte_flow_action_type) 5221 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5222 .conf = cp_mreg, 5223 }; 5224 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5225 sizeof(*ext_actions) * (actions_n - encap_idx)); 5226 } 5227 return 0; 5228 } 5229 5230 /** 5231 * Check the match action from the action list. 5232 * 5233 * @param[in] actions 5234 * Pointer to the list of actions. 5235 * @param[in] attr 5236 * Flow rule attributes. 5237 * @param[in] action 5238 * The action to be check if exist. 5239 * @param[out] match_action_pos 5240 * Pointer to the position of the matched action if exists, otherwise is -1. 5241 * @param[out] qrss_action_pos 5242 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5243 * @param[out] modify_after_mirror 5244 * Pointer to the flag of modify action after FDB mirroring. 5245 * 5246 * @return 5247 * > 0 the total number of actions. 5248 * 0 if not found match action in action list. 5249 */ 5250 static int 5251 flow_check_match_action(const struct rte_flow_action actions[], 5252 const struct rte_flow_attr *attr, 5253 enum rte_flow_action_type action, 5254 int *match_action_pos, int *qrss_action_pos, 5255 int *modify_after_mirror) 5256 { 5257 const struct rte_flow_action_sample *sample; 5258 int actions_n = 0; 5259 uint32_t ratio = 0; 5260 int sub_type = 0; 5261 int flag = 0; 5262 int fdb_mirror = 0; 5263 5264 *match_action_pos = -1; 5265 *qrss_action_pos = -1; 5266 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5267 if (actions->type == action) { 5268 flag = 1; 5269 *match_action_pos = actions_n; 5270 } 5271 switch (actions->type) { 5272 case RTE_FLOW_ACTION_TYPE_QUEUE: 5273 case RTE_FLOW_ACTION_TYPE_RSS: 5274 *qrss_action_pos = actions_n; 5275 break; 5276 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5277 sample = actions->conf; 5278 ratio = sample->ratio; 5279 sub_type = ((const struct rte_flow_action *) 5280 (sample->actions))->type; 5281 if (ratio == 1 && attr->transfer) 5282 fdb_mirror = 1; 5283 break; 5284 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5285 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5286 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5287 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5288 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5289 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5290 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5291 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5292 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5293 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5294 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5295 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5296 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5297 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5298 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5299 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5300 case RTE_FLOW_ACTION_TYPE_FLAG: 5301 case RTE_FLOW_ACTION_TYPE_MARK: 5302 case RTE_FLOW_ACTION_TYPE_SET_META: 5303 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5304 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5305 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5306 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5307 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5308 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5309 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5310 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5311 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5312 case RTE_FLOW_ACTION_TYPE_METER: 5313 if (fdb_mirror) 5314 *modify_after_mirror = 1; 5315 break; 5316 default: 5317 break; 5318 } 5319 actions_n++; 5320 } 5321 if (flag && fdb_mirror && !*modify_after_mirror) { 5322 /* FDB mirroring uses the destination array to implement 5323 * instead of FLOW_SAMPLER object. 5324 */ 5325 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5326 flag = 0; 5327 } 5328 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5329 return flag ? actions_n + 1 : 0; 5330 } 5331 5332 #define SAMPLE_SUFFIX_ITEM 2 5333 5334 /** 5335 * Split the sample flow. 5336 * 5337 * As sample flow will split to two sub flow, sample flow with 5338 * sample action, the other actions will move to new suffix flow. 5339 * 5340 * Also add unique tag id with tag action in the sample flow, 5341 * the same tag id will be as match in the suffix flow. 5342 * 5343 * @param dev 5344 * Pointer to Ethernet device. 5345 * @param[in] add_tag 5346 * Add extra tag action flag. 5347 * @param[out] sfx_items 5348 * Suffix flow match items (list terminated by the END pattern item). 5349 * @param[in] actions 5350 * Associated actions (list terminated by the END action). 5351 * @param[out] actions_sfx 5352 * Suffix flow actions. 5353 * @param[out] actions_pre 5354 * Prefix flow actions. 5355 * @param[in] actions_n 5356 * The total number of actions. 5357 * @param[in] sample_action_pos 5358 * The sample action position. 5359 * @param[in] qrss_action_pos 5360 * The Queue/RSS action position. 5361 * @param[in] jump_table 5362 * Add extra jump action flag. 5363 * @param[out] error 5364 * Perform verbose error reporting if not NULL. 5365 * 5366 * @return 5367 * 0 on success, or unique flow_id, a negative errno value 5368 * otherwise and rte_errno is set. 5369 */ 5370 static int 5371 flow_sample_split_prep(struct rte_eth_dev *dev, 5372 int add_tag, 5373 struct rte_flow_item sfx_items[], 5374 const struct rte_flow_action actions[], 5375 struct rte_flow_action actions_sfx[], 5376 struct rte_flow_action actions_pre[], 5377 int actions_n, 5378 int sample_action_pos, 5379 int qrss_action_pos, 5380 int jump_table, 5381 struct rte_flow_error *error) 5382 { 5383 struct mlx5_priv *priv = dev->data->dev_private; 5384 struct mlx5_rte_flow_action_set_tag *set_tag; 5385 struct mlx5_rte_flow_item_tag *tag_spec; 5386 struct mlx5_rte_flow_item_tag *tag_mask; 5387 struct rte_flow_action_jump *jump_action; 5388 uint32_t tag_id = 0; 5389 int index; 5390 int append_index = 0; 5391 int ret; 5392 5393 if (sample_action_pos < 0) 5394 return rte_flow_error_set(error, EINVAL, 5395 RTE_FLOW_ERROR_TYPE_ACTION, 5396 NULL, "invalid position of sample " 5397 "action in list"); 5398 /* Prepare the actions for prefix and suffix flow. */ 5399 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5400 index = qrss_action_pos; 5401 /* Put the preceding the Queue/RSS action into prefix flow. */ 5402 if (index != 0) 5403 memcpy(actions_pre, actions, 5404 sizeof(struct rte_flow_action) * index); 5405 /* Put others preceding the sample action into prefix flow. */ 5406 if (sample_action_pos > index + 1) 5407 memcpy(actions_pre + index, actions + index + 1, 5408 sizeof(struct rte_flow_action) * 5409 (sample_action_pos - index - 1)); 5410 index = sample_action_pos - 1; 5411 /* Put Queue/RSS action into Suffix flow. */ 5412 memcpy(actions_sfx, actions + qrss_action_pos, 5413 sizeof(struct rte_flow_action)); 5414 actions_sfx++; 5415 } else { 5416 index = sample_action_pos; 5417 if (index != 0) 5418 memcpy(actions_pre, actions, 5419 sizeof(struct rte_flow_action) * index); 5420 } 5421 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5422 * For CX6DX and above, metadata registers Cx preserve their value, 5423 * add an extra tag action for NIC-RX and E-Switch Domain. 5424 */ 5425 if (add_tag) { 5426 /* Prepare the prefix tag action. */ 5427 append_index++; 5428 set_tag = (void *)(actions_pre + actions_n + append_index); 5429 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5430 if (ret < 0) 5431 return ret; 5432 mlx5_ipool_malloc(priv->sh->ipool 5433 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5434 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5435 .id = ret, 5436 .data = tag_id, 5437 }; 5438 /* Prepare the suffix subflow items. */ 5439 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5440 tag_spec->data = tag_id; 5441 tag_spec->id = set_tag->id; 5442 tag_mask = tag_spec + 1; 5443 tag_mask->data = UINT32_MAX; 5444 sfx_items[0] = (struct rte_flow_item){ 5445 .type = (enum rte_flow_item_type) 5446 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5447 .spec = tag_spec, 5448 .last = NULL, 5449 .mask = tag_mask, 5450 }; 5451 sfx_items[1] = (struct rte_flow_item){ 5452 .type = (enum rte_flow_item_type) 5453 RTE_FLOW_ITEM_TYPE_END, 5454 }; 5455 /* Prepare the tag action in prefix subflow. */ 5456 actions_pre[index++] = 5457 (struct rte_flow_action){ 5458 .type = (enum rte_flow_action_type) 5459 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5460 .conf = set_tag, 5461 }; 5462 } 5463 memcpy(actions_pre + index, actions + sample_action_pos, 5464 sizeof(struct rte_flow_action)); 5465 index += 1; 5466 /* For the modify action after the sample action in E-Switch mirroring, 5467 * Add the extra jump action in prefix subflow and jump into the next 5468 * table, then do the modify action in the new table. 5469 */ 5470 if (jump_table) { 5471 /* Prepare the prefix jump action. */ 5472 append_index++; 5473 jump_action = (void *)(actions_pre + actions_n + append_index); 5474 jump_action->group = jump_table; 5475 actions_pre[index++] = 5476 (struct rte_flow_action){ 5477 .type = (enum rte_flow_action_type) 5478 RTE_FLOW_ACTION_TYPE_JUMP, 5479 .conf = jump_action, 5480 }; 5481 } 5482 actions_pre[index] = (struct rte_flow_action){ 5483 .type = (enum rte_flow_action_type) 5484 RTE_FLOW_ACTION_TYPE_END, 5485 }; 5486 /* Put the actions after sample into Suffix flow. */ 5487 memcpy(actions_sfx, actions + sample_action_pos + 1, 5488 sizeof(struct rte_flow_action) * 5489 (actions_n - sample_action_pos - 1)); 5490 return tag_id; 5491 } 5492 5493 /** 5494 * The splitting for metadata feature. 5495 * 5496 * - Q/RSS action on NIC Rx should be split in order to pass by 5497 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5498 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5499 * 5500 * - All the actions on NIC Tx should have a mreg copy action to 5501 * copy reg_a from WQE to reg_c[0]. 5502 * 5503 * @param dev 5504 * Pointer to Ethernet device. 5505 * @param[in] flow 5506 * Parent flow structure pointer. 5507 * @param[in] attr 5508 * Flow rule attributes. 5509 * @param[in] items 5510 * Pattern specification (list terminated by the END pattern item). 5511 * @param[in] actions 5512 * Associated actions (list terminated by the END action). 5513 * @param[in] flow_split_info 5514 * Pointer to flow split info structure. 5515 * @param[out] error 5516 * Perform verbose error reporting if not NULL. 5517 * @return 5518 * 0 on success, negative value otherwise 5519 */ 5520 static int 5521 flow_create_split_metadata(struct rte_eth_dev *dev, 5522 struct rte_flow *flow, 5523 const struct rte_flow_attr *attr, 5524 const struct rte_flow_item items[], 5525 const struct rte_flow_action actions[], 5526 struct mlx5_flow_split_info *flow_split_info, 5527 struct rte_flow_error *error) 5528 { 5529 struct mlx5_priv *priv = dev->data->dev_private; 5530 struct mlx5_dev_config *config = &priv->config; 5531 const struct rte_flow_action *qrss = NULL; 5532 struct rte_flow_action *ext_actions = NULL; 5533 struct mlx5_flow *dev_flow = NULL; 5534 uint32_t qrss_id = 0; 5535 int mtr_sfx = 0; 5536 size_t act_size; 5537 int actions_n; 5538 int encap_idx; 5539 int ret; 5540 5541 /* Check whether extensive metadata feature is engaged. */ 5542 if (!config->dv_flow_en || 5543 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5544 !mlx5_flow_ext_mreg_supported(dev)) 5545 return flow_create_split_inner(dev, flow, NULL, attr, items, 5546 actions, flow_split_info, error); 5547 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5548 &encap_idx); 5549 if (qrss) { 5550 /* Exclude hairpin flows from splitting. */ 5551 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5552 const struct rte_flow_action_queue *queue; 5553 5554 queue = qrss->conf; 5555 if (mlx5_rxq_get_type(dev, queue->index) == 5556 MLX5_RXQ_TYPE_HAIRPIN) 5557 qrss = NULL; 5558 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5559 const struct rte_flow_action_rss *rss; 5560 5561 rss = qrss->conf; 5562 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5563 MLX5_RXQ_TYPE_HAIRPIN) 5564 qrss = NULL; 5565 } 5566 } 5567 if (qrss) { 5568 /* Check if it is in meter suffix table. */ 5569 mtr_sfx = attr->group == (attr->transfer ? 5570 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5571 MLX5_FLOW_TABLE_LEVEL_METER); 5572 /* 5573 * Q/RSS action on NIC Rx should be split in order to pass by 5574 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5575 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5576 */ 5577 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5578 sizeof(struct rte_flow_action_set_tag) + 5579 sizeof(struct rte_flow_action_jump); 5580 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5581 SOCKET_ID_ANY); 5582 if (!ext_actions) 5583 return rte_flow_error_set(error, ENOMEM, 5584 RTE_FLOW_ERROR_TYPE_ACTION, 5585 NULL, "no memory to split " 5586 "metadata flow"); 5587 /* 5588 * If we are the suffix flow of meter, tag already exist. 5589 * Set the tag action to void. 5590 */ 5591 if (mtr_sfx) 5592 ext_actions[qrss - actions].type = 5593 RTE_FLOW_ACTION_TYPE_VOID; 5594 else 5595 ext_actions[qrss - actions].type = 5596 (enum rte_flow_action_type) 5597 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5598 /* 5599 * Create the new actions list with removed Q/RSS action 5600 * and appended set tag and jump to register copy table 5601 * (RX_CP_TBL). We should preallocate unique tag ID here 5602 * in advance, because it is needed for set tag action. 5603 */ 5604 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5605 qrss, actions_n, error); 5606 if (!mtr_sfx && !qrss_id) { 5607 ret = -rte_errno; 5608 goto exit; 5609 } 5610 } else if (attr->egress && !attr->transfer) { 5611 /* 5612 * All the actions on NIC Tx should have a metadata register 5613 * copy action to copy reg_a from WQE to reg_c[meta] 5614 */ 5615 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5616 sizeof(struct mlx5_flow_action_copy_mreg); 5617 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5618 SOCKET_ID_ANY); 5619 if (!ext_actions) 5620 return rte_flow_error_set(error, ENOMEM, 5621 RTE_FLOW_ERROR_TYPE_ACTION, 5622 NULL, "no memory to split " 5623 "metadata flow"); 5624 /* Create the action list appended with copy register. */ 5625 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5626 actions_n, error, encap_idx); 5627 if (ret < 0) 5628 goto exit; 5629 } 5630 /* Add the unmodified original or prefix subflow. */ 5631 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5632 items, ext_actions ? ext_actions : 5633 actions, flow_split_info, error); 5634 if (ret < 0) 5635 goto exit; 5636 MLX5_ASSERT(dev_flow); 5637 if (qrss) { 5638 const struct rte_flow_attr q_attr = { 5639 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5640 .ingress = 1, 5641 }; 5642 /* Internal PMD action to set register. */ 5643 struct mlx5_rte_flow_item_tag q_tag_spec = { 5644 .data = qrss_id, 5645 .id = REG_NON, 5646 }; 5647 struct rte_flow_item q_items[] = { 5648 { 5649 .type = (enum rte_flow_item_type) 5650 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5651 .spec = &q_tag_spec, 5652 .last = NULL, 5653 .mask = NULL, 5654 }, 5655 { 5656 .type = RTE_FLOW_ITEM_TYPE_END, 5657 }, 5658 }; 5659 struct rte_flow_action q_actions[] = { 5660 { 5661 .type = qrss->type, 5662 .conf = qrss->conf, 5663 }, 5664 { 5665 .type = RTE_FLOW_ACTION_TYPE_END, 5666 }, 5667 }; 5668 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5669 5670 /* 5671 * Configure the tag item only if there is no meter subflow. 5672 * Since tag is already marked in the meter suffix subflow 5673 * we can just use the meter suffix items as is. 5674 */ 5675 if (qrss_id) { 5676 /* Not meter subflow. */ 5677 MLX5_ASSERT(!mtr_sfx); 5678 /* 5679 * Put unique id in prefix flow due to it is destroyed 5680 * after suffix flow and id will be freed after there 5681 * is no actual flows with this id and identifier 5682 * reallocation becomes possible (for example, for 5683 * other flows in other threads). 5684 */ 5685 dev_flow->handle->split_flow_id = qrss_id; 5686 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5687 error); 5688 if (ret < 0) 5689 goto exit; 5690 q_tag_spec.id = ret; 5691 } 5692 dev_flow = NULL; 5693 /* Add suffix subflow to execute Q/RSS. */ 5694 flow_split_info->prefix_layers = layers; 5695 flow_split_info->prefix_mark = 0; 5696 ret = flow_create_split_inner(dev, flow, &dev_flow, 5697 &q_attr, mtr_sfx ? items : 5698 q_items, q_actions, 5699 flow_split_info, error); 5700 if (ret < 0) 5701 goto exit; 5702 /* qrss ID should be freed if failed. */ 5703 qrss_id = 0; 5704 MLX5_ASSERT(dev_flow); 5705 } 5706 5707 exit: 5708 /* 5709 * We do not destroy the partially created sub_flows in case of error. 5710 * These ones are included into parent flow list and will be destroyed 5711 * by flow_drv_destroy. 5712 */ 5713 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5714 qrss_id); 5715 mlx5_free(ext_actions); 5716 return ret; 5717 } 5718 5719 /** 5720 * Create meter internal drop flow with the original pattern. 5721 * 5722 * @param dev 5723 * Pointer to Ethernet device. 5724 * @param[in] flow 5725 * Parent flow structure pointer. 5726 * @param[in] attr 5727 * Flow rule attributes. 5728 * @param[in] items 5729 * Pattern specification (list terminated by the END pattern item). 5730 * @param[in] flow_split_info 5731 * Pointer to flow split info structure. 5732 * @param[in] fm 5733 * Pointer to flow meter structure. 5734 * @param[out] error 5735 * Perform verbose error reporting if not NULL. 5736 * @return 5737 * 0 on success, negative value otherwise 5738 */ 5739 static uint32_t 5740 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5741 struct rte_flow *flow, 5742 const struct rte_flow_attr *attr, 5743 const struct rte_flow_item items[], 5744 struct mlx5_flow_split_info *flow_split_info, 5745 struct mlx5_flow_meter_info *fm, 5746 struct rte_flow_error *error) 5747 { 5748 struct mlx5_flow *dev_flow = NULL; 5749 struct rte_flow_attr drop_attr = *attr; 5750 struct rte_flow_action drop_actions[3]; 5751 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5752 5753 MLX5_ASSERT(fm->drop_cnt); 5754 drop_actions[0].type = 5755 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5756 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5757 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5758 drop_actions[1].conf = NULL; 5759 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5760 drop_actions[2].conf = NULL; 5761 drop_split_info.external = false; 5762 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5763 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5764 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5765 return flow_create_split_inner(dev, flow, &dev_flow, 5766 &drop_attr, items, drop_actions, 5767 &drop_split_info, error); 5768 } 5769 5770 /** 5771 * The splitting for meter feature. 5772 * 5773 * - The meter flow will be split to two flows as prefix and 5774 * suffix flow. The packets make sense only it pass the prefix 5775 * meter action. 5776 * 5777 * - Reg_C_5 is used for the packet to match betweend prefix and 5778 * suffix flow. 5779 * 5780 * @param dev 5781 * Pointer to Ethernet device. 5782 * @param[in] flow 5783 * Parent flow structure pointer. 5784 * @param[in] attr 5785 * Flow rule attributes. 5786 * @param[in] items 5787 * Pattern specification (list terminated by the END pattern item). 5788 * @param[in] actions 5789 * Associated actions (list terminated by the END action). 5790 * @param[in] flow_split_info 5791 * Pointer to flow split info structure. 5792 * @param[out] error 5793 * Perform verbose error reporting if not NULL. 5794 * @return 5795 * 0 on success, negative value otherwise 5796 */ 5797 static int 5798 flow_create_split_meter(struct rte_eth_dev *dev, 5799 struct rte_flow *flow, 5800 const struct rte_flow_attr *attr, 5801 const struct rte_flow_item items[], 5802 const struct rte_flow_action actions[], 5803 struct mlx5_flow_split_info *flow_split_info, 5804 struct rte_flow_error *error) 5805 { 5806 struct mlx5_priv *priv = dev->data->dev_private; 5807 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5808 struct rte_flow_action *sfx_actions = NULL; 5809 struct rte_flow_action *pre_actions = NULL; 5810 struct rte_flow_item *sfx_items = NULL; 5811 struct mlx5_flow *dev_flow = NULL; 5812 struct rte_flow_attr sfx_attr = *attr; 5813 struct mlx5_flow_meter_info *fm = NULL; 5814 uint8_t skip_scale_restore; 5815 bool has_mtr = false; 5816 bool has_modify = false; 5817 bool set_mtr_reg = true; 5818 bool is_mtr_hierarchy = false; 5819 uint32_t meter_id = 0; 5820 uint32_t mtr_idx = 0; 5821 uint32_t mtr_flow_id = 0; 5822 size_t act_size; 5823 size_t item_size; 5824 int actions_n = 0; 5825 int ret = 0; 5826 5827 if (priv->mtr_en) 5828 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 5829 &has_modify, &meter_id); 5830 if (has_mtr) { 5831 if (flow->meter) { 5832 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 5833 if (!fm) 5834 return rte_flow_error_set(error, EINVAL, 5835 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5836 NULL, "Meter not found."); 5837 } else { 5838 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 5839 if (!fm) 5840 return rte_flow_error_set(error, EINVAL, 5841 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5842 NULL, "Meter not found."); 5843 ret = mlx5_flow_meter_attach(priv, fm, 5844 &sfx_attr, error); 5845 if (ret) 5846 return -rte_errno; 5847 flow->meter = mtr_idx; 5848 } 5849 MLX5_ASSERT(wks); 5850 wks->fm = fm; 5851 if (!fm->def_policy) { 5852 wks->policy = mlx5_flow_meter_policy_find(dev, 5853 fm->policy_id, 5854 NULL); 5855 MLX5_ASSERT(wks->policy); 5856 if (wks->policy->is_hierarchy) { 5857 wks->final_policy = 5858 mlx5_flow_meter_hierarchy_get_final_policy(dev, 5859 wks->policy); 5860 if (!wks->final_policy) 5861 return rte_flow_error_set(error, 5862 EINVAL, 5863 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5864 "Failed to find terminal policy of hierarchy."); 5865 is_mtr_hierarchy = true; 5866 } 5867 } 5868 /* 5869 * If it isn't default-policy Meter, and 5870 * 1. There's no action in flow to change 5871 * packet (modify/encap/decap etc.), OR 5872 * 2. No drop count needed for this meter. 5873 * 3. It's not meter hierarchy. 5874 * Then no need to use regC to save meter id anymore. 5875 */ 5876 if (!fm->def_policy && !is_mtr_hierarchy && 5877 (!has_modify || !fm->drop_cnt)) 5878 set_mtr_reg = false; 5879 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 5880 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 5881 sizeof(struct mlx5_rte_flow_action_set_tag); 5882 /* Suffix items: tag, vlan, port id, end. */ 5883 #define METER_SUFFIX_ITEM 4 5884 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5885 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5886 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5887 0, SOCKET_ID_ANY); 5888 if (!sfx_actions) 5889 return rte_flow_error_set(error, ENOMEM, 5890 RTE_FLOW_ERROR_TYPE_ACTION, 5891 NULL, "no memory to split " 5892 "meter flow"); 5893 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5894 act_size); 5895 /* There's no suffix flow for meter of non-default policy. */ 5896 if (!fm->def_policy) 5897 pre_actions = sfx_actions + 1; 5898 else 5899 pre_actions = sfx_actions + actions_n; 5900 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 5901 items, sfx_items, actions, 5902 sfx_actions, pre_actions, 5903 (set_mtr_reg ? &mtr_flow_id : NULL), 5904 error); 5905 if (ret) { 5906 ret = -rte_errno; 5907 goto exit; 5908 } 5909 /* Add the prefix subflow. */ 5910 flow_split_info->prefix_mark = 0; 5911 skip_scale_restore = flow_split_info->skip_scale; 5912 flow_split_info->skip_scale |= 5913 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5914 ret = flow_create_split_inner(dev, flow, &dev_flow, 5915 attr, items, pre_actions, 5916 flow_split_info, error); 5917 flow_split_info->skip_scale = skip_scale_restore; 5918 if (ret) { 5919 if (mtr_flow_id) 5920 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 5921 ret = -rte_errno; 5922 goto exit; 5923 } 5924 if (mtr_flow_id) { 5925 dev_flow->handle->split_flow_id = mtr_flow_id; 5926 dev_flow->handle->is_meter_flow_id = 1; 5927 } 5928 if (!fm->def_policy) { 5929 if (!set_mtr_reg && fm->drop_cnt) 5930 ret = 5931 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 5932 &sfx_attr, items, 5933 flow_split_info, 5934 fm, error); 5935 goto exit; 5936 } 5937 /* Setting the sfx group atrr. */ 5938 sfx_attr.group = sfx_attr.transfer ? 5939 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5940 MLX5_FLOW_TABLE_LEVEL_METER; 5941 flow_split_info->prefix_layers = 5942 flow_get_prefix_layer_flags(dev_flow); 5943 flow_split_info->prefix_mark = dev_flow->handle->mark; 5944 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 5945 } 5946 /* Add the prefix subflow. */ 5947 ret = flow_create_split_metadata(dev, flow, 5948 &sfx_attr, sfx_items ? 5949 sfx_items : items, 5950 sfx_actions ? sfx_actions : actions, 5951 flow_split_info, error); 5952 exit: 5953 if (sfx_actions) 5954 mlx5_free(sfx_actions); 5955 return ret; 5956 } 5957 5958 /** 5959 * The splitting for sample feature. 5960 * 5961 * Once Sample action is detected in the action list, the flow actions should 5962 * be split into prefix sub flow and suffix sub flow. 5963 * 5964 * The original items remain in the prefix sub flow, all actions preceding the 5965 * sample action and the sample action itself will be copied to the prefix 5966 * sub flow, the actions following the sample action will be copied to the 5967 * suffix sub flow, Queue action always be located in the suffix sub flow. 5968 * 5969 * In order to make the packet from prefix sub flow matches with suffix sub 5970 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5971 * flow uses tag item with the unique flow id. 5972 * 5973 * @param dev 5974 * Pointer to Ethernet device. 5975 * @param[in] flow 5976 * Parent flow structure pointer. 5977 * @param[in] attr 5978 * Flow rule attributes. 5979 * @param[in] items 5980 * Pattern specification (list terminated by the END pattern item). 5981 * @param[in] actions 5982 * Associated actions (list terminated by the END action). 5983 * @param[in] flow_split_info 5984 * Pointer to flow split info structure. 5985 * @param[out] error 5986 * Perform verbose error reporting if not NULL. 5987 * @return 5988 * 0 on success, negative value otherwise 5989 */ 5990 static int 5991 flow_create_split_sample(struct rte_eth_dev *dev, 5992 struct rte_flow *flow, 5993 const struct rte_flow_attr *attr, 5994 const struct rte_flow_item items[], 5995 const struct rte_flow_action actions[], 5996 struct mlx5_flow_split_info *flow_split_info, 5997 struct rte_flow_error *error) 5998 { 5999 struct mlx5_priv *priv = dev->data->dev_private; 6000 struct rte_flow_action *sfx_actions = NULL; 6001 struct rte_flow_action *pre_actions = NULL; 6002 struct rte_flow_item *sfx_items = NULL; 6003 struct mlx5_flow *dev_flow = NULL; 6004 struct rte_flow_attr sfx_attr = *attr; 6005 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6006 struct mlx5_flow_dv_sample_resource *sample_res; 6007 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6008 struct mlx5_flow_tbl_resource *sfx_tbl; 6009 #endif 6010 size_t act_size; 6011 size_t item_size; 6012 uint32_t fdb_tx = 0; 6013 int32_t tag_id = 0; 6014 int actions_n = 0; 6015 int sample_action_pos; 6016 int qrss_action_pos; 6017 int add_tag = 0; 6018 int modify_after_mirror = 0; 6019 uint16_t jump_table = 0; 6020 const uint32_t next_ft_step = 1; 6021 int ret = 0; 6022 6023 if (priv->sampler_en) 6024 actions_n = flow_check_match_action(actions, attr, 6025 RTE_FLOW_ACTION_TYPE_SAMPLE, 6026 &sample_action_pos, &qrss_action_pos, 6027 &modify_after_mirror); 6028 if (actions_n) { 6029 /* The prefix actions must includes sample, tag, end. */ 6030 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6031 + sizeof(struct mlx5_rte_flow_action_set_tag); 6032 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6033 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6034 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6035 item_size), 0, SOCKET_ID_ANY); 6036 if (!sfx_actions) 6037 return rte_flow_error_set(error, ENOMEM, 6038 RTE_FLOW_ERROR_TYPE_ACTION, 6039 NULL, "no memory to split " 6040 "sample flow"); 6041 /* The representor_id is UINT16_MAX for uplink. */ 6042 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6043 /* 6044 * When reg_c_preserve is set, metadata registers Cx preserve 6045 * their value even through packet duplication. 6046 */ 6047 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 6048 if (add_tag) 6049 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6050 + act_size); 6051 if (modify_after_mirror) 6052 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6053 next_ft_step; 6054 pre_actions = sfx_actions + actions_n; 6055 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6056 actions, sfx_actions, 6057 pre_actions, actions_n, 6058 sample_action_pos, 6059 qrss_action_pos, jump_table, 6060 error); 6061 if (tag_id < 0 || (add_tag && !tag_id)) { 6062 ret = -rte_errno; 6063 goto exit; 6064 } 6065 if (modify_after_mirror) 6066 flow_split_info->skip_scale = 6067 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6068 /* Add the prefix subflow. */ 6069 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6070 items, pre_actions, 6071 flow_split_info, error); 6072 if (ret) { 6073 ret = -rte_errno; 6074 goto exit; 6075 } 6076 dev_flow->handle->split_flow_id = tag_id; 6077 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6078 if (!modify_after_mirror) { 6079 /* Set the sfx group attr. */ 6080 sample_res = (struct mlx5_flow_dv_sample_resource *) 6081 dev_flow->dv.sample_res; 6082 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6083 sample_res->normal_path_tbl; 6084 sfx_tbl_data = container_of(sfx_tbl, 6085 struct mlx5_flow_tbl_data_entry, 6086 tbl); 6087 sfx_attr.group = sfx_attr.transfer ? 6088 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6089 } else { 6090 MLX5_ASSERT(attr->transfer); 6091 sfx_attr.group = jump_table; 6092 } 6093 flow_split_info->prefix_layers = 6094 flow_get_prefix_layer_flags(dev_flow); 6095 flow_split_info->prefix_mark = dev_flow->handle->mark; 6096 /* Suffix group level already be scaled with factor, set 6097 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6098 * again in translation. 6099 */ 6100 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6101 #endif 6102 } 6103 /* Add the suffix subflow. */ 6104 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6105 sfx_items ? sfx_items : items, 6106 sfx_actions ? sfx_actions : actions, 6107 flow_split_info, error); 6108 exit: 6109 if (sfx_actions) 6110 mlx5_free(sfx_actions); 6111 return ret; 6112 } 6113 6114 /** 6115 * Split the flow to subflow set. The splitters might be linked 6116 * in the chain, like this: 6117 * flow_create_split_outer() calls: 6118 * flow_create_split_meter() calls: 6119 * flow_create_split_metadata(meter_subflow_0) calls: 6120 * flow_create_split_inner(metadata_subflow_0) 6121 * flow_create_split_inner(metadata_subflow_1) 6122 * flow_create_split_inner(metadata_subflow_2) 6123 * flow_create_split_metadata(meter_subflow_1) calls: 6124 * flow_create_split_inner(metadata_subflow_0) 6125 * flow_create_split_inner(metadata_subflow_1) 6126 * flow_create_split_inner(metadata_subflow_2) 6127 * 6128 * This provide flexible way to add new levels of flow splitting. 6129 * The all of successfully created subflows are included to the 6130 * parent flow dev_flow list. 6131 * 6132 * @param dev 6133 * Pointer to Ethernet device. 6134 * @param[in] flow 6135 * Parent flow structure pointer. 6136 * @param[in] attr 6137 * Flow rule attributes. 6138 * @param[in] items 6139 * Pattern specification (list terminated by the END pattern item). 6140 * @param[in] actions 6141 * Associated actions (list terminated by the END action). 6142 * @param[in] flow_split_info 6143 * Pointer to flow split info structure. 6144 * @param[out] error 6145 * Perform verbose error reporting if not NULL. 6146 * @return 6147 * 0 on success, negative value otherwise 6148 */ 6149 static int 6150 flow_create_split_outer(struct rte_eth_dev *dev, 6151 struct rte_flow *flow, 6152 const struct rte_flow_attr *attr, 6153 const struct rte_flow_item items[], 6154 const struct rte_flow_action actions[], 6155 struct mlx5_flow_split_info *flow_split_info, 6156 struct rte_flow_error *error) 6157 { 6158 int ret; 6159 6160 ret = flow_create_split_sample(dev, flow, attr, items, 6161 actions, flow_split_info, error); 6162 MLX5_ASSERT(ret <= 0); 6163 return ret; 6164 } 6165 6166 static inline struct mlx5_flow_tunnel * 6167 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6168 { 6169 struct mlx5_flow_tunnel *tunnel; 6170 6171 #pragma GCC diagnostic push 6172 #pragma GCC diagnostic ignored "-Wcast-qual" 6173 tunnel = (typeof(tunnel))flow->tunnel; 6174 #pragma GCC diagnostic pop 6175 6176 return tunnel; 6177 } 6178 6179 /** 6180 * Adjust flow RSS workspace if needed. 6181 * 6182 * @param wks 6183 * Pointer to thread flow work space. 6184 * @param rss_desc 6185 * Pointer to RSS descriptor. 6186 * @param[in] nrssq_num 6187 * New RSS queue number. 6188 * 6189 * @return 6190 * 0 on success, -1 otherwise and rte_errno is set. 6191 */ 6192 static int 6193 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6194 struct mlx5_flow_rss_desc *rss_desc, 6195 uint32_t nrssq_num) 6196 { 6197 if (likely(nrssq_num <= wks->rssq_num)) 6198 return 0; 6199 rss_desc->queue = realloc(rss_desc->queue, 6200 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6201 if (!rss_desc->queue) { 6202 rte_errno = ENOMEM; 6203 return -1; 6204 } 6205 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6206 return 0; 6207 } 6208 6209 /** 6210 * Create a flow and add it to @p list. 6211 * 6212 * @param dev 6213 * Pointer to Ethernet device. 6214 * @param list 6215 * Pointer to a TAILQ flow list. If this parameter NULL, 6216 * no list insertion occurred, flow is just created, 6217 * this is caller's responsibility to track the 6218 * created flow. 6219 * @param[in] attr 6220 * Flow rule attributes. 6221 * @param[in] items 6222 * Pattern specification (list terminated by the END pattern item). 6223 * @param[in] actions 6224 * Associated actions (list terminated by the END action). 6225 * @param[in] external 6226 * This flow rule is created by request external to PMD. 6227 * @param[out] error 6228 * Perform verbose error reporting if not NULL. 6229 * 6230 * @return 6231 * A flow index on success, 0 otherwise and rte_errno is set. 6232 */ 6233 static uint32_t 6234 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6235 const struct rte_flow_attr *attr, 6236 const struct rte_flow_item items[], 6237 const struct rte_flow_action original_actions[], 6238 bool external, struct rte_flow_error *error) 6239 { 6240 struct mlx5_priv *priv = dev->data->dev_private; 6241 struct rte_flow *flow = NULL; 6242 struct mlx5_flow *dev_flow; 6243 const struct rte_flow_action_rss *rss = NULL; 6244 struct mlx5_translated_action_handle 6245 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6246 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6247 union { 6248 struct mlx5_flow_expand_rss buf; 6249 uint8_t buffer[2048]; 6250 } expand_buffer; 6251 union { 6252 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6253 uint8_t buffer[2048]; 6254 } actions_rx; 6255 union { 6256 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6257 uint8_t buffer[2048]; 6258 } actions_hairpin_tx; 6259 union { 6260 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6261 uint8_t buffer[2048]; 6262 } items_tx; 6263 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6264 struct mlx5_flow_rss_desc *rss_desc; 6265 const struct rte_flow_action *p_actions_rx; 6266 uint32_t i; 6267 uint32_t idx = 0; 6268 int hairpin_flow; 6269 struct rte_flow_attr attr_tx = { .priority = 0 }; 6270 const struct rte_flow_action *actions; 6271 struct rte_flow_action *translated_actions = NULL; 6272 struct mlx5_flow_tunnel *tunnel; 6273 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6274 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6275 struct mlx5_flow_split_info flow_split_info = { 6276 .external = !!external, 6277 .skip_scale = 0, 6278 .flow_idx = 0, 6279 .prefix_mark = 0, 6280 .prefix_layers = 0, 6281 .table_id = 0 6282 }; 6283 int ret; 6284 6285 MLX5_ASSERT(wks); 6286 rss_desc = &wks->rss_desc; 6287 ret = flow_action_handles_translate(dev, original_actions, 6288 indir_actions, 6289 &indir_actions_n, 6290 &translated_actions, error); 6291 if (ret < 0) { 6292 MLX5_ASSERT(translated_actions == NULL); 6293 return 0; 6294 } 6295 actions = translated_actions ? translated_actions : original_actions; 6296 p_actions_rx = actions; 6297 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6298 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6299 external, hairpin_flow, error); 6300 if (ret < 0) 6301 goto error_before_hairpin_split; 6302 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6303 if (!flow) { 6304 rte_errno = ENOMEM; 6305 goto error_before_hairpin_split; 6306 } 6307 if (hairpin_flow > 0) { 6308 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6309 rte_errno = EINVAL; 6310 goto error_before_hairpin_split; 6311 } 6312 flow_hairpin_split(dev, actions, actions_rx.actions, 6313 actions_hairpin_tx.actions, items_tx.items, 6314 idx); 6315 p_actions_rx = actions_rx.actions; 6316 } 6317 flow_split_info.flow_idx = idx; 6318 flow->drv_type = flow_get_drv_type(dev, attr); 6319 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6320 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6321 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6322 /* RSS Action only works on NIC RX domain */ 6323 if (attr->ingress && !attr->transfer) 6324 rss = flow_get_rss_action(dev, p_actions_rx); 6325 if (rss) { 6326 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6327 return 0; 6328 /* 6329 * The following information is required by 6330 * mlx5_flow_hashfields_adjust() in advance. 6331 */ 6332 rss_desc->level = rss->level; 6333 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 6334 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 6335 } 6336 flow->dev_handles = 0; 6337 if (rss && rss->types) { 6338 unsigned int graph_root; 6339 6340 graph_root = find_graph_root(items, rss->level); 6341 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6342 items, rss->types, 6343 mlx5_support_expansion, graph_root); 6344 MLX5_ASSERT(ret > 0 && 6345 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6346 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6347 for (i = 0; i < buf->entries; ++i) 6348 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6349 } 6350 } else { 6351 buf->entries = 1; 6352 buf->entry[0].pattern = (void *)(uintptr_t)items; 6353 } 6354 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6355 indir_actions_n); 6356 for (i = 0; i < buf->entries; ++i) { 6357 /* Initialize flow split data. */ 6358 flow_split_info.prefix_layers = 0; 6359 flow_split_info.prefix_mark = 0; 6360 flow_split_info.skip_scale = 0; 6361 /* 6362 * The splitter may create multiple dev_flows, 6363 * depending on configuration. In the simplest 6364 * case it just creates unmodified original flow. 6365 */ 6366 ret = flow_create_split_outer(dev, flow, attr, 6367 buf->entry[i].pattern, 6368 p_actions_rx, &flow_split_info, 6369 error); 6370 if (ret < 0) 6371 goto error; 6372 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6373 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6374 p_actions_rx, 6375 idx, 6376 wks->flows[0].tunnel, 6377 &default_miss_ctx, 6378 error); 6379 if (ret < 0) { 6380 mlx5_free(default_miss_ctx.queue); 6381 goto error; 6382 } 6383 } 6384 } 6385 /* Create the tx flow. */ 6386 if (hairpin_flow) { 6387 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6388 attr_tx.ingress = 0; 6389 attr_tx.egress = 1; 6390 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6391 actions_hairpin_tx.actions, 6392 idx, error); 6393 if (!dev_flow) 6394 goto error; 6395 dev_flow->flow = flow; 6396 dev_flow->external = 0; 6397 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6398 dev_flow->handle, next); 6399 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6400 items_tx.items, 6401 actions_hairpin_tx.actions, error); 6402 if (ret < 0) 6403 goto error; 6404 } 6405 /* 6406 * Update the metadata register copy table. If extensive 6407 * metadata feature is enabled and registers are supported 6408 * we might create the extra rte_flow for each unique 6409 * MARK/FLAG action ID. 6410 * 6411 * The table is updated for ingress Flows only, because 6412 * the egress Flows belong to the different device and 6413 * copy table should be updated in peer NIC Rx domain. 6414 */ 6415 if (attr->ingress && 6416 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6417 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6418 if (ret) 6419 goto error; 6420 } 6421 /* 6422 * If the flow is external (from application) OR device is started, 6423 * OR mreg discover, then apply immediately. 6424 */ 6425 if (external || dev->data->dev_started || 6426 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6427 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6428 ret = flow_drv_apply(dev, flow, error); 6429 if (ret < 0) 6430 goto error; 6431 } 6432 flow->type = type; 6433 flow_rxq_flags_set(dev, flow); 6434 rte_free(translated_actions); 6435 tunnel = flow_tunnel_from_rule(wks->flows); 6436 if (tunnel) { 6437 flow->tunnel = 1; 6438 flow->tunnel_id = tunnel->tunnel_id; 6439 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6440 mlx5_free(default_miss_ctx.queue); 6441 } 6442 mlx5_flow_pop_thread_workspace(); 6443 return idx; 6444 error: 6445 MLX5_ASSERT(flow); 6446 ret = rte_errno; /* Save rte_errno before cleanup. */ 6447 flow_mreg_del_copy_action(dev, flow); 6448 flow_drv_destroy(dev, flow); 6449 if (rss_desc->shared_rss) 6450 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6451 mlx5_ipool_get 6452 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6453 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6454 mlx5_ipool_free(priv->flows[type], idx); 6455 rte_errno = ret; /* Restore rte_errno. */ 6456 ret = rte_errno; 6457 rte_errno = ret; 6458 mlx5_flow_pop_thread_workspace(); 6459 error_before_hairpin_split: 6460 rte_free(translated_actions); 6461 return 0; 6462 } 6463 6464 /** 6465 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6466 * incoming packets to table 1. 6467 * 6468 * Other flow rules, requested for group n, will be created in 6469 * e-switch table n+1. 6470 * Jump action to e-switch group n will be created to group n+1. 6471 * 6472 * Used when working in switchdev mode, to utilise advantages of table 1 6473 * and above. 6474 * 6475 * @param dev 6476 * Pointer to Ethernet device. 6477 * 6478 * @return 6479 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6480 */ 6481 struct rte_flow * 6482 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6483 { 6484 const struct rte_flow_attr attr = { 6485 .group = 0, 6486 .priority = 0, 6487 .ingress = 1, 6488 .egress = 0, 6489 .transfer = 1, 6490 }; 6491 const struct rte_flow_item pattern = { 6492 .type = RTE_FLOW_ITEM_TYPE_END, 6493 }; 6494 struct rte_flow_action_jump jump = { 6495 .group = 1, 6496 }; 6497 const struct rte_flow_action actions[] = { 6498 { 6499 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6500 .conf = &jump, 6501 }, 6502 { 6503 .type = RTE_FLOW_ACTION_TYPE_END, 6504 }, 6505 }; 6506 struct rte_flow_error error; 6507 6508 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6509 &attr, &pattern, 6510 actions, false, &error); 6511 } 6512 6513 /** 6514 * Validate a flow supported by the NIC. 6515 * 6516 * @see rte_flow_validate() 6517 * @see rte_flow_ops 6518 */ 6519 int 6520 mlx5_flow_validate(struct rte_eth_dev *dev, 6521 const struct rte_flow_attr *attr, 6522 const struct rte_flow_item items[], 6523 const struct rte_flow_action original_actions[], 6524 struct rte_flow_error *error) 6525 { 6526 int hairpin_flow; 6527 struct mlx5_translated_action_handle 6528 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6529 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6530 const struct rte_flow_action *actions; 6531 struct rte_flow_action *translated_actions = NULL; 6532 int ret = flow_action_handles_translate(dev, original_actions, 6533 indir_actions, 6534 &indir_actions_n, 6535 &translated_actions, error); 6536 6537 if (ret) 6538 return ret; 6539 actions = translated_actions ? translated_actions : original_actions; 6540 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6541 ret = flow_drv_validate(dev, attr, items, actions, 6542 true, hairpin_flow, error); 6543 rte_free(translated_actions); 6544 return ret; 6545 } 6546 6547 /** 6548 * Create a flow. 6549 * 6550 * @see rte_flow_create() 6551 * @see rte_flow_ops 6552 */ 6553 struct rte_flow * 6554 mlx5_flow_create(struct rte_eth_dev *dev, 6555 const struct rte_flow_attr *attr, 6556 const struct rte_flow_item items[], 6557 const struct rte_flow_action actions[], 6558 struct rte_flow_error *error) 6559 { 6560 /* 6561 * If the device is not started yet, it is not allowed to created a 6562 * flow from application. PMD default flows and traffic control flows 6563 * are not affected. 6564 */ 6565 if (unlikely(!dev->data->dev_started)) { 6566 DRV_LOG(DEBUG, "port %u is not started when " 6567 "inserting a flow", dev->data->port_id); 6568 rte_flow_error_set(error, ENODEV, 6569 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6570 NULL, 6571 "port not started"); 6572 return NULL; 6573 } 6574 6575 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6576 attr, items, actions, 6577 true, error); 6578 } 6579 6580 /** 6581 * Destroy a flow in a list. 6582 * 6583 * @param dev 6584 * Pointer to Ethernet device. 6585 * @param[in] flow_idx 6586 * Index of flow to destroy. 6587 */ 6588 static void 6589 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6590 uint32_t flow_idx) 6591 { 6592 struct mlx5_priv *priv = dev->data->dev_private; 6593 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6594 6595 if (!flow) 6596 return; 6597 MLX5_ASSERT(flow->type == type); 6598 /* 6599 * Update RX queue flags only if port is started, otherwise it is 6600 * already clean. 6601 */ 6602 if (dev->data->dev_started) 6603 flow_rxq_flags_trim(dev, flow); 6604 flow_drv_destroy(dev, flow); 6605 if (flow->tunnel) { 6606 struct mlx5_flow_tunnel *tunnel; 6607 6608 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6609 RTE_VERIFY(tunnel); 6610 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6611 mlx5_flow_tunnel_free(dev, tunnel); 6612 } 6613 flow_mreg_del_copy_action(dev, flow); 6614 mlx5_ipool_free(priv->flows[type], flow_idx); 6615 } 6616 6617 /** 6618 * Destroy all flows. 6619 * 6620 * @param dev 6621 * Pointer to Ethernet device. 6622 * @param type 6623 * Flow type to be flushed. 6624 * @param active 6625 * If flushing is called avtively. 6626 */ 6627 void 6628 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6629 bool active) 6630 { 6631 struct mlx5_priv *priv = dev->data->dev_private; 6632 uint32_t num_flushed = 0, fidx = 1; 6633 struct rte_flow *flow; 6634 6635 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6636 flow_list_destroy(dev, type, fidx); 6637 num_flushed++; 6638 } 6639 if (active) { 6640 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6641 dev->data->port_id, num_flushed); 6642 } 6643 } 6644 6645 /** 6646 * Stop all default actions for flows. 6647 * 6648 * @param dev 6649 * Pointer to Ethernet device. 6650 */ 6651 void 6652 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6653 { 6654 flow_mreg_del_default_copy_action(dev); 6655 flow_rxq_flags_clear(dev); 6656 } 6657 6658 /** 6659 * Start all default actions for flows. 6660 * 6661 * @param dev 6662 * Pointer to Ethernet device. 6663 * @return 6664 * 0 on success, a negative errno value otherwise and rte_errno is set. 6665 */ 6666 int 6667 mlx5_flow_start_default(struct rte_eth_dev *dev) 6668 { 6669 struct rte_flow_error error; 6670 6671 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6672 return flow_mreg_add_default_copy_action(dev, &error); 6673 } 6674 6675 /** 6676 * Release key of thread specific flow workspace data. 6677 */ 6678 void 6679 flow_release_workspace(void *data) 6680 { 6681 struct mlx5_flow_workspace *wks = data; 6682 struct mlx5_flow_workspace *next; 6683 6684 while (wks) { 6685 next = wks->next; 6686 free(wks->rss_desc.queue); 6687 free(wks); 6688 wks = next; 6689 } 6690 } 6691 6692 /** 6693 * Get thread specific current flow workspace. 6694 * 6695 * @return pointer to thread specific flow workspace data, NULL on error. 6696 */ 6697 struct mlx5_flow_workspace* 6698 mlx5_flow_get_thread_workspace(void) 6699 { 6700 struct mlx5_flow_workspace *data; 6701 6702 data = mlx5_flow_os_get_specific_workspace(); 6703 MLX5_ASSERT(data && data->inuse); 6704 if (!data || !data->inuse) 6705 DRV_LOG(ERR, "flow workspace not initialized."); 6706 return data; 6707 } 6708 6709 /** 6710 * Allocate and init new flow workspace. 6711 * 6712 * @return pointer to flow workspace data, NULL on error. 6713 */ 6714 static struct mlx5_flow_workspace* 6715 flow_alloc_thread_workspace(void) 6716 { 6717 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6718 6719 if (!data) { 6720 DRV_LOG(ERR, "Failed to allocate flow workspace " 6721 "memory."); 6722 return NULL; 6723 } 6724 data->rss_desc.queue = calloc(1, 6725 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6726 if (!data->rss_desc.queue) 6727 goto err; 6728 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6729 return data; 6730 err: 6731 if (data->rss_desc.queue) 6732 free(data->rss_desc.queue); 6733 free(data); 6734 return NULL; 6735 } 6736 6737 /** 6738 * Get new thread specific flow workspace. 6739 * 6740 * If current workspace inuse, create new one and set as current. 6741 * 6742 * @return pointer to thread specific flow workspace data, NULL on error. 6743 */ 6744 static struct mlx5_flow_workspace* 6745 mlx5_flow_push_thread_workspace(void) 6746 { 6747 struct mlx5_flow_workspace *curr; 6748 struct mlx5_flow_workspace *data; 6749 6750 curr = mlx5_flow_os_get_specific_workspace(); 6751 if (!curr) { 6752 data = flow_alloc_thread_workspace(); 6753 if (!data) 6754 return NULL; 6755 } else if (!curr->inuse) { 6756 data = curr; 6757 } else if (curr->next) { 6758 data = curr->next; 6759 } else { 6760 data = flow_alloc_thread_workspace(); 6761 if (!data) 6762 return NULL; 6763 curr->next = data; 6764 data->prev = curr; 6765 } 6766 data->inuse = 1; 6767 data->flow_idx = 0; 6768 /* Set as current workspace */ 6769 if (mlx5_flow_os_set_specific_workspace(data)) 6770 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6771 return data; 6772 } 6773 6774 /** 6775 * Close current thread specific flow workspace. 6776 * 6777 * If previous workspace available, set it as current. 6778 * 6779 * @return pointer to thread specific flow workspace data, NULL on error. 6780 */ 6781 static void 6782 mlx5_flow_pop_thread_workspace(void) 6783 { 6784 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6785 6786 if (!data) 6787 return; 6788 if (!data->inuse) { 6789 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6790 return; 6791 } 6792 data->inuse = 0; 6793 if (!data->prev) 6794 return; 6795 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6796 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6797 } 6798 6799 /** 6800 * Verify the flow list is empty 6801 * 6802 * @param dev 6803 * Pointer to Ethernet device. 6804 * 6805 * @return the number of flows not released. 6806 */ 6807 int 6808 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 6809 { 6810 struct mlx5_priv *priv = dev->data->dev_private; 6811 struct rte_flow *flow; 6812 uint32_t idx = 0; 6813 int ret = 0, i; 6814 6815 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 6816 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 6817 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6818 dev->data->port_id, (void *)flow); 6819 ret++; 6820 } 6821 } 6822 return ret; 6823 } 6824 6825 /** 6826 * Enable default hairpin egress flow. 6827 * 6828 * @param dev 6829 * Pointer to Ethernet device. 6830 * @param queue 6831 * The queue index. 6832 * 6833 * @return 6834 * 0 on success, a negative errno value otherwise and rte_errno is set. 6835 */ 6836 int 6837 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6838 uint32_t queue) 6839 { 6840 const struct rte_flow_attr attr = { 6841 .egress = 1, 6842 .priority = 0, 6843 }; 6844 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6845 .queue = queue, 6846 }; 6847 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6848 .queue = UINT32_MAX, 6849 }; 6850 struct rte_flow_item items[] = { 6851 { 6852 .type = (enum rte_flow_item_type) 6853 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6854 .spec = &queue_spec, 6855 .last = NULL, 6856 .mask = &queue_mask, 6857 }, 6858 { 6859 .type = RTE_FLOW_ITEM_TYPE_END, 6860 }, 6861 }; 6862 struct rte_flow_action_jump jump = { 6863 .group = MLX5_HAIRPIN_TX_TABLE, 6864 }; 6865 struct rte_flow_action actions[2]; 6866 uint32_t flow_idx; 6867 struct rte_flow_error error; 6868 6869 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6870 actions[0].conf = &jump; 6871 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6872 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6873 &attr, items, actions, false, &error); 6874 if (!flow_idx) { 6875 DRV_LOG(DEBUG, 6876 "Failed to create ctrl flow: rte_errno(%d)," 6877 " type(%d), message(%s)", 6878 rte_errno, error.type, 6879 error.message ? error.message : " (no stated reason)"); 6880 return -rte_errno; 6881 } 6882 return 0; 6883 } 6884 6885 /** 6886 * Enable a control flow configured from the control plane. 6887 * 6888 * @param dev 6889 * Pointer to Ethernet device. 6890 * @param eth_spec 6891 * An Ethernet flow spec to apply. 6892 * @param eth_mask 6893 * An Ethernet flow mask to apply. 6894 * @param vlan_spec 6895 * A VLAN flow spec to apply. 6896 * @param vlan_mask 6897 * A VLAN flow mask to apply. 6898 * 6899 * @return 6900 * 0 on success, a negative errno value otherwise and rte_errno is set. 6901 */ 6902 int 6903 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6904 struct rte_flow_item_eth *eth_spec, 6905 struct rte_flow_item_eth *eth_mask, 6906 struct rte_flow_item_vlan *vlan_spec, 6907 struct rte_flow_item_vlan *vlan_mask) 6908 { 6909 struct mlx5_priv *priv = dev->data->dev_private; 6910 const struct rte_flow_attr attr = { 6911 .ingress = 1, 6912 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6913 }; 6914 struct rte_flow_item items[] = { 6915 { 6916 .type = RTE_FLOW_ITEM_TYPE_ETH, 6917 .spec = eth_spec, 6918 .last = NULL, 6919 .mask = eth_mask, 6920 }, 6921 { 6922 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6923 RTE_FLOW_ITEM_TYPE_END, 6924 .spec = vlan_spec, 6925 .last = NULL, 6926 .mask = vlan_mask, 6927 }, 6928 { 6929 .type = RTE_FLOW_ITEM_TYPE_END, 6930 }, 6931 }; 6932 uint16_t queue[priv->reta_idx_n]; 6933 struct rte_flow_action_rss action_rss = { 6934 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6935 .level = 0, 6936 .types = priv->rss_conf.rss_hf, 6937 .key_len = priv->rss_conf.rss_key_len, 6938 .queue_num = priv->reta_idx_n, 6939 .key = priv->rss_conf.rss_key, 6940 .queue = queue, 6941 }; 6942 struct rte_flow_action actions[] = { 6943 { 6944 .type = RTE_FLOW_ACTION_TYPE_RSS, 6945 .conf = &action_rss, 6946 }, 6947 { 6948 .type = RTE_FLOW_ACTION_TYPE_END, 6949 }, 6950 }; 6951 uint32_t flow_idx; 6952 struct rte_flow_error error; 6953 unsigned int i; 6954 6955 if (!priv->reta_idx_n || !priv->rxqs_n) { 6956 return 0; 6957 } 6958 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6959 action_rss.types = 0; 6960 for (i = 0; i != priv->reta_idx_n; ++i) 6961 queue[i] = (*priv->reta_idx)[i]; 6962 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6963 &attr, items, actions, false, &error); 6964 if (!flow_idx) 6965 return -rte_errno; 6966 return 0; 6967 } 6968 6969 /** 6970 * Enable a flow control configured from the control plane. 6971 * 6972 * @param dev 6973 * Pointer to Ethernet device. 6974 * @param eth_spec 6975 * An Ethernet flow spec to apply. 6976 * @param eth_mask 6977 * An Ethernet flow mask to apply. 6978 * 6979 * @return 6980 * 0 on success, a negative errno value otherwise and rte_errno is set. 6981 */ 6982 int 6983 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6984 struct rte_flow_item_eth *eth_spec, 6985 struct rte_flow_item_eth *eth_mask) 6986 { 6987 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6988 } 6989 6990 /** 6991 * Create default miss flow rule matching lacp traffic 6992 * 6993 * @param dev 6994 * Pointer to Ethernet device. 6995 * @param eth_spec 6996 * An Ethernet flow spec to apply. 6997 * 6998 * @return 6999 * 0 on success, a negative errno value otherwise and rte_errno is set. 7000 */ 7001 int 7002 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7003 { 7004 /* 7005 * The LACP matching is done by only using ether type since using 7006 * a multicast dst mac causes kernel to give low priority to this flow. 7007 */ 7008 static const struct rte_flow_item_eth lacp_spec = { 7009 .type = RTE_BE16(0x8809), 7010 }; 7011 static const struct rte_flow_item_eth lacp_mask = { 7012 .type = 0xffff, 7013 }; 7014 const struct rte_flow_attr attr = { 7015 .ingress = 1, 7016 }; 7017 struct rte_flow_item items[] = { 7018 { 7019 .type = RTE_FLOW_ITEM_TYPE_ETH, 7020 .spec = &lacp_spec, 7021 .mask = &lacp_mask, 7022 }, 7023 { 7024 .type = RTE_FLOW_ITEM_TYPE_END, 7025 }, 7026 }; 7027 struct rte_flow_action actions[] = { 7028 { 7029 .type = (enum rte_flow_action_type) 7030 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7031 }, 7032 { 7033 .type = RTE_FLOW_ACTION_TYPE_END, 7034 }, 7035 }; 7036 struct rte_flow_error error; 7037 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7038 &attr, items, actions, 7039 false, &error); 7040 7041 if (!flow_idx) 7042 return -rte_errno; 7043 return 0; 7044 } 7045 7046 /** 7047 * Destroy a flow. 7048 * 7049 * @see rte_flow_destroy() 7050 * @see rte_flow_ops 7051 */ 7052 int 7053 mlx5_flow_destroy(struct rte_eth_dev *dev, 7054 struct rte_flow *flow, 7055 struct rte_flow_error *error __rte_unused) 7056 { 7057 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7058 (uintptr_t)(void *)flow); 7059 return 0; 7060 } 7061 7062 /** 7063 * Destroy all flows. 7064 * 7065 * @see rte_flow_flush() 7066 * @see rte_flow_ops 7067 */ 7068 int 7069 mlx5_flow_flush(struct rte_eth_dev *dev, 7070 struct rte_flow_error *error __rte_unused) 7071 { 7072 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7073 return 0; 7074 } 7075 7076 /** 7077 * Isolated mode. 7078 * 7079 * @see rte_flow_isolate() 7080 * @see rte_flow_ops 7081 */ 7082 int 7083 mlx5_flow_isolate(struct rte_eth_dev *dev, 7084 int enable, 7085 struct rte_flow_error *error) 7086 { 7087 struct mlx5_priv *priv = dev->data->dev_private; 7088 7089 if (dev->data->dev_started) { 7090 rte_flow_error_set(error, EBUSY, 7091 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7092 NULL, 7093 "port must be stopped first"); 7094 return -rte_errno; 7095 } 7096 priv->isolated = !!enable; 7097 if (enable) 7098 dev->dev_ops = &mlx5_dev_ops_isolate; 7099 else 7100 dev->dev_ops = &mlx5_dev_ops; 7101 7102 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7103 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7104 7105 return 0; 7106 } 7107 7108 /** 7109 * Query a flow. 7110 * 7111 * @see rte_flow_query() 7112 * @see rte_flow_ops 7113 */ 7114 static int 7115 flow_drv_query(struct rte_eth_dev *dev, 7116 uint32_t flow_idx, 7117 const struct rte_flow_action *actions, 7118 void *data, 7119 struct rte_flow_error *error) 7120 { 7121 struct mlx5_priv *priv = dev->data->dev_private; 7122 const struct mlx5_flow_driver_ops *fops; 7123 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7124 flow_idx); 7125 enum mlx5_flow_drv_type ftype; 7126 7127 if (!flow) { 7128 return rte_flow_error_set(error, ENOENT, 7129 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7130 NULL, 7131 "invalid flow handle"); 7132 } 7133 ftype = flow->drv_type; 7134 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7135 fops = flow_get_drv_ops(ftype); 7136 7137 return fops->query(dev, flow, actions, data, error); 7138 } 7139 7140 /** 7141 * Query a flow. 7142 * 7143 * @see rte_flow_query() 7144 * @see rte_flow_ops 7145 */ 7146 int 7147 mlx5_flow_query(struct rte_eth_dev *dev, 7148 struct rte_flow *flow, 7149 const struct rte_flow_action *actions, 7150 void *data, 7151 struct rte_flow_error *error) 7152 { 7153 int ret; 7154 7155 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7156 error); 7157 if (ret < 0) 7158 return ret; 7159 return 0; 7160 } 7161 7162 /** 7163 * Get rte_flow callbacks. 7164 * 7165 * @param dev 7166 * Pointer to Ethernet device structure. 7167 * @param ops 7168 * Pointer to operation-specific structure. 7169 * 7170 * @return 0 7171 */ 7172 int 7173 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7174 const struct rte_flow_ops **ops) 7175 { 7176 *ops = &mlx5_flow_ops; 7177 return 0; 7178 } 7179 7180 /** 7181 * Validate meter policy actions. 7182 * Dispatcher for action type specific validation. 7183 * 7184 * @param[in] dev 7185 * Pointer to the Ethernet device structure. 7186 * @param[in] action 7187 * The meter policy action object to validate. 7188 * @param[in] attr 7189 * Attributes of flow to determine steering domain. 7190 * @param[out] is_rss 7191 * Is RSS or not. 7192 * @param[out] domain_bitmap 7193 * Domain bitmap. 7194 * @param[out] is_def_policy 7195 * Is default policy or not. 7196 * @param[out] error 7197 * Perform verbose error reporting if not NULL. Initialized in case of 7198 * error only. 7199 * 7200 * @return 7201 * 0 on success, otherwise negative errno value. 7202 */ 7203 int 7204 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7205 const struct rte_flow_action *actions[RTE_COLORS], 7206 struct rte_flow_attr *attr, 7207 bool *is_rss, 7208 uint8_t *domain_bitmap, 7209 uint8_t *policy_mode, 7210 struct rte_mtr_error *error) 7211 { 7212 const struct mlx5_flow_driver_ops *fops; 7213 7214 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7215 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7216 domain_bitmap, policy_mode, error); 7217 } 7218 7219 /** 7220 * Destroy the meter table set. 7221 * 7222 * @param[in] dev 7223 * Pointer to Ethernet device. 7224 * @param[in] mtr_policy 7225 * Meter policy struct. 7226 */ 7227 void 7228 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7229 struct mlx5_flow_meter_policy *mtr_policy) 7230 { 7231 const struct mlx5_flow_driver_ops *fops; 7232 7233 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7234 fops->destroy_mtr_acts(dev, mtr_policy); 7235 } 7236 7237 /** 7238 * Create policy action, lock free, 7239 * (mutex should be acquired by caller). 7240 * Dispatcher for action type specific call. 7241 * 7242 * @param[in] dev 7243 * Pointer to the Ethernet device structure. 7244 * @param[in] mtr_policy 7245 * Meter policy struct. 7246 * @param[in] action 7247 * Action specification used to create meter actions. 7248 * @param[out] error 7249 * Perform verbose error reporting if not NULL. Initialized in case of 7250 * error only. 7251 * 7252 * @return 7253 * 0 on success, otherwise negative errno value. 7254 */ 7255 int 7256 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7257 struct mlx5_flow_meter_policy *mtr_policy, 7258 const struct rte_flow_action *actions[RTE_COLORS], 7259 struct rte_mtr_error *error) 7260 { 7261 const struct mlx5_flow_driver_ops *fops; 7262 7263 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7264 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7265 } 7266 7267 /** 7268 * Create policy rules, lock free, 7269 * (mutex should be acquired by caller). 7270 * Dispatcher for action type specific call. 7271 * 7272 * @param[in] dev 7273 * Pointer to the Ethernet device structure. 7274 * @param[in] mtr_policy 7275 * Meter policy struct. 7276 * 7277 * @return 7278 * 0 on success, -1 otherwise. 7279 */ 7280 int 7281 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7282 struct mlx5_flow_meter_policy *mtr_policy) 7283 { 7284 const struct mlx5_flow_driver_ops *fops; 7285 7286 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7287 return fops->create_policy_rules(dev, mtr_policy); 7288 } 7289 7290 /** 7291 * Destroy policy rules, lock free, 7292 * (mutex should be acquired by caller). 7293 * Dispatcher for action type specific call. 7294 * 7295 * @param[in] dev 7296 * Pointer to the Ethernet device structure. 7297 * @param[in] mtr_policy 7298 * Meter policy struct. 7299 */ 7300 void 7301 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7302 struct mlx5_flow_meter_policy *mtr_policy) 7303 { 7304 const struct mlx5_flow_driver_ops *fops; 7305 7306 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7307 fops->destroy_policy_rules(dev, mtr_policy); 7308 } 7309 7310 /** 7311 * Destroy the default policy table set. 7312 * 7313 * @param[in] dev 7314 * Pointer to Ethernet device. 7315 */ 7316 void 7317 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7318 { 7319 const struct mlx5_flow_driver_ops *fops; 7320 7321 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7322 fops->destroy_def_policy(dev); 7323 } 7324 7325 /** 7326 * Destroy the default policy table set. 7327 * 7328 * @param[in] dev 7329 * Pointer to Ethernet device. 7330 * 7331 * @return 7332 * 0 on success, -1 otherwise. 7333 */ 7334 int 7335 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7336 { 7337 const struct mlx5_flow_driver_ops *fops; 7338 7339 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7340 return fops->create_def_policy(dev); 7341 } 7342 7343 /** 7344 * Create the needed meter and suffix tables. 7345 * 7346 * @param[in] dev 7347 * Pointer to Ethernet device. 7348 * 7349 * @return 7350 * 0 on success, -1 otherwise. 7351 */ 7352 int 7353 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7354 struct mlx5_flow_meter_info *fm, 7355 uint32_t mtr_idx, 7356 uint8_t domain_bitmap) 7357 { 7358 const struct mlx5_flow_driver_ops *fops; 7359 7360 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7361 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7362 } 7363 7364 /** 7365 * Destroy the meter table set. 7366 * 7367 * @param[in] dev 7368 * Pointer to Ethernet device. 7369 * @param[in] tbl 7370 * Pointer to the meter table set. 7371 */ 7372 void 7373 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7374 struct mlx5_flow_meter_info *fm) 7375 { 7376 const struct mlx5_flow_driver_ops *fops; 7377 7378 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7379 fops->destroy_mtr_tbls(dev, fm); 7380 } 7381 7382 /** 7383 * Destroy the global meter drop table. 7384 * 7385 * @param[in] dev 7386 * Pointer to Ethernet device. 7387 */ 7388 void 7389 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7390 { 7391 const struct mlx5_flow_driver_ops *fops; 7392 7393 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7394 fops->destroy_mtr_drop_tbls(dev); 7395 } 7396 7397 /** 7398 * Destroy the sub policy table with RX queue. 7399 * 7400 * @param[in] dev 7401 * Pointer to Ethernet device. 7402 * @param[in] mtr_policy 7403 * Pointer to meter policy table. 7404 */ 7405 void 7406 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7407 struct mlx5_flow_meter_policy *mtr_policy) 7408 { 7409 const struct mlx5_flow_driver_ops *fops; 7410 7411 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7412 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7413 } 7414 7415 /** 7416 * Allocate the needed aso flow meter id. 7417 * 7418 * @param[in] dev 7419 * Pointer to Ethernet device. 7420 * 7421 * @return 7422 * Index to aso flow meter on success, NULL otherwise. 7423 */ 7424 uint32_t 7425 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7426 { 7427 const struct mlx5_flow_driver_ops *fops; 7428 7429 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7430 return fops->create_meter(dev); 7431 } 7432 7433 /** 7434 * Free the aso flow meter id. 7435 * 7436 * @param[in] dev 7437 * Pointer to Ethernet device. 7438 * @param[in] mtr_idx 7439 * Index to aso flow meter to be free. 7440 * 7441 * @return 7442 * 0 on success. 7443 */ 7444 void 7445 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7446 { 7447 const struct mlx5_flow_driver_ops *fops; 7448 7449 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7450 fops->free_meter(dev, mtr_idx); 7451 } 7452 7453 /** 7454 * Allocate a counter. 7455 * 7456 * @param[in] dev 7457 * Pointer to Ethernet device structure. 7458 * 7459 * @return 7460 * Index to allocated counter on success, 0 otherwise. 7461 */ 7462 uint32_t 7463 mlx5_counter_alloc(struct rte_eth_dev *dev) 7464 { 7465 const struct mlx5_flow_driver_ops *fops; 7466 struct rte_flow_attr attr = { .transfer = 0 }; 7467 7468 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7469 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7470 return fops->counter_alloc(dev); 7471 } 7472 DRV_LOG(ERR, 7473 "port %u counter allocate is not supported.", 7474 dev->data->port_id); 7475 return 0; 7476 } 7477 7478 /** 7479 * Free a counter. 7480 * 7481 * @param[in] dev 7482 * Pointer to Ethernet device structure. 7483 * @param[in] cnt 7484 * Index to counter to be free. 7485 */ 7486 void 7487 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7488 { 7489 const struct mlx5_flow_driver_ops *fops; 7490 struct rte_flow_attr attr = { .transfer = 0 }; 7491 7492 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7493 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7494 fops->counter_free(dev, cnt); 7495 return; 7496 } 7497 DRV_LOG(ERR, 7498 "port %u counter free is not supported.", 7499 dev->data->port_id); 7500 } 7501 7502 /** 7503 * Query counter statistics. 7504 * 7505 * @param[in] dev 7506 * Pointer to Ethernet device structure. 7507 * @param[in] cnt 7508 * Index to counter to query. 7509 * @param[in] clear 7510 * Set to clear counter statistics. 7511 * @param[out] pkts 7512 * The counter hits packets number to save. 7513 * @param[out] bytes 7514 * The counter hits bytes number to save. 7515 * 7516 * @return 7517 * 0 on success, a negative errno value otherwise. 7518 */ 7519 int 7520 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7521 bool clear, uint64_t *pkts, uint64_t *bytes) 7522 { 7523 const struct mlx5_flow_driver_ops *fops; 7524 struct rte_flow_attr attr = { .transfer = 0 }; 7525 7526 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7527 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7528 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7529 } 7530 DRV_LOG(ERR, 7531 "port %u counter query is not supported.", 7532 dev->data->port_id); 7533 return -ENOTSUP; 7534 } 7535 7536 /** 7537 * Allocate a new memory for the counter values wrapped by all the needed 7538 * management. 7539 * 7540 * @param[in] sh 7541 * Pointer to mlx5_dev_ctx_shared object. 7542 * 7543 * @return 7544 * 0 on success, a negative errno value otherwise. 7545 */ 7546 static int 7547 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7548 { 7549 struct mlx5_devx_mkey_attr mkey_attr; 7550 struct mlx5_counter_stats_mem_mng *mem_mng; 7551 volatile struct flow_counter_stats *raw_data; 7552 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7553 int size = (sizeof(struct flow_counter_stats) * 7554 MLX5_COUNTERS_PER_POOL + 7555 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7556 sizeof(struct mlx5_counter_stats_mem_mng); 7557 size_t pgsize = rte_mem_page_size(); 7558 uint8_t *mem; 7559 int i; 7560 7561 if (pgsize == (size_t)-1) { 7562 DRV_LOG(ERR, "Failed to get mem page size"); 7563 rte_errno = ENOMEM; 7564 return -ENOMEM; 7565 } 7566 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7567 if (!mem) { 7568 rte_errno = ENOMEM; 7569 return -ENOMEM; 7570 } 7571 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7572 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7573 mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size, 7574 IBV_ACCESS_LOCAL_WRITE); 7575 if (!mem_mng->umem) { 7576 rte_errno = errno; 7577 mlx5_free(mem); 7578 return -rte_errno; 7579 } 7580 memset(&mkey_attr, 0, sizeof(mkey_attr)); 7581 mkey_attr.addr = (uintptr_t)mem; 7582 mkey_attr.size = size; 7583 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7584 mkey_attr.pd = sh->pdn; 7585 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 7586 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 7587 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7588 if (!mem_mng->dm) { 7589 mlx5_os_umem_dereg(mem_mng->umem); 7590 rte_errno = errno; 7591 mlx5_free(mem); 7592 return -rte_errno; 7593 } 7594 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7595 raw_data = (volatile struct flow_counter_stats *)mem; 7596 for (i = 0; i < raws_n; ++i) { 7597 mem_mng->raws[i].mem_mng = mem_mng; 7598 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7599 } 7600 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7601 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7602 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7603 next); 7604 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7605 sh->cmng.mem_mng = mem_mng; 7606 return 0; 7607 } 7608 7609 /** 7610 * Set the statistic memory to the new counter pool. 7611 * 7612 * @param[in] sh 7613 * Pointer to mlx5_dev_ctx_shared object. 7614 * @param[in] pool 7615 * Pointer to the pool to set the statistic memory. 7616 * 7617 * @return 7618 * 0 on success, a negative errno value otherwise. 7619 */ 7620 static int 7621 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7622 struct mlx5_flow_counter_pool *pool) 7623 { 7624 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7625 /* Resize statistic memory once used out. */ 7626 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7627 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7628 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7629 return -1; 7630 } 7631 rte_spinlock_lock(&pool->sl); 7632 pool->raw = cmng->mem_mng->raws + pool->index % 7633 MLX5_CNT_CONTAINER_RESIZE; 7634 rte_spinlock_unlock(&pool->sl); 7635 pool->raw_hw = NULL; 7636 return 0; 7637 } 7638 7639 #define MLX5_POOL_QUERY_FREQ_US 1000000 7640 7641 /** 7642 * Set the periodic procedure for triggering asynchronous batch queries for all 7643 * the counter pools. 7644 * 7645 * @param[in] sh 7646 * Pointer to mlx5_dev_ctx_shared object. 7647 */ 7648 void 7649 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7650 { 7651 uint32_t pools_n, us; 7652 7653 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7654 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7655 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7656 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7657 sh->cmng.query_thread_on = 0; 7658 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7659 } else { 7660 sh->cmng.query_thread_on = 1; 7661 } 7662 } 7663 7664 /** 7665 * The periodic procedure for triggering asynchronous batch queries for all the 7666 * counter pools. This function is probably called by the host thread. 7667 * 7668 * @param[in] arg 7669 * The parameter for the alarm process. 7670 */ 7671 void 7672 mlx5_flow_query_alarm(void *arg) 7673 { 7674 struct mlx5_dev_ctx_shared *sh = arg; 7675 int ret; 7676 uint16_t pool_index = sh->cmng.pool_index; 7677 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7678 struct mlx5_flow_counter_pool *pool; 7679 uint16_t n_valid; 7680 7681 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7682 goto set_alarm; 7683 rte_spinlock_lock(&cmng->pool_update_sl); 7684 pool = cmng->pools[pool_index]; 7685 n_valid = cmng->n_valid; 7686 rte_spinlock_unlock(&cmng->pool_update_sl); 7687 /* Set the statistic memory to the new created pool. */ 7688 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7689 goto set_alarm; 7690 if (pool->raw_hw) 7691 /* There is a pool query in progress. */ 7692 goto set_alarm; 7693 pool->raw_hw = 7694 LIST_FIRST(&sh->cmng.free_stat_raws); 7695 if (!pool->raw_hw) 7696 /* No free counter statistics raw memory. */ 7697 goto set_alarm; 7698 /* 7699 * Identify the counters released between query trigger and query 7700 * handle more efficiently. The counter released in this gap period 7701 * should wait for a new round of query as the new arrived packets 7702 * will not be taken into account. 7703 */ 7704 pool->query_gen++; 7705 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7706 MLX5_COUNTERS_PER_POOL, 7707 NULL, NULL, 7708 pool->raw_hw->mem_mng->dm->id, 7709 (void *)(uintptr_t) 7710 pool->raw_hw->data, 7711 sh->devx_comp, 7712 (uint64_t)(uintptr_t)pool); 7713 if (ret) { 7714 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7715 " %d", pool->min_dcs->id); 7716 pool->raw_hw = NULL; 7717 goto set_alarm; 7718 } 7719 LIST_REMOVE(pool->raw_hw, next); 7720 sh->cmng.pending_queries++; 7721 pool_index++; 7722 if (pool_index >= n_valid) 7723 pool_index = 0; 7724 set_alarm: 7725 sh->cmng.pool_index = pool_index; 7726 mlx5_set_query_alarm(sh); 7727 } 7728 7729 /** 7730 * Check and callback event for new aged flow in the counter pool 7731 * 7732 * @param[in] sh 7733 * Pointer to mlx5_dev_ctx_shared object. 7734 * @param[in] pool 7735 * Pointer to Current counter pool. 7736 */ 7737 static void 7738 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7739 struct mlx5_flow_counter_pool *pool) 7740 { 7741 struct mlx5_priv *priv; 7742 struct mlx5_flow_counter *cnt; 7743 struct mlx5_age_info *age_info; 7744 struct mlx5_age_param *age_param; 7745 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7746 struct mlx5_counter_stats_raw *prev = pool->raw; 7747 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7748 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7749 uint16_t expected = AGE_CANDIDATE; 7750 uint32_t i; 7751 7752 pool->time_of_last_age_check = curr_time; 7753 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7754 cnt = MLX5_POOL_GET_CNT(pool, i); 7755 age_param = MLX5_CNT_TO_AGE(cnt); 7756 if (__atomic_load_n(&age_param->state, 7757 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7758 continue; 7759 if (cur->data[i].hits != prev->data[i].hits) { 7760 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7761 __ATOMIC_RELAXED); 7762 continue; 7763 } 7764 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7765 time_delta, 7766 __ATOMIC_RELAXED) <= age_param->timeout) 7767 continue; 7768 /** 7769 * Hold the lock first, or if between the 7770 * state AGE_TMOUT and tailq operation the 7771 * release happened, the release procedure 7772 * may delete a non-existent tailq node. 7773 */ 7774 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7775 age_info = GET_PORT_AGE_INFO(priv); 7776 rte_spinlock_lock(&age_info->aged_sl); 7777 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7778 AGE_TMOUT, false, 7779 __ATOMIC_RELAXED, 7780 __ATOMIC_RELAXED)) { 7781 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7782 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7783 } 7784 rte_spinlock_unlock(&age_info->aged_sl); 7785 } 7786 mlx5_age_event_prepare(sh); 7787 } 7788 7789 /** 7790 * Handler for the HW respond about ready values from an asynchronous batch 7791 * query. This function is probably called by the host thread. 7792 * 7793 * @param[in] sh 7794 * The pointer to the shared device context. 7795 * @param[in] async_id 7796 * The Devx async ID. 7797 * @param[in] status 7798 * The status of the completion. 7799 */ 7800 void 7801 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7802 uint64_t async_id, int status) 7803 { 7804 struct mlx5_flow_counter_pool *pool = 7805 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7806 struct mlx5_counter_stats_raw *raw_to_free; 7807 uint8_t query_gen = pool->query_gen ^ 1; 7808 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7809 enum mlx5_counter_type cnt_type = 7810 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7811 MLX5_COUNTER_TYPE_ORIGIN; 7812 7813 if (unlikely(status)) { 7814 raw_to_free = pool->raw_hw; 7815 } else { 7816 raw_to_free = pool->raw; 7817 if (pool->is_aged) 7818 mlx5_flow_aging_check(sh, pool); 7819 rte_spinlock_lock(&pool->sl); 7820 pool->raw = pool->raw_hw; 7821 rte_spinlock_unlock(&pool->sl); 7822 /* Be sure the new raw counters data is updated in memory. */ 7823 rte_io_wmb(); 7824 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7825 rte_spinlock_lock(&cmng->csl[cnt_type]); 7826 TAILQ_CONCAT(&cmng->counters[cnt_type], 7827 &pool->counters[query_gen], next); 7828 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7829 } 7830 } 7831 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7832 pool->raw_hw = NULL; 7833 sh->cmng.pending_queries--; 7834 } 7835 7836 static int 7837 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7838 const struct flow_grp_info *grp_info, 7839 struct rte_flow_error *error) 7840 { 7841 if (grp_info->transfer && grp_info->external && 7842 grp_info->fdb_def_rule) { 7843 if (group == UINT32_MAX) 7844 return rte_flow_error_set 7845 (error, EINVAL, 7846 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7847 NULL, 7848 "group index not supported"); 7849 *table = group + 1; 7850 } else { 7851 *table = group; 7852 } 7853 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7854 return 0; 7855 } 7856 7857 /** 7858 * Translate the rte_flow group index to HW table value. 7859 * 7860 * If tunnel offload is disabled, all group ids converted to flow table 7861 * id using the standard method. 7862 * If tunnel offload is enabled, group id can be converted using the 7863 * standard or tunnel conversion method. Group conversion method 7864 * selection depends on flags in `grp_info` parameter: 7865 * - Internal (grp_info.external == 0) groups conversion uses the 7866 * standard method. 7867 * - Group ids in JUMP action converted with the tunnel conversion. 7868 * - Group id in rule attribute conversion depends on a rule type and 7869 * group id value: 7870 * ** non zero group attributes converted with the tunnel method 7871 * ** zero group attribute in non-tunnel rule is converted using the 7872 * standard method - there's only one root table 7873 * ** zero group attribute in steer tunnel rule is converted with the 7874 * standard method - single root table 7875 * ** zero group attribute in match tunnel rule is a special OvS 7876 * case: that value is used for portability reasons. That group 7877 * id is converted with the tunnel conversion method. 7878 * 7879 * @param[in] dev 7880 * Port device 7881 * @param[in] tunnel 7882 * PMD tunnel offload object 7883 * @param[in] group 7884 * rte_flow group index value. 7885 * @param[out] table 7886 * HW table value. 7887 * @param[in] grp_info 7888 * flags used for conversion 7889 * @param[out] error 7890 * Pointer to error structure. 7891 * 7892 * @return 7893 * 0 on success, a negative errno value otherwise and rte_errno is set. 7894 */ 7895 int 7896 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7897 const struct mlx5_flow_tunnel *tunnel, 7898 uint32_t group, uint32_t *table, 7899 const struct flow_grp_info *grp_info, 7900 struct rte_flow_error *error) 7901 { 7902 int ret; 7903 bool standard_translation; 7904 7905 if (!grp_info->skip_scale && grp_info->external && 7906 group < MLX5_MAX_TABLES_EXTERNAL) 7907 group *= MLX5_FLOW_TABLE_FACTOR; 7908 if (is_tunnel_offload_active(dev)) { 7909 standard_translation = !grp_info->external || 7910 grp_info->std_tbl_fix; 7911 } else { 7912 standard_translation = true; 7913 } 7914 DRV_LOG(DEBUG, 7915 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 7916 dev->data->port_id, group, grp_info->transfer, 7917 grp_info->external, grp_info->fdb_def_rule, 7918 standard_translation ? "STANDARD" : "TUNNEL"); 7919 if (standard_translation) 7920 ret = flow_group_to_table(dev->data->port_id, group, table, 7921 grp_info, error); 7922 else 7923 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7924 table, error); 7925 7926 return ret; 7927 } 7928 7929 /** 7930 * Discover availability of metadata reg_c's. 7931 * 7932 * Iteratively use test flows to check availability. 7933 * 7934 * @param[in] dev 7935 * Pointer to the Ethernet device structure. 7936 * 7937 * @return 7938 * 0 on success, a negative errno value otherwise and rte_errno is set. 7939 */ 7940 int 7941 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7942 { 7943 struct mlx5_priv *priv = dev->data->dev_private; 7944 struct mlx5_dev_config *config = &priv->config; 7945 enum modify_reg idx; 7946 int n = 0; 7947 7948 /* reg_c[0] and reg_c[1] are reserved. */ 7949 config->flow_mreg_c[n++] = REG_C_0; 7950 config->flow_mreg_c[n++] = REG_C_1; 7951 /* Discover availability of other reg_c's. */ 7952 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7953 struct rte_flow_attr attr = { 7954 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7955 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7956 .ingress = 1, 7957 }; 7958 struct rte_flow_item items[] = { 7959 [0] = { 7960 .type = RTE_FLOW_ITEM_TYPE_END, 7961 }, 7962 }; 7963 struct rte_flow_action actions[] = { 7964 [0] = { 7965 .type = (enum rte_flow_action_type) 7966 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7967 .conf = &(struct mlx5_flow_action_copy_mreg){ 7968 .src = REG_C_1, 7969 .dst = idx, 7970 }, 7971 }, 7972 [1] = { 7973 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7974 .conf = &(struct rte_flow_action_jump){ 7975 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7976 }, 7977 }, 7978 [2] = { 7979 .type = RTE_FLOW_ACTION_TYPE_END, 7980 }, 7981 }; 7982 uint32_t flow_idx; 7983 struct rte_flow *flow; 7984 struct rte_flow_error error; 7985 7986 if (!config->dv_flow_en) 7987 break; 7988 /* Create internal flow, validation skips copy action. */ 7989 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 7990 items, actions, false, &error); 7991 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7992 flow_idx); 7993 if (!flow) 7994 continue; 7995 config->flow_mreg_c[n++] = idx; 7996 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 7997 } 7998 for (; n < MLX5_MREG_C_NUM; ++n) 7999 config->flow_mreg_c[n] = REG_NON; 8000 return 0; 8001 } 8002 8003 int 8004 save_dump_file(const uint8_t *data, uint32_t size, 8005 uint32_t type, uint32_t id, void *arg, FILE *file) 8006 { 8007 char line[BUF_SIZE]; 8008 uint32_t out = 0; 8009 uint32_t k; 8010 uint32_t actions_num; 8011 struct rte_flow_query_count *count; 8012 8013 memset(line, 0, BUF_SIZE); 8014 switch (type) { 8015 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8016 actions_num = *(uint32_t *)(arg); 8017 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,", 8018 type, id, actions_num); 8019 break; 8020 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8021 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,", 8022 type, id); 8023 break; 8024 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8025 count = (struct rte_flow_query_count *)arg; 8026 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type, 8027 id, count->hits, count->bytes); 8028 return 0; 8029 default: 8030 return -1; 8031 } 8032 8033 for (k = 0; k < size; k++) { 8034 /* Make sure we do not overrun the line buffer length. */ 8035 if (out >= BUF_SIZE - 4) { 8036 line[out] = '\0'; 8037 break; 8038 } 8039 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8040 (data[k]) & 0xff); 8041 } 8042 fprintf(file, "%s\n", line); 8043 return 0; 8044 } 8045 8046 int 8047 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8048 struct rte_flow_query_count *count, struct rte_flow_error *error) 8049 { 8050 struct rte_flow_action action[2]; 8051 enum mlx5_flow_drv_type ftype; 8052 const struct mlx5_flow_driver_ops *fops; 8053 8054 if (!flow) { 8055 return rte_flow_error_set(error, ENOENT, 8056 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8057 NULL, 8058 "invalid flow handle"); 8059 } 8060 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8061 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8062 if (flow->counter) { 8063 memset(count, 0, sizeof(struct rte_flow_query_count)); 8064 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8065 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8066 ftype < MLX5_FLOW_TYPE_MAX); 8067 fops = flow_get_drv_ops(ftype); 8068 return fops->query(dev, flow, action, count, error); 8069 } 8070 return -1; 8071 } 8072 8073 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8074 /** 8075 * Dump flow ipool data to file 8076 * 8077 * @param[in] dev 8078 * The pointer to Ethernet device. 8079 * @param[in] file 8080 * A pointer to a file for output. 8081 * @param[out] error 8082 * Perform verbose error reporting if not NULL. PMDs initialize this 8083 * structure in case of error only. 8084 * @return 8085 * 0 on success, a negative value otherwise. 8086 */ 8087 int 8088 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8089 struct rte_flow *flow, FILE *file, 8090 struct rte_flow_error *error) 8091 { 8092 struct mlx5_priv *priv = dev->data->dev_private; 8093 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8094 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8095 uint32_t handle_idx; 8096 struct mlx5_flow_handle *dh; 8097 struct rte_flow_query_count count; 8098 uint32_t actions_num; 8099 const uint8_t *data; 8100 size_t size; 8101 uint32_t id; 8102 uint32_t type; 8103 8104 if (!flow) { 8105 return rte_flow_error_set(error, ENOENT, 8106 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8107 NULL, 8108 "invalid flow handle"); 8109 } 8110 handle_idx = flow->dev_handles; 8111 while (handle_idx) { 8112 dh = mlx5_ipool_get(priv->sh->ipool 8113 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8114 if (!dh) 8115 continue; 8116 handle_idx = dh->next.next; 8117 id = (uint32_t)(uintptr_t)dh->drv_flow; 8118 8119 /* query counter */ 8120 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8121 if (!mlx5_flow_query_counter(dev, flow, &count, error)) 8122 save_dump_file(NULL, 0, type, 8123 id, (void *)&count, file); 8124 8125 /* Get modify_hdr and encap_decap buf from ipools. */ 8126 encap_decap = NULL; 8127 modify_hdr = dh->dvh.modify_hdr; 8128 8129 if (dh->dvh.rix_encap_decap) { 8130 encap_decap = mlx5_ipool_get(priv->sh->ipool 8131 [MLX5_IPOOL_DECAP_ENCAP], 8132 dh->dvh.rix_encap_decap); 8133 } 8134 if (modify_hdr) { 8135 data = (const uint8_t *)modify_hdr->actions; 8136 size = (size_t)(modify_hdr->actions_num) * 8; 8137 actions_num = modify_hdr->actions_num; 8138 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8139 save_dump_file(data, size, type, id, 8140 (void *)(&actions_num), file); 8141 } 8142 if (encap_decap) { 8143 data = encap_decap->buf; 8144 size = encap_decap->size; 8145 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8146 save_dump_file(data, size, type, 8147 id, NULL, file); 8148 } 8149 } 8150 return 0; 8151 } 8152 #endif 8153 8154 /** 8155 * Dump flow raw hw data to file 8156 * 8157 * @param[in] dev 8158 * The pointer to Ethernet device. 8159 * @param[in] file 8160 * A pointer to a file for output. 8161 * @param[out] error 8162 * Perform verbose error reporting if not NULL. PMDs initialize this 8163 * structure in case of error only. 8164 * @return 8165 * 0 on success, a nagative value otherwise. 8166 */ 8167 int 8168 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8169 FILE *file, 8170 struct rte_flow_error *error __rte_unused) 8171 { 8172 struct mlx5_priv *priv = dev->data->dev_private; 8173 struct mlx5_dev_ctx_shared *sh = priv->sh; 8174 uint32_t handle_idx; 8175 int ret; 8176 struct mlx5_flow_handle *dh; 8177 struct rte_flow *flow; 8178 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8179 uint32_t idx; 8180 #endif 8181 8182 if (!priv->config.dv_flow_en) { 8183 if (fputs("device dv flow disabled\n", file) <= 0) 8184 return -errno; 8185 return -ENOTSUP; 8186 } 8187 8188 /* dump all */ 8189 if (!flow_idx) { 8190 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8191 MLX5_IPOOL_FOREACH(priv->flows[MLX5_FLOW_TYPE_GEN], idx, flow) 8192 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8193 #endif 8194 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8195 sh->rx_domain, 8196 sh->tx_domain, file); 8197 } 8198 /* dump one */ 8199 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8200 (uintptr_t)(void *)flow_idx); 8201 if (!flow) 8202 return -ENOENT; 8203 8204 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8205 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8206 #endif 8207 handle_idx = flow->dev_handles; 8208 while (handle_idx) { 8209 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8210 handle_idx); 8211 if (!dh) 8212 return -ENOENT; 8213 if (dh->drv_flow) { 8214 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8215 file); 8216 if (ret) 8217 return -ENOENT; 8218 } 8219 handle_idx = dh->next.next; 8220 } 8221 return 0; 8222 } 8223 8224 /** 8225 * Get aged-out flows. 8226 * 8227 * @param[in] dev 8228 * Pointer to the Ethernet device structure. 8229 * @param[in] context 8230 * The address of an array of pointers to the aged-out flows contexts. 8231 * @param[in] nb_countexts 8232 * The length of context array pointers. 8233 * @param[out] error 8234 * Perform verbose error reporting if not NULL. Initialized in case of 8235 * error only. 8236 * 8237 * @return 8238 * how many contexts get in success, otherwise negative errno value. 8239 * if nb_contexts is 0, return the amount of all aged contexts. 8240 * if nb_contexts is not 0 , return the amount of aged flows reported 8241 * in the context array. 8242 */ 8243 int 8244 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8245 uint32_t nb_contexts, struct rte_flow_error *error) 8246 { 8247 const struct mlx5_flow_driver_ops *fops; 8248 struct rte_flow_attr attr = { .transfer = 0 }; 8249 8250 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8251 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8252 return fops->get_aged_flows(dev, contexts, nb_contexts, 8253 error); 8254 } 8255 DRV_LOG(ERR, 8256 "port %u get aged flows is not supported.", 8257 dev->data->port_id); 8258 return -ENOTSUP; 8259 } 8260 8261 /* Wrapper for driver action_validate op callback */ 8262 static int 8263 flow_drv_action_validate(struct rte_eth_dev *dev, 8264 const struct rte_flow_indir_action_conf *conf, 8265 const struct rte_flow_action *action, 8266 const struct mlx5_flow_driver_ops *fops, 8267 struct rte_flow_error *error) 8268 { 8269 static const char err_msg[] = "indirect action validation unsupported"; 8270 8271 if (!fops->action_validate) { 8272 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8273 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8274 NULL, err_msg); 8275 return -rte_errno; 8276 } 8277 return fops->action_validate(dev, conf, action, error); 8278 } 8279 8280 /** 8281 * Destroys the shared action by handle. 8282 * 8283 * @param dev 8284 * Pointer to Ethernet device structure. 8285 * @param[in] handle 8286 * Handle for the indirect action object to be destroyed. 8287 * @param[out] error 8288 * Perform verbose error reporting if not NULL. PMDs initialize this 8289 * structure in case of error only. 8290 * 8291 * @return 8292 * 0 on success, a negative errno value otherwise and rte_errno is set. 8293 * 8294 * @note: wrapper for driver action_create op callback. 8295 */ 8296 static int 8297 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8298 struct rte_flow_action_handle *handle, 8299 struct rte_flow_error *error) 8300 { 8301 static const char err_msg[] = "indirect action destruction unsupported"; 8302 struct rte_flow_attr attr = { .transfer = 0 }; 8303 const struct mlx5_flow_driver_ops *fops = 8304 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8305 8306 if (!fops->action_destroy) { 8307 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8308 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8309 NULL, err_msg); 8310 return -rte_errno; 8311 } 8312 return fops->action_destroy(dev, handle, error); 8313 } 8314 8315 /* Wrapper for driver action_destroy op callback */ 8316 static int 8317 flow_drv_action_update(struct rte_eth_dev *dev, 8318 struct rte_flow_action_handle *handle, 8319 const void *update, 8320 const struct mlx5_flow_driver_ops *fops, 8321 struct rte_flow_error *error) 8322 { 8323 static const char err_msg[] = "indirect action update unsupported"; 8324 8325 if (!fops->action_update) { 8326 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8327 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8328 NULL, err_msg); 8329 return -rte_errno; 8330 } 8331 return fops->action_update(dev, handle, update, error); 8332 } 8333 8334 /* Wrapper for driver action_destroy op callback */ 8335 static int 8336 flow_drv_action_query(struct rte_eth_dev *dev, 8337 const struct rte_flow_action_handle *handle, 8338 void *data, 8339 const struct mlx5_flow_driver_ops *fops, 8340 struct rte_flow_error *error) 8341 { 8342 static const char err_msg[] = "indirect action query unsupported"; 8343 8344 if (!fops->action_query) { 8345 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8346 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8347 NULL, err_msg); 8348 return -rte_errno; 8349 } 8350 return fops->action_query(dev, handle, data, error); 8351 } 8352 8353 /** 8354 * Create indirect action for reuse in multiple flow rules. 8355 * 8356 * @param dev 8357 * Pointer to Ethernet device structure. 8358 * @param conf 8359 * Pointer to indirect action object configuration. 8360 * @param[in] action 8361 * Action configuration for indirect action object creation. 8362 * @param[out] error 8363 * Perform verbose error reporting if not NULL. PMDs initialize this 8364 * structure in case of error only. 8365 * @return 8366 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8367 */ 8368 static struct rte_flow_action_handle * 8369 mlx5_action_handle_create(struct rte_eth_dev *dev, 8370 const struct rte_flow_indir_action_conf *conf, 8371 const struct rte_flow_action *action, 8372 struct rte_flow_error *error) 8373 { 8374 static const char err_msg[] = "indirect action creation unsupported"; 8375 struct rte_flow_attr attr = { .transfer = 0 }; 8376 const struct mlx5_flow_driver_ops *fops = 8377 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8378 8379 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8380 return NULL; 8381 if (!fops->action_create) { 8382 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8383 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8384 NULL, err_msg); 8385 return NULL; 8386 } 8387 return fops->action_create(dev, conf, action, error); 8388 } 8389 8390 /** 8391 * Updates inplace the indirect action configuration pointed by *handle* 8392 * with the configuration provided as *update* argument. 8393 * The update of the indirect action configuration effects all flow rules 8394 * reusing the action via handle. 8395 * 8396 * @param dev 8397 * Pointer to Ethernet device structure. 8398 * @param[in] handle 8399 * Handle for the indirect action to be updated. 8400 * @param[in] update 8401 * Action specification used to modify the action pointed by handle. 8402 * *update* could be of same type with the action pointed by the *handle* 8403 * handle argument, or some other structures like a wrapper, depending on 8404 * the indirect action type. 8405 * @param[out] error 8406 * Perform verbose error reporting if not NULL. PMDs initialize this 8407 * structure in case of error only. 8408 * 8409 * @return 8410 * 0 on success, a negative errno value otherwise and rte_errno is set. 8411 */ 8412 static int 8413 mlx5_action_handle_update(struct rte_eth_dev *dev, 8414 struct rte_flow_action_handle *handle, 8415 const void *update, 8416 struct rte_flow_error *error) 8417 { 8418 struct rte_flow_attr attr = { .transfer = 0 }; 8419 const struct mlx5_flow_driver_ops *fops = 8420 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8421 int ret; 8422 8423 ret = flow_drv_action_validate(dev, NULL, 8424 (const struct rte_flow_action *)update, fops, error); 8425 if (ret) 8426 return ret; 8427 return flow_drv_action_update(dev, handle, update, fops, 8428 error); 8429 } 8430 8431 /** 8432 * Query the indirect action by handle. 8433 * 8434 * This function allows retrieving action-specific data such as counters. 8435 * Data is gathered by special action which may be present/referenced in 8436 * more than one flow rule definition. 8437 * 8438 * see @RTE_FLOW_ACTION_TYPE_COUNT 8439 * 8440 * @param dev 8441 * Pointer to Ethernet device structure. 8442 * @param[in] handle 8443 * Handle for the indirect action to query. 8444 * @param[in, out] data 8445 * Pointer to storage for the associated query data type. 8446 * @param[out] error 8447 * Perform verbose error reporting if not NULL. PMDs initialize this 8448 * structure in case of error only. 8449 * 8450 * @return 8451 * 0 on success, a negative errno value otherwise and rte_errno is set. 8452 */ 8453 static int 8454 mlx5_action_handle_query(struct rte_eth_dev *dev, 8455 const struct rte_flow_action_handle *handle, 8456 void *data, 8457 struct rte_flow_error *error) 8458 { 8459 struct rte_flow_attr attr = { .transfer = 0 }; 8460 const struct mlx5_flow_driver_ops *fops = 8461 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8462 8463 return flow_drv_action_query(dev, handle, data, fops, error); 8464 } 8465 8466 /** 8467 * Destroy all indirect actions (shared RSS). 8468 * 8469 * @param dev 8470 * Pointer to Ethernet device. 8471 * 8472 * @return 8473 * 0 on success, a negative errno value otherwise and rte_errno is set. 8474 */ 8475 int 8476 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8477 { 8478 struct rte_flow_error error; 8479 struct mlx5_priv *priv = dev->data->dev_private; 8480 struct mlx5_shared_action_rss *shared_rss; 8481 int ret = 0; 8482 uint32_t idx; 8483 8484 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8485 priv->rss_shared_actions, idx, shared_rss, next) { 8486 ret |= mlx5_action_handle_destroy(dev, 8487 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8488 } 8489 return ret; 8490 } 8491 8492 #ifndef HAVE_MLX5DV_DR 8493 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8494 #else 8495 #define MLX5_DOMAIN_SYNC_FLOW \ 8496 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8497 #endif 8498 8499 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8500 { 8501 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8502 const struct mlx5_flow_driver_ops *fops; 8503 int ret; 8504 struct rte_flow_attr attr = { .transfer = 0 }; 8505 8506 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8507 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8508 if (ret > 0) 8509 ret = -ret; 8510 return ret; 8511 } 8512 8513 const struct mlx5_flow_tunnel * 8514 mlx5_get_tof(const struct rte_flow_item *item, 8515 const struct rte_flow_action *action, 8516 enum mlx5_tof_rule_type *rule_type) 8517 { 8518 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 8519 if (item->type == (typeof(item->type)) 8520 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 8521 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 8522 return flow_items_to_tunnel(item); 8523 } 8524 } 8525 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 8526 if (action->type == (typeof(action->type)) 8527 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 8528 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 8529 return flow_actions_to_tunnel(action); 8530 } 8531 } 8532 return NULL; 8533 } 8534 8535 /** 8536 * tunnel offload functionalilty is defined for DV environment only 8537 */ 8538 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8539 __extension__ 8540 union tunnel_offload_mark { 8541 uint32_t val; 8542 struct { 8543 uint32_t app_reserve:8; 8544 uint32_t table_id:15; 8545 uint32_t transfer:1; 8546 uint32_t _unused_:8; 8547 }; 8548 }; 8549 8550 static bool 8551 mlx5_access_tunnel_offload_db 8552 (struct rte_eth_dev *dev, 8553 bool (*match)(struct rte_eth_dev *, 8554 struct mlx5_flow_tunnel *, const void *), 8555 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8556 void (*miss)(struct rte_eth_dev *, void *), 8557 void *ctx, bool lock_op); 8558 8559 static int 8560 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 8561 struct rte_flow *flow, 8562 const struct rte_flow_attr *attr, 8563 const struct rte_flow_action *app_actions, 8564 uint32_t flow_idx, 8565 const struct mlx5_flow_tunnel *tunnel, 8566 struct tunnel_default_miss_ctx *ctx, 8567 struct rte_flow_error *error) 8568 { 8569 struct mlx5_priv *priv = dev->data->dev_private; 8570 struct mlx5_flow *dev_flow; 8571 struct rte_flow_attr miss_attr = *attr; 8572 const struct rte_flow_item miss_items[2] = { 8573 { 8574 .type = RTE_FLOW_ITEM_TYPE_ETH, 8575 .spec = NULL, 8576 .last = NULL, 8577 .mask = NULL 8578 }, 8579 { 8580 .type = RTE_FLOW_ITEM_TYPE_END, 8581 .spec = NULL, 8582 .last = NULL, 8583 .mask = NULL 8584 } 8585 }; 8586 union tunnel_offload_mark mark_id; 8587 struct rte_flow_action_mark miss_mark; 8588 struct rte_flow_action miss_actions[3] = { 8589 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 8590 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 8591 }; 8592 const struct rte_flow_action_jump *jump_data; 8593 uint32_t i, flow_table = 0; /* prevent compilation warning */ 8594 struct flow_grp_info grp_info = { 8595 .external = 1, 8596 .transfer = attr->transfer, 8597 .fdb_def_rule = !!priv->fdb_def_rule, 8598 .std_tbl_fix = 0, 8599 }; 8600 int ret; 8601 8602 if (!attr->transfer) { 8603 uint32_t q_size; 8604 8605 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 8606 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 8607 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 8608 0, SOCKET_ID_ANY); 8609 if (!ctx->queue) 8610 return rte_flow_error_set 8611 (error, ENOMEM, 8612 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8613 NULL, "invalid default miss RSS"); 8614 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8615 ctx->action_rss.level = 0, 8616 ctx->action_rss.types = priv->rss_conf.rss_hf, 8617 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 8618 ctx->action_rss.queue_num = priv->reta_idx_n, 8619 ctx->action_rss.key = priv->rss_conf.rss_key, 8620 ctx->action_rss.queue = ctx->queue; 8621 if (!priv->reta_idx_n || !priv->rxqs_n) 8622 return rte_flow_error_set 8623 (error, EINVAL, 8624 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8625 NULL, "invalid port configuration"); 8626 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 8627 ctx->action_rss.types = 0; 8628 for (i = 0; i != priv->reta_idx_n; ++i) 8629 ctx->queue[i] = (*priv->reta_idx)[i]; 8630 } else { 8631 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 8632 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 8633 } 8634 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 8635 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 8636 jump_data = app_actions->conf; 8637 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 8638 miss_attr.group = jump_data->group; 8639 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 8640 &flow_table, &grp_info, error); 8641 if (ret) 8642 return rte_flow_error_set(error, EINVAL, 8643 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8644 NULL, "invalid tunnel id"); 8645 mark_id.app_reserve = 0; 8646 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 8647 mark_id.transfer = !!attr->transfer; 8648 mark_id._unused_ = 0; 8649 miss_mark.id = mark_id.val; 8650 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 8651 miss_items, miss_actions, flow_idx, error); 8652 if (!dev_flow) 8653 return -rte_errno; 8654 dev_flow->flow = flow; 8655 dev_flow->external = true; 8656 dev_flow->tunnel = tunnel; 8657 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 8658 /* Subflow object was created, we must include one in the list. */ 8659 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 8660 dev_flow->handle, next); 8661 DRV_LOG(DEBUG, 8662 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 8663 dev->data->port_id, tunnel->app_tunnel.type, 8664 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 8665 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 8666 miss_actions, error); 8667 if (!ret) 8668 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 8669 error); 8670 8671 return ret; 8672 } 8673 8674 static const struct mlx5_flow_tbl_data_entry * 8675 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 8676 { 8677 struct mlx5_priv *priv = dev->data->dev_private; 8678 struct mlx5_dev_ctx_shared *sh = priv->sh; 8679 struct mlx5_list_entry *he; 8680 union tunnel_offload_mark mbits = { .val = mark }; 8681 union mlx5_flow_tbl_key table_key = { 8682 { 8683 .level = tunnel_id_to_flow_tbl(mbits.table_id), 8684 .id = 0, 8685 .reserved = 0, 8686 .dummy = 0, 8687 .is_fdb = !!mbits.transfer, 8688 .is_egress = 0, 8689 } 8690 }; 8691 struct mlx5_flow_cb_ctx ctx = { 8692 .data = &table_key.v64, 8693 }; 8694 8695 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 8696 return he ? 8697 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 8698 } 8699 8700 static void 8701 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 8702 struct mlx5_list_entry *entry) 8703 { 8704 struct mlx5_dev_ctx_shared *sh = tool_ctx; 8705 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8706 8707 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8708 tunnel_flow_tbl_to_id(tte->flow_table)); 8709 mlx5_free(tte); 8710 } 8711 8712 static int 8713 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 8714 struct mlx5_list_entry *entry, void *cb_ctx) 8715 { 8716 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 8717 union tunnel_tbl_key tbl = { 8718 .val = *(uint64_t *)(ctx->data), 8719 }; 8720 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8721 8722 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 8723 } 8724 8725 static struct mlx5_list_entry * 8726 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 8727 { 8728 struct mlx5_dev_ctx_shared *sh = tool_ctx; 8729 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 8730 struct tunnel_tbl_entry *tte; 8731 union tunnel_tbl_key tbl = { 8732 .val = *(uint64_t *)(ctx->data), 8733 }; 8734 8735 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 8736 sizeof(*tte), 0, 8737 SOCKET_ID_ANY); 8738 if (!tte) 8739 goto err; 8740 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8741 &tte->flow_table); 8742 if (tte->flow_table >= MLX5_MAX_TABLES) { 8743 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 8744 tte->flow_table); 8745 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8746 tte->flow_table); 8747 goto err; 8748 } else if (!tte->flow_table) { 8749 goto err; 8750 } 8751 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 8752 tte->tunnel_id = tbl.tunnel_id; 8753 tte->group = tbl.group; 8754 return &tte->hash; 8755 err: 8756 if (tte) 8757 mlx5_free(tte); 8758 return NULL; 8759 } 8760 8761 static struct mlx5_list_entry * 8762 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 8763 struct mlx5_list_entry *oentry, 8764 void *cb_ctx __rte_unused) 8765 { 8766 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 8767 0, SOCKET_ID_ANY); 8768 8769 if (!tte) 8770 return NULL; 8771 memcpy(tte, oentry, sizeof(*tte)); 8772 return &tte->hash; 8773 } 8774 8775 static void 8776 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 8777 struct mlx5_list_entry *entry) 8778 { 8779 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8780 8781 mlx5_free(tte); 8782 } 8783 8784 static uint32_t 8785 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 8786 const struct mlx5_flow_tunnel *tunnel, 8787 uint32_t group, uint32_t *table, 8788 struct rte_flow_error *error) 8789 { 8790 struct mlx5_list_entry *he; 8791 struct tunnel_tbl_entry *tte; 8792 union tunnel_tbl_key key = { 8793 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 8794 .group = group 8795 }; 8796 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8797 struct mlx5_hlist *group_hash; 8798 struct mlx5_flow_cb_ctx ctx = { 8799 .data = &key.val, 8800 }; 8801 8802 group_hash = tunnel ? tunnel->groups : thub->groups; 8803 he = mlx5_hlist_register(group_hash, key.val, &ctx); 8804 if (!he) 8805 return rte_flow_error_set(error, EINVAL, 8806 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8807 NULL, 8808 "tunnel group index not supported"); 8809 tte = container_of(he, typeof(*tte), hash); 8810 *table = tte->flow_table; 8811 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 8812 dev->data->port_id, key.tunnel_id, group, *table); 8813 return 0; 8814 } 8815 8816 static void 8817 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8818 struct mlx5_flow_tunnel *tunnel) 8819 { 8820 struct mlx5_priv *priv = dev->data->dev_private; 8821 struct mlx5_indexed_pool *ipool; 8822 8823 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8824 dev->data->port_id, tunnel->tunnel_id); 8825 LIST_REMOVE(tunnel, chain); 8826 mlx5_hlist_destroy(tunnel->groups); 8827 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8828 mlx5_ipool_free(ipool, tunnel->tunnel_id); 8829 } 8830 8831 static bool 8832 mlx5_access_tunnel_offload_db 8833 (struct rte_eth_dev *dev, 8834 bool (*match)(struct rte_eth_dev *, 8835 struct mlx5_flow_tunnel *, const void *), 8836 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8837 void (*miss)(struct rte_eth_dev *, void *), 8838 void *ctx, bool lock_op) 8839 { 8840 bool verdict = false; 8841 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8842 struct mlx5_flow_tunnel *tunnel; 8843 8844 rte_spinlock_lock(&thub->sl); 8845 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 8846 verdict = match(dev, tunnel, (const void *)ctx); 8847 if (verdict) 8848 break; 8849 } 8850 if (!lock_op) 8851 rte_spinlock_unlock(&thub->sl); 8852 if (verdict && hit) 8853 hit(dev, tunnel, ctx); 8854 if (!verdict && miss) 8855 miss(dev, ctx); 8856 if (lock_op) 8857 rte_spinlock_unlock(&thub->sl); 8858 8859 return verdict; 8860 } 8861 8862 struct tunnel_db_find_tunnel_id_ctx { 8863 uint32_t tunnel_id; 8864 struct mlx5_flow_tunnel *tunnel; 8865 }; 8866 8867 static bool 8868 find_tunnel_id_match(struct rte_eth_dev *dev, 8869 struct mlx5_flow_tunnel *tunnel, const void *x) 8870 { 8871 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8872 8873 RTE_SET_USED(dev); 8874 return tunnel->tunnel_id == ctx->tunnel_id; 8875 } 8876 8877 static void 8878 find_tunnel_id_hit(struct rte_eth_dev *dev, 8879 struct mlx5_flow_tunnel *tunnel, void *x) 8880 { 8881 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8882 RTE_SET_USED(dev); 8883 ctx->tunnel = tunnel; 8884 } 8885 8886 static struct mlx5_flow_tunnel * 8887 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 8888 { 8889 struct tunnel_db_find_tunnel_id_ctx ctx = { 8890 .tunnel_id = id, 8891 }; 8892 8893 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 8894 find_tunnel_id_hit, NULL, &ctx, true); 8895 8896 return ctx.tunnel; 8897 } 8898 8899 static struct mlx5_flow_tunnel * 8900 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 8901 const struct rte_flow_tunnel *app_tunnel) 8902 { 8903 struct mlx5_priv *priv = dev->data->dev_private; 8904 struct mlx5_indexed_pool *ipool; 8905 struct mlx5_flow_tunnel *tunnel; 8906 uint32_t id; 8907 8908 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8909 tunnel = mlx5_ipool_zmalloc(ipool, &id); 8910 if (!tunnel) 8911 return NULL; 8912 if (id >= MLX5_MAX_TUNNELS) { 8913 mlx5_ipool_free(ipool, id); 8914 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 8915 return NULL; 8916 } 8917 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 8918 priv->sh, 8919 mlx5_flow_tunnel_grp2tbl_create_cb, 8920 mlx5_flow_tunnel_grp2tbl_match_cb, 8921 mlx5_flow_tunnel_grp2tbl_remove_cb, 8922 mlx5_flow_tunnel_grp2tbl_clone_cb, 8923 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 8924 if (!tunnel->groups) { 8925 mlx5_ipool_free(ipool, id); 8926 return NULL; 8927 } 8928 /* initiate new PMD tunnel */ 8929 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8930 tunnel->tunnel_id = id; 8931 tunnel->action.type = (typeof(tunnel->action.type)) 8932 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8933 tunnel->action.conf = tunnel; 8934 tunnel->item.type = (typeof(tunnel->item.type)) 8935 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8936 tunnel->item.spec = tunnel; 8937 tunnel->item.last = NULL; 8938 tunnel->item.mask = NULL; 8939 8940 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8941 dev->data->port_id, tunnel->tunnel_id); 8942 8943 return tunnel; 8944 } 8945 8946 struct tunnel_db_get_tunnel_ctx { 8947 const struct rte_flow_tunnel *app_tunnel; 8948 struct mlx5_flow_tunnel *tunnel; 8949 }; 8950 8951 static bool get_tunnel_match(struct rte_eth_dev *dev, 8952 struct mlx5_flow_tunnel *tunnel, const void *x) 8953 { 8954 const struct tunnel_db_get_tunnel_ctx *ctx = x; 8955 8956 RTE_SET_USED(dev); 8957 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 8958 sizeof(*ctx->app_tunnel)); 8959 } 8960 8961 static void get_tunnel_hit(struct rte_eth_dev *dev, 8962 struct mlx5_flow_tunnel *tunnel, void *x) 8963 { 8964 /* called under tunnel spinlock protection */ 8965 struct tunnel_db_get_tunnel_ctx *ctx = x; 8966 8967 RTE_SET_USED(dev); 8968 tunnel->refctn++; 8969 ctx->tunnel = tunnel; 8970 } 8971 8972 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 8973 { 8974 /* called under tunnel spinlock protection */ 8975 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8976 struct tunnel_db_get_tunnel_ctx *ctx = x; 8977 8978 rte_spinlock_unlock(&thub->sl); 8979 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 8980 rte_spinlock_lock(&thub->sl); 8981 if (ctx->tunnel) { 8982 ctx->tunnel->refctn = 1; 8983 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 8984 } 8985 } 8986 8987 8988 static int 8989 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 8990 const struct rte_flow_tunnel *app_tunnel, 8991 struct mlx5_flow_tunnel **tunnel) 8992 { 8993 struct tunnel_db_get_tunnel_ctx ctx = { 8994 .app_tunnel = app_tunnel, 8995 }; 8996 8997 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 8998 get_tunnel_miss, &ctx, true); 8999 *tunnel = ctx.tunnel; 9000 return ctx.tunnel ? 0 : -ENOMEM; 9001 } 9002 9003 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9004 { 9005 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9006 9007 if (!thub) 9008 return; 9009 if (!LIST_EMPTY(&thub->tunnels)) 9010 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9011 mlx5_hlist_destroy(thub->groups); 9012 mlx5_free(thub); 9013 } 9014 9015 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9016 { 9017 int err; 9018 struct mlx5_flow_tunnel_hub *thub; 9019 9020 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9021 0, SOCKET_ID_ANY); 9022 if (!thub) 9023 return -ENOMEM; 9024 LIST_INIT(&thub->tunnels); 9025 rte_spinlock_init(&thub->sl); 9026 thub->groups = mlx5_hlist_create("flow groups", 64, 9027 false, true, sh, 9028 mlx5_flow_tunnel_grp2tbl_create_cb, 9029 mlx5_flow_tunnel_grp2tbl_match_cb, 9030 mlx5_flow_tunnel_grp2tbl_remove_cb, 9031 mlx5_flow_tunnel_grp2tbl_clone_cb, 9032 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9033 if (!thub->groups) { 9034 err = -rte_errno; 9035 goto err; 9036 } 9037 sh->tunnel_hub = thub; 9038 9039 return 0; 9040 9041 err: 9042 if (thub->groups) 9043 mlx5_hlist_destroy(thub->groups); 9044 if (thub) 9045 mlx5_free(thub); 9046 return err; 9047 } 9048 9049 static inline bool 9050 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9051 struct rte_flow_tunnel *tunnel, 9052 const char *err_msg) 9053 { 9054 err_msg = NULL; 9055 if (!is_tunnel_offload_active(dev)) { 9056 err_msg = "tunnel offload was not activated"; 9057 goto out; 9058 } else if (!tunnel) { 9059 err_msg = "no application tunnel"; 9060 goto out; 9061 } 9062 9063 switch (tunnel->type) { 9064 default: 9065 err_msg = "unsupported tunnel type"; 9066 goto out; 9067 case RTE_FLOW_ITEM_TYPE_VXLAN: 9068 break; 9069 } 9070 9071 out: 9072 return !err_msg; 9073 } 9074 9075 static int 9076 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9077 struct rte_flow_tunnel *app_tunnel, 9078 struct rte_flow_action **actions, 9079 uint32_t *num_of_actions, 9080 struct rte_flow_error *error) 9081 { 9082 int ret; 9083 struct mlx5_flow_tunnel *tunnel; 9084 const char *err_msg = NULL; 9085 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 9086 9087 if (!verdict) 9088 return rte_flow_error_set(error, EINVAL, 9089 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9090 err_msg); 9091 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9092 if (ret < 0) { 9093 return rte_flow_error_set(error, ret, 9094 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9095 "failed to initialize pmd tunnel"); 9096 } 9097 *actions = &tunnel->action; 9098 *num_of_actions = 1; 9099 return 0; 9100 } 9101 9102 static int 9103 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9104 struct rte_flow_tunnel *app_tunnel, 9105 struct rte_flow_item **items, 9106 uint32_t *num_of_items, 9107 struct rte_flow_error *error) 9108 { 9109 int ret; 9110 struct mlx5_flow_tunnel *tunnel; 9111 const char *err_msg = NULL; 9112 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 9113 9114 if (!verdict) 9115 return rte_flow_error_set(error, EINVAL, 9116 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9117 err_msg); 9118 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9119 if (ret < 0) { 9120 return rte_flow_error_set(error, ret, 9121 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9122 "failed to initialize pmd tunnel"); 9123 } 9124 *items = &tunnel->item; 9125 *num_of_items = 1; 9126 return 0; 9127 } 9128 9129 struct tunnel_db_element_release_ctx { 9130 struct rte_flow_item *items; 9131 struct rte_flow_action *actions; 9132 uint32_t num_elements; 9133 struct rte_flow_error *error; 9134 int ret; 9135 }; 9136 9137 static bool 9138 tunnel_element_release_match(struct rte_eth_dev *dev, 9139 struct mlx5_flow_tunnel *tunnel, const void *x) 9140 { 9141 const struct tunnel_db_element_release_ctx *ctx = x; 9142 9143 RTE_SET_USED(dev); 9144 if (ctx->num_elements != 1) 9145 return false; 9146 else if (ctx->items) 9147 return ctx->items == &tunnel->item; 9148 else if (ctx->actions) 9149 return ctx->actions == &tunnel->action; 9150 9151 return false; 9152 } 9153 9154 static void 9155 tunnel_element_release_hit(struct rte_eth_dev *dev, 9156 struct mlx5_flow_tunnel *tunnel, void *x) 9157 { 9158 struct tunnel_db_element_release_ctx *ctx = x; 9159 ctx->ret = 0; 9160 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9161 mlx5_flow_tunnel_free(dev, tunnel); 9162 } 9163 9164 static void 9165 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9166 { 9167 struct tunnel_db_element_release_ctx *ctx = x; 9168 RTE_SET_USED(dev); 9169 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9170 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9171 "invalid argument"); 9172 } 9173 9174 static int 9175 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9176 struct rte_flow_item *pmd_items, 9177 uint32_t num_items, struct rte_flow_error *err) 9178 { 9179 struct tunnel_db_element_release_ctx ctx = { 9180 .items = pmd_items, 9181 .actions = NULL, 9182 .num_elements = num_items, 9183 .error = err, 9184 }; 9185 9186 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9187 tunnel_element_release_hit, 9188 tunnel_element_release_miss, &ctx, false); 9189 9190 return ctx.ret; 9191 } 9192 9193 static int 9194 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9195 struct rte_flow_action *pmd_actions, 9196 uint32_t num_actions, struct rte_flow_error *err) 9197 { 9198 struct tunnel_db_element_release_ctx ctx = { 9199 .items = NULL, 9200 .actions = pmd_actions, 9201 .num_elements = num_actions, 9202 .error = err, 9203 }; 9204 9205 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9206 tunnel_element_release_hit, 9207 tunnel_element_release_miss, &ctx, false); 9208 9209 return ctx.ret; 9210 } 9211 9212 static int 9213 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9214 struct rte_mbuf *m, 9215 struct rte_flow_restore_info *info, 9216 struct rte_flow_error *err) 9217 { 9218 uint64_t ol_flags = m->ol_flags; 9219 const struct mlx5_flow_tbl_data_entry *tble; 9220 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 9221 9222 if (!is_tunnel_offload_active(dev)) { 9223 info->flags = 0; 9224 return 0; 9225 } 9226 9227 if ((ol_flags & mask) != mask) 9228 goto err; 9229 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9230 if (!tble) { 9231 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9232 dev->data->port_id, m->hash.fdir.hi); 9233 goto err; 9234 } 9235 MLX5_ASSERT(tble->tunnel); 9236 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9237 info->group_id = tble->group_id; 9238 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9239 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9240 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9241 9242 return 0; 9243 9244 err: 9245 return rte_flow_error_set(err, EINVAL, 9246 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9247 "failed to get restore info"); 9248 } 9249 9250 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9251 static int 9252 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9253 __rte_unused struct rte_flow_tunnel *app_tunnel, 9254 __rte_unused struct rte_flow_action **actions, 9255 __rte_unused uint32_t *num_of_actions, 9256 __rte_unused struct rte_flow_error *error) 9257 { 9258 return -ENOTSUP; 9259 } 9260 9261 static int 9262 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9263 __rte_unused struct rte_flow_tunnel *app_tunnel, 9264 __rte_unused struct rte_flow_item **items, 9265 __rte_unused uint32_t *num_of_items, 9266 __rte_unused struct rte_flow_error *error) 9267 { 9268 return -ENOTSUP; 9269 } 9270 9271 static int 9272 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9273 __rte_unused struct rte_flow_item *pmd_items, 9274 __rte_unused uint32_t num_items, 9275 __rte_unused struct rte_flow_error *err) 9276 { 9277 return -ENOTSUP; 9278 } 9279 9280 static int 9281 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9282 __rte_unused struct rte_flow_action *pmd_action, 9283 __rte_unused uint32_t num_actions, 9284 __rte_unused struct rte_flow_error *err) 9285 { 9286 return -ENOTSUP; 9287 } 9288 9289 static int 9290 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9291 __rte_unused struct rte_mbuf *m, 9292 __rte_unused struct rte_flow_restore_info *i, 9293 __rte_unused struct rte_flow_error *err) 9294 { 9295 return -ENOTSUP; 9296 } 9297 9298 static int 9299 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9300 __rte_unused struct rte_flow *flow, 9301 __rte_unused const struct rte_flow_attr *attr, 9302 __rte_unused const struct rte_flow_action *actions, 9303 __rte_unused uint32_t flow_idx, 9304 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9305 __rte_unused struct tunnel_default_miss_ctx *ctx, 9306 __rte_unused struct rte_flow_error *error) 9307 { 9308 return -ENOTSUP; 9309 } 9310 9311 static struct mlx5_flow_tunnel * 9312 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9313 __rte_unused uint32_t id) 9314 { 9315 return NULL; 9316 } 9317 9318 static void 9319 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9320 __rte_unused struct mlx5_flow_tunnel *tunnel) 9321 { 9322 } 9323 9324 static uint32_t 9325 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9326 __rte_unused const struct mlx5_flow_tunnel *t, 9327 __rte_unused uint32_t group, 9328 __rte_unused uint32_t *table, 9329 struct rte_flow_error *error) 9330 { 9331 return rte_flow_error_set(error, ENOTSUP, 9332 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9333 "tunnel offload requires DV support"); 9334 } 9335 9336 void 9337 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9338 __rte_unused uint16_t port_id) 9339 { 9340 } 9341 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9342 9343 static void 9344 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9345 { 9346 int ret; 9347 struct rte_flow_error error; 9348 9349 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9350 char *item_name; 9351 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9352 sizeof(item_name), 9353 (void *)(uintptr_t)item->type, &error); 9354 if (ret > 0) 9355 printf("%s ", item_name); 9356 else 9357 printf("%d\n", (int)item->type); 9358 } 9359 printf("END\n"); 9360 } 9361