1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 #endif 80 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 81 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 82 }; 83 84 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 85 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 86 (const int []){ \ 87 __VA_ARGS__, 0, \ 88 } 89 90 /** Node object of input graph for mlx5_flow_expand_rss(). */ 91 struct mlx5_flow_expand_node { 92 const int *const next; 93 /**< 94 * List of next node indexes. Index 0 is interpreted as a terminator. 95 */ 96 const enum rte_flow_item_type type; 97 /**< Pattern item type of current node. */ 98 uint64_t rss_types; 99 /**< 100 * RSS types bit-field associated with this node 101 * (see ETH_RSS_* definitions). 102 */ 103 uint8_t optional; 104 /**< optional expand field. Default 0 to expand, 1 not go deeper. */ 105 }; 106 107 /** Object returned by mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_rss { 109 uint32_t entries; 110 /**< Number of entries @p patterns and @p priorities. */ 111 struct { 112 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 113 uint32_t priority; /**< Priority offset for each expansion. */ 114 } entry[]; 115 }; 116 117 static void 118 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 119 120 static bool 121 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 122 { 123 switch (item->type) { 124 case RTE_FLOW_ITEM_TYPE_ETH: 125 case RTE_FLOW_ITEM_TYPE_VLAN: 126 case RTE_FLOW_ITEM_TYPE_IPV4: 127 case RTE_FLOW_ITEM_TYPE_IPV6: 128 case RTE_FLOW_ITEM_TYPE_UDP: 129 case RTE_FLOW_ITEM_TYPE_TCP: 130 case RTE_FLOW_ITEM_TYPE_VXLAN: 131 case RTE_FLOW_ITEM_TYPE_NVGRE: 132 case RTE_FLOW_ITEM_TYPE_GRE: 133 case RTE_FLOW_ITEM_TYPE_GENEVE: 134 case RTE_FLOW_ITEM_TYPE_MPLS: 135 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 136 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 137 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 138 case RTE_FLOW_ITEM_TYPE_GTP: 139 return true; 140 default: 141 break; 142 } 143 return false; 144 } 145 146 static enum rte_flow_item_type 147 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 148 { 149 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 150 uint16_t ether_type = 0; 151 uint16_t ether_type_m; 152 uint8_t ip_next_proto = 0; 153 uint8_t ip_next_proto_m; 154 155 if (item == NULL || item->spec == NULL) 156 return ret; 157 switch (item->type) { 158 case RTE_FLOW_ITEM_TYPE_ETH: 159 if (item->mask) 160 ether_type_m = ((const struct rte_flow_item_eth *) 161 (item->mask))->type; 162 else 163 ether_type_m = rte_flow_item_eth_mask.type; 164 if (ether_type_m != RTE_BE16(0xFFFF)) 165 break; 166 ether_type = ((const struct rte_flow_item_eth *) 167 (item->spec))->type; 168 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 169 ret = RTE_FLOW_ITEM_TYPE_IPV4; 170 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 171 ret = RTE_FLOW_ITEM_TYPE_IPV6; 172 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 173 ret = RTE_FLOW_ITEM_TYPE_VLAN; 174 else 175 ret = RTE_FLOW_ITEM_TYPE_END; 176 break; 177 case RTE_FLOW_ITEM_TYPE_VLAN: 178 if (item->mask) 179 ether_type_m = ((const struct rte_flow_item_vlan *) 180 (item->mask))->inner_type; 181 else 182 ether_type_m = rte_flow_item_vlan_mask.inner_type; 183 if (ether_type_m != RTE_BE16(0xFFFF)) 184 break; 185 ether_type = ((const struct rte_flow_item_vlan *) 186 (item->spec))->inner_type; 187 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 188 ret = RTE_FLOW_ITEM_TYPE_IPV4; 189 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 190 ret = RTE_FLOW_ITEM_TYPE_IPV6; 191 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 192 ret = RTE_FLOW_ITEM_TYPE_VLAN; 193 else 194 ret = RTE_FLOW_ITEM_TYPE_END; 195 break; 196 case RTE_FLOW_ITEM_TYPE_IPV4: 197 if (item->mask) 198 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 199 (item->mask))->hdr.next_proto_id; 200 else 201 ip_next_proto_m = 202 rte_flow_item_ipv4_mask.hdr.next_proto_id; 203 if (ip_next_proto_m != 0xFF) 204 break; 205 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 206 (item->spec))->hdr.next_proto_id; 207 if (ip_next_proto == IPPROTO_UDP) 208 ret = RTE_FLOW_ITEM_TYPE_UDP; 209 else if (ip_next_proto == IPPROTO_TCP) 210 ret = RTE_FLOW_ITEM_TYPE_TCP; 211 else if (ip_next_proto == IPPROTO_IP) 212 ret = RTE_FLOW_ITEM_TYPE_IPV4; 213 else if (ip_next_proto == IPPROTO_IPV6) 214 ret = RTE_FLOW_ITEM_TYPE_IPV6; 215 else 216 ret = RTE_FLOW_ITEM_TYPE_END; 217 break; 218 case RTE_FLOW_ITEM_TYPE_IPV6: 219 if (item->mask) 220 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 221 (item->mask))->hdr.proto; 222 else 223 ip_next_proto_m = 224 rte_flow_item_ipv6_mask.hdr.proto; 225 if (ip_next_proto_m != 0xFF) 226 break; 227 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 228 (item->spec))->hdr.proto; 229 if (ip_next_proto == IPPROTO_UDP) 230 ret = RTE_FLOW_ITEM_TYPE_UDP; 231 else if (ip_next_proto == IPPROTO_TCP) 232 ret = RTE_FLOW_ITEM_TYPE_TCP; 233 else if (ip_next_proto == IPPROTO_IP) 234 ret = RTE_FLOW_ITEM_TYPE_IPV4; 235 else if (ip_next_proto == IPPROTO_IPV6) 236 ret = RTE_FLOW_ITEM_TYPE_IPV6; 237 else 238 ret = RTE_FLOW_ITEM_TYPE_END; 239 break; 240 default: 241 ret = RTE_FLOW_ITEM_TYPE_VOID; 242 break; 243 } 244 return ret; 245 } 246 247 #define MLX5_RSS_EXP_ELT_N 16 248 249 /** 250 * Expand RSS flows into several possible flows according to the RSS hash 251 * fields requested and the driver capabilities. 252 * 253 * @param[out] buf 254 * Buffer to store the result expansion. 255 * @param[in] size 256 * Buffer size in bytes. If 0, @p buf can be NULL. 257 * @param[in] pattern 258 * User flow pattern. 259 * @param[in] types 260 * RSS types to expand (see ETH_RSS_* definitions). 261 * @param[in] graph 262 * Input graph to expand @p pattern according to @p types. 263 * @param[in] graph_root_index 264 * Index of root node in @p graph, typically 0. 265 * 266 * @return 267 * A positive value representing the size of @p buf in bytes regardless of 268 * @p size on success, a negative errno value otherwise and rte_errno is 269 * set, the following errors are defined: 270 * 271 * -E2BIG: graph-depth @p graph is too deep. 272 * -EINVAL: @p size has not enough space for expanded pattern. 273 */ 274 static int 275 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 276 const struct rte_flow_item *pattern, uint64_t types, 277 const struct mlx5_flow_expand_node graph[], 278 int graph_root_index) 279 { 280 const struct rte_flow_item *item; 281 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 282 const int *next_node; 283 const int *stack[MLX5_RSS_EXP_ELT_N]; 284 int stack_pos = 0; 285 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 286 unsigned int i; 287 size_t lsize; 288 size_t user_pattern_size = 0; 289 void *addr = NULL; 290 const struct mlx5_flow_expand_node *next = NULL; 291 struct rte_flow_item missed_item; 292 int missed = 0; 293 int elt = 0; 294 const struct rte_flow_item *last_item = NULL; 295 296 memset(&missed_item, 0, sizeof(missed_item)); 297 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 298 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 299 if (lsize > size) 300 return -EINVAL; 301 buf->entry[0].priority = 0; 302 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 303 buf->entries = 0; 304 addr = buf->entry[0].pattern; 305 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 306 if (!mlx5_flow_is_rss_expandable_item(item)) { 307 user_pattern_size += sizeof(*item); 308 continue; 309 } 310 last_item = item; 311 for (i = 0; node->next && node->next[i]; ++i) { 312 next = &graph[node->next[i]]; 313 if (next->type == item->type) 314 break; 315 } 316 if (next) 317 node = next; 318 user_pattern_size += sizeof(*item); 319 } 320 user_pattern_size += sizeof(*item); /* Handle END item. */ 321 lsize += user_pattern_size; 322 if (lsize > size) 323 return -EINVAL; 324 /* Copy the user pattern in the first entry of the buffer. */ 325 rte_memcpy(addr, pattern, user_pattern_size); 326 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 327 buf->entries = 1; 328 /* Start expanding. */ 329 memset(flow_items, 0, sizeof(flow_items)); 330 user_pattern_size -= sizeof(*item); 331 /* 332 * Check if the last valid item has spec set, need complete pattern, 333 * and the pattern can be used for expansion. 334 */ 335 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 336 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 337 /* Item type END indicates expansion is not required. */ 338 return lsize; 339 } 340 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 341 next = NULL; 342 missed = 1; 343 for (i = 0; node->next && node->next[i]; ++i) { 344 next = &graph[node->next[i]]; 345 if (next->type == missed_item.type) { 346 flow_items[0].type = missed_item.type; 347 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 348 break; 349 } 350 next = NULL; 351 } 352 } 353 if (next && missed) { 354 elt = 2; /* missed item + item end. */ 355 node = next; 356 lsize += elt * sizeof(*item) + user_pattern_size; 357 if (lsize > size) 358 return -EINVAL; 359 if (node->rss_types & types) { 360 buf->entry[buf->entries].priority = 1; 361 buf->entry[buf->entries].pattern = addr; 362 buf->entries++; 363 rte_memcpy(addr, buf->entry[0].pattern, 364 user_pattern_size); 365 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 366 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 367 addr = (void *)(((uintptr_t)addr) + 368 elt * sizeof(*item)); 369 } 370 } 371 memset(flow_items, 0, sizeof(flow_items)); 372 next_node = node->next; 373 stack[stack_pos] = next_node; 374 node = next_node ? &graph[*next_node] : NULL; 375 while (node) { 376 flow_items[stack_pos].type = node->type; 377 if (node->rss_types & types) { 378 size_t n; 379 /* 380 * compute the number of items to copy from the 381 * expansion and copy it. 382 * When the stack_pos is 0, there are 1 element in it, 383 * plus the addition END item. 384 */ 385 elt = stack_pos + 2; 386 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 387 lsize += elt * sizeof(*item) + user_pattern_size; 388 if (lsize > size) 389 return -EINVAL; 390 n = elt * sizeof(*item); 391 buf->entry[buf->entries].priority = 392 stack_pos + 1 + missed; 393 buf->entry[buf->entries].pattern = addr; 394 buf->entries++; 395 rte_memcpy(addr, buf->entry[0].pattern, 396 user_pattern_size); 397 addr = (void *)(((uintptr_t)addr) + 398 user_pattern_size); 399 rte_memcpy(addr, &missed_item, 400 missed * sizeof(*item)); 401 addr = (void *)(((uintptr_t)addr) + 402 missed * sizeof(*item)); 403 rte_memcpy(addr, flow_items, n); 404 addr = (void *)(((uintptr_t)addr) + n); 405 } 406 /* Go deeper. */ 407 if (!node->optional && node->next) { 408 next_node = node->next; 409 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 410 rte_errno = E2BIG; 411 return -rte_errno; 412 } 413 stack[stack_pos] = next_node; 414 } else if (*(next_node + 1)) { 415 /* Follow up with the next possibility. */ 416 ++next_node; 417 } else { 418 /* Move to the next path. */ 419 if (stack_pos) 420 next_node = stack[--stack_pos]; 421 next_node++; 422 stack[stack_pos] = next_node; 423 } 424 node = *next_node ? &graph[*next_node] : NULL; 425 }; 426 return lsize; 427 } 428 429 enum mlx5_expansion { 430 MLX5_EXPANSION_ROOT, 431 MLX5_EXPANSION_ROOT_OUTER, 432 MLX5_EXPANSION_ROOT_ETH_VLAN, 433 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 434 MLX5_EXPANSION_OUTER_ETH, 435 MLX5_EXPANSION_OUTER_ETH_VLAN, 436 MLX5_EXPANSION_OUTER_VLAN, 437 MLX5_EXPANSION_OUTER_IPV4, 438 MLX5_EXPANSION_OUTER_IPV4_UDP, 439 MLX5_EXPANSION_OUTER_IPV4_TCP, 440 MLX5_EXPANSION_OUTER_IPV6, 441 MLX5_EXPANSION_OUTER_IPV6_UDP, 442 MLX5_EXPANSION_OUTER_IPV6_TCP, 443 MLX5_EXPANSION_VXLAN, 444 MLX5_EXPANSION_VXLAN_GPE, 445 MLX5_EXPANSION_GRE, 446 MLX5_EXPANSION_NVGRE, 447 MLX5_EXPANSION_GRE_KEY, 448 MLX5_EXPANSION_MPLS, 449 MLX5_EXPANSION_ETH, 450 MLX5_EXPANSION_ETH_VLAN, 451 MLX5_EXPANSION_VLAN, 452 MLX5_EXPANSION_IPV4, 453 MLX5_EXPANSION_IPV4_UDP, 454 MLX5_EXPANSION_IPV4_TCP, 455 MLX5_EXPANSION_IPV6, 456 MLX5_EXPANSION_IPV6_UDP, 457 MLX5_EXPANSION_IPV6_TCP, 458 MLX5_EXPANSION_IPV6_FRAG_EXT, 459 MLX5_EXPANSION_GTP 460 }; 461 462 /** Supported expansion of items. */ 463 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 464 [MLX5_EXPANSION_ROOT] = { 465 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 466 MLX5_EXPANSION_IPV4, 467 MLX5_EXPANSION_IPV6), 468 .type = RTE_FLOW_ITEM_TYPE_END, 469 }, 470 [MLX5_EXPANSION_ROOT_OUTER] = { 471 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 472 MLX5_EXPANSION_OUTER_IPV4, 473 MLX5_EXPANSION_OUTER_IPV6), 474 .type = RTE_FLOW_ITEM_TYPE_END, 475 }, 476 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 477 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 478 .type = RTE_FLOW_ITEM_TYPE_END, 479 }, 480 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 481 .next = MLX5_FLOW_EXPAND_RSS_NEXT 482 (MLX5_EXPANSION_OUTER_ETH_VLAN), 483 .type = RTE_FLOW_ITEM_TYPE_END, 484 }, 485 [MLX5_EXPANSION_OUTER_ETH] = { 486 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 487 MLX5_EXPANSION_OUTER_IPV6), 488 .type = RTE_FLOW_ITEM_TYPE_ETH, 489 .rss_types = 0, 490 }, 491 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 492 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 493 .type = RTE_FLOW_ITEM_TYPE_ETH, 494 .rss_types = 0, 495 }, 496 [MLX5_EXPANSION_OUTER_VLAN] = { 497 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 498 MLX5_EXPANSION_OUTER_IPV6), 499 .type = RTE_FLOW_ITEM_TYPE_VLAN, 500 }, 501 [MLX5_EXPANSION_OUTER_IPV4] = { 502 .next = MLX5_FLOW_EXPAND_RSS_NEXT 503 (MLX5_EXPANSION_OUTER_IPV4_UDP, 504 MLX5_EXPANSION_OUTER_IPV4_TCP, 505 MLX5_EXPANSION_GRE, 506 MLX5_EXPANSION_NVGRE, 507 MLX5_EXPANSION_IPV4, 508 MLX5_EXPANSION_IPV6), 509 .type = RTE_FLOW_ITEM_TYPE_IPV4, 510 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 511 ETH_RSS_NONFRAG_IPV4_OTHER, 512 }, 513 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 514 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 515 MLX5_EXPANSION_VXLAN_GPE, 516 MLX5_EXPANSION_MPLS, 517 MLX5_EXPANSION_GTP), 518 .type = RTE_FLOW_ITEM_TYPE_UDP, 519 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 520 }, 521 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 522 .type = RTE_FLOW_ITEM_TYPE_TCP, 523 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 524 }, 525 [MLX5_EXPANSION_OUTER_IPV6] = { 526 .next = MLX5_FLOW_EXPAND_RSS_NEXT 527 (MLX5_EXPANSION_OUTER_IPV6_UDP, 528 MLX5_EXPANSION_OUTER_IPV6_TCP, 529 MLX5_EXPANSION_IPV4, 530 MLX5_EXPANSION_IPV6, 531 MLX5_EXPANSION_GRE, 532 MLX5_EXPANSION_NVGRE), 533 .type = RTE_FLOW_ITEM_TYPE_IPV6, 534 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 535 ETH_RSS_NONFRAG_IPV6_OTHER, 536 }, 537 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 538 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 539 MLX5_EXPANSION_VXLAN_GPE, 540 MLX5_EXPANSION_MPLS, 541 MLX5_EXPANSION_GTP), 542 .type = RTE_FLOW_ITEM_TYPE_UDP, 543 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 544 }, 545 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 546 .type = RTE_FLOW_ITEM_TYPE_TCP, 547 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 548 }, 549 [MLX5_EXPANSION_VXLAN] = { 550 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 551 MLX5_EXPANSION_IPV4, 552 MLX5_EXPANSION_IPV6), 553 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 554 }, 555 [MLX5_EXPANSION_VXLAN_GPE] = { 556 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 557 MLX5_EXPANSION_IPV4, 558 MLX5_EXPANSION_IPV6), 559 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 560 }, 561 [MLX5_EXPANSION_GRE] = { 562 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 563 MLX5_EXPANSION_IPV6, 564 MLX5_EXPANSION_GRE_KEY, 565 MLX5_EXPANSION_MPLS), 566 .type = RTE_FLOW_ITEM_TYPE_GRE, 567 }, 568 [MLX5_EXPANSION_GRE_KEY] = { 569 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 570 MLX5_EXPANSION_IPV6, 571 MLX5_EXPANSION_MPLS), 572 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 573 .optional = 1, 574 }, 575 [MLX5_EXPANSION_NVGRE] = { 576 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 577 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 578 }, 579 [MLX5_EXPANSION_MPLS] = { 580 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6, 582 MLX5_EXPANSION_ETH), 583 .type = RTE_FLOW_ITEM_TYPE_MPLS, 584 .optional = 1, 585 }, 586 [MLX5_EXPANSION_ETH] = { 587 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 588 MLX5_EXPANSION_IPV6), 589 .type = RTE_FLOW_ITEM_TYPE_ETH, 590 }, 591 [MLX5_EXPANSION_ETH_VLAN] = { 592 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 593 .type = RTE_FLOW_ITEM_TYPE_ETH, 594 }, 595 [MLX5_EXPANSION_VLAN] = { 596 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 597 MLX5_EXPANSION_IPV6), 598 .type = RTE_FLOW_ITEM_TYPE_VLAN, 599 }, 600 [MLX5_EXPANSION_IPV4] = { 601 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 602 MLX5_EXPANSION_IPV4_TCP), 603 .type = RTE_FLOW_ITEM_TYPE_IPV4, 604 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 605 ETH_RSS_NONFRAG_IPV4_OTHER, 606 }, 607 [MLX5_EXPANSION_IPV4_UDP] = { 608 .type = RTE_FLOW_ITEM_TYPE_UDP, 609 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 610 }, 611 [MLX5_EXPANSION_IPV4_TCP] = { 612 .type = RTE_FLOW_ITEM_TYPE_TCP, 613 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 614 }, 615 [MLX5_EXPANSION_IPV6] = { 616 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 617 MLX5_EXPANSION_IPV6_TCP, 618 MLX5_EXPANSION_IPV6_FRAG_EXT), 619 .type = RTE_FLOW_ITEM_TYPE_IPV6, 620 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 621 ETH_RSS_NONFRAG_IPV6_OTHER, 622 }, 623 [MLX5_EXPANSION_IPV6_UDP] = { 624 .type = RTE_FLOW_ITEM_TYPE_UDP, 625 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 626 }, 627 [MLX5_EXPANSION_IPV6_TCP] = { 628 .type = RTE_FLOW_ITEM_TYPE_TCP, 629 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 630 }, 631 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 632 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 633 }, 634 [MLX5_EXPANSION_GTP] = { 635 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 636 MLX5_EXPANSION_IPV6), 637 .type = RTE_FLOW_ITEM_TYPE_GTP 638 } 639 }; 640 641 static struct rte_flow_action_handle * 642 mlx5_action_handle_create(struct rte_eth_dev *dev, 643 const struct rte_flow_indir_action_conf *conf, 644 const struct rte_flow_action *action, 645 struct rte_flow_error *error); 646 static int mlx5_action_handle_destroy 647 (struct rte_eth_dev *dev, 648 struct rte_flow_action_handle *handle, 649 struct rte_flow_error *error); 650 static int mlx5_action_handle_update 651 (struct rte_eth_dev *dev, 652 struct rte_flow_action_handle *handle, 653 const void *update, 654 struct rte_flow_error *error); 655 static int mlx5_action_handle_query 656 (struct rte_eth_dev *dev, 657 const struct rte_flow_action_handle *handle, 658 void *data, 659 struct rte_flow_error *error); 660 static int 661 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 662 struct rte_flow_tunnel *app_tunnel, 663 struct rte_flow_action **actions, 664 uint32_t *num_of_actions, 665 struct rte_flow_error *error); 666 static int 667 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 668 struct rte_flow_tunnel *app_tunnel, 669 struct rte_flow_item **items, 670 uint32_t *num_of_items, 671 struct rte_flow_error *error); 672 static int 673 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 674 struct rte_flow_item *pmd_items, 675 uint32_t num_items, struct rte_flow_error *err); 676 static int 677 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 678 struct rte_flow_action *pmd_actions, 679 uint32_t num_actions, 680 struct rte_flow_error *err); 681 static int 682 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 683 struct rte_mbuf *m, 684 struct rte_flow_restore_info *info, 685 struct rte_flow_error *err); 686 687 static const struct rte_flow_ops mlx5_flow_ops = { 688 .validate = mlx5_flow_validate, 689 .create = mlx5_flow_create, 690 .destroy = mlx5_flow_destroy, 691 .flush = mlx5_flow_flush, 692 .isolate = mlx5_flow_isolate, 693 .query = mlx5_flow_query, 694 .dev_dump = mlx5_flow_dev_dump, 695 .get_aged_flows = mlx5_flow_get_aged_flows, 696 .action_handle_create = mlx5_action_handle_create, 697 .action_handle_destroy = mlx5_action_handle_destroy, 698 .action_handle_update = mlx5_action_handle_update, 699 .action_handle_query = mlx5_action_handle_query, 700 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 701 .tunnel_match = mlx5_flow_tunnel_match, 702 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 703 .tunnel_item_release = mlx5_flow_tunnel_item_release, 704 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 705 }; 706 707 /* Tunnel information. */ 708 struct mlx5_flow_tunnel_info { 709 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 710 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 711 }; 712 713 static struct mlx5_flow_tunnel_info tunnels_info[] = { 714 { 715 .tunnel = MLX5_FLOW_LAYER_VXLAN, 716 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 717 }, 718 { 719 .tunnel = MLX5_FLOW_LAYER_GENEVE, 720 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 721 }, 722 { 723 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 724 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 725 }, 726 { 727 .tunnel = MLX5_FLOW_LAYER_GRE, 728 .ptype = RTE_PTYPE_TUNNEL_GRE, 729 }, 730 { 731 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 732 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 733 }, 734 { 735 .tunnel = MLX5_FLOW_LAYER_MPLS, 736 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 737 }, 738 { 739 .tunnel = MLX5_FLOW_LAYER_NVGRE, 740 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 741 }, 742 { 743 .tunnel = MLX5_FLOW_LAYER_IPIP, 744 .ptype = RTE_PTYPE_TUNNEL_IP, 745 }, 746 { 747 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 748 .ptype = RTE_PTYPE_TUNNEL_IP, 749 }, 750 { 751 .tunnel = MLX5_FLOW_LAYER_GTP, 752 .ptype = RTE_PTYPE_TUNNEL_GTPU, 753 }, 754 }; 755 756 757 758 /** 759 * Translate tag ID to register. 760 * 761 * @param[in] dev 762 * Pointer to the Ethernet device structure. 763 * @param[in] feature 764 * The feature that request the register. 765 * @param[in] id 766 * The request register ID. 767 * @param[out] error 768 * Error description in case of any. 769 * 770 * @return 771 * The request register on success, a negative errno 772 * value otherwise and rte_errno is set. 773 */ 774 int 775 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 776 enum mlx5_feature_name feature, 777 uint32_t id, 778 struct rte_flow_error *error) 779 { 780 struct mlx5_priv *priv = dev->data->dev_private; 781 struct mlx5_dev_config *config = &priv->config; 782 enum modify_reg start_reg; 783 bool skip_mtr_reg = false; 784 785 switch (feature) { 786 case MLX5_HAIRPIN_RX: 787 return REG_B; 788 case MLX5_HAIRPIN_TX: 789 return REG_A; 790 case MLX5_METADATA_RX: 791 switch (config->dv_xmeta_en) { 792 case MLX5_XMETA_MODE_LEGACY: 793 return REG_B; 794 case MLX5_XMETA_MODE_META16: 795 return REG_C_0; 796 case MLX5_XMETA_MODE_META32: 797 return REG_C_1; 798 } 799 break; 800 case MLX5_METADATA_TX: 801 return REG_A; 802 case MLX5_METADATA_FDB: 803 switch (config->dv_xmeta_en) { 804 case MLX5_XMETA_MODE_LEGACY: 805 return REG_NON; 806 case MLX5_XMETA_MODE_META16: 807 return REG_C_0; 808 case MLX5_XMETA_MODE_META32: 809 return REG_C_1; 810 } 811 break; 812 case MLX5_FLOW_MARK: 813 switch (config->dv_xmeta_en) { 814 case MLX5_XMETA_MODE_LEGACY: 815 return REG_NON; 816 case MLX5_XMETA_MODE_META16: 817 return REG_C_1; 818 case MLX5_XMETA_MODE_META32: 819 return REG_C_0; 820 } 821 break; 822 case MLX5_MTR_ID: 823 /* 824 * If meter color and meter id share one register, flow match 825 * should use the meter color register for match. 826 */ 827 if (priv->mtr_reg_share) 828 return priv->mtr_color_reg; 829 else 830 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 831 REG_C_3; 832 case MLX5_MTR_COLOR: 833 case MLX5_ASO_FLOW_HIT: 834 case MLX5_ASO_CONNTRACK: 835 /* All features use the same REG_C. */ 836 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 837 return priv->mtr_color_reg; 838 case MLX5_COPY_MARK: 839 /* 840 * Metadata COPY_MARK register using is in meter suffix sub 841 * flow while with meter. It's safe to share the same register. 842 */ 843 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 844 case MLX5_APP_TAG: 845 /* 846 * If meter is enable, it will engage the register for color 847 * match and flow match. If meter color match is not using the 848 * REG_C_2, need to skip the REG_C_x be used by meter color 849 * match. 850 * If meter is disable, free to use all available registers. 851 */ 852 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 853 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 854 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 855 if (id > (uint32_t)(REG_C_7 - start_reg)) 856 return rte_flow_error_set(error, EINVAL, 857 RTE_FLOW_ERROR_TYPE_ITEM, 858 NULL, "invalid tag id"); 859 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 860 return rte_flow_error_set(error, ENOTSUP, 861 RTE_FLOW_ERROR_TYPE_ITEM, 862 NULL, "unsupported tag id"); 863 /* 864 * This case means meter is using the REG_C_x great than 2. 865 * Take care not to conflict with meter color REG_C_x. 866 * If the available index REG_C_y >= REG_C_x, skip the 867 * color register. 868 */ 869 if (skip_mtr_reg && config->flow_mreg_c 870 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 871 if (id >= (uint32_t)(REG_C_7 - start_reg)) 872 return rte_flow_error_set(error, EINVAL, 873 RTE_FLOW_ERROR_TYPE_ITEM, 874 NULL, "invalid tag id"); 875 if (config->flow_mreg_c 876 [id + 1 + start_reg - REG_C_0] != REG_NON) 877 return config->flow_mreg_c 878 [id + 1 + start_reg - REG_C_0]; 879 return rte_flow_error_set(error, ENOTSUP, 880 RTE_FLOW_ERROR_TYPE_ITEM, 881 NULL, "unsupported tag id"); 882 } 883 return config->flow_mreg_c[id + start_reg - REG_C_0]; 884 } 885 MLX5_ASSERT(false); 886 return rte_flow_error_set(error, EINVAL, 887 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 888 NULL, "invalid feature name"); 889 } 890 891 /** 892 * Check extensive flow metadata register support. 893 * 894 * @param dev 895 * Pointer to rte_eth_dev structure. 896 * 897 * @return 898 * True if device supports extensive flow metadata register, otherwise false. 899 */ 900 bool 901 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 902 { 903 struct mlx5_priv *priv = dev->data->dev_private; 904 struct mlx5_dev_config *config = &priv->config; 905 906 /* 907 * Having available reg_c can be regarded inclusively as supporting 908 * extensive flow metadata register, which could mean, 909 * - metadata register copy action by modify header. 910 * - 16 modify header actions is supported. 911 * - reg_c's are preserved across different domain (FDB and NIC) on 912 * packet loopback by flow lookup miss. 913 */ 914 return config->flow_mreg_c[2] != REG_NON; 915 } 916 917 /** 918 * Get the lowest priority. 919 * 920 * @param[in] dev 921 * Pointer to the Ethernet device structure. 922 * @param[in] attributes 923 * Pointer to device flow rule attributes. 924 * 925 * @return 926 * The value of lowest priority of flow. 927 */ 928 uint32_t 929 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 930 const struct rte_flow_attr *attr) 931 { 932 struct mlx5_priv *priv = dev->data->dev_private; 933 934 if (!attr->group && !attr->transfer) 935 return priv->config.flow_prio - 2; 936 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 937 } 938 939 /** 940 * Calculate matcher priority of the flow. 941 * 942 * @param[in] dev 943 * Pointer to the Ethernet device structure. 944 * @param[in] attr 945 * Pointer to device flow rule attributes. 946 * @param[in] subpriority 947 * The priority based on the items. 948 * @return 949 * The matcher priority of the flow. 950 */ 951 uint16_t 952 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 953 const struct rte_flow_attr *attr, 954 uint32_t subpriority) 955 { 956 uint16_t priority = (uint16_t)attr->priority; 957 struct mlx5_priv *priv = dev->data->dev_private; 958 959 if (!attr->group && !attr->transfer) { 960 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 961 priority = priv->config.flow_prio - 1; 962 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 963 } 964 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 965 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 966 return priority * 3 + subpriority; 967 } 968 969 /** 970 * Verify the @p item specifications (spec, last, mask) are compatible with the 971 * NIC capabilities. 972 * 973 * @param[in] item 974 * Item specification. 975 * @param[in] mask 976 * @p item->mask or flow default bit-masks. 977 * @param[in] nic_mask 978 * Bit-masks covering supported fields by the NIC to compare with user mask. 979 * @param[in] size 980 * Bit-masks size in bytes. 981 * @param[in] range_accepted 982 * True if range of values is accepted for specific fields, false otherwise. 983 * @param[out] error 984 * Pointer to error structure. 985 * 986 * @return 987 * 0 on success, a negative errno value otherwise and rte_errno is set. 988 */ 989 int 990 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 991 const uint8_t *mask, 992 const uint8_t *nic_mask, 993 unsigned int size, 994 bool range_accepted, 995 struct rte_flow_error *error) 996 { 997 unsigned int i; 998 999 MLX5_ASSERT(nic_mask); 1000 for (i = 0; i < size; ++i) 1001 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1002 return rte_flow_error_set(error, ENOTSUP, 1003 RTE_FLOW_ERROR_TYPE_ITEM, 1004 item, 1005 "mask enables non supported" 1006 " bits"); 1007 if (!item->spec && (item->mask || item->last)) 1008 return rte_flow_error_set(error, EINVAL, 1009 RTE_FLOW_ERROR_TYPE_ITEM, item, 1010 "mask/last without a spec is not" 1011 " supported"); 1012 if (item->spec && item->last && !range_accepted) { 1013 uint8_t spec[size]; 1014 uint8_t last[size]; 1015 unsigned int i; 1016 int ret; 1017 1018 for (i = 0; i < size; ++i) { 1019 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1020 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1021 } 1022 ret = memcmp(spec, last, size); 1023 if (ret != 0) 1024 return rte_flow_error_set(error, EINVAL, 1025 RTE_FLOW_ERROR_TYPE_ITEM, 1026 item, 1027 "range is not valid"); 1028 } 1029 return 0; 1030 } 1031 1032 /** 1033 * Adjust the hash fields according to the @p flow information. 1034 * 1035 * @param[in] dev_flow. 1036 * Pointer to the mlx5_flow. 1037 * @param[in] tunnel 1038 * 1 when the hash field is for a tunnel item. 1039 * @param[in] layer_types 1040 * ETH_RSS_* types. 1041 * @param[in] hash_fields 1042 * Item hash fields. 1043 * 1044 * @return 1045 * The hash fields that should be used. 1046 */ 1047 uint64_t 1048 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1049 int tunnel __rte_unused, uint64_t layer_types, 1050 uint64_t hash_fields) 1051 { 1052 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1053 int rss_request_inner = rss_desc->level >= 2; 1054 1055 /* Check RSS hash level for tunnel. */ 1056 if (tunnel && rss_request_inner) 1057 hash_fields |= IBV_RX_HASH_INNER; 1058 else if (tunnel || rss_request_inner) 1059 return 0; 1060 #endif 1061 /* Check if requested layer matches RSS hash fields. */ 1062 if (!(rss_desc->types & layer_types)) 1063 return 0; 1064 return hash_fields; 1065 } 1066 1067 /** 1068 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1069 * if several tunnel rules are used on this queue, the tunnel ptype will be 1070 * cleared. 1071 * 1072 * @param rxq_ctrl 1073 * Rx queue to update. 1074 */ 1075 static void 1076 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1077 { 1078 unsigned int i; 1079 uint32_t tunnel_ptype = 0; 1080 1081 /* Look up for the ptype to use. */ 1082 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1083 if (!rxq_ctrl->flow_tunnels_n[i]) 1084 continue; 1085 if (!tunnel_ptype) { 1086 tunnel_ptype = tunnels_info[i].ptype; 1087 } else { 1088 tunnel_ptype = 0; 1089 break; 1090 } 1091 } 1092 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1093 } 1094 1095 /** 1096 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1097 * flow. 1098 * 1099 * @param[in] dev 1100 * Pointer to the Ethernet device structure. 1101 * @param[in] dev_handle 1102 * Pointer to device flow handle structure. 1103 */ 1104 void 1105 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1106 struct mlx5_flow_handle *dev_handle) 1107 { 1108 struct mlx5_priv *priv = dev->data->dev_private; 1109 const int mark = dev_handle->mark; 1110 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1111 struct mlx5_ind_table_obj *ind_tbl = NULL; 1112 unsigned int i; 1113 1114 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1115 struct mlx5_hrxq *hrxq; 1116 1117 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1118 dev_handle->rix_hrxq); 1119 if (hrxq) 1120 ind_tbl = hrxq->ind_table; 1121 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1122 struct mlx5_shared_action_rss *shared_rss; 1123 1124 shared_rss = mlx5_ipool_get 1125 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1126 dev_handle->rix_srss); 1127 if (shared_rss) 1128 ind_tbl = shared_rss->ind_tbl; 1129 } 1130 if (!ind_tbl) 1131 return; 1132 for (i = 0; i != ind_tbl->queues_n; ++i) { 1133 int idx = ind_tbl->queues[i]; 1134 struct mlx5_rxq_ctrl *rxq_ctrl = 1135 container_of((*priv->rxqs)[idx], 1136 struct mlx5_rxq_ctrl, rxq); 1137 1138 /* 1139 * To support metadata register copy on Tx loopback, 1140 * this must be always enabled (metadata may arive 1141 * from other port - not from local flows only. 1142 */ 1143 if (priv->config.dv_flow_en && 1144 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1145 mlx5_flow_ext_mreg_supported(dev)) { 1146 rxq_ctrl->rxq.mark = 1; 1147 rxq_ctrl->flow_mark_n = 1; 1148 } else if (mark) { 1149 rxq_ctrl->rxq.mark = 1; 1150 rxq_ctrl->flow_mark_n++; 1151 } 1152 if (tunnel) { 1153 unsigned int j; 1154 1155 /* Increase the counter matching the flow. */ 1156 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1157 if ((tunnels_info[j].tunnel & 1158 dev_handle->layers) == 1159 tunnels_info[j].tunnel) { 1160 rxq_ctrl->flow_tunnels_n[j]++; 1161 break; 1162 } 1163 } 1164 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1165 } 1166 } 1167 } 1168 1169 /** 1170 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1171 * 1172 * @param[in] dev 1173 * Pointer to the Ethernet device structure. 1174 * @param[in] flow 1175 * Pointer to flow structure. 1176 */ 1177 static void 1178 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1179 { 1180 struct mlx5_priv *priv = dev->data->dev_private; 1181 uint32_t handle_idx; 1182 struct mlx5_flow_handle *dev_handle; 1183 1184 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1185 handle_idx, dev_handle, next) 1186 flow_drv_rxq_flags_set(dev, dev_handle); 1187 } 1188 1189 /** 1190 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1191 * device flow if no other flow uses it with the same kind of request. 1192 * 1193 * @param dev 1194 * Pointer to Ethernet device. 1195 * @param[in] dev_handle 1196 * Pointer to the device flow handle structure. 1197 */ 1198 static void 1199 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1200 struct mlx5_flow_handle *dev_handle) 1201 { 1202 struct mlx5_priv *priv = dev->data->dev_private; 1203 const int mark = dev_handle->mark; 1204 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1205 struct mlx5_ind_table_obj *ind_tbl = NULL; 1206 unsigned int i; 1207 1208 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1209 struct mlx5_hrxq *hrxq; 1210 1211 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1212 dev_handle->rix_hrxq); 1213 if (hrxq) 1214 ind_tbl = hrxq->ind_table; 1215 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1216 struct mlx5_shared_action_rss *shared_rss; 1217 1218 shared_rss = mlx5_ipool_get 1219 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1220 dev_handle->rix_srss); 1221 if (shared_rss) 1222 ind_tbl = shared_rss->ind_tbl; 1223 } 1224 if (!ind_tbl) 1225 return; 1226 MLX5_ASSERT(dev->data->dev_started); 1227 for (i = 0; i != ind_tbl->queues_n; ++i) { 1228 int idx = ind_tbl->queues[i]; 1229 struct mlx5_rxq_ctrl *rxq_ctrl = 1230 container_of((*priv->rxqs)[idx], 1231 struct mlx5_rxq_ctrl, rxq); 1232 1233 if (priv->config.dv_flow_en && 1234 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1235 mlx5_flow_ext_mreg_supported(dev)) { 1236 rxq_ctrl->rxq.mark = 1; 1237 rxq_ctrl->flow_mark_n = 1; 1238 } else if (mark) { 1239 rxq_ctrl->flow_mark_n--; 1240 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1241 } 1242 if (tunnel) { 1243 unsigned int j; 1244 1245 /* Decrease the counter matching the flow. */ 1246 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1247 if ((tunnels_info[j].tunnel & 1248 dev_handle->layers) == 1249 tunnels_info[j].tunnel) { 1250 rxq_ctrl->flow_tunnels_n[j]--; 1251 break; 1252 } 1253 } 1254 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1255 } 1256 } 1257 } 1258 1259 /** 1260 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1261 * @p flow if no other flow uses it with the same kind of request. 1262 * 1263 * @param dev 1264 * Pointer to Ethernet device. 1265 * @param[in] flow 1266 * Pointer to the flow. 1267 */ 1268 static void 1269 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1270 { 1271 struct mlx5_priv *priv = dev->data->dev_private; 1272 uint32_t handle_idx; 1273 struct mlx5_flow_handle *dev_handle; 1274 1275 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1276 handle_idx, dev_handle, next) 1277 flow_drv_rxq_flags_trim(dev, dev_handle); 1278 } 1279 1280 /** 1281 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1282 * 1283 * @param dev 1284 * Pointer to Ethernet device. 1285 */ 1286 static void 1287 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1288 { 1289 struct mlx5_priv *priv = dev->data->dev_private; 1290 unsigned int i; 1291 1292 for (i = 0; i != priv->rxqs_n; ++i) { 1293 struct mlx5_rxq_ctrl *rxq_ctrl; 1294 unsigned int j; 1295 1296 if (!(*priv->rxqs)[i]) 1297 continue; 1298 rxq_ctrl = container_of((*priv->rxqs)[i], 1299 struct mlx5_rxq_ctrl, rxq); 1300 rxq_ctrl->flow_mark_n = 0; 1301 rxq_ctrl->rxq.mark = 0; 1302 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1303 rxq_ctrl->flow_tunnels_n[j] = 0; 1304 rxq_ctrl->rxq.tunnel = 0; 1305 } 1306 } 1307 1308 /** 1309 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1310 * 1311 * @param[in] dev 1312 * Pointer to the Ethernet device structure. 1313 */ 1314 void 1315 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1316 { 1317 struct mlx5_priv *priv = dev->data->dev_private; 1318 struct mlx5_rxq_data *data; 1319 unsigned int i; 1320 1321 for (i = 0; i != priv->rxqs_n; ++i) { 1322 if (!(*priv->rxqs)[i]) 1323 continue; 1324 data = (*priv->rxqs)[i]; 1325 if (!rte_flow_dynf_metadata_avail()) { 1326 data->dynf_meta = 0; 1327 data->flow_meta_mask = 0; 1328 data->flow_meta_offset = -1; 1329 data->flow_meta_port_mask = 0; 1330 } else { 1331 data->dynf_meta = 1; 1332 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1333 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1334 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1335 } 1336 } 1337 } 1338 1339 /* 1340 * return a pointer to the desired action in the list of actions. 1341 * 1342 * @param[in] actions 1343 * The list of actions to search the action in. 1344 * @param[in] action 1345 * The action to find. 1346 * 1347 * @return 1348 * Pointer to the action in the list, if found. NULL otherwise. 1349 */ 1350 const struct rte_flow_action * 1351 mlx5_flow_find_action(const struct rte_flow_action *actions, 1352 enum rte_flow_action_type action) 1353 { 1354 if (actions == NULL) 1355 return NULL; 1356 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1357 if (actions->type == action) 1358 return actions; 1359 return NULL; 1360 } 1361 1362 /* 1363 * Validate the flag action. 1364 * 1365 * @param[in] action_flags 1366 * Bit-fields that holds the actions detected until now. 1367 * @param[in] attr 1368 * Attributes of flow that includes this action. 1369 * @param[out] error 1370 * Pointer to error structure. 1371 * 1372 * @return 1373 * 0 on success, a negative errno value otherwise and rte_errno is set. 1374 */ 1375 int 1376 mlx5_flow_validate_action_flag(uint64_t action_flags, 1377 const struct rte_flow_attr *attr, 1378 struct rte_flow_error *error) 1379 { 1380 if (action_flags & MLX5_FLOW_ACTION_MARK) 1381 return rte_flow_error_set(error, EINVAL, 1382 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1383 "can't mark and flag in same flow"); 1384 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1385 return rte_flow_error_set(error, EINVAL, 1386 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1387 "can't have 2 flag" 1388 " actions in same flow"); 1389 if (attr->egress) 1390 return rte_flow_error_set(error, ENOTSUP, 1391 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1392 "flag action not supported for " 1393 "egress"); 1394 return 0; 1395 } 1396 1397 /* 1398 * Validate the mark action. 1399 * 1400 * @param[in] action 1401 * Pointer to the queue action. 1402 * @param[in] action_flags 1403 * Bit-fields that holds the actions detected until now. 1404 * @param[in] attr 1405 * Attributes of flow that includes this action. 1406 * @param[out] error 1407 * Pointer to error structure. 1408 * 1409 * @return 1410 * 0 on success, a negative errno value otherwise and rte_errno is set. 1411 */ 1412 int 1413 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1414 uint64_t action_flags, 1415 const struct rte_flow_attr *attr, 1416 struct rte_flow_error *error) 1417 { 1418 const struct rte_flow_action_mark *mark = action->conf; 1419 1420 if (!mark) 1421 return rte_flow_error_set(error, EINVAL, 1422 RTE_FLOW_ERROR_TYPE_ACTION, 1423 action, 1424 "configuration cannot be null"); 1425 if (mark->id >= MLX5_FLOW_MARK_MAX) 1426 return rte_flow_error_set(error, EINVAL, 1427 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1428 &mark->id, 1429 "mark id must in 0 <= id < " 1430 RTE_STR(MLX5_FLOW_MARK_MAX)); 1431 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1432 return rte_flow_error_set(error, EINVAL, 1433 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1434 "can't flag and mark in same flow"); 1435 if (action_flags & MLX5_FLOW_ACTION_MARK) 1436 return rte_flow_error_set(error, EINVAL, 1437 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1438 "can't have 2 mark actions in same" 1439 " flow"); 1440 if (attr->egress) 1441 return rte_flow_error_set(error, ENOTSUP, 1442 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1443 "mark action not supported for " 1444 "egress"); 1445 return 0; 1446 } 1447 1448 /* 1449 * Validate the drop action. 1450 * 1451 * @param[in] action_flags 1452 * Bit-fields that holds the actions detected until now. 1453 * @param[in] attr 1454 * Attributes of flow that includes this action. 1455 * @param[out] error 1456 * Pointer to error structure. 1457 * 1458 * @return 1459 * 0 on success, a negative errno value otherwise and rte_errno is set. 1460 */ 1461 int 1462 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1463 const struct rte_flow_attr *attr, 1464 struct rte_flow_error *error) 1465 { 1466 if (attr->egress) 1467 return rte_flow_error_set(error, ENOTSUP, 1468 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1469 "drop action not supported for " 1470 "egress"); 1471 return 0; 1472 } 1473 1474 /* 1475 * Validate the queue action. 1476 * 1477 * @param[in] action 1478 * Pointer to the queue action. 1479 * @param[in] action_flags 1480 * Bit-fields that holds the actions detected until now. 1481 * @param[in] dev 1482 * Pointer to the Ethernet device structure. 1483 * @param[in] attr 1484 * Attributes of flow that includes this action. 1485 * @param[out] error 1486 * Pointer to error structure. 1487 * 1488 * @return 1489 * 0 on success, a negative errno value otherwise and rte_errno is set. 1490 */ 1491 int 1492 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1493 uint64_t action_flags, 1494 struct rte_eth_dev *dev, 1495 const struct rte_flow_attr *attr, 1496 struct rte_flow_error *error) 1497 { 1498 struct mlx5_priv *priv = dev->data->dev_private; 1499 const struct rte_flow_action_queue *queue = action->conf; 1500 1501 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1502 return rte_flow_error_set(error, EINVAL, 1503 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1504 "can't have 2 fate actions in" 1505 " same flow"); 1506 if (!priv->rxqs_n) 1507 return rte_flow_error_set(error, EINVAL, 1508 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1509 NULL, "No Rx queues configured"); 1510 if (queue->index >= priv->rxqs_n) 1511 return rte_flow_error_set(error, EINVAL, 1512 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1513 &queue->index, 1514 "queue index out of range"); 1515 if (!(*priv->rxqs)[queue->index]) 1516 return rte_flow_error_set(error, EINVAL, 1517 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1518 &queue->index, 1519 "queue is not configured"); 1520 if (attr->egress) 1521 return rte_flow_error_set(error, ENOTSUP, 1522 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1523 "queue action not supported for " 1524 "egress"); 1525 return 0; 1526 } 1527 1528 /* 1529 * Validate the rss action. 1530 * 1531 * @param[in] dev 1532 * Pointer to the Ethernet device structure. 1533 * @param[in] action 1534 * Pointer to the queue action. 1535 * @param[out] error 1536 * Pointer to error structure. 1537 * 1538 * @return 1539 * 0 on success, a negative errno value otherwise and rte_errno is set. 1540 */ 1541 int 1542 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1543 const struct rte_flow_action *action, 1544 struct rte_flow_error *error) 1545 { 1546 struct mlx5_priv *priv = dev->data->dev_private; 1547 const struct rte_flow_action_rss *rss = action->conf; 1548 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1549 unsigned int i; 1550 1551 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1552 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1553 return rte_flow_error_set(error, ENOTSUP, 1554 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1555 &rss->func, 1556 "RSS hash function not supported"); 1557 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1558 if (rss->level > 2) 1559 #else 1560 if (rss->level > 1) 1561 #endif 1562 return rte_flow_error_set(error, ENOTSUP, 1563 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1564 &rss->level, 1565 "tunnel RSS is not supported"); 1566 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1567 if (rss->key_len == 0 && rss->key != NULL) 1568 return rte_flow_error_set(error, ENOTSUP, 1569 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1570 &rss->key_len, 1571 "RSS hash key length 0"); 1572 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1573 return rte_flow_error_set(error, ENOTSUP, 1574 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1575 &rss->key_len, 1576 "RSS hash key too small"); 1577 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1578 return rte_flow_error_set(error, ENOTSUP, 1579 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1580 &rss->key_len, 1581 "RSS hash key too large"); 1582 if (rss->queue_num > priv->config.ind_table_max_size) 1583 return rte_flow_error_set(error, ENOTSUP, 1584 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1585 &rss->queue_num, 1586 "number of queues too large"); 1587 if (rss->types & MLX5_RSS_HF_MASK) 1588 return rte_flow_error_set(error, ENOTSUP, 1589 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1590 &rss->types, 1591 "some RSS protocols are not" 1592 " supported"); 1593 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1594 !(rss->types & ETH_RSS_IP)) 1595 return rte_flow_error_set(error, EINVAL, 1596 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1597 "L3 partial RSS requested but L3 RSS" 1598 " type not specified"); 1599 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1600 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1601 return rte_flow_error_set(error, EINVAL, 1602 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1603 "L4 partial RSS requested but L4 RSS" 1604 " type not specified"); 1605 if (!priv->rxqs_n) 1606 return rte_flow_error_set(error, EINVAL, 1607 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1608 NULL, "No Rx queues configured"); 1609 if (!rss->queue_num) 1610 return rte_flow_error_set(error, EINVAL, 1611 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1612 NULL, "No queues configured"); 1613 for (i = 0; i != rss->queue_num; ++i) { 1614 struct mlx5_rxq_ctrl *rxq_ctrl; 1615 1616 if (rss->queue[i] >= priv->rxqs_n) 1617 return rte_flow_error_set 1618 (error, EINVAL, 1619 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1620 &rss->queue[i], "queue index out of range"); 1621 if (!(*priv->rxqs)[rss->queue[i]]) 1622 return rte_flow_error_set 1623 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1624 &rss->queue[i], "queue is not configured"); 1625 rxq_ctrl = container_of((*priv->rxqs)[rss->queue[i]], 1626 struct mlx5_rxq_ctrl, rxq); 1627 if (i == 0) 1628 rxq_type = rxq_ctrl->type; 1629 if (rxq_type != rxq_ctrl->type) 1630 return rte_flow_error_set 1631 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1632 &rss->queue[i], 1633 "combining hairpin and regular RSS queues is not supported"); 1634 } 1635 return 0; 1636 } 1637 1638 /* 1639 * Validate the rss action. 1640 * 1641 * @param[in] action 1642 * Pointer to the queue action. 1643 * @param[in] action_flags 1644 * Bit-fields that holds the actions detected until now. 1645 * @param[in] dev 1646 * Pointer to the Ethernet device structure. 1647 * @param[in] attr 1648 * Attributes of flow that includes this action. 1649 * @param[in] item_flags 1650 * Items that were detected. 1651 * @param[out] error 1652 * Pointer to error structure. 1653 * 1654 * @return 1655 * 0 on success, a negative errno value otherwise and rte_errno is set. 1656 */ 1657 int 1658 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1659 uint64_t action_flags, 1660 struct rte_eth_dev *dev, 1661 const struct rte_flow_attr *attr, 1662 uint64_t item_flags, 1663 struct rte_flow_error *error) 1664 { 1665 const struct rte_flow_action_rss *rss = action->conf; 1666 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1667 int ret; 1668 1669 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1670 return rte_flow_error_set(error, EINVAL, 1671 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1672 "can't have 2 fate actions" 1673 " in same flow"); 1674 ret = mlx5_validate_action_rss(dev, action, error); 1675 if (ret) 1676 return ret; 1677 if (attr->egress) 1678 return rte_flow_error_set(error, ENOTSUP, 1679 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1680 "rss action not supported for " 1681 "egress"); 1682 if (rss->level > 1 && !tunnel) 1683 return rte_flow_error_set(error, EINVAL, 1684 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1685 "inner RSS is not supported for " 1686 "non-tunnel flows"); 1687 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1688 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1689 return rte_flow_error_set(error, EINVAL, 1690 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1691 "RSS on eCPRI is not supported now"); 1692 } 1693 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1694 !(item_flags & 1695 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1696 rss->level > 1) 1697 return rte_flow_error_set(error, EINVAL, 1698 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1699 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1700 return 0; 1701 } 1702 1703 /* 1704 * Validate the default miss action. 1705 * 1706 * @param[in] action_flags 1707 * Bit-fields that holds the actions detected until now. 1708 * @param[out] error 1709 * Pointer to error structure. 1710 * 1711 * @return 1712 * 0 on success, a negative errno value otherwise and rte_errno is set. 1713 */ 1714 int 1715 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1716 const struct rte_flow_attr *attr, 1717 struct rte_flow_error *error) 1718 { 1719 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1720 return rte_flow_error_set(error, EINVAL, 1721 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1722 "can't have 2 fate actions in" 1723 " same flow"); 1724 if (attr->egress) 1725 return rte_flow_error_set(error, ENOTSUP, 1726 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1727 "default miss action not supported " 1728 "for egress"); 1729 if (attr->group) 1730 return rte_flow_error_set(error, ENOTSUP, 1731 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1732 "only group 0 is supported"); 1733 if (attr->transfer) 1734 return rte_flow_error_set(error, ENOTSUP, 1735 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1736 NULL, "transfer is not supported"); 1737 return 0; 1738 } 1739 1740 /* 1741 * Validate the count action. 1742 * 1743 * @param[in] dev 1744 * Pointer to the Ethernet device structure. 1745 * @param[in] attr 1746 * Attributes of flow that includes this action. 1747 * @param[out] error 1748 * Pointer to error structure. 1749 * 1750 * @return 1751 * 0 on success, a negative errno value otherwise and rte_errno is set. 1752 */ 1753 int 1754 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1755 const struct rte_flow_attr *attr, 1756 struct rte_flow_error *error) 1757 { 1758 if (attr->egress) 1759 return rte_flow_error_set(error, ENOTSUP, 1760 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1761 "count action not supported for " 1762 "egress"); 1763 return 0; 1764 } 1765 1766 /* 1767 * Validate the ASO CT action. 1768 * 1769 * @param[in] dev 1770 * Pointer to the Ethernet device structure. 1771 * @param[in] conntrack 1772 * Pointer to the CT action profile. 1773 * @param[out] error 1774 * Pointer to error structure. 1775 * 1776 * @return 1777 * 0 on success, a negative errno value otherwise and rte_errno is set. 1778 */ 1779 int 1780 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1781 const struct rte_flow_action_conntrack *conntrack, 1782 struct rte_flow_error *error) 1783 { 1784 RTE_SET_USED(dev); 1785 1786 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1787 return rte_flow_error_set(error, EINVAL, 1788 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1789 "Invalid CT state"); 1790 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1791 return rte_flow_error_set(error, EINVAL, 1792 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1793 "Invalid last TCP packet flag"); 1794 return 0; 1795 } 1796 1797 /** 1798 * Verify the @p attributes will be correctly understood by the NIC and store 1799 * them in the @p flow if everything is correct. 1800 * 1801 * @param[in] dev 1802 * Pointer to the Ethernet device structure. 1803 * @param[in] attributes 1804 * Pointer to flow attributes 1805 * @param[out] error 1806 * Pointer to error structure. 1807 * 1808 * @return 1809 * 0 on success, a negative errno value otherwise and rte_errno is set. 1810 */ 1811 int 1812 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1813 const struct rte_flow_attr *attributes, 1814 struct rte_flow_error *error) 1815 { 1816 struct mlx5_priv *priv = dev->data->dev_private; 1817 uint32_t priority_max = priv->config.flow_prio - 1; 1818 1819 if (attributes->group) 1820 return rte_flow_error_set(error, ENOTSUP, 1821 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1822 NULL, "groups is not supported"); 1823 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 1824 attributes->priority >= priority_max) 1825 return rte_flow_error_set(error, ENOTSUP, 1826 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1827 NULL, "priority out of range"); 1828 if (attributes->egress) 1829 return rte_flow_error_set(error, ENOTSUP, 1830 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1831 "egress is not supported"); 1832 if (attributes->transfer && !priv->config.dv_esw_en) 1833 return rte_flow_error_set(error, ENOTSUP, 1834 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1835 NULL, "transfer is not supported"); 1836 if (!attributes->ingress) 1837 return rte_flow_error_set(error, EINVAL, 1838 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1839 NULL, 1840 "ingress attribute is mandatory"); 1841 return 0; 1842 } 1843 1844 /** 1845 * Validate ICMP6 item. 1846 * 1847 * @param[in] item 1848 * Item specification. 1849 * @param[in] item_flags 1850 * Bit-fields that holds the items detected until now. 1851 * @param[in] ext_vlan_sup 1852 * Whether extended VLAN features are supported or not. 1853 * @param[out] error 1854 * Pointer to error structure. 1855 * 1856 * @return 1857 * 0 on success, a negative errno value otherwise and rte_errno is set. 1858 */ 1859 int 1860 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1861 uint64_t item_flags, 1862 uint8_t target_protocol, 1863 struct rte_flow_error *error) 1864 { 1865 const struct rte_flow_item_icmp6 *mask = item->mask; 1866 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1867 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1868 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1869 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1870 MLX5_FLOW_LAYER_OUTER_L4; 1871 int ret; 1872 1873 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1874 return rte_flow_error_set(error, EINVAL, 1875 RTE_FLOW_ERROR_TYPE_ITEM, item, 1876 "protocol filtering not compatible" 1877 " with ICMP6 layer"); 1878 if (!(item_flags & l3m)) 1879 return rte_flow_error_set(error, EINVAL, 1880 RTE_FLOW_ERROR_TYPE_ITEM, item, 1881 "IPv6 is mandatory to filter on" 1882 " ICMP6"); 1883 if (item_flags & l4m) 1884 return rte_flow_error_set(error, EINVAL, 1885 RTE_FLOW_ERROR_TYPE_ITEM, item, 1886 "multiple L4 layers not supported"); 1887 if (!mask) 1888 mask = &rte_flow_item_icmp6_mask; 1889 ret = mlx5_flow_item_acceptable 1890 (item, (const uint8_t *)mask, 1891 (const uint8_t *)&rte_flow_item_icmp6_mask, 1892 sizeof(struct rte_flow_item_icmp6), 1893 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1894 if (ret < 0) 1895 return ret; 1896 return 0; 1897 } 1898 1899 /** 1900 * Validate ICMP item. 1901 * 1902 * @param[in] item 1903 * Item specification. 1904 * @param[in] item_flags 1905 * Bit-fields that holds the items detected until now. 1906 * @param[out] error 1907 * Pointer to error structure. 1908 * 1909 * @return 1910 * 0 on success, a negative errno value otherwise and rte_errno is set. 1911 */ 1912 int 1913 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1914 uint64_t item_flags, 1915 uint8_t target_protocol, 1916 struct rte_flow_error *error) 1917 { 1918 const struct rte_flow_item_icmp *mask = item->mask; 1919 const struct rte_flow_item_icmp nic_mask = { 1920 .hdr.icmp_type = 0xff, 1921 .hdr.icmp_code = 0xff, 1922 .hdr.icmp_ident = RTE_BE16(0xffff), 1923 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1924 }; 1925 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1926 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1927 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1928 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1929 MLX5_FLOW_LAYER_OUTER_L4; 1930 int ret; 1931 1932 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1933 return rte_flow_error_set(error, EINVAL, 1934 RTE_FLOW_ERROR_TYPE_ITEM, item, 1935 "protocol filtering not compatible" 1936 " with ICMP layer"); 1937 if (!(item_flags & l3m)) 1938 return rte_flow_error_set(error, EINVAL, 1939 RTE_FLOW_ERROR_TYPE_ITEM, item, 1940 "IPv4 is mandatory to filter" 1941 " on ICMP"); 1942 if (item_flags & l4m) 1943 return rte_flow_error_set(error, EINVAL, 1944 RTE_FLOW_ERROR_TYPE_ITEM, item, 1945 "multiple L4 layers not supported"); 1946 if (!mask) 1947 mask = &nic_mask; 1948 ret = mlx5_flow_item_acceptable 1949 (item, (const uint8_t *)mask, 1950 (const uint8_t *)&nic_mask, 1951 sizeof(struct rte_flow_item_icmp), 1952 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1953 if (ret < 0) 1954 return ret; 1955 return 0; 1956 } 1957 1958 /** 1959 * Validate Ethernet item. 1960 * 1961 * @param[in] item 1962 * Item specification. 1963 * @param[in] item_flags 1964 * Bit-fields that holds the items detected until now. 1965 * @param[out] error 1966 * Pointer to error structure. 1967 * 1968 * @return 1969 * 0 on success, a negative errno value otherwise and rte_errno is set. 1970 */ 1971 int 1972 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1973 uint64_t item_flags, bool ext_vlan_sup, 1974 struct rte_flow_error *error) 1975 { 1976 const struct rte_flow_item_eth *mask = item->mask; 1977 const struct rte_flow_item_eth nic_mask = { 1978 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1979 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1980 .type = RTE_BE16(0xffff), 1981 .has_vlan = ext_vlan_sup ? 1 : 0, 1982 }; 1983 int ret; 1984 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1985 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1986 MLX5_FLOW_LAYER_OUTER_L2; 1987 1988 if (item_flags & ethm) 1989 return rte_flow_error_set(error, ENOTSUP, 1990 RTE_FLOW_ERROR_TYPE_ITEM, item, 1991 "multiple L2 layers not supported"); 1992 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1993 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1994 return rte_flow_error_set(error, EINVAL, 1995 RTE_FLOW_ERROR_TYPE_ITEM, item, 1996 "L2 layer should not follow " 1997 "L3 layers"); 1998 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1999 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2000 return rte_flow_error_set(error, EINVAL, 2001 RTE_FLOW_ERROR_TYPE_ITEM, item, 2002 "L2 layer should not follow VLAN"); 2003 if (item_flags & MLX5_FLOW_LAYER_GTP) 2004 return rte_flow_error_set(error, EINVAL, 2005 RTE_FLOW_ERROR_TYPE_ITEM, item, 2006 "L2 layer should not follow GTP"); 2007 if (!mask) 2008 mask = &rte_flow_item_eth_mask; 2009 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2010 (const uint8_t *)&nic_mask, 2011 sizeof(struct rte_flow_item_eth), 2012 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2013 return ret; 2014 } 2015 2016 /** 2017 * Validate VLAN item. 2018 * 2019 * @param[in] item 2020 * Item specification. 2021 * @param[in] item_flags 2022 * Bit-fields that holds the items detected until now. 2023 * @param[in] dev 2024 * Ethernet device flow is being created on. 2025 * @param[out] error 2026 * Pointer to error structure. 2027 * 2028 * @return 2029 * 0 on success, a negative errno value otherwise and rte_errno is set. 2030 */ 2031 int 2032 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2033 uint64_t item_flags, 2034 struct rte_eth_dev *dev, 2035 struct rte_flow_error *error) 2036 { 2037 const struct rte_flow_item_vlan *spec = item->spec; 2038 const struct rte_flow_item_vlan *mask = item->mask; 2039 const struct rte_flow_item_vlan nic_mask = { 2040 .tci = RTE_BE16(UINT16_MAX), 2041 .inner_type = RTE_BE16(UINT16_MAX), 2042 }; 2043 uint16_t vlan_tag = 0; 2044 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2045 int ret; 2046 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2047 MLX5_FLOW_LAYER_INNER_L4) : 2048 (MLX5_FLOW_LAYER_OUTER_L3 | 2049 MLX5_FLOW_LAYER_OUTER_L4); 2050 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2051 MLX5_FLOW_LAYER_OUTER_VLAN; 2052 2053 if (item_flags & vlanm) 2054 return rte_flow_error_set(error, EINVAL, 2055 RTE_FLOW_ERROR_TYPE_ITEM, item, 2056 "multiple VLAN layers not supported"); 2057 else if ((item_flags & l34m) != 0) 2058 return rte_flow_error_set(error, EINVAL, 2059 RTE_FLOW_ERROR_TYPE_ITEM, item, 2060 "VLAN cannot follow L3/L4 layer"); 2061 if (!mask) 2062 mask = &rte_flow_item_vlan_mask; 2063 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2064 (const uint8_t *)&nic_mask, 2065 sizeof(struct rte_flow_item_vlan), 2066 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2067 if (ret) 2068 return ret; 2069 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2070 struct mlx5_priv *priv = dev->data->dev_private; 2071 2072 if (priv->vmwa_context) { 2073 /* 2074 * Non-NULL context means we have a virtual machine 2075 * and SR-IOV enabled, we have to create VLAN interface 2076 * to make hypervisor to setup E-Switch vport 2077 * context correctly. We avoid creating the multiple 2078 * VLAN interfaces, so we cannot support VLAN tag mask. 2079 */ 2080 return rte_flow_error_set(error, EINVAL, 2081 RTE_FLOW_ERROR_TYPE_ITEM, 2082 item, 2083 "VLAN tag mask is not" 2084 " supported in virtual" 2085 " environment"); 2086 } 2087 } 2088 if (spec) { 2089 vlan_tag = spec->tci; 2090 vlan_tag &= mask->tci; 2091 } 2092 /* 2093 * From verbs perspective an empty VLAN is equivalent 2094 * to a packet without VLAN layer. 2095 */ 2096 if (!vlan_tag) 2097 return rte_flow_error_set(error, EINVAL, 2098 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2099 item->spec, 2100 "VLAN cannot be empty"); 2101 return 0; 2102 } 2103 2104 /** 2105 * Validate IPV4 item. 2106 * 2107 * @param[in] item 2108 * Item specification. 2109 * @param[in] item_flags 2110 * Bit-fields that holds the items detected until now. 2111 * @param[in] last_item 2112 * Previous validated item in the pattern items. 2113 * @param[in] ether_type 2114 * Type in the ethernet layer header (including dot1q). 2115 * @param[in] acc_mask 2116 * Acceptable mask, if NULL default internal default mask 2117 * will be used to check whether item fields are supported. 2118 * @param[in] range_accepted 2119 * True if range of values is accepted for specific fields, false otherwise. 2120 * @param[out] error 2121 * Pointer to error structure. 2122 * 2123 * @return 2124 * 0 on success, a negative errno value otherwise and rte_errno is set. 2125 */ 2126 int 2127 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2128 uint64_t item_flags, 2129 uint64_t last_item, 2130 uint16_t ether_type, 2131 const struct rte_flow_item_ipv4 *acc_mask, 2132 bool range_accepted, 2133 struct rte_flow_error *error) 2134 { 2135 const struct rte_flow_item_ipv4 *mask = item->mask; 2136 const struct rte_flow_item_ipv4 *spec = item->spec; 2137 const struct rte_flow_item_ipv4 nic_mask = { 2138 .hdr = { 2139 .src_addr = RTE_BE32(0xffffffff), 2140 .dst_addr = RTE_BE32(0xffffffff), 2141 .type_of_service = 0xff, 2142 .next_proto_id = 0xff, 2143 }, 2144 }; 2145 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2146 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2147 MLX5_FLOW_LAYER_OUTER_L3; 2148 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2149 MLX5_FLOW_LAYER_OUTER_L4; 2150 int ret; 2151 uint8_t next_proto = 0xFF; 2152 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2153 MLX5_FLOW_LAYER_OUTER_VLAN | 2154 MLX5_FLOW_LAYER_INNER_VLAN); 2155 2156 if ((last_item & l2_vlan) && ether_type && 2157 ether_type != RTE_ETHER_TYPE_IPV4) 2158 return rte_flow_error_set(error, EINVAL, 2159 RTE_FLOW_ERROR_TYPE_ITEM, item, 2160 "IPv4 cannot follow L2/VLAN layer " 2161 "which ether type is not IPv4"); 2162 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2163 if (mask && spec) 2164 next_proto = mask->hdr.next_proto_id & 2165 spec->hdr.next_proto_id; 2166 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2167 return rte_flow_error_set(error, EINVAL, 2168 RTE_FLOW_ERROR_TYPE_ITEM, 2169 item, 2170 "multiple tunnel " 2171 "not supported"); 2172 } 2173 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2174 return rte_flow_error_set(error, EINVAL, 2175 RTE_FLOW_ERROR_TYPE_ITEM, item, 2176 "wrong tunnel type - IPv6 specified " 2177 "but IPv4 item provided"); 2178 if (item_flags & l3m) 2179 return rte_flow_error_set(error, ENOTSUP, 2180 RTE_FLOW_ERROR_TYPE_ITEM, item, 2181 "multiple L3 layers not supported"); 2182 else if (item_flags & l4m) 2183 return rte_flow_error_set(error, EINVAL, 2184 RTE_FLOW_ERROR_TYPE_ITEM, item, 2185 "L3 cannot follow an L4 layer."); 2186 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2187 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2188 return rte_flow_error_set(error, EINVAL, 2189 RTE_FLOW_ERROR_TYPE_ITEM, item, 2190 "L3 cannot follow an NVGRE layer."); 2191 if (!mask) 2192 mask = &rte_flow_item_ipv4_mask; 2193 else if (mask->hdr.next_proto_id != 0 && 2194 mask->hdr.next_proto_id != 0xff) 2195 return rte_flow_error_set(error, EINVAL, 2196 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2197 "partial mask is not supported" 2198 " for protocol"); 2199 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2200 acc_mask ? (const uint8_t *)acc_mask 2201 : (const uint8_t *)&nic_mask, 2202 sizeof(struct rte_flow_item_ipv4), 2203 range_accepted, error); 2204 if (ret < 0) 2205 return ret; 2206 return 0; 2207 } 2208 2209 /** 2210 * Validate IPV6 item. 2211 * 2212 * @param[in] item 2213 * Item specification. 2214 * @param[in] item_flags 2215 * Bit-fields that holds the items detected until now. 2216 * @param[in] last_item 2217 * Previous validated item in the pattern items. 2218 * @param[in] ether_type 2219 * Type in the ethernet layer header (including dot1q). 2220 * @param[in] acc_mask 2221 * Acceptable mask, if NULL default internal default mask 2222 * will be used to check whether item fields are supported. 2223 * @param[out] error 2224 * Pointer to error structure. 2225 * 2226 * @return 2227 * 0 on success, a negative errno value otherwise and rte_errno is set. 2228 */ 2229 int 2230 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2231 uint64_t item_flags, 2232 uint64_t last_item, 2233 uint16_t ether_type, 2234 const struct rte_flow_item_ipv6 *acc_mask, 2235 struct rte_flow_error *error) 2236 { 2237 const struct rte_flow_item_ipv6 *mask = item->mask; 2238 const struct rte_flow_item_ipv6 *spec = item->spec; 2239 const struct rte_flow_item_ipv6 nic_mask = { 2240 .hdr = { 2241 .src_addr = 2242 "\xff\xff\xff\xff\xff\xff\xff\xff" 2243 "\xff\xff\xff\xff\xff\xff\xff\xff", 2244 .dst_addr = 2245 "\xff\xff\xff\xff\xff\xff\xff\xff" 2246 "\xff\xff\xff\xff\xff\xff\xff\xff", 2247 .vtc_flow = RTE_BE32(0xffffffff), 2248 .proto = 0xff, 2249 }, 2250 }; 2251 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2252 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2253 MLX5_FLOW_LAYER_OUTER_L3; 2254 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2255 MLX5_FLOW_LAYER_OUTER_L4; 2256 int ret; 2257 uint8_t next_proto = 0xFF; 2258 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2259 MLX5_FLOW_LAYER_OUTER_VLAN | 2260 MLX5_FLOW_LAYER_INNER_VLAN); 2261 2262 if ((last_item & l2_vlan) && ether_type && 2263 ether_type != RTE_ETHER_TYPE_IPV6) 2264 return rte_flow_error_set(error, EINVAL, 2265 RTE_FLOW_ERROR_TYPE_ITEM, item, 2266 "IPv6 cannot follow L2/VLAN layer " 2267 "which ether type is not IPv6"); 2268 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2269 next_proto = spec->hdr.proto; 2270 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2271 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2272 return rte_flow_error_set(error, EINVAL, 2273 RTE_FLOW_ERROR_TYPE_ITEM, 2274 item, 2275 "multiple tunnel " 2276 "not supported"); 2277 } 2278 if (next_proto == IPPROTO_HOPOPTS || 2279 next_proto == IPPROTO_ROUTING || 2280 next_proto == IPPROTO_FRAGMENT || 2281 next_proto == IPPROTO_ESP || 2282 next_proto == IPPROTO_AH || 2283 next_proto == IPPROTO_DSTOPTS) 2284 return rte_flow_error_set(error, EINVAL, 2285 RTE_FLOW_ERROR_TYPE_ITEM, item, 2286 "IPv6 proto (next header) should " 2287 "not be set as extension header"); 2288 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2289 return rte_flow_error_set(error, EINVAL, 2290 RTE_FLOW_ERROR_TYPE_ITEM, item, 2291 "wrong tunnel type - IPv4 specified " 2292 "but IPv6 item provided"); 2293 if (item_flags & l3m) 2294 return rte_flow_error_set(error, ENOTSUP, 2295 RTE_FLOW_ERROR_TYPE_ITEM, item, 2296 "multiple L3 layers not supported"); 2297 else if (item_flags & l4m) 2298 return rte_flow_error_set(error, EINVAL, 2299 RTE_FLOW_ERROR_TYPE_ITEM, item, 2300 "L3 cannot follow an L4 layer."); 2301 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2302 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2303 return rte_flow_error_set(error, EINVAL, 2304 RTE_FLOW_ERROR_TYPE_ITEM, item, 2305 "L3 cannot follow an NVGRE layer."); 2306 if (!mask) 2307 mask = &rte_flow_item_ipv6_mask; 2308 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2309 acc_mask ? (const uint8_t *)acc_mask 2310 : (const uint8_t *)&nic_mask, 2311 sizeof(struct rte_flow_item_ipv6), 2312 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2313 if (ret < 0) 2314 return ret; 2315 return 0; 2316 } 2317 2318 /** 2319 * Validate UDP item. 2320 * 2321 * @param[in] item 2322 * Item specification. 2323 * @param[in] item_flags 2324 * Bit-fields that holds the items detected until now. 2325 * @param[in] target_protocol 2326 * The next protocol in the previous item. 2327 * @param[in] flow_mask 2328 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2329 * @param[out] error 2330 * Pointer to error structure. 2331 * 2332 * @return 2333 * 0 on success, a negative errno value otherwise and rte_errno is set. 2334 */ 2335 int 2336 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2337 uint64_t item_flags, 2338 uint8_t target_protocol, 2339 struct rte_flow_error *error) 2340 { 2341 const struct rte_flow_item_udp *mask = item->mask; 2342 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2343 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2344 MLX5_FLOW_LAYER_OUTER_L3; 2345 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2346 MLX5_FLOW_LAYER_OUTER_L4; 2347 int ret; 2348 2349 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2350 return rte_flow_error_set(error, EINVAL, 2351 RTE_FLOW_ERROR_TYPE_ITEM, item, 2352 "protocol filtering not compatible" 2353 " with UDP layer"); 2354 if (!(item_flags & l3m)) 2355 return rte_flow_error_set(error, EINVAL, 2356 RTE_FLOW_ERROR_TYPE_ITEM, item, 2357 "L3 is mandatory to filter on L4"); 2358 if (item_flags & l4m) 2359 return rte_flow_error_set(error, EINVAL, 2360 RTE_FLOW_ERROR_TYPE_ITEM, item, 2361 "multiple L4 layers not supported"); 2362 if (!mask) 2363 mask = &rte_flow_item_udp_mask; 2364 ret = mlx5_flow_item_acceptable 2365 (item, (const uint8_t *)mask, 2366 (const uint8_t *)&rte_flow_item_udp_mask, 2367 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2368 error); 2369 if (ret < 0) 2370 return ret; 2371 return 0; 2372 } 2373 2374 /** 2375 * Validate TCP item. 2376 * 2377 * @param[in] item 2378 * Item specification. 2379 * @param[in] item_flags 2380 * Bit-fields that holds the items detected until now. 2381 * @param[in] target_protocol 2382 * The next protocol in the previous item. 2383 * @param[out] error 2384 * Pointer to error structure. 2385 * 2386 * @return 2387 * 0 on success, a negative errno value otherwise and rte_errno is set. 2388 */ 2389 int 2390 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2391 uint64_t item_flags, 2392 uint8_t target_protocol, 2393 const struct rte_flow_item_tcp *flow_mask, 2394 struct rte_flow_error *error) 2395 { 2396 const struct rte_flow_item_tcp *mask = item->mask; 2397 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2398 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2399 MLX5_FLOW_LAYER_OUTER_L3; 2400 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2401 MLX5_FLOW_LAYER_OUTER_L4; 2402 int ret; 2403 2404 MLX5_ASSERT(flow_mask); 2405 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2406 return rte_flow_error_set(error, EINVAL, 2407 RTE_FLOW_ERROR_TYPE_ITEM, item, 2408 "protocol filtering not compatible" 2409 " with TCP layer"); 2410 if (!(item_flags & l3m)) 2411 return rte_flow_error_set(error, EINVAL, 2412 RTE_FLOW_ERROR_TYPE_ITEM, item, 2413 "L3 is mandatory to filter on L4"); 2414 if (item_flags & l4m) 2415 return rte_flow_error_set(error, EINVAL, 2416 RTE_FLOW_ERROR_TYPE_ITEM, item, 2417 "multiple L4 layers not supported"); 2418 if (!mask) 2419 mask = &rte_flow_item_tcp_mask; 2420 ret = mlx5_flow_item_acceptable 2421 (item, (const uint8_t *)mask, 2422 (const uint8_t *)flow_mask, 2423 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2424 error); 2425 if (ret < 0) 2426 return ret; 2427 return 0; 2428 } 2429 2430 /** 2431 * Validate VXLAN item. 2432 * 2433 * @param[in] dev 2434 * Pointer to the Ethernet device structure. 2435 * @param[in] udp_dport 2436 * UDP destination port 2437 * @param[in] item 2438 * Item specification. 2439 * @param[in] item_flags 2440 * Bit-fields that holds the items detected until now. 2441 * @param[in] attr 2442 * Flow rule attributes. 2443 * @param[out] error 2444 * Pointer to error structure. 2445 * 2446 * @return 2447 * 0 on success, a negative errno value otherwise and rte_errno is set. 2448 */ 2449 int 2450 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2451 uint16_t udp_dport, 2452 const struct rte_flow_item *item, 2453 uint64_t item_flags, 2454 const struct rte_flow_attr *attr, 2455 struct rte_flow_error *error) 2456 { 2457 const struct rte_flow_item_vxlan *spec = item->spec; 2458 const struct rte_flow_item_vxlan *mask = item->mask; 2459 int ret; 2460 struct mlx5_priv *priv = dev->data->dev_private; 2461 union vni { 2462 uint32_t vlan_id; 2463 uint8_t vni[4]; 2464 } id = { .vlan_id = 0, }; 2465 const struct rte_flow_item_vxlan nic_mask = { 2466 .vni = "\xff\xff\xff", 2467 .rsvd1 = 0xff, 2468 }; 2469 const struct rte_flow_item_vxlan *valid_mask; 2470 2471 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2472 return rte_flow_error_set(error, ENOTSUP, 2473 RTE_FLOW_ERROR_TYPE_ITEM, item, 2474 "multiple tunnel layers not" 2475 " supported"); 2476 valid_mask = &rte_flow_item_vxlan_mask; 2477 /* 2478 * Verify only UDPv4 is present as defined in 2479 * https://tools.ietf.org/html/rfc7348 2480 */ 2481 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2482 return rte_flow_error_set(error, EINVAL, 2483 RTE_FLOW_ERROR_TYPE_ITEM, item, 2484 "no outer UDP layer found"); 2485 if (!mask) 2486 mask = &rte_flow_item_vxlan_mask; 2487 2488 if (priv->sh->steering_format_version != 2489 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2490 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2491 /* FDB domain & NIC domain non-zero group */ 2492 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2493 valid_mask = &nic_mask; 2494 /* Group zero in NIC domain */ 2495 if (!attr->group && !attr->transfer && 2496 priv->sh->tunnel_header_0_1) 2497 valid_mask = &nic_mask; 2498 } 2499 ret = mlx5_flow_item_acceptable 2500 (item, (const uint8_t *)mask, 2501 (const uint8_t *)valid_mask, 2502 sizeof(struct rte_flow_item_vxlan), 2503 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2504 if (ret < 0) 2505 return ret; 2506 if (spec) { 2507 memcpy(&id.vni[1], spec->vni, 3); 2508 memcpy(&id.vni[1], mask->vni, 3); 2509 } 2510 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2511 return rte_flow_error_set(error, ENOTSUP, 2512 RTE_FLOW_ERROR_TYPE_ITEM, item, 2513 "VXLAN tunnel must be fully defined"); 2514 return 0; 2515 } 2516 2517 /** 2518 * Validate VXLAN_GPE item. 2519 * 2520 * @param[in] item 2521 * Item specification. 2522 * @param[in] item_flags 2523 * Bit-fields that holds the items detected until now. 2524 * @param[in] priv 2525 * Pointer to the private data structure. 2526 * @param[in] target_protocol 2527 * The next protocol in the previous item. 2528 * @param[out] error 2529 * Pointer to error structure. 2530 * 2531 * @return 2532 * 0 on success, a negative errno value otherwise and rte_errno is set. 2533 */ 2534 int 2535 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2536 uint64_t item_flags, 2537 struct rte_eth_dev *dev, 2538 struct rte_flow_error *error) 2539 { 2540 struct mlx5_priv *priv = dev->data->dev_private; 2541 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2542 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2543 int ret; 2544 union vni { 2545 uint32_t vlan_id; 2546 uint8_t vni[4]; 2547 } id = { .vlan_id = 0, }; 2548 2549 if (!priv->config.l3_vxlan_en) 2550 return rte_flow_error_set(error, ENOTSUP, 2551 RTE_FLOW_ERROR_TYPE_ITEM, item, 2552 "L3 VXLAN is not enabled by device" 2553 " parameter and/or not configured in" 2554 " firmware"); 2555 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2556 return rte_flow_error_set(error, ENOTSUP, 2557 RTE_FLOW_ERROR_TYPE_ITEM, item, 2558 "multiple tunnel layers not" 2559 " supported"); 2560 /* 2561 * Verify only UDPv4 is present as defined in 2562 * https://tools.ietf.org/html/rfc7348 2563 */ 2564 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2565 return rte_flow_error_set(error, EINVAL, 2566 RTE_FLOW_ERROR_TYPE_ITEM, item, 2567 "no outer UDP layer found"); 2568 if (!mask) 2569 mask = &rte_flow_item_vxlan_gpe_mask; 2570 ret = mlx5_flow_item_acceptable 2571 (item, (const uint8_t *)mask, 2572 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2573 sizeof(struct rte_flow_item_vxlan_gpe), 2574 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2575 if (ret < 0) 2576 return ret; 2577 if (spec) { 2578 if (spec->protocol) 2579 return rte_flow_error_set(error, ENOTSUP, 2580 RTE_FLOW_ERROR_TYPE_ITEM, 2581 item, 2582 "VxLAN-GPE protocol" 2583 " not supported"); 2584 memcpy(&id.vni[1], spec->vni, 3); 2585 memcpy(&id.vni[1], mask->vni, 3); 2586 } 2587 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2588 return rte_flow_error_set(error, ENOTSUP, 2589 RTE_FLOW_ERROR_TYPE_ITEM, item, 2590 "VXLAN-GPE tunnel must be fully" 2591 " defined"); 2592 return 0; 2593 } 2594 /** 2595 * Validate GRE Key item. 2596 * 2597 * @param[in] item 2598 * Item specification. 2599 * @param[in] item_flags 2600 * Bit flags to mark detected items. 2601 * @param[in] gre_item 2602 * Pointer to gre_item 2603 * @param[out] error 2604 * Pointer to error structure. 2605 * 2606 * @return 2607 * 0 on success, a negative errno value otherwise and rte_errno is set. 2608 */ 2609 int 2610 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2611 uint64_t item_flags, 2612 const struct rte_flow_item *gre_item, 2613 struct rte_flow_error *error) 2614 { 2615 const rte_be32_t *mask = item->mask; 2616 int ret = 0; 2617 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2618 const struct rte_flow_item_gre *gre_spec; 2619 const struct rte_flow_item_gre *gre_mask; 2620 2621 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2622 return rte_flow_error_set(error, ENOTSUP, 2623 RTE_FLOW_ERROR_TYPE_ITEM, item, 2624 "Multiple GRE key not support"); 2625 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2626 return rte_flow_error_set(error, ENOTSUP, 2627 RTE_FLOW_ERROR_TYPE_ITEM, item, 2628 "No preceding GRE header"); 2629 if (item_flags & MLX5_FLOW_LAYER_INNER) 2630 return rte_flow_error_set(error, ENOTSUP, 2631 RTE_FLOW_ERROR_TYPE_ITEM, item, 2632 "GRE key following a wrong item"); 2633 gre_mask = gre_item->mask; 2634 if (!gre_mask) 2635 gre_mask = &rte_flow_item_gre_mask; 2636 gre_spec = gre_item->spec; 2637 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2638 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2639 return rte_flow_error_set(error, EINVAL, 2640 RTE_FLOW_ERROR_TYPE_ITEM, item, 2641 "Key bit must be on"); 2642 2643 if (!mask) 2644 mask = &gre_key_default_mask; 2645 ret = mlx5_flow_item_acceptable 2646 (item, (const uint8_t *)mask, 2647 (const uint8_t *)&gre_key_default_mask, 2648 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2649 return ret; 2650 } 2651 2652 /** 2653 * Validate GRE item. 2654 * 2655 * @param[in] item 2656 * Item specification. 2657 * @param[in] item_flags 2658 * Bit flags to mark detected items. 2659 * @param[in] target_protocol 2660 * The next protocol in the previous item. 2661 * @param[out] error 2662 * Pointer to error structure. 2663 * 2664 * @return 2665 * 0 on success, a negative errno value otherwise and rte_errno is set. 2666 */ 2667 int 2668 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2669 uint64_t item_flags, 2670 uint8_t target_protocol, 2671 struct rte_flow_error *error) 2672 { 2673 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2674 const struct rte_flow_item_gre *mask = item->mask; 2675 int ret; 2676 const struct rte_flow_item_gre nic_mask = { 2677 .c_rsvd0_ver = RTE_BE16(0xB000), 2678 .protocol = RTE_BE16(UINT16_MAX), 2679 }; 2680 2681 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2682 return rte_flow_error_set(error, EINVAL, 2683 RTE_FLOW_ERROR_TYPE_ITEM, item, 2684 "protocol filtering not compatible" 2685 " with this GRE layer"); 2686 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2687 return rte_flow_error_set(error, ENOTSUP, 2688 RTE_FLOW_ERROR_TYPE_ITEM, item, 2689 "multiple tunnel layers not" 2690 " supported"); 2691 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2692 return rte_flow_error_set(error, ENOTSUP, 2693 RTE_FLOW_ERROR_TYPE_ITEM, item, 2694 "L3 Layer is missing"); 2695 if (!mask) 2696 mask = &rte_flow_item_gre_mask; 2697 ret = mlx5_flow_item_acceptable 2698 (item, (const uint8_t *)mask, 2699 (const uint8_t *)&nic_mask, 2700 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2701 error); 2702 if (ret < 0) 2703 return ret; 2704 #ifndef HAVE_MLX5DV_DR 2705 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2706 if (spec && (spec->protocol & mask->protocol)) 2707 return rte_flow_error_set(error, ENOTSUP, 2708 RTE_FLOW_ERROR_TYPE_ITEM, item, 2709 "without MPLS support the" 2710 " specification cannot be used for" 2711 " filtering"); 2712 #endif 2713 #endif 2714 return 0; 2715 } 2716 2717 /** 2718 * Validate Geneve item. 2719 * 2720 * @param[in] item 2721 * Item specification. 2722 * @param[in] itemFlags 2723 * Bit-fields that holds the items detected until now. 2724 * @param[in] enPriv 2725 * Pointer to the private data structure. 2726 * @param[out] error 2727 * Pointer to error structure. 2728 * 2729 * @return 2730 * 0 on success, a negative errno value otherwise and rte_errno is set. 2731 */ 2732 2733 int 2734 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2735 uint64_t item_flags, 2736 struct rte_eth_dev *dev, 2737 struct rte_flow_error *error) 2738 { 2739 struct mlx5_priv *priv = dev->data->dev_private; 2740 const struct rte_flow_item_geneve *spec = item->spec; 2741 const struct rte_flow_item_geneve *mask = item->mask; 2742 int ret; 2743 uint16_t gbhdr; 2744 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2745 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2746 const struct rte_flow_item_geneve nic_mask = { 2747 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2748 .vni = "\xff\xff\xff", 2749 .protocol = RTE_BE16(UINT16_MAX), 2750 }; 2751 2752 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2753 return rte_flow_error_set(error, ENOTSUP, 2754 RTE_FLOW_ERROR_TYPE_ITEM, item, 2755 "L3 Geneve is not enabled by device" 2756 " parameter and/or not configured in" 2757 " firmware"); 2758 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2759 return rte_flow_error_set(error, ENOTSUP, 2760 RTE_FLOW_ERROR_TYPE_ITEM, item, 2761 "multiple tunnel layers not" 2762 " supported"); 2763 /* 2764 * Verify only UDPv4 is present as defined in 2765 * https://tools.ietf.org/html/rfc7348 2766 */ 2767 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2768 return rte_flow_error_set(error, EINVAL, 2769 RTE_FLOW_ERROR_TYPE_ITEM, item, 2770 "no outer UDP layer found"); 2771 if (!mask) 2772 mask = &rte_flow_item_geneve_mask; 2773 ret = mlx5_flow_item_acceptable 2774 (item, (const uint8_t *)mask, 2775 (const uint8_t *)&nic_mask, 2776 sizeof(struct rte_flow_item_geneve), 2777 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2778 if (ret) 2779 return ret; 2780 if (spec) { 2781 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2782 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2783 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2784 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2785 return rte_flow_error_set(error, ENOTSUP, 2786 RTE_FLOW_ERROR_TYPE_ITEM, 2787 item, 2788 "Geneve protocol unsupported" 2789 " fields are being used"); 2790 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2791 return rte_flow_error_set 2792 (error, ENOTSUP, 2793 RTE_FLOW_ERROR_TYPE_ITEM, 2794 item, 2795 "Unsupported Geneve options length"); 2796 } 2797 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2798 return rte_flow_error_set 2799 (error, ENOTSUP, 2800 RTE_FLOW_ERROR_TYPE_ITEM, item, 2801 "Geneve tunnel must be fully defined"); 2802 return 0; 2803 } 2804 2805 /** 2806 * Validate Geneve TLV option item. 2807 * 2808 * @param[in] item 2809 * Item specification. 2810 * @param[in] last_item 2811 * Previous validated item in the pattern items. 2812 * @param[in] geneve_item 2813 * Previous GENEVE item specification. 2814 * @param[in] dev 2815 * Pointer to the rte_eth_dev structure. 2816 * @param[out] error 2817 * Pointer to error structure. 2818 * 2819 * @return 2820 * 0 on success, a negative errno value otherwise and rte_errno is set. 2821 */ 2822 int 2823 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 2824 uint64_t last_item, 2825 const struct rte_flow_item *geneve_item, 2826 struct rte_eth_dev *dev, 2827 struct rte_flow_error *error) 2828 { 2829 struct mlx5_priv *priv = dev->data->dev_private; 2830 struct mlx5_dev_ctx_shared *sh = priv->sh; 2831 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 2832 struct mlx5_hca_attr *hca_attr = &priv->config.hca_attr; 2833 uint8_t data_max_supported = 2834 hca_attr->max_geneve_tlv_option_data_len * 4; 2835 struct mlx5_dev_config *config = &priv->config; 2836 const struct rte_flow_item_geneve *geneve_spec; 2837 const struct rte_flow_item_geneve *geneve_mask; 2838 const struct rte_flow_item_geneve_opt *spec = item->spec; 2839 const struct rte_flow_item_geneve_opt *mask = item->mask; 2840 unsigned int i; 2841 unsigned int data_len; 2842 uint8_t tlv_option_len; 2843 uint16_t optlen_m, optlen_v; 2844 const struct rte_flow_item_geneve_opt full_mask = { 2845 .option_class = RTE_BE16(0xffff), 2846 .option_type = 0xff, 2847 .option_len = 0x1f, 2848 }; 2849 2850 if (!mask) 2851 mask = &rte_flow_item_geneve_opt_mask; 2852 if (!spec) 2853 return rte_flow_error_set 2854 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2855 "Geneve TLV opt class/type/length must be specified"); 2856 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 2857 return rte_flow_error_set 2858 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2859 "Geneve TLV opt length exceeeds the limit (31)"); 2860 /* Check if class type and length masks are full. */ 2861 if (full_mask.option_class != mask->option_class || 2862 full_mask.option_type != mask->option_type || 2863 full_mask.option_len != (mask->option_len & full_mask.option_len)) 2864 return rte_flow_error_set 2865 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2866 "Geneve TLV opt class/type/length masks must be full"); 2867 /* Check if length is supported */ 2868 if ((uint32_t)spec->option_len > 2869 config->hca_attr.max_geneve_tlv_option_data_len) 2870 return rte_flow_error_set 2871 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2872 "Geneve TLV opt length not supported"); 2873 if (config->hca_attr.max_geneve_tlv_options > 1) 2874 DRV_LOG(DEBUG, 2875 "max_geneve_tlv_options supports more than 1 option"); 2876 /* Check GENEVE item preceding. */ 2877 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 2878 return rte_flow_error_set 2879 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2880 "Geneve opt item must be preceded with Geneve item"); 2881 geneve_spec = geneve_item->spec; 2882 geneve_mask = geneve_item->mask ? geneve_item->mask : 2883 &rte_flow_item_geneve_mask; 2884 /* Check if GENEVE TLV option size doesn't exceed option length */ 2885 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 2886 geneve_spec->ver_opt_len_o_c_rsvd0)) { 2887 tlv_option_len = spec->option_len & mask->option_len; 2888 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 2889 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 2890 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 2891 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 2892 if ((optlen_v & optlen_m) <= tlv_option_len) 2893 return rte_flow_error_set 2894 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2895 "GENEVE TLV option length exceeds optlen"); 2896 } 2897 /* Check if length is 0 or data is 0. */ 2898 if (spec->data == NULL || spec->option_len == 0) 2899 return rte_flow_error_set 2900 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2901 "Geneve TLV opt with zero data/length not supported"); 2902 /* Check not all data & mask are 0. */ 2903 data_len = spec->option_len * 4; 2904 if (mask->data == NULL) { 2905 for (i = 0; i < data_len; i++) 2906 if (spec->data[i]) 2907 break; 2908 if (i == data_len) 2909 return rte_flow_error_set(error, ENOTSUP, 2910 RTE_FLOW_ERROR_TYPE_ITEM, item, 2911 "Can't match on Geneve option data 0"); 2912 } else { 2913 for (i = 0; i < data_len; i++) 2914 if (spec->data[i] & mask->data[i]) 2915 break; 2916 if (i == data_len) 2917 return rte_flow_error_set(error, ENOTSUP, 2918 RTE_FLOW_ERROR_TYPE_ITEM, item, 2919 "Can't match on Geneve option data and mask 0"); 2920 /* Check data mask supported. */ 2921 for (i = data_max_supported; i < data_len ; i++) 2922 if (mask->data[i]) 2923 return rte_flow_error_set(error, ENOTSUP, 2924 RTE_FLOW_ERROR_TYPE_ITEM, item, 2925 "Data mask is of unsupported size"); 2926 } 2927 /* Check GENEVE option is supported in NIC. */ 2928 if (!config->hca_attr.geneve_tlv_opt) 2929 return rte_flow_error_set 2930 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 2931 "Geneve TLV opt not supported"); 2932 /* Check if we already have geneve option with different type/class. */ 2933 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 2934 geneve_opt_resource = sh->geneve_tlv_option_resource; 2935 if (geneve_opt_resource != NULL) 2936 if (geneve_opt_resource->option_class != spec->option_class || 2937 geneve_opt_resource->option_type != spec->option_type || 2938 geneve_opt_resource->length != spec->option_len) { 2939 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2940 return rte_flow_error_set(error, ENOTSUP, 2941 RTE_FLOW_ERROR_TYPE_ITEM, item, 2942 "Only one Geneve TLV option supported"); 2943 } 2944 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 2945 return 0; 2946 } 2947 2948 /** 2949 * Validate MPLS item. 2950 * 2951 * @param[in] dev 2952 * Pointer to the rte_eth_dev structure. 2953 * @param[in] item 2954 * Item specification. 2955 * @param[in] item_flags 2956 * Bit-fields that holds the items detected until now. 2957 * @param[in] prev_layer 2958 * The protocol layer indicated in previous item. 2959 * @param[out] error 2960 * Pointer to error structure. 2961 * 2962 * @return 2963 * 0 on success, a negative errno value otherwise and rte_errno is set. 2964 */ 2965 int 2966 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2967 const struct rte_flow_item *item __rte_unused, 2968 uint64_t item_flags __rte_unused, 2969 uint64_t prev_layer __rte_unused, 2970 struct rte_flow_error *error) 2971 { 2972 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2973 const struct rte_flow_item_mpls *mask = item->mask; 2974 struct mlx5_priv *priv = dev->data->dev_private; 2975 int ret; 2976 2977 if (!priv->config.mpls_en) 2978 return rte_flow_error_set(error, ENOTSUP, 2979 RTE_FLOW_ERROR_TYPE_ITEM, item, 2980 "MPLS not supported or" 2981 " disabled in firmware" 2982 " configuration."); 2983 /* MPLS over UDP, GRE is allowed */ 2984 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 2985 MLX5_FLOW_LAYER_GRE | 2986 MLX5_FLOW_LAYER_GRE_KEY))) 2987 return rte_flow_error_set(error, EINVAL, 2988 RTE_FLOW_ERROR_TYPE_ITEM, item, 2989 "protocol filtering not compatible" 2990 " with MPLS layer"); 2991 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2992 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2993 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2994 return rte_flow_error_set(error, ENOTSUP, 2995 RTE_FLOW_ERROR_TYPE_ITEM, item, 2996 "multiple tunnel layers not" 2997 " supported"); 2998 if (!mask) 2999 mask = &rte_flow_item_mpls_mask; 3000 ret = mlx5_flow_item_acceptable 3001 (item, (const uint8_t *)mask, 3002 (const uint8_t *)&rte_flow_item_mpls_mask, 3003 sizeof(struct rte_flow_item_mpls), 3004 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3005 if (ret < 0) 3006 return ret; 3007 return 0; 3008 #else 3009 return rte_flow_error_set(error, ENOTSUP, 3010 RTE_FLOW_ERROR_TYPE_ITEM, item, 3011 "MPLS is not supported by Verbs, please" 3012 " update."); 3013 #endif 3014 } 3015 3016 /** 3017 * Validate NVGRE item. 3018 * 3019 * @param[in] item 3020 * Item specification. 3021 * @param[in] item_flags 3022 * Bit flags to mark detected items. 3023 * @param[in] target_protocol 3024 * The next protocol in the previous item. 3025 * @param[out] error 3026 * Pointer to error structure. 3027 * 3028 * @return 3029 * 0 on success, a negative errno value otherwise and rte_errno is set. 3030 */ 3031 int 3032 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3033 uint64_t item_flags, 3034 uint8_t target_protocol, 3035 struct rte_flow_error *error) 3036 { 3037 const struct rte_flow_item_nvgre *mask = item->mask; 3038 int ret; 3039 3040 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3041 return rte_flow_error_set(error, EINVAL, 3042 RTE_FLOW_ERROR_TYPE_ITEM, item, 3043 "protocol filtering not compatible" 3044 " with this GRE layer"); 3045 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3046 return rte_flow_error_set(error, ENOTSUP, 3047 RTE_FLOW_ERROR_TYPE_ITEM, item, 3048 "multiple tunnel layers not" 3049 " supported"); 3050 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3051 return rte_flow_error_set(error, ENOTSUP, 3052 RTE_FLOW_ERROR_TYPE_ITEM, item, 3053 "L3 Layer is missing"); 3054 if (!mask) 3055 mask = &rte_flow_item_nvgre_mask; 3056 ret = mlx5_flow_item_acceptable 3057 (item, (const uint8_t *)mask, 3058 (const uint8_t *)&rte_flow_item_nvgre_mask, 3059 sizeof(struct rte_flow_item_nvgre), 3060 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3061 if (ret < 0) 3062 return ret; 3063 return 0; 3064 } 3065 3066 /** 3067 * Validate eCPRI item. 3068 * 3069 * @param[in] item 3070 * Item specification. 3071 * @param[in] item_flags 3072 * Bit-fields that holds the items detected until now. 3073 * @param[in] last_item 3074 * Previous validated item in the pattern items. 3075 * @param[in] ether_type 3076 * Type in the ethernet layer header (including dot1q). 3077 * @param[in] acc_mask 3078 * Acceptable mask, if NULL default internal default mask 3079 * will be used to check whether item fields are supported. 3080 * @param[out] error 3081 * Pointer to error structure. 3082 * 3083 * @return 3084 * 0 on success, a negative errno value otherwise and rte_errno is set. 3085 */ 3086 int 3087 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3088 uint64_t item_flags, 3089 uint64_t last_item, 3090 uint16_t ether_type, 3091 const struct rte_flow_item_ecpri *acc_mask, 3092 struct rte_flow_error *error) 3093 { 3094 const struct rte_flow_item_ecpri *mask = item->mask; 3095 const struct rte_flow_item_ecpri nic_mask = { 3096 .hdr = { 3097 .common = { 3098 .u32 = 3099 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3100 .type = 0xFF, 3101 }).u32), 3102 }, 3103 .dummy[0] = 0xFFFFFFFF, 3104 }, 3105 }; 3106 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3107 MLX5_FLOW_LAYER_OUTER_VLAN); 3108 struct rte_flow_item_ecpri mask_lo; 3109 3110 if (!(last_item & outer_l2_vlan) && 3111 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3112 return rte_flow_error_set(error, EINVAL, 3113 RTE_FLOW_ERROR_TYPE_ITEM, item, 3114 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3115 if ((last_item & outer_l2_vlan) && ether_type && 3116 ether_type != RTE_ETHER_TYPE_ECPRI) 3117 return rte_flow_error_set(error, EINVAL, 3118 RTE_FLOW_ERROR_TYPE_ITEM, item, 3119 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3120 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3121 return rte_flow_error_set(error, EINVAL, 3122 RTE_FLOW_ERROR_TYPE_ITEM, item, 3123 "eCPRI with tunnel is not supported right now"); 3124 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3125 return rte_flow_error_set(error, ENOTSUP, 3126 RTE_FLOW_ERROR_TYPE_ITEM, item, 3127 "multiple L3 layers not supported"); 3128 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3129 return rte_flow_error_set(error, EINVAL, 3130 RTE_FLOW_ERROR_TYPE_ITEM, item, 3131 "eCPRI cannot coexist with a TCP layer"); 3132 /* In specification, eCPRI could be over UDP layer. */ 3133 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3134 return rte_flow_error_set(error, EINVAL, 3135 RTE_FLOW_ERROR_TYPE_ITEM, item, 3136 "eCPRI over UDP layer is not yet supported right now"); 3137 /* Mask for type field in common header could be zero. */ 3138 if (!mask) 3139 mask = &rte_flow_item_ecpri_mask; 3140 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3141 /* Input mask is in big-endian format. */ 3142 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3143 return rte_flow_error_set(error, EINVAL, 3144 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3145 "partial mask is not supported for protocol"); 3146 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3147 return rte_flow_error_set(error, EINVAL, 3148 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3149 "message header mask must be after a type mask"); 3150 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3151 acc_mask ? (const uint8_t *)acc_mask 3152 : (const uint8_t *)&nic_mask, 3153 sizeof(struct rte_flow_item_ecpri), 3154 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3155 } 3156 3157 static int 3158 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3159 const struct rte_flow_attr *attr __rte_unused, 3160 const struct rte_flow_item items[] __rte_unused, 3161 const struct rte_flow_action actions[] __rte_unused, 3162 bool external __rte_unused, 3163 int hairpin __rte_unused, 3164 struct rte_flow_error *error) 3165 { 3166 return rte_flow_error_set(error, ENOTSUP, 3167 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3168 } 3169 3170 static struct mlx5_flow * 3171 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3172 const struct rte_flow_attr *attr __rte_unused, 3173 const struct rte_flow_item items[] __rte_unused, 3174 const struct rte_flow_action actions[] __rte_unused, 3175 struct rte_flow_error *error) 3176 { 3177 rte_flow_error_set(error, ENOTSUP, 3178 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3179 return NULL; 3180 } 3181 3182 static int 3183 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3184 struct mlx5_flow *dev_flow __rte_unused, 3185 const struct rte_flow_attr *attr __rte_unused, 3186 const struct rte_flow_item items[] __rte_unused, 3187 const struct rte_flow_action actions[] __rte_unused, 3188 struct rte_flow_error *error) 3189 { 3190 return rte_flow_error_set(error, ENOTSUP, 3191 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3192 } 3193 3194 static int 3195 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3196 struct rte_flow *flow __rte_unused, 3197 struct rte_flow_error *error) 3198 { 3199 return rte_flow_error_set(error, ENOTSUP, 3200 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3201 } 3202 3203 static void 3204 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3205 struct rte_flow *flow __rte_unused) 3206 { 3207 } 3208 3209 static void 3210 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3211 struct rte_flow *flow __rte_unused) 3212 { 3213 } 3214 3215 static int 3216 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3217 struct rte_flow *flow __rte_unused, 3218 const struct rte_flow_action *actions __rte_unused, 3219 void *data __rte_unused, 3220 struct rte_flow_error *error) 3221 { 3222 return rte_flow_error_set(error, ENOTSUP, 3223 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3224 } 3225 3226 static int 3227 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3228 uint32_t domains __rte_unused, 3229 uint32_t flags __rte_unused) 3230 { 3231 return 0; 3232 } 3233 3234 /* Void driver to protect from null pointer reference. */ 3235 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3236 .validate = flow_null_validate, 3237 .prepare = flow_null_prepare, 3238 .translate = flow_null_translate, 3239 .apply = flow_null_apply, 3240 .remove = flow_null_remove, 3241 .destroy = flow_null_destroy, 3242 .query = flow_null_query, 3243 .sync_domain = flow_null_sync_domain, 3244 }; 3245 3246 /** 3247 * Select flow driver type according to flow attributes and device 3248 * configuration. 3249 * 3250 * @param[in] dev 3251 * Pointer to the dev structure. 3252 * @param[in] attr 3253 * Pointer to the flow attributes. 3254 * 3255 * @return 3256 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3257 */ 3258 static enum mlx5_flow_drv_type 3259 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3260 { 3261 struct mlx5_priv *priv = dev->data->dev_private; 3262 /* The OS can determine first a specific flow type (DV, VERBS) */ 3263 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3264 3265 if (type != MLX5_FLOW_TYPE_MAX) 3266 return type; 3267 /* If no OS specific type - continue with DV/VERBS selection */ 3268 if (attr->transfer && priv->config.dv_esw_en) 3269 type = MLX5_FLOW_TYPE_DV; 3270 if (!attr->transfer) 3271 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3272 MLX5_FLOW_TYPE_VERBS; 3273 return type; 3274 } 3275 3276 #define flow_get_drv_ops(type) flow_drv_ops[type] 3277 3278 /** 3279 * Flow driver validation API. This abstracts calling driver specific functions. 3280 * The type of flow driver is determined according to flow attributes. 3281 * 3282 * @param[in] dev 3283 * Pointer to the dev structure. 3284 * @param[in] attr 3285 * Pointer to the flow attributes. 3286 * @param[in] items 3287 * Pointer to the list of items. 3288 * @param[in] actions 3289 * Pointer to the list of actions. 3290 * @param[in] external 3291 * This flow rule is created by request external to PMD. 3292 * @param[in] hairpin 3293 * Number of hairpin TX actions, 0 means classic flow. 3294 * @param[out] error 3295 * Pointer to the error structure. 3296 * 3297 * @return 3298 * 0 on success, a negative errno value otherwise and rte_errno is set. 3299 */ 3300 static inline int 3301 flow_drv_validate(struct rte_eth_dev *dev, 3302 const struct rte_flow_attr *attr, 3303 const struct rte_flow_item items[], 3304 const struct rte_flow_action actions[], 3305 bool external, int hairpin, struct rte_flow_error *error) 3306 { 3307 const struct mlx5_flow_driver_ops *fops; 3308 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3309 3310 fops = flow_get_drv_ops(type); 3311 return fops->validate(dev, attr, items, actions, external, 3312 hairpin, error); 3313 } 3314 3315 /** 3316 * Flow driver preparation API. This abstracts calling driver specific 3317 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3318 * calculates the size of memory required for device flow, allocates the memory, 3319 * initializes the device flow and returns the pointer. 3320 * 3321 * @note 3322 * This function initializes device flow structure such as dv or verbs in 3323 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3324 * rest. For example, adding returning device flow to flow->dev_flow list and 3325 * setting backward reference to the flow should be done out of this function. 3326 * layers field is not filled either. 3327 * 3328 * @param[in] dev 3329 * Pointer to the dev structure. 3330 * @param[in] attr 3331 * Pointer to the flow attributes. 3332 * @param[in] items 3333 * Pointer to the list of items. 3334 * @param[in] actions 3335 * Pointer to the list of actions. 3336 * @param[in] flow_idx 3337 * This memory pool index to the flow. 3338 * @param[out] error 3339 * Pointer to the error structure. 3340 * 3341 * @return 3342 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3343 */ 3344 static inline struct mlx5_flow * 3345 flow_drv_prepare(struct rte_eth_dev *dev, 3346 const struct rte_flow *flow, 3347 const struct rte_flow_attr *attr, 3348 const struct rte_flow_item items[], 3349 const struct rte_flow_action actions[], 3350 uint32_t flow_idx, 3351 struct rte_flow_error *error) 3352 { 3353 const struct mlx5_flow_driver_ops *fops; 3354 enum mlx5_flow_drv_type type = flow->drv_type; 3355 struct mlx5_flow *mlx5_flow = NULL; 3356 3357 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3358 fops = flow_get_drv_ops(type); 3359 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3360 if (mlx5_flow) 3361 mlx5_flow->flow_idx = flow_idx; 3362 return mlx5_flow; 3363 } 3364 3365 /** 3366 * Flow driver translation API. This abstracts calling driver specific 3367 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3368 * translates a generic flow into a driver flow. flow_drv_prepare() must 3369 * precede. 3370 * 3371 * @note 3372 * dev_flow->layers could be filled as a result of parsing during translation 3373 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3374 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3375 * flow->actions could be overwritten even though all the expanded dev_flows 3376 * have the same actions. 3377 * 3378 * @param[in] dev 3379 * Pointer to the rte dev structure. 3380 * @param[in, out] dev_flow 3381 * Pointer to the mlx5 flow. 3382 * @param[in] attr 3383 * Pointer to the flow attributes. 3384 * @param[in] items 3385 * Pointer to the list of items. 3386 * @param[in] actions 3387 * Pointer to the list of actions. 3388 * @param[out] error 3389 * Pointer to the error structure. 3390 * 3391 * @return 3392 * 0 on success, a negative errno value otherwise and rte_errno is set. 3393 */ 3394 static inline int 3395 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3396 const struct rte_flow_attr *attr, 3397 const struct rte_flow_item items[], 3398 const struct rte_flow_action actions[], 3399 struct rte_flow_error *error) 3400 { 3401 const struct mlx5_flow_driver_ops *fops; 3402 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3403 3404 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3405 fops = flow_get_drv_ops(type); 3406 return fops->translate(dev, dev_flow, attr, items, actions, error); 3407 } 3408 3409 /** 3410 * Flow driver apply API. This abstracts calling driver specific functions. 3411 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3412 * translated driver flows on to device. flow_drv_translate() must precede. 3413 * 3414 * @param[in] dev 3415 * Pointer to Ethernet device structure. 3416 * @param[in, out] flow 3417 * Pointer to flow structure. 3418 * @param[out] error 3419 * Pointer to error structure. 3420 * 3421 * @return 3422 * 0 on success, a negative errno value otherwise and rte_errno is set. 3423 */ 3424 static inline int 3425 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3426 struct rte_flow_error *error) 3427 { 3428 const struct mlx5_flow_driver_ops *fops; 3429 enum mlx5_flow_drv_type type = flow->drv_type; 3430 3431 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3432 fops = flow_get_drv_ops(type); 3433 return fops->apply(dev, flow, error); 3434 } 3435 3436 /** 3437 * Flow driver destroy API. This abstracts calling driver specific functions. 3438 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3439 * on device and releases resources of the flow. 3440 * 3441 * @param[in] dev 3442 * Pointer to Ethernet device. 3443 * @param[in, out] flow 3444 * Pointer to flow structure. 3445 */ 3446 static inline void 3447 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3448 { 3449 const struct mlx5_flow_driver_ops *fops; 3450 enum mlx5_flow_drv_type type = flow->drv_type; 3451 3452 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3453 fops = flow_get_drv_ops(type); 3454 fops->destroy(dev, flow); 3455 } 3456 3457 /** 3458 * Flow driver find RSS policy tbl API. This abstracts calling driver 3459 * specific functions. Parent flow (rte_flow) should have driver 3460 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3461 * 3462 * @param[in] dev 3463 * Pointer to Ethernet device. 3464 * @param[in, out] flow 3465 * Pointer to flow structure. 3466 * @param[in] policy 3467 * Pointer to meter policy table. 3468 * @param[in] rss_desc 3469 * Pointer to rss_desc 3470 */ 3471 static struct mlx5_flow_meter_sub_policy * 3472 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3473 struct rte_flow *flow, 3474 struct mlx5_flow_meter_policy *policy, 3475 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3476 { 3477 const struct mlx5_flow_driver_ops *fops; 3478 enum mlx5_flow_drv_type type = flow->drv_type; 3479 3480 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3481 fops = flow_get_drv_ops(type); 3482 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3483 } 3484 3485 /** 3486 * Flow driver color tag rule API. This abstracts calling driver 3487 * specific functions. Parent flow (rte_flow) should have driver 3488 * type (drv_type). It will create the color tag rules in hierarchy meter. 3489 * 3490 * @param[in] dev 3491 * Pointer to Ethernet device. 3492 * @param[in, out] flow 3493 * Pointer to flow structure. 3494 * @param[in] fm 3495 * Pointer to flow meter structure. 3496 * @param[in] src_port 3497 * The src port this extra rule should use. 3498 * @param[in] item 3499 * The src port id match item. 3500 * @param[out] error 3501 * Pointer to error structure. 3502 */ 3503 static int 3504 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3505 struct rte_flow *flow, 3506 struct mlx5_flow_meter_info *fm, 3507 int32_t src_port, 3508 const struct rte_flow_item *item, 3509 struct rte_flow_error *error) 3510 { 3511 const struct mlx5_flow_driver_ops *fops; 3512 enum mlx5_flow_drv_type type = flow->drv_type; 3513 3514 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3515 fops = flow_get_drv_ops(type); 3516 return fops->meter_hierarchy_rule_create(dev, fm, 3517 src_port, item, error); 3518 } 3519 3520 /** 3521 * Get RSS action from the action list. 3522 * 3523 * @param[in] dev 3524 * Pointer to Ethernet device. 3525 * @param[in] actions 3526 * Pointer to the list of actions. 3527 * @param[in] flow 3528 * Parent flow structure pointer. 3529 * 3530 * @return 3531 * Pointer to the RSS action if exist, else return NULL. 3532 */ 3533 static const struct rte_flow_action_rss* 3534 flow_get_rss_action(struct rte_eth_dev *dev, 3535 const struct rte_flow_action actions[]) 3536 { 3537 struct mlx5_priv *priv = dev->data->dev_private; 3538 const struct rte_flow_action_rss *rss = NULL; 3539 3540 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3541 switch (actions->type) { 3542 case RTE_FLOW_ACTION_TYPE_RSS: 3543 rss = actions->conf; 3544 break; 3545 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3546 { 3547 const struct rte_flow_action_sample *sample = 3548 actions->conf; 3549 const struct rte_flow_action *act = sample->actions; 3550 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3551 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3552 rss = act->conf; 3553 break; 3554 } 3555 case RTE_FLOW_ACTION_TYPE_METER: 3556 { 3557 uint32_t mtr_idx; 3558 struct mlx5_flow_meter_info *fm; 3559 struct mlx5_flow_meter_policy *policy; 3560 const struct rte_flow_action_meter *mtr = actions->conf; 3561 3562 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3563 if (fm && !fm->def_policy) { 3564 policy = mlx5_flow_meter_policy_find(dev, 3565 fm->policy_id, NULL); 3566 MLX5_ASSERT(policy); 3567 if (policy->is_hierarchy) { 3568 policy = 3569 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3570 policy); 3571 if (!policy) 3572 return NULL; 3573 } 3574 if (policy->is_rss) 3575 rss = 3576 policy->act_cnt[RTE_COLOR_GREEN].rss->conf; 3577 } 3578 break; 3579 } 3580 default: 3581 break; 3582 } 3583 } 3584 return rss; 3585 } 3586 3587 /** 3588 * Get ASO age action by index. 3589 * 3590 * @param[in] dev 3591 * Pointer to the Ethernet device structure. 3592 * @param[in] age_idx 3593 * Index to the ASO age action. 3594 * 3595 * @return 3596 * The specified ASO age action. 3597 */ 3598 struct mlx5_aso_age_action* 3599 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3600 { 3601 uint16_t pool_idx = age_idx & UINT16_MAX; 3602 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3603 struct mlx5_priv *priv = dev->data->dev_private; 3604 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3605 struct mlx5_aso_age_pool *pool = mng->pools[pool_idx]; 3606 3607 return &pool->actions[offset - 1]; 3608 } 3609 3610 /* maps indirect action to translated direct in some actions array */ 3611 struct mlx5_translated_action_handle { 3612 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3613 int index; /**< Index in related array of rte_flow_action. */ 3614 }; 3615 3616 /** 3617 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3618 * direct action if translation possible. 3619 * This functionality used to run same execution path for both direct and 3620 * indirect actions on flow create. All necessary preparations for indirect 3621 * action handling should be performed on *handle* actions list returned 3622 * from this call. 3623 * 3624 * @param[in] dev 3625 * Pointer to Ethernet device. 3626 * @param[in] actions 3627 * List of actions to translate. 3628 * @param[out] handle 3629 * List to store translated indirect action object handles. 3630 * @param[in, out] indir_n 3631 * Size of *handle* array. On return should be updated with number of 3632 * indirect actions retrieved from the *actions* list. 3633 * @param[out] translated_actions 3634 * List of actions where all indirect actions were translated to direct 3635 * if possible. NULL if no translation took place. 3636 * @param[out] error 3637 * Pointer to the error structure. 3638 * 3639 * @return 3640 * 0 on success, a negative errno value otherwise and rte_errno is set. 3641 */ 3642 static int 3643 flow_action_handles_translate(struct rte_eth_dev *dev, 3644 const struct rte_flow_action actions[], 3645 struct mlx5_translated_action_handle *handle, 3646 int *indir_n, 3647 struct rte_flow_action **translated_actions, 3648 struct rte_flow_error *error) 3649 { 3650 struct mlx5_priv *priv = dev->data->dev_private; 3651 struct rte_flow_action *translated = NULL; 3652 size_t actions_size; 3653 int n; 3654 int copied_n = 0; 3655 struct mlx5_translated_action_handle *handle_end = NULL; 3656 3657 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3658 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3659 continue; 3660 if (copied_n == *indir_n) { 3661 return rte_flow_error_set 3662 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3663 NULL, "too many shared actions"); 3664 } 3665 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3666 sizeof(actions[n].conf)); 3667 handle[copied_n].index = n; 3668 copied_n++; 3669 } 3670 n++; 3671 *indir_n = copied_n; 3672 if (!copied_n) 3673 return 0; 3674 actions_size = sizeof(struct rte_flow_action) * n; 3675 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3676 if (!translated) { 3677 rte_errno = ENOMEM; 3678 return -ENOMEM; 3679 } 3680 memcpy(translated, actions, actions_size); 3681 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3682 struct mlx5_shared_action_rss *shared_rss; 3683 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3684 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3685 uint32_t idx = act_idx & 3686 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3687 3688 switch (type) { 3689 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3690 shared_rss = mlx5_ipool_get 3691 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3692 translated[handle->index].type = 3693 RTE_FLOW_ACTION_TYPE_RSS; 3694 translated[handle->index].conf = 3695 &shared_rss->origin; 3696 break; 3697 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3698 translated[handle->index].type = 3699 (enum rte_flow_action_type) 3700 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3701 translated[handle->index].conf = (void *)(uintptr_t)idx; 3702 break; 3703 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3704 if (priv->sh->flow_hit_aso_en) { 3705 translated[handle->index].type = 3706 (enum rte_flow_action_type) 3707 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3708 translated[handle->index].conf = 3709 (void *)(uintptr_t)idx; 3710 break; 3711 } 3712 /* Fall-through */ 3713 case MLX5_INDIRECT_ACTION_TYPE_CT: 3714 if (priv->sh->ct_aso_en) { 3715 translated[handle->index].type = 3716 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3717 translated[handle->index].conf = 3718 (void *)(uintptr_t)idx; 3719 break; 3720 } 3721 /* Fall-through */ 3722 default: 3723 mlx5_free(translated); 3724 return rte_flow_error_set 3725 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3726 NULL, "invalid indirect action type"); 3727 } 3728 } 3729 *translated_actions = translated; 3730 return 0; 3731 } 3732 3733 /** 3734 * Get Shared RSS action from the action list. 3735 * 3736 * @param[in] dev 3737 * Pointer to Ethernet device. 3738 * @param[in] shared 3739 * Pointer to the list of actions. 3740 * @param[in] shared_n 3741 * Actions list length. 3742 * 3743 * @return 3744 * The MLX5 RSS action ID if exists, otherwise return 0. 3745 */ 3746 static uint32_t 3747 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3748 struct mlx5_translated_action_handle *handle, 3749 int shared_n) 3750 { 3751 struct mlx5_translated_action_handle *handle_end; 3752 struct mlx5_priv *priv = dev->data->dev_private; 3753 struct mlx5_shared_action_rss *shared_rss; 3754 3755 3756 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3757 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3758 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3759 uint32_t idx = act_idx & 3760 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3761 switch (type) { 3762 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3763 shared_rss = mlx5_ipool_get 3764 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3765 idx); 3766 __atomic_add_fetch(&shared_rss->refcnt, 1, 3767 __ATOMIC_RELAXED); 3768 return idx; 3769 default: 3770 break; 3771 } 3772 } 3773 return 0; 3774 } 3775 3776 static unsigned int 3777 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3778 { 3779 const struct rte_flow_item *item; 3780 unsigned int has_vlan = 0; 3781 3782 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3783 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3784 has_vlan = 1; 3785 break; 3786 } 3787 } 3788 if (has_vlan) 3789 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3790 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3791 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3792 MLX5_EXPANSION_ROOT_OUTER; 3793 } 3794 3795 /** 3796 * Get layer flags from the prefix flow. 3797 * 3798 * Some flows may be split to several subflows, the prefix subflow gets the 3799 * match items and the suffix sub flow gets the actions. 3800 * Some actions need the user defined match item flags to get the detail for 3801 * the action. 3802 * This function helps the suffix flow to get the item layer flags from prefix 3803 * subflow. 3804 * 3805 * @param[in] dev_flow 3806 * Pointer the created preifx subflow. 3807 * 3808 * @return 3809 * The layers get from prefix subflow. 3810 */ 3811 static inline uint64_t 3812 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3813 { 3814 uint64_t layers = 0; 3815 3816 /* 3817 * Layers bits could be localization, but usually the compiler will 3818 * help to do the optimization work for source code. 3819 * If no decap actions, use the layers directly. 3820 */ 3821 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3822 return dev_flow->handle->layers; 3823 /* Convert L3 layers with decap action. */ 3824 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3825 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3826 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3827 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3828 /* Convert L4 layers with decap action. */ 3829 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3830 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3831 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3832 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3833 return layers; 3834 } 3835 3836 /** 3837 * Get metadata split action information. 3838 * 3839 * @param[in] actions 3840 * Pointer to the list of actions. 3841 * @param[out] qrss 3842 * Pointer to the return pointer. 3843 * @param[out] qrss_type 3844 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3845 * if no QUEUE/RSS is found. 3846 * @param[out] encap_idx 3847 * Pointer to the index of the encap action if exists, otherwise the last 3848 * action index. 3849 * 3850 * @return 3851 * Total number of actions. 3852 */ 3853 static int 3854 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3855 const struct rte_flow_action **qrss, 3856 int *encap_idx) 3857 { 3858 const struct rte_flow_action_raw_encap *raw_encap; 3859 int actions_n = 0; 3860 int raw_decap_idx = -1; 3861 3862 *encap_idx = -1; 3863 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3864 switch (actions->type) { 3865 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3866 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3867 *encap_idx = actions_n; 3868 break; 3869 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3870 raw_decap_idx = actions_n; 3871 break; 3872 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3873 raw_encap = actions->conf; 3874 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3875 *encap_idx = raw_decap_idx != -1 ? 3876 raw_decap_idx : actions_n; 3877 break; 3878 case RTE_FLOW_ACTION_TYPE_QUEUE: 3879 case RTE_FLOW_ACTION_TYPE_RSS: 3880 *qrss = actions; 3881 break; 3882 default: 3883 break; 3884 } 3885 actions_n++; 3886 } 3887 if (*encap_idx == -1) 3888 *encap_idx = actions_n; 3889 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3890 return actions_n + 1; 3891 } 3892 3893 /** 3894 * Check if the action will change packet. 3895 * 3896 * @param dev 3897 * Pointer to Ethernet device. 3898 * @param[in] type 3899 * action type. 3900 * 3901 * @return 3902 * true if action will change packet, false otherwise. 3903 */ 3904 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 3905 enum rte_flow_action_type type) 3906 { 3907 struct mlx5_priv *priv = dev->data->dev_private; 3908 3909 switch (type) { 3910 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3911 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3912 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3913 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3914 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3915 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3916 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3917 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3918 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3919 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3920 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3921 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3922 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3923 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3924 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 3925 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 3926 case RTE_FLOW_ACTION_TYPE_SET_META: 3927 case RTE_FLOW_ACTION_TYPE_SET_TAG: 3928 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 3929 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3930 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3931 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3932 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3933 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3934 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3935 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3936 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3937 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3938 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 3939 return true; 3940 case RTE_FLOW_ACTION_TYPE_FLAG: 3941 case RTE_FLOW_ACTION_TYPE_MARK: 3942 if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 3943 return true; 3944 else 3945 return false; 3946 default: 3947 return false; 3948 } 3949 } 3950 3951 /** 3952 * Check meter action from the action list. 3953 * 3954 * @param dev 3955 * Pointer to Ethernet device. 3956 * @param[in] actions 3957 * Pointer to the list of actions. 3958 * @param[out] has_mtr 3959 * Pointer to the meter exist flag. 3960 * @param[out] has_modify 3961 * Pointer to the flag showing there's packet change action. 3962 * @param[out] meter_id 3963 * Pointer to the meter id. 3964 * 3965 * @return 3966 * Total number of actions. 3967 */ 3968 static int 3969 flow_check_meter_action(struct rte_eth_dev *dev, 3970 const struct rte_flow_action actions[], 3971 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 3972 { 3973 const struct rte_flow_action_meter *mtr = NULL; 3974 int actions_n = 0; 3975 3976 MLX5_ASSERT(has_mtr); 3977 *has_mtr = false; 3978 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3979 switch (actions->type) { 3980 case RTE_FLOW_ACTION_TYPE_METER: 3981 mtr = actions->conf; 3982 *meter_id = mtr->mtr_id; 3983 *has_mtr = true; 3984 break; 3985 default: 3986 break; 3987 } 3988 if (!*has_mtr) 3989 *has_modify |= flow_check_modify_action_type(dev, 3990 actions->type); 3991 actions_n++; 3992 } 3993 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3994 return actions_n + 1; 3995 } 3996 3997 /** 3998 * Check if the flow should be split due to hairpin. 3999 * The reason for the split is that in current HW we can't 4000 * support encap and push-vlan on Rx, so if a flow contains 4001 * these actions we move it to Tx. 4002 * 4003 * @param dev 4004 * Pointer to Ethernet device. 4005 * @param[in] attr 4006 * Flow rule attributes. 4007 * @param[in] actions 4008 * Associated actions (list terminated by the END action). 4009 * 4010 * @return 4011 * > 0 the number of actions and the flow should be split, 4012 * 0 when no split required. 4013 */ 4014 static int 4015 flow_check_hairpin_split(struct rte_eth_dev *dev, 4016 const struct rte_flow_attr *attr, 4017 const struct rte_flow_action actions[]) 4018 { 4019 int queue_action = 0; 4020 int action_n = 0; 4021 int split = 0; 4022 const struct rte_flow_action_queue *queue; 4023 const struct rte_flow_action_rss *rss; 4024 const struct rte_flow_action_raw_encap *raw_encap; 4025 const struct rte_eth_hairpin_conf *conf; 4026 4027 if (!attr->ingress) 4028 return 0; 4029 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4030 switch (actions->type) { 4031 case RTE_FLOW_ACTION_TYPE_QUEUE: 4032 queue = actions->conf; 4033 if (queue == NULL) 4034 return 0; 4035 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4036 if (conf == NULL || conf->tx_explicit != 0) 4037 return 0; 4038 queue_action = 1; 4039 action_n++; 4040 break; 4041 case RTE_FLOW_ACTION_TYPE_RSS: 4042 rss = actions->conf; 4043 if (rss == NULL || rss->queue_num == 0) 4044 return 0; 4045 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4046 if (conf == NULL || conf->tx_explicit != 0) 4047 return 0; 4048 queue_action = 1; 4049 action_n++; 4050 break; 4051 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4052 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4053 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4054 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4055 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4056 split++; 4057 action_n++; 4058 break; 4059 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4060 raw_encap = actions->conf; 4061 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4062 split++; 4063 action_n++; 4064 break; 4065 default: 4066 action_n++; 4067 break; 4068 } 4069 } 4070 if (split && queue_action) 4071 return action_n; 4072 return 0; 4073 } 4074 4075 /* Declare flow create/destroy prototype in advance. */ 4076 static uint32_t 4077 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4078 const struct rte_flow_attr *attr, 4079 const struct rte_flow_item items[], 4080 const struct rte_flow_action actions[], 4081 bool external, struct rte_flow_error *error); 4082 4083 static void 4084 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4085 uint32_t flow_idx); 4086 4087 int 4088 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4089 struct mlx5_list_entry *entry, void *cb_ctx) 4090 { 4091 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4092 struct mlx5_flow_mreg_copy_resource *mcp_res = 4093 container_of(entry, typeof(*mcp_res), hlist_ent); 4094 4095 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4096 } 4097 4098 struct mlx5_list_entry * 4099 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4100 { 4101 struct rte_eth_dev *dev = tool_ctx; 4102 struct mlx5_priv *priv = dev->data->dev_private; 4103 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4104 struct mlx5_flow_mreg_copy_resource *mcp_res; 4105 struct rte_flow_error *error = ctx->error; 4106 uint32_t idx = 0; 4107 int ret; 4108 uint32_t mark_id = *(uint32_t *)(ctx->data); 4109 struct rte_flow_attr attr = { 4110 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4111 .ingress = 1, 4112 }; 4113 struct mlx5_rte_flow_item_tag tag_spec = { 4114 .data = mark_id, 4115 }; 4116 struct rte_flow_item items[] = { 4117 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4118 }; 4119 struct rte_flow_action_mark ftag = { 4120 .id = mark_id, 4121 }; 4122 struct mlx5_flow_action_copy_mreg cp_mreg = { 4123 .dst = REG_B, 4124 .src = REG_NON, 4125 }; 4126 struct rte_flow_action_jump jump = { 4127 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4128 }; 4129 struct rte_flow_action actions[] = { 4130 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4131 }; 4132 4133 /* Fill the register fileds in the flow. */ 4134 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4135 if (ret < 0) 4136 return NULL; 4137 tag_spec.id = ret; 4138 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4139 if (ret < 0) 4140 return NULL; 4141 cp_mreg.src = ret; 4142 /* Provide the full width of FLAG specific value. */ 4143 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4144 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4145 /* Build a new flow. */ 4146 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4147 items[0] = (struct rte_flow_item){ 4148 .type = (enum rte_flow_item_type) 4149 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4150 .spec = &tag_spec, 4151 }; 4152 items[1] = (struct rte_flow_item){ 4153 .type = RTE_FLOW_ITEM_TYPE_END, 4154 }; 4155 actions[0] = (struct rte_flow_action){ 4156 .type = (enum rte_flow_action_type) 4157 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4158 .conf = &ftag, 4159 }; 4160 actions[1] = (struct rte_flow_action){ 4161 .type = (enum rte_flow_action_type) 4162 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4163 .conf = &cp_mreg, 4164 }; 4165 actions[2] = (struct rte_flow_action){ 4166 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4167 .conf = &jump, 4168 }; 4169 actions[3] = (struct rte_flow_action){ 4170 .type = RTE_FLOW_ACTION_TYPE_END, 4171 }; 4172 } else { 4173 /* Default rule, wildcard match. */ 4174 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4175 items[0] = (struct rte_flow_item){ 4176 .type = RTE_FLOW_ITEM_TYPE_END, 4177 }; 4178 actions[0] = (struct rte_flow_action){ 4179 .type = (enum rte_flow_action_type) 4180 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4181 .conf = &cp_mreg, 4182 }; 4183 actions[1] = (struct rte_flow_action){ 4184 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4185 .conf = &jump, 4186 }; 4187 actions[2] = (struct rte_flow_action){ 4188 .type = RTE_FLOW_ACTION_TYPE_END, 4189 }; 4190 } 4191 /* Build a new entry. */ 4192 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4193 if (!mcp_res) { 4194 rte_errno = ENOMEM; 4195 return NULL; 4196 } 4197 mcp_res->idx = idx; 4198 mcp_res->mark_id = mark_id; 4199 /* 4200 * The copy Flows are not included in any list. There 4201 * ones are referenced from other Flows and can not 4202 * be applied, removed, deleted in ardbitrary order 4203 * by list traversing. 4204 */ 4205 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4206 &attr, items, actions, false, error); 4207 if (!mcp_res->rix_flow) { 4208 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4209 return NULL; 4210 } 4211 return &mcp_res->hlist_ent; 4212 } 4213 4214 struct mlx5_list_entry * 4215 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4216 void *cb_ctx __rte_unused) 4217 { 4218 struct rte_eth_dev *dev = tool_ctx; 4219 struct mlx5_priv *priv = dev->data->dev_private; 4220 struct mlx5_flow_mreg_copy_resource *mcp_res; 4221 uint32_t idx = 0; 4222 4223 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4224 if (!mcp_res) { 4225 rte_errno = ENOMEM; 4226 return NULL; 4227 } 4228 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4229 mcp_res->idx = idx; 4230 return &mcp_res->hlist_ent; 4231 } 4232 4233 void 4234 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4235 { 4236 struct mlx5_flow_mreg_copy_resource *mcp_res = 4237 container_of(entry, typeof(*mcp_res), hlist_ent); 4238 struct rte_eth_dev *dev = tool_ctx; 4239 struct mlx5_priv *priv = dev->data->dev_private; 4240 4241 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4242 } 4243 4244 /** 4245 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4246 * 4247 * As mark_id is unique, if there's already a registered flow for the mark_id, 4248 * return by increasing the reference counter of the resource. Otherwise, create 4249 * the resource (mcp_res) and flow. 4250 * 4251 * Flow looks like, 4252 * - If ingress port is ANY and reg_c[1] is mark_id, 4253 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4254 * 4255 * For default flow (zero mark_id), flow is like, 4256 * - If ingress port is ANY, 4257 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4258 * 4259 * @param dev 4260 * Pointer to Ethernet device. 4261 * @param mark_id 4262 * ID of MARK action, zero means default flow for META. 4263 * @param[out] error 4264 * Perform verbose error reporting if not NULL. 4265 * 4266 * @return 4267 * Associated resource on success, NULL otherwise and rte_errno is set. 4268 */ 4269 static struct mlx5_flow_mreg_copy_resource * 4270 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4271 struct rte_flow_error *error) 4272 { 4273 struct mlx5_priv *priv = dev->data->dev_private; 4274 struct mlx5_list_entry *entry; 4275 struct mlx5_flow_cb_ctx ctx = { 4276 .dev = dev, 4277 .error = error, 4278 .data = &mark_id, 4279 }; 4280 4281 /* Check if already registered. */ 4282 MLX5_ASSERT(priv->mreg_cp_tbl); 4283 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4284 if (!entry) 4285 return NULL; 4286 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4287 hlist_ent); 4288 } 4289 4290 void 4291 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4292 { 4293 struct mlx5_flow_mreg_copy_resource *mcp_res = 4294 container_of(entry, typeof(*mcp_res), hlist_ent); 4295 struct rte_eth_dev *dev = tool_ctx; 4296 struct mlx5_priv *priv = dev->data->dev_private; 4297 4298 MLX5_ASSERT(mcp_res->rix_flow); 4299 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4300 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4301 } 4302 4303 /** 4304 * Release flow in RX_CP_TBL. 4305 * 4306 * @param dev 4307 * Pointer to Ethernet device. 4308 * @flow 4309 * Parent flow for wich copying is provided. 4310 */ 4311 static void 4312 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4313 struct rte_flow *flow) 4314 { 4315 struct mlx5_flow_mreg_copy_resource *mcp_res; 4316 struct mlx5_priv *priv = dev->data->dev_private; 4317 4318 if (!flow->rix_mreg_copy) 4319 return; 4320 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4321 flow->rix_mreg_copy); 4322 if (!mcp_res || !priv->mreg_cp_tbl) 4323 return; 4324 MLX5_ASSERT(mcp_res->rix_flow); 4325 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4326 flow->rix_mreg_copy = 0; 4327 } 4328 4329 /** 4330 * Remove the default copy action from RX_CP_TBL. 4331 * 4332 * This functions is called in the mlx5_dev_start(). No thread safe 4333 * is guaranteed. 4334 * 4335 * @param dev 4336 * Pointer to Ethernet device. 4337 */ 4338 static void 4339 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4340 { 4341 struct mlx5_list_entry *entry; 4342 struct mlx5_priv *priv = dev->data->dev_private; 4343 struct mlx5_flow_cb_ctx ctx; 4344 uint32_t mark_id; 4345 4346 /* Check if default flow is registered. */ 4347 if (!priv->mreg_cp_tbl) 4348 return; 4349 mark_id = MLX5_DEFAULT_COPY_ID; 4350 ctx.data = &mark_id; 4351 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4352 if (!entry) 4353 return; 4354 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4355 } 4356 4357 /** 4358 * Add the default copy action in in RX_CP_TBL. 4359 * 4360 * This functions is called in the mlx5_dev_start(). No thread safe 4361 * is guaranteed. 4362 * 4363 * @param dev 4364 * Pointer to Ethernet device. 4365 * @param[out] error 4366 * Perform verbose error reporting if not NULL. 4367 * 4368 * @return 4369 * 0 for success, negative value otherwise and rte_errno is set. 4370 */ 4371 static int 4372 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4373 struct rte_flow_error *error) 4374 { 4375 struct mlx5_priv *priv = dev->data->dev_private; 4376 struct mlx5_flow_mreg_copy_resource *mcp_res; 4377 struct mlx5_flow_cb_ctx ctx; 4378 uint32_t mark_id; 4379 4380 /* Check whether extensive metadata feature is engaged. */ 4381 if (!priv->config.dv_flow_en || 4382 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4383 !mlx5_flow_ext_mreg_supported(dev) || 4384 !priv->sh->dv_regc0_mask) 4385 return 0; 4386 /* 4387 * Add default mreg copy flow may be called multiple time, but 4388 * only be called once in stop. Avoid register it twice. 4389 */ 4390 mark_id = MLX5_DEFAULT_COPY_ID; 4391 ctx.data = &mark_id; 4392 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4393 return 0; 4394 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4395 if (!mcp_res) 4396 return -rte_errno; 4397 return 0; 4398 } 4399 4400 /** 4401 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4402 * 4403 * All the flow having Q/RSS action should be split by 4404 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4405 * performs the following, 4406 * - CQE->flow_tag := reg_c[1] (MARK) 4407 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4408 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4409 * but there should be a flow per each MARK ID set by MARK action. 4410 * 4411 * For the aforementioned reason, if there's a MARK action in flow's action 4412 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4413 * the MARK ID to CQE's flow_tag like, 4414 * - If reg_c[1] is mark_id, 4415 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4416 * 4417 * For SET_META action which stores value in reg_c[0], as the destination is 4418 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4419 * MARK ID means the default flow. The default flow looks like, 4420 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4421 * 4422 * @param dev 4423 * Pointer to Ethernet device. 4424 * @param flow 4425 * Pointer to flow structure. 4426 * @param[in] actions 4427 * Pointer to the list of actions. 4428 * @param[out] error 4429 * Perform verbose error reporting if not NULL. 4430 * 4431 * @return 4432 * 0 on success, negative value otherwise and rte_errno is set. 4433 */ 4434 static int 4435 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4436 struct rte_flow *flow, 4437 const struct rte_flow_action *actions, 4438 struct rte_flow_error *error) 4439 { 4440 struct mlx5_priv *priv = dev->data->dev_private; 4441 struct mlx5_dev_config *config = &priv->config; 4442 struct mlx5_flow_mreg_copy_resource *mcp_res; 4443 const struct rte_flow_action_mark *mark; 4444 4445 /* Check whether extensive metadata feature is engaged. */ 4446 if (!config->dv_flow_en || 4447 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4448 !mlx5_flow_ext_mreg_supported(dev) || 4449 !priv->sh->dv_regc0_mask) 4450 return 0; 4451 /* Find MARK action. */ 4452 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4453 switch (actions->type) { 4454 case RTE_FLOW_ACTION_TYPE_FLAG: 4455 mcp_res = flow_mreg_add_copy_action 4456 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4457 if (!mcp_res) 4458 return -rte_errno; 4459 flow->rix_mreg_copy = mcp_res->idx; 4460 return 0; 4461 case RTE_FLOW_ACTION_TYPE_MARK: 4462 mark = (const struct rte_flow_action_mark *) 4463 actions->conf; 4464 mcp_res = 4465 flow_mreg_add_copy_action(dev, mark->id, error); 4466 if (!mcp_res) 4467 return -rte_errno; 4468 flow->rix_mreg_copy = mcp_res->idx; 4469 return 0; 4470 default: 4471 break; 4472 } 4473 } 4474 return 0; 4475 } 4476 4477 #define MLX5_MAX_SPLIT_ACTIONS 24 4478 #define MLX5_MAX_SPLIT_ITEMS 24 4479 4480 /** 4481 * Split the hairpin flow. 4482 * Since HW can't support encap and push-vlan on Rx, we move these 4483 * actions to Tx. 4484 * If the count action is after the encap then we also 4485 * move the count action. in this case the count will also measure 4486 * the outer bytes. 4487 * 4488 * @param dev 4489 * Pointer to Ethernet device. 4490 * @param[in] actions 4491 * Associated actions (list terminated by the END action). 4492 * @param[out] actions_rx 4493 * Rx flow actions. 4494 * @param[out] actions_tx 4495 * Tx flow actions.. 4496 * @param[out] pattern_tx 4497 * The pattern items for the Tx flow. 4498 * @param[out] flow_id 4499 * The flow ID connected to this flow. 4500 * 4501 * @return 4502 * 0 on success. 4503 */ 4504 static int 4505 flow_hairpin_split(struct rte_eth_dev *dev, 4506 const struct rte_flow_action actions[], 4507 struct rte_flow_action actions_rx[], 4508 struct rte_flow_action actions_tx[], 4509 struct rte_flow_item pattern_tx[], 4510 uint32_t flow_id) 4511 { 4512 const struct rte_flow_action_raw_encap *raw_encap; 4513 const struct rte_flow_action_raw_decap *raw_decap; 4514 struct mlx5_rte_flow_action_set_tag *set_tag; 4515 struct rte_flow_action *tag_action; 4516 struct mlx5_rte_flow_item_tag *tag_item; 4517 struct rte_flow_item *item; 4518 char *addr; 4519 int encap = 0; 4520 4521 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4522 switch (actions->type) { 4523 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4524 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4525 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4526 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4527 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4528 rte_memcpy(actions_tx, actions, 4529 sizeof(struct rte_flow_action)); 4530 actions_tx++; 4531 break; 4532 case RTE_FLOW_ACTION_TYPE_COUNT: 4533 if (encap) { 4534 rte_memcpy(actions_tx, actions, 4535 sizeof(struct rte_flow_action)); 4536 actions_tx++; 4537 } else { 4538 rte_memcpy(actions_rx, actions, 4539 sizeof(struct rte_flow_action)); 4540 actions_rx++; 4541 } 4542 break; 4543 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4544 raw_encap = actions->conf; 4545 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4546 memcpy(actions_tx, actions, 4547 sizeof(struct rte_flow_action)); 4548 actions_tx++; 4549 encap = 1; 4550 } else { 4551 rte_memcpy(actions_rx, actions, 4552 sizeof(struct rte_flow_action)); 4553 actions_rx++; 4554 } 4555 break; 4556 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4557 raw_decap = actions->conf; 4558 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4559 memcpy(actions_tx, actions, 4560 sizeof(struct rte_flow_action)); 4561 actions_tx++; 4562 } else { 4563 rte_memcpy(actions_rx, actions, 4564 sizeof(struct rte_flow_action)); 4565 actions_rx++; 4566 } 4567 break; 4568 default: 4569 rte_memcpy(actions_rx, actions, 4570 sizeof(struct rte_flow_action)); 4571 actions_rx++; 4572 break; 4573 } 4574 } 4575 /* Add set meta action and end action for the Rx flow. */ 4576 tag_action = actions_rx; 4577 tag_action->type = (enum rte_flow_action_type) 4578 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4579 actions_rx++; 4580 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4581 actions_rx++; 4582 set_tag = (void *)actions_rx; 4583 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4584 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4585 .data = flow_id, 4586 }; 4587 MLX5_ASSERT(set_tag->id > REG_NON); 4588 tag_action->conf = set_tag; 4589 /* Create Tx item list. */ 4590 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4591 addr = (void *)&pattern_tx[2]; 4592 item = pattern_tx; 4593 item->type = (enum rte_flow_item_type) 4594 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4595 tag_item = (void *)addr; 4596 tag_item->data = flow_id; 4597 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4598 MLX5_ASSERT(set_tag->id > REG_NON); 4599 item->spec = tag_item; 4600 addr += sizeof(struct mlx5_rte_flow_item_tag); 4601 tag_item = (void *)addr; 4602 tag_item->data = UINT32_MAX; 4603 tag_item->id = UINT16_MAX; 4604 item->mask = tag_item; 4605 item->last = NULL; 4606 item++; 4607 item->type = RTE_FLOW_ITEM_TYPE_END; 4608 return 0; 4609 } 4610 4611 /** 4612 * The last stage of splitting chain, just creates the subflow 4613 * without any modification. 4614 * 4615 * @param[in] dev 4616 * Pointer to Ethernet device. 4617 * @param[in] flow 4618 * Parent flow structure pointer. 4619 * @param[in, out] sub_flow 4620 * Pointer to return the created subflow, may be NULL. 4621 * @param[in] attr 4622 * Flow rule attributes. 4623 * @param[in] items 4624 * Pattern specification (list terminated by the END pattern item). 4625 * @param[in] actions 4626 * Associated actions (list terminated by the END action). 4627 * @param[in] flow_split_info 4628 * Pointer to flow split info structure. 4629 * @param[out] error 4630 * Perform verbose error reporting if not NULL. 4631 * @return 4632 * 0 on success, negative value otherwise 4633 */ 4634 static int 4635 flow_create_split_inner(struct rte_eth_dev *dev, 4636 struct rte_flow *flow, 4637 struct mlx5_flow **sub_flow, 4638 const struct rte_flow_attr *attr, 4639 const struct rte_flow_item items[], 4640 const struct rte_flow_action actions[], 4641 struct mlx5_flow_split_info *flow_split_info, 4642 struct rte_flow_error *error) 4643 { 4644 struct mlx5_flow *dev_flow; 4645 4646 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4647 flow_split_info->flow_idx, error); 4648 if (!dev_flow) 4649 return -rte_errno; 4650 dev_flow->flow = flow; 4651 dev_flow->external = flow_split_info->external; 4652 dev_flow->skip_scale = flow_split_info->skip_scale; 4653 /* Subflow object was created, we must include one in the list. */ 4654 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4655 dev_flow->handle, next); 4656 /* 4657 * If dev_flow is as one of the suffix flow, some actions in suffix 4658 * flow may need some user defined item layer flags, and pass the 4659 * Metadate rxq mark flag to suffix flow as well. 4660 */ 4661 if (flow_split_info->prefix_layers) 4662 dev_flow->handle->layers = flow_split_info->prefix_layers; 4663 if (flow_split_info->prefix_mark) 4664 dev_flow->handle->mark = 1; 4665 if (sub_flow) 4666 *sub_flow = dev_flow; 4667 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4668 dev_flow->dv.table_id = flow_split_info->table_id; 4669 #endif 4670 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4671 } 4672 4673 /** 4674 * Get the sub policy of a meter. 4675 * 4676 * @param[in] dev 4677 * Pointer to Ethernet device. 4678 * @param[in] flow 4679 * Parent flow structure pointer. 4680 * @param wks 4681 * Pointer to thread flow work space. 4682 * @param[in] attr 4683 * Flow rule attributes. 4684 * @param[in] items 4685 * Pattern specification (list terminated by the END pattern item). 4686 * @param[out] error 4687 * Perform verbose error reporting if not NULL. 4688 * 4689 * @return 4690 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4691 */ 4692 static struct mlx5_flow_meter_sub_policy * 4693 get_meter_sub_policy(struct rte_eth_dev *dev, 4694 struct rte_flow *flow, 4695 struct mlx5_flow_workspace *wks, 4696 const struct rte_flow_attr *attr, 4697 const struct rte_flow_item items[], 4698 struct rte_flow_error *error) 4699 { 4700 struct mlx5_flow_meter_policy *policy; 4701 struct mlx5_flow_meter_policy *final_policy; 4702 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4703 4704 policy = wks->policy; 4705 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4706 if (final_policy->is_rss || final_policy->is_queue) { 4707 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4708 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4709 uint32_t i; 4710 4711 /* 4712 * This is a tmp dev_flow, 4713 * no need to register any matcher for it in translate. 4714 */ 4715 wks->skip_matcher_reg = 1; 4716 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4717 struct mlx5_flow dev_flow = {0}; 4718 struct mlx5_flow_handle dev_handle = { {0} }; 4719 uint8_t fate = final_policy->act_cnt[i].fate_action; 4720 4721 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4722 const struct rte_flow_action_rss *rss_act = 4723 final_policy->act_cnt[i].rss->conf; 4724 struct rte_flow_action rss_actions[2] = { 4725 [0] = { 4726 .type = RTE_FLOW_ACTION_TYPE_RSS, 4727 .conf = rss_act, 4728 }, 4729 [1] = { 4730 .type = RTE_FLOW_ACTION_TYPE_END, 4731 .conf = NULL, 4732 } 4733 }; 4734 4735 dev_flow.handle = &dev_handle; 4736 dev_flow.ingress = attr->ingress; 4737 dev_flow.flow = flow; 4738 dev_flow.external = 0; 4739 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4740 dev_flow.dv.transfer = attr->transfer; 4741 #endif 4742 /** 4743 * Translate RSS action to get rss hash fields. 4744 */ 4745 if (flow_drv_translate(dev, &dev_flow, attr, 4746 items, rss_actions, error)) 4747 goto exit; 4748 rss_desc_v[i] = wks->rss_desc; 4749 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4750 rss_desc_v[i].hash_fields = 4751 dev_flow.hash_fields; 4752 rss_desc_v[i].queue_num = 4753 rss_desc_v[i].hash_fields ? 4754 rss_desc_v[i].queue_num : 1; 4755 rss_desc_v[i].tunnel = 4756 !!(dev_flow.handle->layers & 4757 MLX5_FLOW_LAYER_TUNNEL); 4758 /* Use the RSS queues in the containers. */ 4759 rss_desc_v[i].queue = 4760 (uint16_t *)(uintptr_t)rss_act->queue; 4761 rss_desc[i] = &rss_desc_v[i]; 4762 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4763 /* This is queue action. */ 4764 rss_desc_v[i] = wks->rss_desc; 4765 rss_desc_v[i].key_len = 0; 4766 rss_desc_v[i].hash_fields = 0; 4767 rss_desc_v[i].queue = 4768 &final_policy->act_cnt[i].queue; 4769 rss_desc_v[i].queue_num = 1; 4770 rss_desc[i] = &rss_desc_v[i]; 4771 } else { 4772 rss_desc[i] = NULL; 4773 } 4774 } 4775 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4776 flow, policy, rss_desc); 4777 } else { 4778 enum mlx5_meter_domain mtr_domain = 4779 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4780 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4781 MLX5_MTR_DOMAIN_INGRESS); 4782 sub_policy = policy->sub_policys[mtr_domain][0]; 4783 } 4784 if (!sub_policy) 4785 rte_flow_error_set(error, EINVAL, 4786 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4787 "Failed to get meter sub-policy."); 4788 exit: 4789 return sub_policy; 4790 } 4791 4792 /** 4793 * Split the meter flow. 4794 * 4795 * As meter flow will split to three sub flow, other than meter 4796 * action, the other actions make sense to only meter accepts 4797 * the packet. If it need to be dropped, no other additional 4798 * actions should be take. 4799 * 4800 * One kind of special action which decapsulates the L3 tunnel 4801 * header will be in the prefix sub flow, as not to take the 4802 * L3 tunnel header into account. 4803 * 4804 * @param[in] dev 4805 * Pointer to Ethernet device. 4806 * @param[in] flow 4807 * Parent flow structure pointer. 4808 * @param wks 4809 * Pointer to thread flow work space. 4810 * @param[in] attr 4811 * Flow rule attributes. 4812 * @param[in] items 4813 * Pattern specification (list terminated by the END pattern item). 4814 * @param[out] sfx_items 4815 * Suffix flow match items (list terminated by the END pattern item). 4816 * @param[in] actions 4817 * Associated actions (list terminated by the END action). 4818 * @param[out] actions_sfx 4819 * Suffix flow actions. 4820 * @param[out] actions_pre 4821 * Prefix flow actions. 4822 * @param[out] mtr_flow_id 4823 * Pointer to meter flow id. 4824 * @param[out] error 4825 * Perform verbose error reporting if not NULL. 4826 * 4827 * @return 4828 * 0 on success, a negative errno value otherwise and rte_errno is set. 4829 */ 4830 static int 4831 flow_meter_split_prep(struct rte_eth_dev *dev, 4832 struct rte_flow *flow, 4833 struct mlx5_flow_workspace *wks, 4834 const struct rte_flow_attr *attr, 4835 const struct rte_flow_item items[], 4836 struct rte_flow_item sfx_items[], 4837 const struct rte_flow_action actions[], 4838 struct rte_flow_action actions_sfx[], 4839 struct rte_flow_action actions_pre[], 4840 uint32_t *mtr_flow_id, 4841 struct rte_flow_error *error) 4842 { 4843 struct mlx5_priv *priv = dev->data->dev_private; 4844 struct mlx5_flow_meter_info *fm = wks->fm; 4845 struct rte_flow_action *tag_action = NULL; 4846 struct rte_flow_item *tag_item; 4847 struct mlx5_rte_flow_action_set_tag *set_tag; 4848 const struct rte_flow_action_raw_encap *raw_encap; 4849 const struct rte_flow_action_raw_decap *raw_decap; 4850 struct mlx5_rte_flow_item_tag *tag_item_spec; 4851 struct mlx5_rte_flow_item_tag *tag_item_mask; 4852 uint32_t tag_id = 0; 4853 struct rte_flow_item *vlan_item_dst = NULL; 4854 const struct rte_flow_item *vlan_item_src = NULL; 4855 struct rte_flow_action *hw_mtr_action; 4856 struct rte_flow_action *action_pre_head = NULL; 4857 int32_t flow_src_port = priv->representor_id; 4858 bool mtr_first; 4859 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 4860 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 4861 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 4862 uint32_t flow_id = 0; 4863 uint32_t flow_id_reversed = 0; 4864 uint8_t flow_id_bits = 0; 4865 int shift; 4866 4867 /* Prepare the suffix subflow items. */ 4868 tag_item = sfx_items++; 4869 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4870 struct mlx5_priv *port_priv; 4871 const struct rte_flow_item_port_id *pid_v; 4872 int item_type = items->type; 4873 4874 switch (item_type) { 4875 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4876 pid_v = items->spec; 4877 MLX5_ASSERT(pid_v); 4878 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 4879 if (!port_priv) 4880 return rte_flow_error_set(error, 4881 rte_errno, 4882 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 4883 pid_v, 4884 "Failed to get port info."); 4885 flow_src_port = port_priv->representor_id; 4886 if (!fm->def_policy && wks->policy->is_hierarchy && 4887 flow_src_port != priv->representor_id) { 4888 if (flow_drv_mtr_hierarchy_rule_create(dev, 4889 flow, fm, 4890 flow_src_port, 4891 items, 4892 error)) 4893 return -rte_errno; 4894 } 4895 memcpy(sfx_items, items, sizeof(*sfx_items)); 4896 sfx_items++; 4897 break; 4898 case RTE_FLOW_ITEM_TYPE_VLAN: 4899 /* Determine if copy vlan item below. */ 4900 vlan_item_src = items; 4901 vlan_item_dst = sfx_items++; 4902 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 4903 break; 4904 default: 4905 break; 4906 } 4907 } 4908 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4909 sfx_items++; 4910 mtr_first = priv->sh->meter_aso_en && 4911 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 4912 /* For ASO meter, meter must be before tag in TX direction. */ 4913 if (mtr_first) { 4914 action_pre_head = actions_pre++; 4915 /* Leave space for tag action. */ 4916 tag_action = actions_pre++; 4917 } 4918 /* Prepare the actions for prefix and suffix flow. */ 4919 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4920 struct rte_flow_action *action_cur = NULL; 4921 4922 switch (actions->type) { 4923 case RTE_FLOW_ACTION_TYPE_METER: 4924 if (mtr_first) { 4925 action_cur = action_pre_head; 4926 } else { 4927 /* Leave space for tag action. */ 4928 tag_action = actions_pre++; 4929 action_cur = actions_pre++; 4930 } 4931 break; 4932 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4933 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4934 action_cur = actions_pre++; 4935 break; 4936 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4937 raw_encap = actions->conf; 4938 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4939 action_cur = actions_pre++; 4940 break; 4941 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4942 raw_decap = actions->conf; 4943 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4944 action_cur = actions_pre++; 4945 break; 4946 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4947 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4948 if (vlan_item_dst && vlan_item_src) { 4949 memcpy(vlan_item_dst, vlan_item_src, 4950 sizeof(*vlan_item_dst)); 4951 /* 4952 * Convert to internal match item, it is used 4953 * for vlan push and set vid. 4954 */ 4955 vlan_item_dst->type = (enum rte_flow_item_type) 4956 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4957 } 4958 break; 4959 default: 4960 break; 4961 } 4962 if (!action_cur) 4963 action_cur = (fm->def_policy) ? 4964 actions_sfx++ : actions_pre++; 4965 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 4966 } 4967 /* Add end action to the actions. */ 4968 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4969 if (priv->sh->meter_aso_en) { 4970 /** 4971 * For ASO meter, need to add an extra jump action explicitly, 4972 * to jump from meter to policer table. 4973 */ 4974 struct mlx5_flow_meter_sub_policy *sub_policy; 4975 struct mlx5_flow_tbl_data_entry *tbl_data; 4976 4977 if (!fm->def_policy) { 4978 sub_policy = get_meter_sub_policy(dev, flow, wks, 4979 attr, items, error); 4980 if (!sub_policy) 4981 return -rte_errno; 4982 } else { 4983 enum mlx5_meter_domain mtr_domain = 4984 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4985 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4986 MLX5_MTR_DOMAIN_INGRESS); 4987 4988 sub_policy = 4989 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 4990 } 4991 tbl_data = container_of(sub_policy->tbl_rsc, 4992 struct mlx5_flow_tbl_data_entry, tbl); 4993 hw_mtr_action = actions_pre++; 4994 hw_mtr_action->type = (enum rte_flow_action_type) 4995 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 4996 hw_mtr_action->conf = tbl_data->jump.action; 4997 } 4998 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4999 actions_pre++; 5000 if (!tag_action) 5001 return rte_flow_error_set(error, ENOMEM, 5002 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5003 NULL, "No tag action space."); 5004 if (!mtr_flow_id) { 5005 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5006 goto exit; 5007 } 5008 /* Only default-policy Meter creates mtr flow id. */ 5009 if (fm->def_policy) { 5010 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5011 if (!tag_id) 5012 return rte_flow_error_set(error, ENOMEM, 5013 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5014 "Failed to allocate meter flow id."); 5015 flow_id = tag_id - 1; 5016 flow_id_bits = (!flow_id) ? 1 : 5017 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5018 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5019 mtr_reg_bits) { 5020 mlx5_ipool_free(fm->flow_ipool, tag_id); 5021 return rte_flow_error_set(error, EINVAL, 5022 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5023 "Meter flow id exceeds max limit."); 5024 } 5025 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5026 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5027 } 5028 /* Build tag actions and items for meter_id/meter flow_id. */ 5029 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5030 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5031 tag_item_mask = tag_item_spec + 1; 5032 /* Both flow_id and meter_id share the same register. */ 5033 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5034 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5035 0, error), 5036 .offset = mtr_id_offset, 5037 .length = mtr_reg_bits, 5038 .data = flow->meter, 5039 }; 5040 /* 5041 * The color Reg bits used by flow_id are growing from 5042 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5043 */ 5044 for (shift = 0; shift < flow_id_bits; shift++) 5045 flow_id_reversed = (flow_id_reversed << 1) | 5046 ((flow_id >> shift) & 0x1); 5047 set_tag->data |= 5048 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5049 tag_item_spec->id = set_tag->id; 5050 tag_item_spec->data = set_tag->data << mtr_id_offset; 5051 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5052 tag_action->type = (enum rte_flow_action_type) 5053 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5054 tag_action->conf = set_tag; 5055 tag_item->type = (enum rte_flow_item_type) 5056 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5057 tag_item->spec = tag_item_spec; 5058 tag_item->last = NULL; 5059 tag_item->mask = tag_item_mask; 5060 exit: 5061 if (mtr_flow_id) 5062 *mtr_flow_id = tag_id; 5063 return 0; 5064 } 5065 5066 /** 5067 * Split action list having QUEUE/RSS for metadata register copy. 5068 * 5069 * Once Q/RSS action is detected in user's action list, the flow action 5070 * should be split in order to copy metadata registers, which will happen in 5071 * RX_CP_TBL like, 5072 * - CQE->flow_tag := reg_c[1] (MARK) 5073 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5074 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5075 * This is because the last action of each flow must be a terminal action 5076 * (QUEUE, RSS or DROP). 5077 * 5078 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5079 * stored and kept in the mlx5_flow structure per each sub_flow. 5080 * 5081 * The Q/RSS action is replaced with, 5082 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5083 * And the following JUMP action is added at the end, 5084 * - JUMP, to RX_CP_TBL. 5085 * 5086 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5087 * flow_create_split_metadata() routine. The flow will look like, 5088 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5089 * 5090 * @param dev 5091 * Pointer to Ethernet device. 5092 * @param[out] split_actions 5093 * Pointer to store split actions to jump to CP_TBL. 5094 * @param[in] actions 5095 * Pointer to the list of original flow actions. 5096 * @param[in] qrss 5097 * Pointer to the Q/RSS action. 5098 * @param[in] actions_n 5099 * Number of original actions. 5100 * @param[out] error 5101 * Perform verbose error reporting if not NULL. 5102 * 5103 * @return 5104 * non-zero unique flow_id on success, otherwise 0 and 5105 * error/rte_error are set. 5106 */ 5107 static uint32_t 5108 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5109 struct rte_flow_action *split_actions, 5110 const struct rte_flow_action *actions, 5111 const struct rte_flow_action *qrss, 5112 int actions_n, struct rte_flow_error *error) 5113 { 5114 struct mlx5_priv *priv = dev->data->dev_private; 5115 struct mlx5_rte_flow_action_set_tag *set_tag; 5116 struct rte_flow_action_jump *jump; 5117 const int qrss_idx = qrss - actions; 5118 uint32_t flow_id = 0; 5119 int ret = 0; 5120 5121 /* 5122 * Given actions will be split 5123 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5124 * - Add jump to mreg CP_TBL. 5125 * As a result, there will be one more action. 5126 */ 5127 ++actions_n; 5128 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5129 set_tag = (void *)(split_actions + actions_n); 5130 /* 5131 * If tag action is not set to void(it means we are not the meter 5132 * suffix flow), add the tag action. Since meter suffix flow already 5133 * has the tag added. 5134 */ 5135 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 5136 /* 5137 * Allocate the new subflow ID. This one is unique within 5138 * device and not shared with representors. Otherwise, 5139 * we would have to resolve multi-thread access synch 5140 * issue. Each flow on the shared device is appended 5141 * with source vport identifier, so the resulting 5142 * flows will be unique in the shared (by master and 5143 * representors) domain even if they have coinciding 5144 * IDs. 5145 */ 5146 mlx5_ipool_malloc(priv->sh->ipool 5147 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5148 if (!flow_id) 5149 return rte_flow_error_set(error, ENOMEM, 5150 RTE_FLOW_ERROR_TYPE_ACTION, 5151 NULL, "can't allocate id " 5152 "for split Q/RSS subflow"); 5153 /* Internal SET_TAG action to set flow ID. */ 5154 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5155 .data = flow_id, 5156 }; 5157 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5158 if (ret < 0) 5159 return ret; 5160 set_tag->id = ret; 5161 /* Construct new actions array. */ 5162 /* Replace QUEUE/RSS action. */ 5163 split_actions[qrss_idx] = (struct rte_flow_action){ 5164 .type = (enum rte_flow_action_type) 5165 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5166 .conf = set_tag, 5167 }; 5168 } 5169 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5170 jump = (void *)(set_tag + 1); 5171 *jump = (struct rte_flow_action_jump){ 5172 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5173 }; 5174 split_actions[actions_n - 2] = (struct rte_flow_action){ 5175 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5176 .conf = jump, 5177 }; 5178 split_actions[actions_n - 1] = (struct rte_flow_action){ 5179 .type = RTE_FLOW_ACTION_TYPE_END, 5180 }; 5181 return flow_id; 5182 } 5183 5184 /** 5185 * Extend the given action list for Tx metadata copy. 5186 * 5187 * Copy the given action list to the ext_actions and add flow metadata register 5188 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5189 * 5190 * @param[out] ext_actions 5191 * Pointer to the extended action list. 5192 * @param[in] actions 5193 * Pointer to the list of actions. 5194 * @param[in] actions_n 5195 * Number of actions in the list. 5196 * @param[out] error 5197 * Perform verbose error reporting if not NULL. 5198 * @param[in] encap_idx 5199 * The encap action inndex. 5200 * 5201 * @return 5202 * 0 on success, negative value otherwise 5203 */ 5204 static int 5205 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5206 struct rte_flow_action *ext_actions, 5207 const struct rte_flow_action *actions, 5208 int actions_n, struct rte_flow_error *error, 5209 int encap_idx) 5210 { 5211 struct mlx5_flow_action_copy_mreg *cp_mreg = 5212 (struct mlx5_flow_action_copy_mreg *) 5213 (ext_actions + actions_n + 1); 5214 int ret; 5215 5216 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5217 if (ret < 0) 5218 return ret; 5219 cp_mreg->dst = ret; 5220 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5221 if (ret < 0) 5222 return ret; 5223 cp_mreg->src = ret; 5224 if (encap_idx != 0) 5225 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5226 if (encap_idx == actions_n - 1) { 5227 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5228 .type = (enum rte_flow_action_type) 5229 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5230 .conf = cp_mreg, 5231 }; 5232 ext_actions[actions_n] = (struct rte_flow_action){ 5233 .type = RTE_FLOW_ACTION_TYPE_END, 5234 }; 5235 } else { 5236 ext_actions[encap_idx] = (struct rte_flow_action){ 5237 .type = (enum rte_flow_action_type) 5238 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5239 .conf = cp_mreg, 5240 }; 5241 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5242 sizeof(*ext_actions) * (actions_n - encap_idx)); 5243 } 5244 return 0; 5245 } 5246 5247 /** 5248 * Check the match action from the action list. 5249 * 5250 * @param[in] actions 5251 * Pointer to the list of actions. 5252 * @param[in] attr 5253 * Flow rule attributes. 5254 * @param[in] action 5255 * The action to be check if exist. 5256 * @param[out] match_action_pos 5257 * Pointer to the position of the matched action if exists, otherwise is -1. 5258 * @param[out] qrss_action_pos 5259 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5260 * @param[out] modify_after_mirror 5261 * Pointer to the flag of modify action after FDB mirroring. 5262 * 5263 * @return 5264 * > 0 the total number of actions. 5265 * 0 if not found match action in action list. 5266 */ 5267 static int 5268 flow_check_match_action(const struct rte_flow_action actions[], 5269 const struct rte_flow_attr *attr, 5270 enum rte_flow_action_type action, 5271 int *match_action_pos, int *qrss_action_pos, 5272 int *modify_after_mirror) 5273 { 5274 const struct rte_flow_action_sample *sample; 5275 const struct rte_flow_action_raw_decap *decap; 5276 int actions_n = 0; 5277 uint32_t ratio = 0; 5278 int sub_type = 0; 5279 int flag = 0; 5280 int fdb_mirror = 0; 5281 5282 *match_action_pos = -1; 5283 *qrss_action_pos = -1; 5284 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5285 if (actions->type == action) { 5286 flag = 1; 5287 *match_action_pos = actions_n; 5288 } 5289 switch (actions->type) { 5290 case RTE_FLOW_ACTION_TYPE_QUEUE: 5291 case RTE_FLOW_ACTION_TYPE_RSS: 5292 *qrss_action_pos = actions_n; 5293 break; 5294 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5295 sample = actions->conf; 5296 ratio = sample->ratio; 5297 sub_type = ((const struct rte_flow_action *) 5298 (sample->actions))->type; 5299 if (ratio == 1 && attr->transfer) 5300 fdb_mirror = 1; 5301 break; 5302 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5303 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5304 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5305 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5306 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5307 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5308 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5309 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5310 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5311 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5312 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5313 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5314 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5315 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5316 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5317 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5318 case RTE_FLOW_ACTION_TYPE_FLAG: 5319 case RTE_FLOW_ACTION_TYPE_MARK: 5320 case RTE_FLOW_ACTION_TYPE_SET_META: 5321 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5322 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5323 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5324 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5325 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5326 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5327 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5328 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5329 case RTE_FLOW_ACTION_TYPE_METER: 5330 if (fdb_mirror) 5331 *modify_after_mirror = 1; 5332 break; 5333 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5334 decap = actions->conf; 5335 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5336 ; 5337 actions_n++; 5338 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5339 const struct rte_flow_action_raw_encap *encap = 5340 actions->conf; 5341 if (decap->size <= 5342 MLX5_ENCAPSULATION_DECISION_SIZE && 5343 encap->size > 5344 MLX5_ENCAPSULATION_DECISION_SIZE) 5345 /* L3 encap. */ 5346 break; 5347 } 5348 if (fdb_mirror) 5349 *modify_after_mirror = 1; 5350 break; 5351 default: 5352 break; 5353 } 5354 actions_n++; 5355 } 5356 if (flag && fdb_mirror && !*modify_after_mirror) { 5357 /* FDB mirroring uses the destination array to implement 5358 * instead of FLOW_SAMPLER object. 5359 */ 5360 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5361 flag = 0; 5362 } 5363 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5364 return flag ? actions_n + 1 : 0; 5365 } 5366 5367 #define SAMPLE_SUFFIX_ITEM 2 5368 5369 /** 5370 * Split the sample flow. 5371 * 5372 * As sample flow will split to two sub flow, sample flow with 5373 * sample action, the other actions will move to new suffix flow. 5374 * 5375 * Also add unique tag id with tag action in the sample flow, 5376 * the same tag id will be as match in the suffix flow. 5377 * 5378 * @param dev 5379 * Pointer to Ethernet device. 5380 * @param[in] add_tag 5381 * Add extra tag action flag. 5382 * @param[out] sfx_items 5383 * Suffix flow match items (list terminated by the END pattern item). 5384 * @param[in] actions 5385 * Associated actions (list terminated by the END action). 5386 * @param[out] actions_sfx 5387 * Suffix flow actions. 5388 * @param[out] actions_pre 5389 * Prefix flow actions. 5390 * @param[in] actions_n 5391 * The total number of actions. 5392 * @param[in] sample_action_pos 5393 * The sample action position. 5394 * @param[in] qrss_action_pos 5395 * The Queue/RSS action position. 5396 * @param[in] jump_table 5397 * Add extra jump action flag. 5398 * @param[out] error 5399 * Perform verbose error reporting if not NULL. 5400 * 5401 * @return 5402 * 0 on success, or unique flow_id, a negative errno value 5403 * otherwise and rte_errno is set. 5404 */ 5405 static int 5406 flow_sample_split_prep(struct rte_eth_dev *dev, 5407 int add_tag, 5408 struct rte_flow_item sfx_items[], 5409 const struct rte_flow_action actions[], 5410 struct rte_flow_action actions_sfx[], 5411 struct rte_flow_action actions_pre[], 5412 int actions_n, 5413 int sample_action_pos, 5414 int qrss_action_pos, 5415 int jump_table, 5416 struct rte_flow_error *error) 5417 { 5418 struct mlx5_priv *priv = dev->data->dev_private; 5419 struct mlx5_rte_flow_action_set_tag *set_tag; 5420 struct mlx5_rte_flow_item_tag *tag_spec; 5421 struct mlx5_rte_flow_item_tag *tag_mask; 5422 struct rte_flow_action_jump *jump_action; 5423 uint32_t tag_id = 0; 5424 int index; 5425 int append_index = 0; 5426 int ret; 5427 5428 if (sample_action_pos < 0) 5429 return rte_flow_error_set(error, EINVAL, 5430 RTE_FLOW_ERROR_TYPE_ACTION, 5431 NULL, "invalid position of sample " 5432 "action in list"); 5433 /* Prepare the actions for prefix and suffix flow. */ 5434 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5435 index = qrss_action_pos; 5436 /* Put the preceding the Queue/RSS action into prefix flow. */ 5437 if (index != 0) 5438 memcpy(actions_pre, actions, 5439 sizeof(struct rte_flow_action) * index); 5440 /* Put others preceding the sample action into prefix flow. */ 5441 if (sample_action_pos > index + 1) 5442 memcpy(actions_pre + index, actions + index + 1, 5443 sizeof(struct rte_flow_action) * 5444 (sample_action_pos - index - 1)); 5445 index = sample_action_pos - 1; 5446 /* Put Queue/RSS action into Suffix flow. */ 5447 memcpy(actions_sfx, actions + qrss_action_pos, 5448 sizeof(struct rte_flow_action)); 5449 actions_sfx++; 5450 } else { 5451 index = sample_action_pos; 5452 if (index != 0) 5453 memcpy(actions_pre, actions, 5454 sizeof(struct rte_flow_action) * index); 5455 } 5456 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5457 * For CX6DX and above, metadata registers Cx preserve their value, 5458 * add an extra tag action for NIC-RX and E-Switch Domain. 5459 */ 5460 if (add_tag) { 5461 /* Prepare the prefix tag action. */ 5462 append_index++; 5463 set_tag = (void *)(actions_pre + actions_n + append_index); 5464 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 5465 if (ret < 0) 5466 return ret; 5467 mlx5_ipool_malloc(priv->sh->ipool 5468 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5469 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5470 .id = ret, 5471 .data = tag_id, 5472 }; 5473 /* Prepare the suffix subflow items. */ 5474 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5475 tag_spec->data = tag_id; 5476 tag_spec->id = set_tag->id; 5477 tag_mask = tag_spec + 1; 5478 tag_mask->data = UINT32_MAX; 5479 sfx_items[0] = (struct rte_flow_item){ 5480 .type = (enum rte_flow_item_type) 5481 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5482 .spec = tag_spec, 5483 .last = NULL, 5484 .mask = tag_mask, 5485 }; 5486 sfx_items[1] = (struct rte_flow_item){ 5487 .type = (enum rte_flow_item_type) 5488 RTE_FLOW_ITEM_TYPE_END, 5489 }; 5490 /* Prepare the tag action in prefix subflow. */ 5491 actions_pre[index++] = 5492 (struct rte_flow_action){ 5493 .type = (enum rte_flow_action_type) 5494 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5495 .conf = set_tag, 5496 }; 5497 } 5498 memcpy(actions_pre + index, actions + sample_action_pos, 5499 sizeof(struct rte_flow_action)); 5500 index += 1; 5501 /* For the modify action after the sample action in E-Switch mirroring, 5502 * Add the extra jump action in prefix subflow and jump into the next 5503 * table, then do the modify action in the new table. 5504 */ 5505 if (jump_table) { 5506 /* Prepare the prefix jump action. */ 5507 append_index++; 5508 jump_action = (void *)(actions_pre + actions_n + append_index); 5509 jump_action->group = jump_table; 5510 actions_pre[index++] = 5511 (struct rte_flow_action){ 5512 .type = (enum rte_flow_action_type) 5513 RTE_FLOW_ACTION_TYPE_JUMP, 5514 .conf = jump_action, 5515 }; 5516 } 5517 actions_pre[index] = (struct rte_flow_action){ 5518 .type = (enum rte_flow_action_type) 5519 RTE_FLOW_ACTION_TYPE_END, 5520 }; 5521 /* Put the actions after sample into Suffix flow. */ 5522 memcpy(actions_sfx, actions + sample_action_pos + 1, 5523 sizeof(struct rte_flow_action) * 5524 (actions_n - sample_action_pos - 1)); 5525 return tag_id; 5526 } 5527 5528 /** 5529 * The splitting for metadata feature. 5530 * 5531 * - Q/RSS action on NIC Rx should be split in order to pass by 5532 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5533 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5534 * 5535 * - All the actions on NIC Tx should have a mreg copy action to 5536 * copy reg_a from WQE to reg_c[0]. 5537 * 5538 * @param dev 5539 * Pointer to Ethernet device. 5540 * @param[in] flow 5541 * Parent flow structure pointer. 5542 * @param[in] attr 5543 * Flow rule attributes. 5544 * @param[in] items 5545 * Pattern specification (list terminated by the END pattern item). 5546 * @param[in] actions 5547 * Associated actions (list terminated by the END action). 5548 * @param[in] flow_split_info 5549 * Pointer to flow split info structure. 5550 * @param[out] error 5551 * Perform verbose error reporting if not NULL. 5552 * @return 5553 * 0 on success, negative value otherwise 5554 */ 5555 static int 5556 flow_create_split_metadata(struct rte_eth_dev *dev, 5557 struct rte_flow *flow, 5558 const struct rte_flow_attr *attr, 5559 const struct rte_flow_item items[], 5560 const struct rte_flow_action actions[], 5561 struct mlx5_flow_split_info *flow_split_info, 5562 struct rte_flow_error *error) 5563 { 5564 struct mlx5_priv *priv = dev->data->dev_private; 5565 struct mlx5_dev_config *config = &priv->config; 5566 const struct rte_flow_action *qrss = NULL; 5567 struct rte_flow_action *ext_actions = NULL; 5568 struct mlx5_flow *dev_flow = NULL; 5569 uint32_t qrss_id = 0; 5570 int mtr_sfx = 0; 5571 size_t act_size; 5572 int actions_n; 5573 int encap_idx; 5574 int ret; 5575 5576 /* Check whether extensive metadata feature is engaged. */ 5577 if (!config->dv_flow_en || 5578 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5579 !mlx5_flow_ext_mreg_supported(dev)) 5580 return flow_create_split_inner(dev, flow, NULL, attr, items, 5581 actions, flow_split_info, error); 5582 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5583 &encap_idx); 5584 if (qrss) { 5585 /* Exclude hairpin flows from splitting. */ 5586 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5587 const struct rte_flow_action_queue *queue; 5588 5589 queue = qrss->conf; 5590 if (mlx5_rxq_get_type(dev, queue->index) == 5591 MLX5_RXQ_TYPE_HAIRPIN) 5592 qrss = NULL; 5593 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5594 const struct rte_flow_action_rss *rss; 5595 5596 rss = qrss->conf; 5597 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5598 MLX5_RXQ_TYPE_HAIRPIN) 5599 qrss = NULL; 5600 } 5601 } 5602 if (qrss) { 5603 /* Check if it is in meter suffix table. */ 5604 mtr_sfx = attr->group == (attr->transfer ? 5605 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5606 MLX5_FLOW_TABLE_LEVEL_METER); 5607 /* 5608 * Q/RSS action on NIC Rx should be split in order to pass by 5609 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5610 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5611 */ 5612 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5613 sizeof(struct rte_flow_action_set_tag) + 5614 sizeof(struct rte_flow_action_jump); 5615 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5616 SOCKET_ID_ANY); 5617 if (!ext_actions) 5618 return rte_flow_error_set(error, ENOMEM, 5619 RTE_FLOW_ERROR_TYPE_ACTION, 5620 NULL, "no memory to split " 5621 "metadata flow"); 5622 /* 5623 * If we are the suffix flow of meter, tag already exist. 5624 * Set the tag action to void. 5625 */ 5626 if (mtr_sfx) 5627 ext_actions[qrss - actions].type = 5628 RTE_FLOW_ACTION_TYPE_VOID; 5629 else 5630 ext_actions[qrss - actions].type = 5631 (enum rte_flow_action_type) 5632 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5633 /* 5634 * Create the new actions list with removed Q/RSS action 5635 * and appended set tag and jump to register copy table 5636 * (RX_CP_TBL). We should preallocate unique tag ID here 5637 * in advance, because it is needed for set tag action. 5638 */ 5639 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5640 qrss, actions_n, error); 5641 if (!mtr_sfx && !qrss_id) { 5642 ret = -rte_errno; 5643 goto exit; 5644 } 5645 } else if (attr->egress && !attr->transfer) { 5646 /* 5647 * All the actions on NIC Tx should have a metadata register 5648 * copy action to copy reg_a from WQE to reg_c[meta] 5649 */ 5650 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5651 sizeof(struct mlx5_flow_action_copy_mreg); 5652 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5653 SOCKET_ID_ANY); 5654 if (!ext_actions) 5655 return rte_flow_error_set(error, ENOMEM, 5656 RTE_FLOW_ERROR_TYPE_ACTION, 5657 NULL, "no memory to split " 5658 "metadata flow"); 5659 /* Create the action list appended with copy register. */ 5660 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5661 actions_n, error, encap_idx); 5662 if (ret < 0) 5663 goto exit; 5664 } 5665 /* Add the unmodified original or prefix subflow. */ 5666 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5667 items, ext_actions ? ext_actions : 5668 actions, flow_split_info, error); 5669 if (ret < 0) 5670 goto exit; 5671 MLX5_ASSERT(dev_flow); 5672 if (qrss) { 5673 const struct rte_flow_attr q_attr = { 5674 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5675 .ingress = 1, 5676 }; 5677 /* Internal PMD action to set register. */ 5678 struct mlx5_rte_flow_item_tag q_tag_spec = { 5679 .data = qrss_id, 5680 .id = REG_NON, 5681 }; 5682 struct rte_flow_item q_items[] = { 5683 { 5684 .type = (enum rte_flow_item_type) 5685 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5686 .spec = &q_tag_spec, 5687 .last = NULL, 5688 .mask = NULL, 5689 }, 5690 { 5691 .type = RTE_FLOW_ITEM_TYPE_END, 5692 }, 5693 }; 5694 struct rte_flow_action q_actions[] = { 5695 { 5696 .type = qrss->type, 5697 .conf = qrss->conf, 5698 }, 5699 { 5700 .type = RTE_FLOW_ACTION_TYPE_END, 5701 }, 5702 }; 5703 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5704 5705 /* 5706 * Configure the tag item only if there is no meter subflow. 5707 * Since tag is already marked in the meter suffix subflow 5708 * we can just use the meter suffix items as is. 5709 */ 5710 if (qrss_id) { 5711 /* Not meter subflow. */ 5712 MLX5_ASSERT(!mtr_sfx); 5713 /* 5714 * Put unique id in prefix flow due to it is destroyed 5715 * after suffix flow and id will be freed after there 5716 * is no actual flows with this id and identifier 5717 * reallocation becomes possible (for example, for 5718 * other flows in other threads). 5719 */ 5720 dev_flow->handle->split_flow_id = qrss_id; 5721 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5722 error); 5723 if (ret < 0) 5724 goto exit; 5725 q_tag_spec.id = ret; 5726 } 5727 dev_flow = NULL; 5728 /* Add suffix subflow to execute Q/RSS. */ 5729 flow_split_info->prefix_layers = layers; 5730 flow_split_info->prefix_mark = 0; 5731 ret = flow_create_split_inner(dev, flow, &dev_flow, 5732 &q_attr, mtr_sfx ? items : 5733 q_items, q_actions, 5734 flow_split_info, error); 5735 if (ret < 0) 5736 goto exit; 5737 /* qrss ID should be freed if failed. */ 5738 qrss_id = 0; 5739 MLX5_ASSERT(dev_flow); 5740 } 5741 5742 exit: 5743 /* 5744 * We do not destroy the partially created sub_flows in case of error. 5745 * These ones are included into parent flow list and will be destroyed 5746 * by flow_drv_destroy. 5747 */ 5748 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5749 qrss_id); 5750 mlx5_free(ext_actions); 5751 return ret; 5752 } 5753 5754 /** 5755 * Create meter internal drop flow with the original pattern. 5756 * 5757 * @param dev 5758 * Pointer to Ethernet device. 5759 * @param[in] flow 5760 * Parent flow structure pointer. 5761 * @param[in] attr 5762 * Flow rule attributes. 5763 * @param[in] items 5764 * Pattern specification (list terminated by the END pattern item). 5765 * @param[in] flow_split_info 5766 * Pointer to flow split info structure. 5767 * @param[in] fm 5768 * Pointer to flow meter structure. 5769 * @param[out] error 5770 * Perform verbose error reporting if not NULL. 5771 * @return 5772 * 0 on success, negative value otherwise 5773 */ 5774 static uint32_t 5775 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5776 struct rte_flow *flow, 5777 const struct rte_flow_attr *attr, 5778 const struct rte_flow_item items[], 5779 struct mlx5_flow_split_info *flow_split_info, 5780 struct mlx5_flow_meter_info *fm, 5781 struct rte_flow_error *error) 5782 { 5783 struct mlx5_flow *dev_flow = NULL; 5784 struct rte_flow_attr drop_attr = *attr; 5785 struct rte_flow_action drop_actions[3]; 5786 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5787 5788 MLX5_ASSERT(fm->drop_cnt); 5789 drop_actions[0].type = 5790 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5791 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5792 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5793 drop_actions[1].conf = NULL; 5794 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5795 drop_actions[2].conf = NULL; 5796 drop_split_info.external = false; 5797 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 5798 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 5799 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 5800 return flow_create_split_inner(dev, flow, &dev_flow, 5801 &drop_attr, items, drop_actions, 5802 &drop_split_info, error); 5803 } 5804 5805 /** 5806 * The splitting for meter feature. 5807 * 5808 * - The meter flow will be split to two flows as prefix and 5809 * suffix flow. The packets make sense only it pass the prefix 5810 * meter action. 5811 * 5812 * - Reg_C_5 is used for the packet to match betweend prefix and 5813 * suffix flow. 5814 * 5815 * @param dev 5816 * Pointer to Ethernet device. 5817 * @param[in] flow 5818 * Parent flow structure pointer. 5819 * @param[in] attr 5820 * Flow rule attributes. 5821 * @param[in] items 5822 * Pattern specification (list terminated by the END pattern item). 5823 * @param[in] actions 5824 * Associated actions (list terminated by the END action). 5825 * @param[in] flow_split_info 5826 * Pointer to flow split info structure. 5827 * @param[out] error 5828 * Perform verbose error reporting if not NULL. 5829 * @return 5830 * 0 on success, negative value otherwise 5831 */ 5832 static int 5833 flow_create_split_meter(struct rte_eth_dev *dev, 5834 struct rte_flow *flow, 5835 const struct rte_flow_attr *attr, 5836 const struct rte_flow_item items[], 5837 const struct rte_flow_action actions[], 5838 struct mlx5_flow_split_info *flow_split_info, 5839 struct rte_flow_error *error) 5840 { 5841 struct mlx5_priv *priv = dev->data->dev_private; 5842 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5843 struct rte_flow_action *sfx_actions = NULL; 5844 struct rte_flow_action *pre_actions = NULL; 5845 struct rte_flow_item *sfx_items = NULL; 5846 struct mlx5_flow *dev_flow = NULL; 5847 struct rte_flow_attr sfx_attr = *attr; 5848 struct mlx5_flow_meter_info *fm = NULL; 5849 uint8_t skip_scale_restore; 5850 bool has_mtr = false; 5851 bool has_modify = false; 5852 bool set_mtr_reg = true; 5853 bool is_mtr_hierarchy = false; 5854 uint32_t meter_id = 0; 5855 uint32_t mtr_idx = 0; 5856 uint32_t mtr_flow_id = 0; 5857 size_t act_size; 5858 size_t item_size; 5859 int actions_n = 0; 5860 int ret = 0; 5861 5862 if (priv->mtr_en) 5863 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 5864 &has_modify, &meter_id); 5865 if (has_mtr) { 5866 if (flow->meter) { 5867 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 5868 if (!fm) 5869 return rte_flow_error_set(error, EINVAL, 5870 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5871 NULL, "Meter not found."); 5872 } else { 5873 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 5874 if (!fm) 5875 return rte_flow_error_set(error, EINVAL, 5876 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5877 NULL, "Meter not found."); 5878 ret = mlx5_flow_meter_attach(priv, fm, 5879 &sfx_attr, error); 5880 if (ret) 5881 return -rte_errno; 5882 flow->meter = mtr_idx; 5883 } 5884 MLX5_ASSERT(wks); 5885 wks->fm = fm; 5886 if (!fm->def_policy) { 5887 wks->policy = mlx5_flow_meter_policy_find(dev, 5888 fm->policy_id, 5889 NULL); 5890 MLX5_ASSERT(wks->policy); 5891 if (wks->policy->is_hierarchy) { 5892 wks->final_policy = 5893 mlx5_flow_meter_hierarchy_get_final_policy(dev, 5894 wks->policy); 5895 if (!wks->final_policy) 5896 return rte_flow_error_set(error, 5897 EINVAL, 5898 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5899 "Failed to find terminal policy of hierarchy."); 5900 is_mtr_hierarchy = true; 5901 } 5902 } 5903 /* 5904 * If it isn't default-policy Meter, and 5905 * 1. There's no action in flow to change 5906 * packet (modify/encap/decap etc.), OR 5907 * 2. No drop count needed for this meter. 5908 * 3. It's not meter hierarchy. 5909 * Then no need to use regC to save meter id anymore. 5910 */ 5911 if (!fm->def_policy && !is_mtr_hierarchy && 5912 (!has_modify || !fm->drop_cnt)) 5913 set_mtr_reg = false; 5914 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 5915 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 5916 sizeof(struct mlx5_rte_flow_action_set_tag); 5917 /* Suffix items: tag, vlan, port id, end. */ 5918 #define METER_SUFFIX_ITEM 4 5919 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5920 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5921 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5922 0, SOCKET_ID_ANY); 5923 if (!sfx_actions) 5924 return rte_flow_error_set(error, ENOMEM, 5925 RTE_FLOW_ERROR_TYPE_ACTION, 5926 NULL, "no memory to split " 5927 "meter flow"); 5928 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5929 act_size); 5930 /* There's no suffix flow for meter of non-default policy. */ 5931 if (!fm->def_policy) 5932 pre_actions = sfx_actions + 1; 5933 else 5934 pre_actions = sfx_actions + actions_n; 5935 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 5936 items, sfx_items, actions, 5937 sfx_actions, pre_actions, 5938 (set_mtr_reg ? &mtr_flow_id : NULL), 5939 error); 5940 if (ret) { 5941 ret = -rte_errno; 5942 goto exit; 5943 } 5944 /* Add the prefix subflow. */ 5945 flow_split_info->prefix_mark = 0; 5946 skip_scale_restore = flow_split_info->skip_scale; 5947 flow_split_info->skip_scale |= 5948 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 5949 ret = flow_create_split_inner(dev, flow, &dev_flow, 5950 attr, items, pre_actions, 5951 flow_split_info, error); 5952 flow_split_info->skip_scale = skip_scale_restore; 5953 if (ret) { 5954 if (mtr_flow_id) 5955 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 5956 ret = -rte_errno; 5957 goto exit; 5958 } 5959 if (mtr_flow_id) { 5960 dev_flow->handle->split_flow_id = mtr_flow_id; 5961 dev_flow->handle->is_meter_flow_id = 1; 5962 } 5963 if (!fm->def_policy) { 5964 if (!set_mtr_reg && fm->drop_cnt) 5965 ret = 5966 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 5967 &sfx_attr, items, 5968 flow_split_info, 5969 fm, error); 5970 goto exit; 5971 } 5972 /* Setting the sfx group atrr. */ 5973 sfx_attr.group = sfx_attr.transfer ? 5974 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5975 MLX5_FLOW_TABLE_LEVEL_METER; 5976 flow_split_info->prefix_layers = 5977 flow_get_prefix_layer_flags(dev_flow); 5978 flow_split_info->prefix_mark = dev_flow->handle->mark; 5979 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 5980 } 5981 /* Add the prefix subflow. */ 5982 ret = flow_create_split_metadata(dev, flow, 5983 &sfx_attr, sfx_items ? 5984 sfx_items : items, 5985 sfx_actions ? sfx_actions : actions, 5986 flow_split_info, error); 5987 exit: 5988 if (sfx_actions) 5989 mlx5_free(sfx_actions); 5990 return ret; 5991 } 5992 5993 /** 5994 * The splitting for sample feature. 5995 * 5996 * Once Sample action is detected in the action list, the flow actions should 5997 * be split into prefix sub flow and suffix sub flow. 5998 * 5999 * The original items remain in the prefix sub flow, all actions preceding the 6000 * sample action and the sample action itself will be copied to the prefix 6001 * sub flow, the actions following the sample action will be copied to the 6002 * suffix sub flow, Queue action always be located in the suffix sub flow. 6003 * 6004 * In order to make the packet from prefix sub flow matches with suffix sub 6005 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6006 * flow uses tag item with the unique flow id. 6007 * 6008 * @param dev 6009 * Pointer to Ethernet device. 6010 * @param[in] flow 6011 * Parent flow structure pointer. 6012 * @param[in] attr 6013 * Flow rule attributes. 6014 * @param[in] items 6015 * Pattern specification (list terminated by the END pattern item). 6016 * @param[in] actions 6017 * Associated actions (list terminated by the END action). 6018 * @param[in] flow_split_info 6019 * Pointer to flow split info structure. 6020 * @param[out] error 6021 * Perform verbose error reporting if not NULL. 6022 * @return 6023 * 0 on success, negative value otherwise 6024 */ 6025 static int 6026 flow_create_split_sample(struct rte_eth_dev *dev, 6027 struct rte_flow *flow, 6028 const struct rte_flow_attr *attr, 6029 const struct rte_flow_item items[], 6030 const struct rte_flow_action actions[], 6031 struct mlx5_flow_split_info *flow_split_info, 6032 struct rte_flow_error *error) 6033 { 6034 struct mlx5_priv *priv = dev->data->dev_private; 6035 struct rte_flow_action *sfx_actions = NULL; 6036 struct rte_flow_action *pre_actions = NULL; 6037 struct rte_flow_item *sfx_items = NULL; 6038 struct mlx5_flow *dev_flow = NULL; 6039 struct rte_flow_attr sfx_attr = *attr; 6040 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6041 struct mlx5_flow_dv_sample_resource *sample_res; 6042 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6043 struct mlx5_flow_tbl_resource *sfx_tbl; 6044 #endif 6045 size_t act_size; 6046 size_t item_size; 6047 uint32_t fdb_tx = 0; 6048 int32_t tag_id = 0; 6049 int actions_n = 0; 6050 int sample_action_pos; 6051 int qrss_action_pos; 6052 int add_tag = 0; 6053 int modify_after_mirror = 0; 6054 uint16_t jump_table = 0; 6055 const uint32_t next_ft_step = 1; 6056 int ret = 0; 6057 6058 if (priv->sampler_en) 6059 actions_n = flow_check_match_action(actions, attr, 6060 RTE_FLOW_ACTION_TYPE_SAMPLE, 6061 &sample_action_pos, &qrss_action_pos, 6062 &modify_after_mirror); 6063 if (actions_n) { 6064 /* The prefix actions must includes sample, tag, end. */ 6065 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6066 + sizeof(struct mlx5_rte_flow_action_set_tag); 6067 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6068 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6069 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6070 item_size), 0, SOCKET_ID_ANY); 6071 if (!sfx_actions) 6072 return rte_flow_error_set(error, ENOMEM, 6073 RTE_FLOW_ERROR_TYPE_ACTION, 6074 NULL, "no memory to split " 6075 "sample flow"); 6076 /* The representor_id is UINT16_MAX for uplink. */ 6077 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6078 /* 6079 * When reg_c_preserve is set, metadata registers Cx preserve 6080 * their value even through packet duplication. 6081 */ 6082 add_tag = (!fdb_tx || priv->config.hca_attr.reg_c_preserve); 6083 if (add_tag) 6084 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6085 + act_size); 6086 if (modify_after_mirror) 6087 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6088 next_ft_step; 6089 pre_actions = sfx_actions + actions_n; 6090 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6091 actions, sfx_actions, 6092 pre_actions, actions_n, 6093 sample_action_pos, 6094 qrss_action_pos, jump_table, 6095 error); 6096 if (tag_id < 0 || (add_tag && !tag_id)) { 6097 ret = -rte_errno; 6098 goto exit; 6099 } 6100 if (modify_after_mirror) 6101 flow_split_info->skip_scale = 6102 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6103 /* Add the prefix subflow. */ 6104 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6105 items, pre_actions, 6106 flow_split_info, error); 6107 if (ret) { 6108 ret = -rte_errno; 6109 goto exit; 6110 } 6111 dev_flow->handle->split_flow_id = tag_id; 6112 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6113 if (!modify_after_mirror) { 6114 /* Set the sfx group attr. */ 6115 sample_res = (struct mlx5_flow_dv_sample_resource *) 6116 dev_flow->dv.sample_res; 6117 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6118 sample_res->normal_path_tbl; 6119 sfx_tbl_data = container_of(sfx_tbl, 6120 struct mlx5_flow_tbl_data_entry, 6121 tbl); 6122 sfx_attr.group = sfx_attr.transfer ? 6123 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6124 } else { 6125 MLX5_ASSERT(attr->transfer); 6126 sfx_attr.group = jump_table; 6127 } 6128 flow_split_info->prefix_layers = 6129 flow_get_prefix_layer_flags(dev_flow); 6130 flow_split_info->prefix_mark = dev_flow->handle->mark; 6131 /* Suffix group level already be scaled with factor, set 6132 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6133 * again in translation. 6134 */ 6135 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6136 #endif 6137 } 6138 /* Add the suffix subflow. */ 6139 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6140 sfx_items ? sfx_items : items, 6141 sfx_actions ? sfx_actions : actions, 6142 flow_split_info, error); 6143 exit: 6144 if (sfx_actions) 6145 mlx5_free(sfx_actions); 6146 return ret; 6147 } 6148 6149 /** 6150 * Split the flow to subflow set. The splitters might be linked 6151 * in the chain, like this: 6152 * flow_create_split_outer() calls: 6153 * flow_create_split_meter() calls: 6154 * flow_create_split_metadata(meter_subflow_0) calls: 6155 * flow_create_split_inner(metadata_subflow_0) 6156 * flow_create_split_inner(metadata_subflow_1) 6157 * flow_create_split_inner(metadata_subflow_2) 6158 * flow_create_split_metadata(meter_subflow_1) calls: 6159 * flow_create_split_inner(metadata_subflow_0) 6160 * flow_create_split_inner(metadata_subflow_1) 6161 * flow_create_split_inner(metadata_subflow_2) 6162 * 6163 * This provide flexible way to add new levels of flow splitting. 6164 * The all of successfully created subflows are included to the 6165 * parent flow dev_flow list. 6166 * 6167 * @param dev 6168 * Pointer to Ethernet device. 6169 * @param[in] flow 6170 * Parent flow structure pointer. 6171 * @param[in] attr 6172 * Flow rule attributes. 6173 * @param[in] items 6174 * Pattern specification (list terminated by the END pattern item). 6175 * @param[in] actions 6176 * Associated actions (list terminated by the END action). 6177 * @param[in] flow_split_info 6178 * Pointer to flow split info structure. 6179 * @param[out] error 6180 * Perform verbose error reporting if not NULL. 6181 * @return 6182 * 0 on success, negative value otherwise 6183 */ 6184 static int 6185 flow_create_split_outer(struct rte_eth_dev *dev, 6186 struct rte_flow *flow, 6187 const struct rte_flow_attr *attr, 6188 const struct rte_flow_item items[], 6189 const struct rte_flow_action actions[], 6190 struct mlx5_flow_split_info *flow_split_info, 6191 struct rte_flow_error *error) 6192 { 6193 int ret; 6194 6195 ret = flow_create_split_sample(dev, flow, attr, items, 6196 actions, flow_split_info, error); 6197 MLX5_ASSERT(ret <= 0); 6198 return ret; 6199 } 6200 6201 static inline struct mlx5_flow_tunnel * 6202 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6203 { 6204 struct mlx5_flow_tunnel *tunnel; 6205 6206 #pragma GCC diagnostic push 6207 #pragma GCC diagnostic ignored "-Wcast-qual" 6208 tunnel = (typeof(tunnel))flow->tunnel; 6209 #pragma GCC diagnostic pop 6210 6211 return tunnel; 6212 } 6213 6214 /** 6215 * Adjust flow RSS workspace if needed. 6216 * 6217 * @param wks 6218 * Pointer to thread flow work space. 6219 * @param rss_desc 6220 * Pointer to RSS descriptor. 6221 * @param[in] nrssq_num 6222 * New RSS queue number. 6223 * 6224 * @return 6225 * 0 on success, -1 otherwise and rte_errno is set. 6226 */ 6227 static int 6228 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6229 struct mlx5_flow_rss_desc *rss_desc, 6230 uint32_t nrssq_num) 6231 { 6232 if (likely(nrssq_num <= wks->rssq_num)) 6233 return 0; 6234 rss_desc->queue = realloc(rss_desc->queue, 6235 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6236 if (!rss_desc->queue) { 6237 rte_errno = ENOMEM; 6238 return -1; 6239 } 6240 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6241 return 0; 6242 } 6243 6244 /** 6245 * Create a flow and add it to @p list. 6246 * 6247 * @param dev 6248 * Pointer to Ethernet device. 6249 * @param list 6250 * Pointer to a TAILQ flow list. If this parameter NULL, 6251 * no list insertion occurred, flow is just created, 6252 * this is caller's responsibility to track the 6253 * created flow. 6254 * @param[in] attr 6255 * Flow rule attributes. 6256 * @param[in] items 6257 * Pattern specification (list terminated by the END pattern item). 6258 * @param[in] actions 6259 * Associated actions (list terminated by the END action). 6260 * @param[in] external 6261 * This flow rule is created by request external to PMD. 6262 * @param[out] error 6263 * Perform verbose error reporting if not NULL. 6264 * 6265 * @return 6266 * A flow index on success, 0 otherwise and rte_errno is set. 6267 */ 6268 static uint32_t 6269 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6270 const struct rte_flow_attr *attr, 6271 const struct rte_flow_item items[], 6272 const struct rte_flow_action original_actions[], 6273 bool external, struct rte_flow_error *error) 6274 { 6275 struct mlx5_priv *priv = dev->data->dev_private; 6276 struct rte_flow *flow = NULL; 6277 struct mlx5_flow *dev_flow; 6278 const struct rte_flow_action_rss *rss = NULL; 6279 struct mlx5_translated_action_handle 6280 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6281 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6282 union { 6283 struct mlx5_flow_expand_rss buf; 6284 uint8_t buffer[2048]; 6285 } expand_buffer; 6286 union { 6287 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6288 uint8_t buffer[2048]; 6289 } actions_rx; 6290 union { 6291 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6292 uint8_t buffer[2048]; 6293 } actions_hairpin_tx; 6294 union { 6295 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6296 uint8_t buffer[2048]; 6297 } items_tx; 6298 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6299 struct mlx5_flow_rss_desc *rss_desc; 6300 const struct rte_flow_action *p_actions_rx; 6301 uint32_t i; 6302 uint32_t idx = 0; 6303 int hairpin_flow; 6304 struct rte_flow_attr attr_tx = { .priority = 0 }; 6305 const struct rte_flow_action *actions; 6306 struct rte_flow_action *translated_actions = NULL; 6307 struct mlx5_flow_tunnel *tunnel; 6308 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6309 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6310 struct mlx5_flow_split_info flow_split_info = { 6311 .external = !!external, 6312 .skip_scale = 0, 6313 .flow_idx = 0, 6314 .prefix_mark = 0, 6315 .prefix_layers = 0, 6316 .table_id = 0 6317 }; 6318 int ret; 6319 6320 MLX5_ASSERT(wks); 6321 rss_desc = &wks->rss_desc; 6322 ret = flow_action_handles_translate(dev, original_actions, 6323 indir_actions, 6324 &indir_actions_n, 6325 &translated_actions, error); 6326 if (ret < 0) { 6327 MLX5_ASSERT(translated_actions == NULL); 6328 return 0; 6329 } 6330 actions = translated_actions ? translated_actions : original_actions; 6331 p_actions_rx = actions; 6332 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6333 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6334 external, hairpin_flow, error); 6335 if (ret < 0) 6336 goto error_before_hairpin_split; 6337 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6338 if (!flow) { 6339 rte_errno = ENOMEM; 6340 goto error_before_hairpin_split; 6341 } 6342 if (hairpin_flow > 0) { 6343 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6344 rte_errno = EINVAL; 6345 goto error_before_hairpin_split; 6346 } 6347 flow_hairpin_split(dev, actions, actions_rx.actions, 6348 actions_hairpin_tx.actions, items_tx.items, 6349 idx); 6350 p_actions_rx = actions_rx.actions; 6351 } 6352 flow_split_info.flow_idx = idx; 6353 flow->drv_type = flow_get_drv_type(dev, attr); 6354 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6355 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6356 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6357 /* RSS Action only works on NIC RX domain */ 6358 if (attr->ingress && !attr->transfer) 6359 rss = flow_get_rss_action(dev, p_actions_rx); 6360 if (rss) { 6361 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6362 return 0; 6363 /* 6364 * The following information is required by 6365 * mlx5_flow_hashfields_adjust() in advance. 6366 */ 6367 rss_desc->level = rss->level; 6368 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 6369 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 6370 } 6371 flow->dev_handles = 0; 6372 if (rss && rss->types) { 6373 unsigned int graph_root; 6374 6375 graph_root = find_graph_root(items, rss->level); 6376 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6377 items, rss->types, 6378 mlx5_support_expansion, graph_root); 6379 MLX5_ASSERT(ret > 0 && 6380 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6381 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6382 for (i = 0; i < buf->entries; ++i) 6383 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6384 } 6385 } else { 6386 buf->entries = 1; 6387 buf->entry[0].pattern = (void *)(uintptr_t)items; 6388 } 6389 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6390 indir_actions_n); 6391 for (i = 0; i < buf->entries; ++i) { 6392 /* Initialize flow split data. */ 6393 flow_split_info.prefix_layers = 0; 6394 flow_split_info.prefix_mark = 0; 6395 flow_split_info.skip_scale = 0; 6396 /* 6397 * The splitter may create multiple dev_flows, 6398 * depending on configuration. In the simplest 6399 * case it just creates unmodified original flow. 6400 */ 6401 ret = flow_create_split_outer(dev, flow, attr, 6402 buf->entry[i].pattern, 6403 p_actions_rx, &flow_split_info, 6404 error); 6405 if (ret < 0) 6406 goto error; 6407 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6408 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6409 p_actions_rx, 6410 idx, 6411 wks->flows[0].tunnel, 6412 &default_miss_ctx, 6413 error); 6414 if (ret < 0) { 6415 mlx5_free(default_miss_ctx.queue); 6416 goto error; 6417 } 6418 } 6419 } 6420 /* Create the tx flow. */ 6421 if (hairpin_flow) { 6422 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6423 attr_tx.ingress = 0; 6424 attr_tx.egress = 1; 6425 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6426 actions_hairpin_tx.actions, 6427 idx, error); 6428 if (!dev_flow) 6429 goto error; 6430 dev_flow->flow = flow; 6431 dev_flow->external = 0; 6432 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6433 dev_flow->handle, next); 6434 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6435 items_tx.items, 6436 actions_hairpin_tx.actions, error); 6437 if (ret < 0) 6438 goto error; 6439 } 6440 /* 6441 * Update the metadata register copy table. If extensive 6442 * metadata feature is enabled and registers are supported 6443 * we might create the extra rte_flow for each unique 6444 * MARK/FLAG action ID. 6445 * 6446 * The table is updated for ingress Flows only, because 6447 * the egress Flows belong to the different device and 6448 * copy table should be updated in peer NIC Rx domain. 6449 */ 6450 if (attr->ingress && 6451 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6452 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6453 if (ret) 6454 goto error; 6455 } 6456 /* 6457 * If the flow is external (from application) OR device is started, 6458 * OR mreg discover, then apply immediately. 6459 */ 6460 if (external || dev->data->dev_started || 6461 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6462 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6463 ret = flow_drv_apply(dev, flow, error); 6464 if (ret < 0) 6465 goto error; 6466 } 6467 flow->type = type; 6468 flow_rxq_flags_set(dev, flow); 6469 rte_free(translated_actions); 6470 tunnel = flow_tunnel_from_rule(wks->flows); 6471 if (tunnel) { 6472 flow->tunnel = 1; 6473 flow->tunnel_id = tunnel->tunnel_id; 6474 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6475 mlx5_free(default_miss_ctx.queue); 6476 } 6477 mlx5_flow_pop_thread_workspace(); 6478 return idx; 6479 error: 6480 MLX5_ASSERT(flow); 6481 ret = rte_errno; /* Save rte_errno before cleanup. */ 6482 flow_mreg_del_copy_action(dev, flow); 6483 flow_drv_destroy(dev, flow); 6484 if (rss_desc->shared_rss) 6485 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6486 mlx5_ipool_get 6487 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6488 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6489 mlx5_ipool_free(priv->flows[type], idx); 6490 rte_errno = ret; /* Restore rte_errno. */ 6491 ret = rte_errno; 6492 rte_errno = ret; 6493 mlx5_flow_pop_thread_workspace(); 6494 error_before_hairpin_split: 6495 rte_free(translated_actions); 6496 return 0; 6497 } 6498 6499 /** 6500 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6501 * incoming packets to table 1. 6502 * 6503 * Other flow rules, requested for group n, will be created in 6504 * e-switch table n+1. 6505 * Jump action to e-switch group n will be created to group n+1. 6506 * 6507 * Used when working in switchdev mode, to utilise advantages of table 1 6508 * and above. 6509 * 6510 * @param dev 6511 * Pointer to Ethernet device. 6512 * 6513 * @return 6514 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6515 */ 6516 struct rte_flow * 6517 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6518 { 6519 const struct rte_flow_attr attr = { 6520 .group = 0, 6521 .priority = 0, 6522 .ingress = 1, 6523 .egress = 0, 6524 .transfer = 1, 6525 }; 6526 const struct rte_flow_item pattern = { 6527 .type = RTE_FLOW_ITEM_TYPE_END, 6528 }; 6529 struct rte_flow_action_jump jump = { 6530 .group = 1, 6531 }; 6532 const struct rte_flow_action actions[] = { 6533 { 6534 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6535 .conf = &jump, 6536 }, 6537 { 6538 .type = RTE_FLOW_ACTION_TYPE_END, 6539 }, 6540 }; 6541 struct rte_flow_error error; 6542 6543 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6544 &attr, &pattern, 6545 actions, false, &error); 6546 } 6547 6548 /** 6549 * Validate a flow supported by the NIC. 6550 * 6551 * @see rte_flow_validate() 6552 * @see rte_flow_ops 6553 */ 6554 int 6555 mlx5_flow_validate(struct rte_eth_dev *dev, 6556 const struct rte_flow_attr *attr, 6557 const struct rte_flow_item items[], 6558 const struct rte_flow_action original_actions[], 6559 struct rte_flow_error *error) 6560 { 6561 int hairpin_flow; 6562 struct mlx5_translated_action_handle 6563 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6564 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6565 const struct rte_flow_action *actions; 6566 struct rte_flow_action *translated_actions = NULL; 6567 int ret = flow_action_handles_translate(dev, original_actions, 6568 indir_actions, 6569 &indir_actions_n, 6570 &translated_actions, error); 6571 6572 if (ret) 6573 return ret; 6574 actions = translated_actions ? translated_actions : original_actions; 6575 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6576 ret = flow_drv_validate(dev, attr, items, actions, 6577 true, hairpin_flow, error); 6578 rte_free(translated_actions); 6579 return ret; 6580 } 6581 6582 /** 6583 * Create a flow. 6584 * 6585 * @see rte_flow_create() 6586 * @see rte_flow_ops 6587 */ 6588 struct rte_flow * 6589 mlx5_flow_create(struct rte_eth_dev *dev, 6590 const struct rte_flow_attr *attr, 6591 const struct rte_flow_item items[], 6592 const struct rte_flow_action actions[], 6593 struct rte_flow_error *error) 6594 { 6595 /* 6596 * If the device is not started yet, it is not allowed to created a 6597 * flow from application. PMD default flows and traffic control flows 6598 * are not affected. 6599 */ 6600 if (unlikely(!dev->data->dev_started)) { 6601 DRV_LOG(DEBUG, "port %u is not started when " 6602 "inserting a flow", dev->data->port_id); 6603 rte_flow_error_set(error, ENODEV, 6604 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6605 NULL, 6606 "port not started"); 6607 return NULL; 6608 } 6609 6610 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6611 attr, items, actions, 6612 true, error); 6613 } 6614 6615 /** 6616 * Destroy a flow in a list. 6617 * 6618 * @param dev 6619 * Pointer to Ethernet device. 6620 * @param[in] flow_idx 6621 * Index of flow to destroy. 6622 */ 6623 static void 6624 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6625 uint32_t flow_idx) 6626 { 6627 struct mlx5_priv *priv = dev->data->dev_private; 6628 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6629 6630 if (!flow) 6631 return; 6632 MLX5_ASSERT(flow->type == type); 6633 /* 6634 * Update RX queue flags only if port is started, otherwise it is 6635 * already clean. 6636 */ 6637 if (dev->data->dev_started) 6638 flow_rxq_flags_trim(dev, flow); 6639 flow_drv_destroy(dev, flow); 6640 if (flow->tunnel) { 6641 struct mlx5_flow_tunnel *tunnel; 6642 6643 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6644 RTE_VERIFY(tunnel); 6645 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6646 mlx5_flow_tunnel_free(dev, tunnel); 6647 } 6648 flow_mreg_del_copy_action(dev, flow); 6649 mlx5_ipool_free(priv->flows[type], flow_idx); 6650 } 6651 6652 /** 6653 * Destroy all flows. 6654 * 6655 * @param dev 6656 * Pointer to Ethernet device. 6657 * @param type 6658 * Flow type to be flushed. 6659 * @param active 6660 * If flushing is called avtively. 6661 */ 6662 void 6663 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6664 bool active) 6665 { 6666 struct mlx5_priv *priv = dev->data->dev_private; 6667 uint32_t num_flushed = 0, fidx = 1; 6668 struct rte_flow *flow; 6669 6670 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6671 flow_list_destroy(dev, type, fidx); 6672 num_flushed++; 6673 } 6674 if (active) { 6675 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6676 dev->data->port_id, num_flushed); 6677 } 6678 } 6679 6680 /** 6681 * Stop all default actions for flows. 6682 * 6683 * @param dev 6684 * Pointer to Ethernet device. 6685 */ 6686 void 6687 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6688 { 6689 flow_mreg_del_default_copy_action(dev); 6690 flow_rxq_flags_clear(dev); 6691 } 6692 6693 /** 6694 * Start all default actions for flows. 6695 * 6696 * @param dev 6697 * Pointer to Ethernet device. 6698 * @return 6699 * 0 on success, a negative errno value otherwise and rte_errno is set. 6700 */ 6701 int 6702 mlx5_flow_start_default(struct rte_eth_dev *dev) 6703 { 6704 struct rte_flow_error error; 6705 6706 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6707 return flow_mreg_add_default_copy_action(dev, &error); 6708 } 6709 6710 /** 6711 * Release key of thread specific flow workspace data. 6712 */ 6713 void 6714 flow_release_workspace(void *data) 6715 { 6716 struct mlx5_flow_workspace *wks = data; 6717 struct mlx5_flow_workspace *next; 6718 6719 while (wks) { 6720 next = wks->next; 6721 free(wks->rss_desc.queue); 6722 free(wks); 6723 wks = next; 6724 } 6725 } 6726 6727 /** 6728 * Get thread specific current flow workspace. 6729 * 6730 * @return pointer to thread specific flow workspace data, NULL on error. 6731 */ 6732 struct mlx5_flow_workspace* 6733 mlx5_flow_get_thread_workspace(void) 6734 { 6735 struct mlx5_flow_workspace *data; 6736 6737 data = mlx5_flow_os_get_specific_workspace(); 6738 MLX5_ASSERT(data && data->inuse); 6739 if (!data || !data->inuse) 6740 DRV_LOG(ERR, "flow workspace not initialized."); 6741 return data; 6742 } 6743 6744 /** 6745 * Allocate and init new flow workspace. 6746 * 6747 * @return pointer to flow workspace data, NULL on error. 6748 */ 6749 static struct mlx5_flow_workspace* 6750 flow_alloc_thread_workspace(void) 6751 { 6752 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 6753 6754 if (!data) { 6755 DRV_LOG(ERR, "Failed to allocate flow workspace " 6756 "memory."); 6757 return NULL; 6758 } 6759 data->rss_desc.queue = calloc(1, 6760 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6761 if (!data->rss_desc.queue) 6762 goto err; 6763 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 6764 return data; 6765 err: 6766 if (data->rss_desc.queue) 6767 free(data->rss_desc.queue); 6768 free(data); 6769 return NULL; 6770 } 6771 6772 /** 6773 * Get new thread specific flow workspace. 6774 * 6775 * If current workspace inuse, create new one and set as current. 6776 * 6777 * @return pointer to thread specific flow workspace data, NULL on error. 6778 */ 6779 static struct mlx5_flow_workspace* 6780 mlx5_flow_push_thread_workspace(void) 6781 { 6782 struct mlx5_flow_workspace *curr; 6783 struct mlx5_flow_workspace *data; 6784 6785 curr = mlx5_flow_os_get_specific_workspace(); 6786 if (!curr) { 6787 data = flow_alloc_thread_workspace(); 6788 if (!data) 6789 return NULL; 6790 } else if (!curr->inuse) { 6791 data = curr; 6792 } else if (curr->next) { 6793 data = curr->next; 6794 } else { 6795 data = flow_alloc_thread_workspace(); 6796 if (!data) 6797 return NULL; 6798 curr->next = data; 6799 data->prev = curr; 6800 } 6801 data->inuse = 1; 6802 data->flow_idx = 0; 6803 /* Set as current workspace */ 6804 if (mlx5_flow_os_set_specific_workspace(data)) 6805 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6806 return data; 6807 } 6808 6809 /** 6810 * Close current thread specific flow workspace. 6811 * 6812 * If previous workspace available, set it as current. 6813 * 6814 * @return pointer to thread specific flow workspace data, NULL on error. 6815 */ 6816 static void 6817 mlx5_flow_pop_thread_workspace(void) 6818 { 6819 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 6820 6821 if (!data) 6822 return; 6823 if (!data->inuse) { 6824 DRV_LOG(ERR, "Failed to close unused flow workspace."); 6825 return; 6826 } 6827 data->inuse = 0; 6828 if (!data->prev) 6829 return; 6830 if (mlx5_flow_os_set_specific_workspace(data->prev)) 6831 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6832 } 6833 6834 /** 6835 * Verify the flow list is empty 6836 * 6837 * @param dev 6838 * Pointer to Ethernet device. 6839 * 6840 * @return the number of flows not released. 6841 */ 6842 int 6843 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 6844 { 6845 struct mlx5_priv *priv = dev->data->dev_private; 6846 struct rte_flow *flow; 6847 uint32_t idx = 0; 6848 int ret = 0, i; 6849 6850 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 6851 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 6852 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6853 dev->data->port_id, (void *)flow); 6854 ret++; 6855 } 6856 } 6857 return ret; 6858 } 6859 6860 /** 6861 * Enable default hairpin egress flow. 6862 * 6863 * @param dev 6864 * Pointer to Ethernet device. 6865 * @param queue 6866 * The queue index. 6867 * 6868 * @return 6869 * 0 on success, a negative errno value otherwise and rte_errno is set. 6870 */ 6871 int 6872 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6873 uint32_t queue) 6874 { 6875 const struct rte_flow_attr attr = { 6876 .egress = 1, 6877 .priority = 0, 6878 }; 6879 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6880 .queue = queue, 6881 }; 6882 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6883 .queue = UINT32_MAX, 6884 }; 6885 struct rte_flow_item items[] = { 6886 { 6887 .type = (enum rte_flow_item_type) 6888 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6889 .spec = &queue_spec, 6890 .last = NULL, 6891 .mask = &queue_mask, 6892 }, 6893 { 6894 .type = RTE_FLOW_ITEM_TYPE_END, 6895 }, 6896 }; 6897 struct rte_flow_action_jump jump = { 6898 .group = MLX5_HAIRPIN_TX_TABLE, 6899 }; 6900 struct rte_flow_action actions[2]; 6901 uint32_t flow_idx; 6902 struct rte_flow_error error; 6903 6904 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6905 actions[0].conf = &jump; 6906 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6907 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6908 &attr, items, actions, false, &error); 6909 if (!flow_idx) { 6910 DRV_LOG(DEBUG, 6911 "Failed to create ctrl flow: rte_errno(%d)," 6912 " type(%d), message(%s)", 6913 rte_errno, error.type, 6914 error.message ? error.message : " (no stated reason)"); 6915 return -rte_errno; 6916 } 6917 return 0; 6918 } 6919 6920 /** 6921 * Enable a control flow configured from the control plane. 6922 * 6923 * @param dev 6924 * Pointer to Ethernet device. 6925 * @param eth_spec 6926 * An Ethernet flow spec to apply. 6927 * @param eth_mask 6928 * An Ethernet flow mask to apply. 6929 * @param vlan_spec 6930 * A VLAN flow spec to apply. 6931 * @param vlan_mask 6932 * A VLAN flow mask to apply. 6933 * 6934 * @return 6935 * 0 on success, a negative errno value otherwise and rte_errno is set. 6936 */ 6937 int 6938 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6939 struct rte_flow_item_eth *eth_spec, 6940 struct rte_flow_item_eth *eth_mask, 6941 struct rte_flow_item_vlan *vlan_spec, 6942 struct rte_flow_item_vlan *vlan_mask) 6943 { 6944 struct mlx5_priv *priv = dev->data->dev_private; 6945 const struct rte_flow_attr attr = { 6946 .ingress = 1, 6947 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6948 }; 6949 struct rte_flow_item items[] = { 6950 { 6951 .type = RTE_FLOW_ITEM_TYPE_ETH, 6952 .spec = eth_spec, 6953 .last = NULL, 6954 .mask = eth_mask, 6955 }, 6956 { 6957 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6958 RTE_FLOW_ITEM_TYPE_END, 6959 .spec = vlan_spec, 6960 .last = NULL, 6961 .mask = vlan_mask, 6962 }, 6963 { 6964 .type = RTE_FLOW_ITEM_TYPE_END, 6965 }, 6966 }; 6967 uint16_t queue[priv->reta_idx_n]; 6968 struct rte_flow_action_rss action_rss = { 6969 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6970 .level = 0, 6971 .types = priv->rss_conf.rss_hf, 6972 .key_len = priv->rss_conf.rss_key_len, 6973 .queue_num = priv->reta_idx_n, 6974 .key = priv->rss_conf.rss_key, 6975 .queue = queue, 6976 }; 6977 struct rte_flow_action actions[] = { 6978 { 6979 .type = RTE_FLOW_ACTION_TYPE_RSS, 6980 .conf = &action_rss, 6981 }, 6982 { 6983 .type = RTE_FLOW_ACTION_TYPE_END, 6984 }, 6985 }; 6986 uint32_t flow_idx; 6987 struct rte_flow_error error; 6988 unsigned int i; 6989 6990 if (!priv->reta_idx_n || !priv->rxqs_n) { 6991 return 0; 6992 } 6993 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6994 action_rss.types = 0; 6995 for (i = 0; i != priv->reta_idx_n; ++i) 6996 queue[i] = (*priv->reta_idx)[i]; 6997 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6998 &attr, items, actions, false, &error); 6999 if (!flow_idx) 7000 return -rte_errno; 7001 return 0; 7002 } 7003 7004 /** 7005 * Enable a flow control configured from the control plane. 7006 * 7007 * @param dev 7008 * Pointer to Ethernet device. 7009 * @param eth_spec 7010 * An Ethernet flow spec to apply. 7011 * @param eth_mask 7012 * An Ethernet flow mask to apply. 7013 * 7014 * @return 7015 * 0 on success, a negative errno value otherwise and rte_errno is set. 7016 */ 7017 int 7018 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7019 struct rte_flow_item_eth *eth_spec, 7020 struct rte_flow_item_eth *eth_mask) 7021 { 7022 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7023 } 7024 7025 /** 7026 * Create default miss flow rule matching lacp traffic 7027 * 7028 * @param dev 7029 * Pointer to Ethernet device. 7030 * @param eth_spec 7031 * An Ethernet flow spec to apply. 7032 * 7033 * @return 7034 * 0 on success, a negative errno value otherwise and rte_errno is set. 7035 */ 7036 int 7037 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7038 { 7039 /* 7040 * The LACP matching is done by only using ether type since using 7041 * a multicast dst mac causes kernel to give low priority to this flow. 7042 */ 7043 static const struct rte_flow_item_eth lacp_spec = { 7044 .type = RTE_BE16(0x8809), 7045 }; 7046 static const struct rte_flow_item_eth lacp_mask = { 7047 .type = 0xffff, 7048 }; 7049 const struct rte_flow_attr attr = { 7050 .ingress = 1, 7051 }; 7052 struct rte_flow_item items[] = { 7053 { 7054 .type = RTE_FLOW_ITEM_TYPE_ETH, 7055 .spec = &lacp_spec, 7056 .mask = &lacp_mask, 7057 }, 7058 { 7059 .type = RTE_FLOW_ITEM_TYPE_END, 7060 }, 7061 }; 7062 struct rte_flow_action actions[] = { 7063 { 7064 .type = (enum rte_flow_action_type) 7065 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7066 }, 7067 { 7068 .type = RTE_FLOW_ACTION_TYPE_END, 7069 }, 7070 }; 7071 struct rte_flow_error error; 7072 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7073 &attr, items, actions, 7074 false, &error); 7075 7076 if (!flow_idx) 7077 return -rte_errno; 7078 return 0; 7079 } 7080 7081 /** 7082 * Destroy a flow. 7083 * 7084 * @see rte_flow_destroy() 7085 * @see rte_flow_ops 7086 */ 7087 int 7088 mlx5_flow_destroy(struct rte_eth_dev *dev, 7089 struct rte_flow *flow, 7090 struct rte_flow_error *error __rte_unused) 7091 { 7092 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7093 (uintptr_t)(void *)flow); 7094 return 0; 7095 } 7096 7097 /** 7098 * Destroy all flows. 7099 * 7100 * @see rte_flow_flush() 7101 * @see rte_flow_ops 7102 */ 7103 int 7104 mlx5_flow_flush(struct rte_eth_dev *dev, 7105 struct rte_flow_error *error __rte_unused) 7106 { 7107 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7108 return 0; 7109 } 7110 7111 /** 7112 * Isolated mode. 7113 * 7114 * @see rte_flow_isolate() 7115 * @see rte_flow_ops 7116 */ 7117 int 7118 mlx5_flow_isolate(struct rte_eth_dev *dev, 7119 int enable, 7120 struct rte_flow_error *error) 7121 { 7122 struct mlx5_priv *priv = dev->data->dev_private; 7123 7124 if (dev->data->dev_started) { 7125 rte_flow_error_set(error, EBUSY, 7126 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7127 NULL, 7128 "port must be stopped first"); 7129 return -rte_errno; 7130 } 7131 priv->isolated = !!enable; 7132 if (enable) 7133 dev->dev_ops = &mlx5_dev_ops_isolate; 7134 else 7135 dev->dev_ops = &mlx5_dev_ops; 7136 7137 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7138 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7139 7140 return 0; 7141 } 7142 7143 /** 7144 * Query a flow. 7145 * 7146 * @see rte_flow_query() 7147 * @see rte_flow_ops 7148 */ 7149 static int 7150 flow_drv_query(struct rte_eth_dev *dev, 7151 uint32_t flow_idx, 7152 const struct rte_flow_action *actions, 7153 void *data, 7154 struct rte_flow_error *error) 7155 { 7156 struct mlx5_priv *priv = dev->data->dev_private; 7157 const struct mlx5_flow_driver_ops *fops; 7158 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7159 flow_idx); 7160 enum mlx5_flow_drv_type ftype; 7161 7162 if (!flow) { 7163 return rte_flow_error_set(error, ENOENT, 7164 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7165 NULL, 7166 "invalid flow handle"); 7167 } 7168 ftype = flow->drv_type; 7169 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7170 fops = flow_get_drv_ops(ftype); 7171 7172 return fops->query(dev, flow, actions, data, error); 7173 } 7174 7175 /** 7176 * Query a flow. 7177 * 7178 * @see rte_flow_query() 7179 * @see rte_flow_ops 7180 */ 7181 int 7182 mlx5_flow_query(struct rte_eth_dev *dev, 7183 struct rte_flow *flow, 7184 const struct rte_flow_action *actions, 7185 void *data, 7186 struct rte_flow_error *error) 7187 { 7188 int ret; 7189 7190 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7191 error); 7192 if (ret < 0) 7193 return ret; 7194 return 0; 7195 } 7196 7197 /** 7198 * Get rte_flow callbacks. 7199 * 7200 * @param dev 7201 * Pointer to Ethernet device structure. 7202 * @param ops 7203 * Pointer to operation-specific structure. 7204 * 7205 * @return 0 7206 */ 7207 int 7208 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7209 const struct rte_flow_ops **ops) 7210 { 7211 *ops = &mlx5_flow_ops; 7212 return 0; 7213 } 7214 7215 /** 7216 * Validate meter policy actions. 7217 * Dispatcher for action type specific validation. 7218 * 7219 * @param[in] dev 7220 * Pointer to the Ethernet device structure. 7221 * @param[in] action 7222 * The meter policy action object to validate. 7223 * @param[in] attr 7224 * Attributes of flow to determine steering domain. 7225 * @param[out] is_rss 7226 * Is RSS or not. 7227 * @param[out] domain_bitmap 7228 * Domain bitmap. 7229 * @param[out] is_def_policy 7230 * Is default policy or not. 7231 * @param[out] error 7232 * Perform verbose error reporting if not NULL. Initialized in case of 7233 * error only. 7234 * 7235 * @return 7236 * 0 on success, otherwise negative errno value. 7237 */ 7238 int 7239 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7240 const struct rte_flow_action *actions[RTE_COLORS], 7241 struct rte_flow_attr *attr, 7242 bool *is_rss, 7243 uint8_t *domain_bitmap, 7244 uint8_t *policy_mode, 7245 struct rte_mtr_error *error) 7246 { 7247 const struct mlx5_flow_driver_ops *fops; 7248 7249 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7250 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7251 domain_bitmap, policy_mode, error); 7252 } 7253 7254 /** 7255 * Destroy the meter table set. 7256 * 7257 * @param[in] dev 7258 * Pointer to Ethernet device. 7259 * @param[in] mtr_policy 7260 * Meter policy struct. 7261 */ 7262 void 7263 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7264 struct mlx5_flow_meter_policy *mtr_policy) 7265 { 7266 const struct mlx5_flow_driver_ops *fops; 7267 7268 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7269 fops->destroy_mtr_acts(dev, mtr_policy); 7270 } 7271 7272 /** 7273 * Create policy action, lock free, 7274 * (mutex should be acquired by caller). 7275 * Dispatcher for action type specific call. 7276 * 7277 * @param[in] dev 7278 * Pointer to the Ethernet device structure. 7279 * @param[in] mtr_policy 7280 * Meter policy struct. 7281 * @param[in] action 7282 * Action specification used to create meter actions. 7283 * @param[out] error 7284 * Perform verbose error reporting if not NULL. Initialized in case of 7285 * error only. 7286 * 7287 * @return 7288 * 0 on success, otherwise negative errno value. 7289 */ 7290 int 7291 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7292 struct mlx5_flow_meter_policy *mtr_policy, 7293 const struct rte_flow_action *actions[RTE_COLORS], 7294 struct rte_mtr_error *error) 7295 { 7296 const struct mlx5_flow_driver_ops *fops; 7297 7298 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7299 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7300 } 7301 7302 /** 7303 * Create policy rules, lock free, 7304 * (mutex should be acquired by caller). 7305 * Dispatcher for action type specific call. 7306 * 7307 * @param[in] dev 7308 * Pointer to the Ethernet device structure. 7309 * @param[in] mtr_policy 7310 * Meter policy struct. 7311 * 7312 * @return 7313 * 0 on success, -1 otherwise. 7314 */ 7315 int 7316 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7317 struct mlx5_flow_meter_policy *mtr_policy) 7318 { 7319 const struct mlx5_flow_driver_ops *fops; 7320 7321 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7322 return fops->create_policy_rules(dev, mtr_policy); 7323 } 7324 7325 /** 7326 * Destroy policy rules, lock free, 7327 * (mutex should be acquired by caller). 7328 * Dispatcher for action type specific call. 7329 * 7330 * @param[in] dev 7331 * Pointer to the Ethernet device structure. 7332 * @param[in] mtr_policy 7333 * Meter policy struct. 7334 */ 7335 void 7336 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7337 struct mlx5_flow_meter_policy *mtr_policy) 7338 { 7339 const struct mlx5_flow_driver_ops *fops; 7340 7341 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7342 fops->destroy_policy_rules(dev, mtr_policy); 7343 } 7344 7345 /** 7346 * Destroy the default policy table set. 7347 * 7348 * @param[in] dev 7349 * Pointer to Ethernet device. 7350 */ 7351 void 7352 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7353 { 7354 const struct mlx5_flow_driver_ops *fops; 7355 7356 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7357 fops->destroy_def_policy(dev); 7358 } 7359 7360 /** 7361 * Destroy the default policy table set. 7362 * 7363 * @param[in] dev 7364 * Pointer to Ethernet device. 7365 * 7366 * @return 7367 * 0 on success, -1 otherwise. 7368 */ 7369 int 7370 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7371 { 7372 const struct mlx5_flow_driver_ops *fops; 7373 7374 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7375 return fops->create_def_policy(dev); 7376 } 7377 7378 /** 7379 * Create the needed meter and suffix tables. 7380 * 7381 * @param[in] dev 7382 * Pointer to Ethernet device. 7383 * 7384 * @return 7385 * 0 on success, -1 otherwise. 7386 */ 7387 int 7388 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7389 struct mlx5_flow_meter_info *fm, 7390 uint32_t mtr_idx, 7391 uint8_t domain_bitmap) 7392 { 7393 const struct mlx5_flow_driver_ops *fops; 7394 7395 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7396 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7397 } 7398 7399 /** 7400 * Destroy the meter table set. 7401 * 7402 * @param[in] dev 7403 * Pointer to Ethernet device. 7404 * @param[in] tbl 7405 * Pointer to the meter table set. 7406 */ 7407 void 7408 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7409 struct mlx5_flow_meter_info *fm) 7410 { 7411 const struct mlx5_flow_driver_ops *fops; 7412 7413 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7414 fops->destroy_mtr_tbls(dev, fm); 7415 } 7416 7417 /** 7418 * Destroy the global meter drop table. 7419 * 7420 * @param[in] dev 7421 * Pointer to Ethernet device. 7422 */ 7423 void 7424 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7425 { 7426 const struct mlx5_flow_driver_ops *fops; 7427 7428 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7429 fops->destroy_mtr_drop_tbls(dev); 7430 } 7431 7432 /** 7433 * Destroy the sub policy table with RX queue. 7434 * 7435 * @param[in] dev 7436 * Pointer to Ethernet device. 7437 * @param[in] mtr_policy 7438 * Pointer to meter policy table. 7439 */ 7440 void 7441 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7442 struct mlx5_flow_meter_policy *mtr_policy) 7443 { 7444 const struct mlx5_flow_driver_ops *fops; 7445 7446 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7447 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7448 } 7449 7450 /** 7451 * Allocate the needed aso flow meter id. 7452 * 7453 * @param[in] dev 7454 * Pointer to Ethernet device. 7455 * 7456 * @return 7457 * Index to aso flow meter on success, NULL otherwise. 7458 */ 7459 uint32_t 7460 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7461 { 7462 const struct mlx5_flow_driver_ops *fops; 7463 7464 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7465 return fops->create_meter(dev); 7466 } 7467 7468 /** 7469 * Free the aso flow meter id. 7470 * 7471 * @param[in] dev 7472 * Pointer to Ethernet device. 7473 * @param[in] mtr_idx 7474 * Index to aso flow meter to be free. 7475 * 7476 * @return 7477 * 0 on success. 7478 */ 7479 void 7480 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7481 { 7482 const struct mlx5_flow_driver_ops *fops; 7483 7484 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7485 fops->free_meter(dev, mtr_idx); 7486 } 7487 7488 /** 7489 * Allocate a counter. 7490 * 7491 * @param[in] dev 7492 * Pointer to Ethernet device structure. 7493 * 7494 * @return 7495 * Index to allocated counter on success, 0 otherwise. 7496 */ 7497 uint32_t 7498 mlx5_counter_alloc(struct rte_eth_dev *dev) 7499 { 7500 const struct mlx5_flow_driver_ops *fops; 7501 struct rte_flow_attr attr = { .transfer = 0 }; 7502 7503 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7504 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7505 return fops->counter_alloc(dev); 7506 } 7507 DRV_LOG(ERR, 7508 "port %u counter allocate is not supported.", 7509 dev->data->port_id); 7510 return 0; 7511 } 7512 7513 /** 7514 * Free a counter. 7515 * 7516 * @param[in] dev 7517 * Pointer to Ethernet device structure. 7518 * @param[in] cnt 7519 * Index to counter to be free. 7520 */ 7521 void 7522 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7523 { 7524 const struct mlx5_flow_driver_ops *fops; 7525 struct rte_flow_attr attr = { .transfer = 0 }; 7526 7527 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7528 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7529 fops->counter_free(dev, cnt); 7530 return; 7531 } 7532 DRV_LOG(ERR, 7533 "port %u counter free is not supported.", 7534 dev->data->port_id); 7535 } 7536 7537 /** 7538 * Query counter statistics. 7539 * 7540 * @param[in] dev 7541 * Pointer to Ethernet device structure. 7542 * @param[in] cnt 7543 * Index to counter to query. 7544 * @param[in] clear 7545 * Set to clear counter statistics. 7546 * @param[out] pkts 7547 * The counter hits packets number to save. 7548 * @param[out] bytes 7549 * The counter hits bytes number to save. 7550 * 7551 * @return 7552 * 0 on success, a negative errno value otherwise. 7553 */ 7554 int 7555 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7556 bool clear, uint64_t *pkts, uint64_t *bytes) 7557 { 7558 const struct mlx5_flow_driver_ops *fops; 7559 struct rte_flow_attr attr = { .transfer = 0 }; 7560 7561 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7562 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7563 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7564 } 7565 DRV_LOG(ERR, 7566 "port %u counter query is not supported.", 7567 dev->data->port_id); 7568 return -ENOTSUP; 7569 } 7570 7571 /** 7572 * Allocate a new memory for the counter values wrapped by all the needed 7573 * management. 7574 * 7575 * @param[in] sh 7576 * Pointer to mlx5_dev_ctx_shared object. 7577 * 7578 * @return 7579 * 0 on success, a negative errno value otherwise. 7580 */ 7581 static int 7582 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7583 { 7584 struct mlx5_devx_mkey_attr mkey_attr; 7585 struct mlx5_counter_stats_mem_mng *mem_mng; 7586 volatile struct flow_counter_stats *raw_data; 7587 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7588 int size = (sizeof(struct flow_counter_stats) * 7589 MLX5_COUNTERS_PER_POOL + 7590 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7591 sizeof(struct mlx5_counter_stats_mem_mng); 7592 size_t pgsize = rte_mem_page_size(); 7593 uint8_t *mem; 7594 int i; 7595 7596 if (pgsize == (size_t)-1) { 7597 DRV_LOG(ERR, "Failed to get mem page size"); 7598 rte_errno = ENOMEM; 7599 return -ENOMEM; 7600 } 7601 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7602 if (!mem) { 7603 rte_errno = ENOMEM; 7604 return -ENOMEM; 7605 } 7606 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7607 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7608 mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size, 7609 IBV_ACCESS_LOCAL_WRITE); 7610 if (!mem_mng->umem) { 7611 rte_errno = errno; 7612 mlx5_free(mem); 7613 return -rte_errno; 7614 } 7615 memset(&mkey_attr, 0, sizeof(mkey_attr)); 7616 mkey_attr.addr = (uintptr_t)mem; 7617 mkey_attr.size = size; 7618 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7619 mkey_attr.pd = sh->pdn; 7620 mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write; 7621 mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read; 7622 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7623 if (!mem_mng->dm) { 7624 mlx5_os_umem_dereg(mem_mng->umem); 7625 rte_errno = errno; 7626 mlx5_free(mem); 7627 return -rte_errno; 7628 } 7629 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7630 raw_data = (volatile struct flow_counter_stats *)mem; 7631 for (i = 0; i < raws_n; ++i) { 7632 mem_mng->raws[i].mem_mng = mem_mng; 7633 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7634 } 7635 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7636 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7637 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7638 next); 7639 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7640 sh->cmng.mem_mng = mem_mng; 7641 return 0; 7642 } 7643 7644 /** 7645 * Set the statistic memory to the new counter pool. 7646 * 7647 * @param[in] sh 7648 * Pointer to mlx5_dev_ctx_shared object. 7649 * @param[in] pool 7650 * Pointer to the pool to set the statistic memory. 7651 * 7652 * @return 7653 * 0 on success, a negative errno value otherwise. 7654 */ 7655 static int 7656 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7657 struct mlx5_flow_counter_pool *pool) 7658 { 7659 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7660 /* Resize statistic memory once used out. */ 7661 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7662 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7663 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7664 return -1; 7665 } 7666 rte_spinlock_lock(&pool->sl); 7667 pool->raw = cmng->mem_mng->raws + pool->index % 7668 MLX5_CNT_CONTAINER_RESIZE; 7669 rte_spinlock_unlock(&pool->sl); 7670 pool->raw_hw = NULL; 7671 return 0; 7672 } 7673 7674 #define MLX5_POOL_QUERY_FREQ_US 1000000 7675 7676 /** 7677 * Set the periodic procedure for triggering asynchronous batch queries for all 7678 * the counter pools. 7679 * 7680 * @param[in] sh 7681 * Pointer to mlx5_dev_ctx_shared object. 7682 */ 7683 void 7684 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7685 { 7686 uint32_t pools_n, us; 7687 7688 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7689 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7690 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7691 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7692 sh->cmng.query_thread_on = 0; 7693 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7694 } else { 7695 sh->cmng.query_thread_on = 1; 7696 } 7697 } 7698 7699 /** 7700 * The periodic procedure for triggering asynchronous batch queries for all the 7701 * counter pools. This function is probably called by the host thread. 7702 * 7703 * @param[in] arg 7704 * The parameter for the alarm process. 7705 */ 7706 void 7707 mlx5_flow_query_alarm(void *arg) 7708 { 7709 struct mlx5_dev_ctx_shared *sh = arg; 7710 int ret; 7711 uint16_t pool_index = sh->cmng.pool_index; 7712 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7713 struct mlx5_flow_counter_pool *pool; 7714 uint16_t n_valid; 7715 7716 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7717 goto set_alarm; 7718 rte_spinlock_lock(&cmng->pool_update_sl); 7719 pool = cmng->pools[pool_index]; 7720 n_valid = cmng->n_valid; 7721 rte_spinlock_unlock(&cmng->pool_update_sl); 7722 /* Set the statistic memory to the new created pool. */ 7723 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7724 goto set_alarm; 7725 if (pool->raw_hw) 7726 /* There is a pool query in progress. */ 7727 goto set_alarm; 7728 pool->raw_hw = 7729 LIST_FIRST(&sh->cmng.free_stat_raws); 7730 if (!pool->raw_hw) 7731 /* No free counter statistics raw memory. */ 7732 goto set_alarm; 7733 /* 7734 * Identify the counters released between query trigger and query 7735 * handle more efficiently. The counter released in this gap period 7736 * should wait for a new round of query as the new arrived packets 7737 * will not be taken into account. 7738 */ 7739 pool->query_gen++; 7740 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7741 MLX5_COUNTERS_PER_POOL, 7742 NULL, NULL, 7743 pool->raw_hw->mem_mng->dm->id, 7744 (void *)(uintptr_t) 7745 pool->raw_hw->data, 7746 sh->devx_comp, 7747 (uint64_t)(uintptr_t)pool); 7748 if (ret) { 7749 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7750 " %d", pool->min_dcs->id); 7751 pool->raw_hw = NULL; 7752 goto set_alarm; 7753 } 7754 LIST_REMOVE(pool->raw_hw, next); 7755 sh->cmng.pending_queries++; 7756 pool_index++; 7757 if (pool_index >= n_valid) 7758 pool_index = 0; 7759 set_alarm: 7760 sh->cmng.pool_index = pool_index; 7761 mlx5_set_query_alarm(sh); 7762 } 7763 7764 /** 7765 * Check and callback event for new aged flow in the counter pool 7766 * 7767 * @param[in] sh 7768 * Pointer to mlx5_dev_ctx_shared object. 7769 * @param[in] pool 7770 * Pointer to Current counter pool. 7771 */ 7772 static void 7773 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7774 struct mlx5_flow_counter_pool *pool) 7775 { 7776 struct mlx5_priv *priv; 7777 struct mlx5_flow_counter *cnt; 7778 struct mlx5_age_info *age_info; 7779 struct mlx5_age_param *age_param; 7780 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7781 struct mlx5_counter_stats_raw *prev = pool->raw; 7782 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7783 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7784 uint16_t expected = AGE_CANDIDATE; 7785 uint32_t i; 7786 7787 pool->time_of_last_age_check = curr_time; 7788 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7789 cnt = MLX5_POOL_GET_CNT(pool, i); 7790 age_param = MLX5_CNT_TO_AGE(cnt); 7791 if (__atomic_load_n(&age_param->state, 7792 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7793 continue; 7794 if (cur->data[i].hits != prev->data[i].hits) { 7795 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7796 __ATOMIC_RELAXED); 7797 continue; 7798 } 7799 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7800 time_delta, 7801 __ATOMIC_RELAXED) <= age_param->timeout) 7802 continue; 7803 /** 7804 * Hold the lock first, or if between the 7805 * state AGE_TMOUT and tailq operation the 7806 * release happened, the release procedure 7807 * may delete a non-existent tailq node. 7808 */ 7809 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7810 age_info = GET_PORT_AGE_INFO(priv); 7811 rte_spinlock_lock(&age_info->aged_sl); 7812 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7813 AGE_TMOUT, false, 7814 __ATOMIC_RELAXED, 7815 __ATOMIC_RELAXED)) { 7816 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7817 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7818 } 7819 rte_spinlock_unlock(&age_info->aged_sl); 7820 } 7821 mlx5_age_event_prepare(sh); 7822 } 7823 7824 /** 7825 * Handler for the HW respond about ready values from an asynchronous batch 7826 * query. This function is probably called by the host thread. 7827 * 7828 * @param[in] sh 7829 * The pointer to the shared device context. 7830 * @param[in] async_id 7831 * The Devx async ID. 7832 * @param[in] status 7833 * The status of the completion. 7834 */ 7835 void 7836 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7837 uint64_t async_id, int status) 7838 { 7839 struct mlx5_flow_counter_pool *pool = 7840 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7841 struct mlx5_counter_stats_raw *raw_to_free; 7842 uint8_t query_gen = pool->query_gen ^ 1; 7843 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7844 enum mlx5_counter_type cnt_type = 7845 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7846 MLX5_COUNTER_TYPE_ORIGIN; 7847 7848 if (unlikely(status)) { 7849 raw_to_free = pool->raw_hw; 7850 } else { 7851 raw_to_free = pool->raw; 7852 if (pool->is_aged) 7853 mlx5_flow_aging_check(sh, pool); 7854 rte_spinlock_lock(&pool->sl); 7855 pool->raw = pool->raw_hw; 7856 rte_spinlock_unlock(&pool->sl); 7857 /* Be sure the new raw counters data is updated in memory. */ 7858 rte_io_wmb(); 7859 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7860 rte_spinlock_lock(&cmng->csl[cnt_type]); 7861 TAILQ_CONCAT(&cmng->counters[cnt_type], 7862 &pool->counters[query_gen], next); 7863 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7864 } 7865 } 7866 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7867 pool->raw_hw = NULL; 7868 sh->cmng.pending_queries--; 7869 } 7870 7871 static int 7872 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7873 const struct flow_grp_info *grp_info, 7874 struct rte_flow_error *error) 7875 { 7876 if (grp_info->transfer && grp_info->external && 7877 grp_info->fdb_def_rule) { 7878 if (group == UINT32_MAX) 7879 return rte_flow_error_set 7880 (error, EINVAL, 7881 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7882 NULL, 7883 "group index not supported"); 7884 *table = group + 1; 7885 } else { 7886 *table = group; 7887 } 7888 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7889 return 0; 7890 } 7891 7892 /** 7893 * Translate the rte_flow group index to HW table value. 7894 * 7895 * If tunnel offload is disabled, all group ids converted to flow table 7896 * id using the standard method. 7897 * If tunnel offload is enabled, group id can be converted using the 7898 * standard or tunnel conversion method. Group conversion method 7899 * selection depends on flags in `grp_info` parameter: 7900 * - Internal (grp_info.external == 0) groups conversion uses the 7901 * standard method. 7902 * - Group ids in JUMP action converted with the tunnel conversion. 7903 * - Group id in rule attribute conversion depends on a rule type and 7904 * group id value: 7905 * ** non zero group attributes converted with the tunnel method 7906 * ** zero group attribute in non-tunnel rule is converted using the 7907 * standard method - there's only one root table 7908 * ** zero group attribute in steer tunnel rule is converted with the 7909 * standard method - single root table 7910 * ** zero group attribute in match tunnel rule is a special OvS 7911 * case: that value is used for portability reasons. That group 7912 * id is converted with the tunnel conversion method. 7913 * 7914 * @param[in] dev 7915 * Port device 7916 * @param[in] tunnel 7917 * PMD tunnel offload object 7918 * @param[in] group 7919 * rte_flow group index value. 7920 * @param[out] table 7921 * HW table value. 7922 * @param[in] grp_info 7923 * flags used for conversion 7924 * @param[out] error 7925 * Pointer to error structure. 7926 * 7927 * @return 7928 * 0 on success, a negative errno value otherwise and rte_errno is set. 7929 */ 7930 int 7931 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7932 const struct mlx5_flow_tunnel *tunnel, 7933 uint32_t group, uint32_t *table, 7934 const struct flow_grp_info *grp_info, 7935 struct rte_flow_error *error) 7936 { 7937 int ret; 7938 bool standard_translation; 7939 7940 if (!grp_info->skip_scale && grp_info->external && 7941 group < MLX5_MAX_TABLES_EXTERNAL) 7942 group *= MLX5_FLOW_TABLE_FACTOR; 7943 if (is_tunnel_offload_active(dev)) { 7944 standard_translation = !grp_info->external || 7945 grp_info->std_tbl_fix; 7946 } else { 7947 standard_translation = true; 7948 } 7949 DRV_LOG(DEBUG, 7950 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 7951 dev->data->port_id, group, grp_info->transfer, 7952 grp_info->external, grp_info->fdb_def_rule, 7953 standard_translation ? "STANDARD" : "TUNNEL"); 7954 if (standard_translation) 7955 ret = flow_group_to_table(dev->data->port_id, group, table, 7956 grp_info, error); 7957 else 7958 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7959 table, error); 7960 7961 return ret; 7962 } 7963 7964 /** 7965 * Discover availability of metadata reg_c's. 7966 * 7967 * Iteratively use test flows to check availability. 7968 * 7969 * @param[in] dev 7970 * Pointer to the Ethernet device structure. 7971 * 7972 * @return 7973 * 0 on success, a negative errno value otherwise and rte_errno is set. 7974 */ 7975 int 7976 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7977 { 7978 struct mlx5_priv *priv = dev->data->dev_private; 7979 struct mlx5_dev_config *config = &priv->config; 7980 enum modify_reg idx; 7981 int n = 0; 7982 7983 /* reg_c[0] and reg_c[1] are reserved. */ 7984 config->flow_mreg_c[n++] = REG_C_0; 7985 config->flow_mreg_c[n++] = REG_C_1; 7986 /* Discover availability of other reg_c's. */ 7987 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7988 struct rte_flow_attr attr = { 7989 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7990 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7991 .ingress = 1, 7992 }; 7993 struct rte_flow_item items[] = { 7994 [0] = { 7995 .type = RTE_FLOW_ITEM_TYPE_END, 7996 }, 7997 }; 7998 struct rte_flow_action actions[] = { 7999 [0] = { 8000 .type = (enum rte_flow_action_type) 8001 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8002 .conf = &(struct mlx5_flow_action_copy_mreg){ 8003 .src = REG_C_1, 8004 .dst = idx, 8005 }, 8006 }, 8007 [1] = { 8008 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8009 .conf = &(struct rte_flow_action_jump){ 8010 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8011 }, 8012 }, 8013 [2] = { 8014 .type = RTE_FLOW_ACTION_TYPE_END, 8015 }, 8016 }; 8017 uint32_t flow_idx; 8018 struct rte_flow *flow; 8019 struct rte_flow_error error; 8020 8021 if (!config->dv_flow_en) 8022 break; 8023 /* Create internal flow, validation skips copy action. */ 8024 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8025 items, actions, false, &error); 8026 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8027 flow_idx); 8028 if (!flow) 8029 continue; 8030 config->flow_mreg_c[n++] = idx; 8031 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8032 } 8033 for (; n < MLX5_MREG_C_NUM; ++n) 8034 config->flow_mreg_c[n] = REG_NON; 8035 return 0; 8036 } 8037 8038 int 8039 save_dump_file(const uint8_t *data, uint32_t size, 8040 uint32_t type, uint32_t id, void *arg, FILE *file) 8041 { 8042 char line[BUF_SIZE]; 8043 uint32_t out = 0; 8044 uint32_t k; 8045 uint32_t actions_num; 8046 struct rte_flow_query_count *count; 8047 8048 memset(line, 0, BUF_SIZE); 8049 switch (type) { 8050 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8051 actions_num = *(uint32_t *)(arg); 8052 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,%d,", 8053 type, id, actions_num); 8054 break; 8055 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8056 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%x,", 8057 type, id); 8058 break; 8059 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8060 count = (struct rte_flow_query_count *)arg; 8061 fprintf(file, "%d,0x%x,%" PRIu64 ",%" PRIu64 "\n", type, 8062 id, count->hits, count->bytes); 8063 return 0; 8064 default: 8065 return -1; 8066 } 8067 8068 for (k = 0; k < size; k++) { 8069 /* Make sure we do not overrun the line buffer length. */ 8070 if (out >= BUF_SIZE - 4) { 8071 line[out] = '\0'; 8072 break; 8073 } 8074 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8075 (data[k]) & 0xff); 8076 } 8077 fprintf(file, "%s\n", line); 8078 return 0; 8079 } 8080 8081 int 8082 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8083 struct rte_flow_query_count *count, struct rte_flow_error *error) 8084 { 8085 struct rte_flow_action action[2]; 8086 enum mlx5_flow_drv_type ftype; 8087 const struct mlx5_flow_driver_ops *fops; 8088 8089 if (!flow) { 8090 return rte_flow_error_set(error, ENOENT, 8091 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8092 NULL, 8093 "invalid flow handle"); 8094 } 8095 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8096 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8097 if (flow->counter) { 8098 memset(count, 0, sizeof(struct rte_flow_query_count)); 8099 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8100 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8101 ftype < MLX5_FLOW_TYPE_MAX); 8102 fops = flow_get_drv_ops(ftype); 8103 return fops->query(dev, flow, action, count, error); 8104 } 8105 return -1; 8106 } 8107 8108 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8109 /** 8110 * Dump flow ipool data to file 8111 * 8112 * @param[in] dev 8113 * The pointer to Ethernet device. 8114 * @param[in] file 8115 * A pointer to a file for output. 8116 * @param[out] error 8117 * Perform verbose error reporting if not NULL. PMDs initialize this 8118 * structure in case of error only. 8119 * @return 8120 * 0 on success, a negative value otherwise. 8121 */ 8122 int 8123 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8124 struct rte_flow *flow, FILE *file, 8125 struct rte_flow_error *error) 8126 { 8127 struct mlx5_priv *priv = dev->data->dev_private; 8128 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8129 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8130 uint32_t handle_idx; 8131 struct mlx5_flow_handle *dh; 8132 struct rte_flow_query_count count; 8133 uint32_t actions_num; 8134 const uint8_t *data; 8135 size_t size; 8136 uint32_t id; 8137 uint32_t type; 8138 8139 if (!flow) { 8140 return rte_flow_error_set(error, ENOENT, 8141 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8142 NULL, 8143 "invalid flow handle"); 8144 } 8145 handle_idx = flow->dev_handles; 8146 while (handle_idx) { 8147 dh = mlx5_ipool_get(priv->sh->ipool 8148 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8149 if (!dh) 8150 continue; 8151 handle_idx = dh->next.next; 8152 id = (uint32_t)(uintptr_t)dh->drv_flow; 8153 8154 /* query counter */ 8155 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8156 if (!mlx5_flow_query_counter(dev, flow, &count, error)) 8157 save_dump_file(NULL, 0, type, 8158 id, (void *)&count, file); 8159 8160 /* Get modify_hdr and encap_decap buf from ipools. */ 8161 encap_decap = NULL; 8162 modify_hdr = dh->dvh.modify_hdr; 8163 8164 if (dh->dvh.rix_encap_decap) { 8165 encap_decap = mlx5_ipool_get(priv->sh->ipool 8166 [MLX5_IPOOL_DECAP_ENCAP], 8167 dh->dvh.rix_encap_decap); 8168 } 8169 if (modify_hdr) { 8170 data = (const uint8_t *)modify_hdr->actions; 8171 size = (size_t)(modify_hdr->actions_num) * 8; 8172 actions_num = modify_hdr->actions_num; 8173 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8174 save_dump_file(data, size, type, id, 8175 (void *)(&actions_num), file); 8176 } 8177 if (encap_decap) { 8178 data = encap_decap->buf; 8179 size = encap_decap->size; 8180 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8181 save_dump_file(data, size, type, 8182 id, NULL, file); 8183 } 8184 } 8185 return 0; 8186 } 8187 #endif 8188 8189 /** 8190 * Dump flow raw hw data to file 8191 * 8192 * @param[in] dev 8193 * The pointer to Ethernet device. 8194 * @param[in] file 8195 * A pointer to a file for output. 8196 * @param[out] error 8197 * Perform verbose error reporting if not NULL. PMDs initialize this 8198 * structure in case of error only. 8199 * @return 8200 * 0 on success, a nagative value otherwise. 8201 */ 8202 int 8203 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8204 FILE *file, 8205 struct rte_flow_error *error __rte_unused) 8206 { 8207 struct mlx5_priv *priv = dev->data->dev_private; 8208 struct mlx5_dev_ctx_shared *sh = priv->sh; 8209 uint32_t handle_idx; 8210 int ret; 8211 struct mlx5_flow_handle *dh; 8212 struct rte_flow *flow; 8213 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8214 uint32_t idx; 8215 #endif 8216 8217 if (!priv->config.dv_flow_en) { 8218 if (fputs("device dv flow disabled\n", file) <= 0) 8219 return -errno; 8220 return -ENOTSUP; 8221 } 8222 8223 /* dump all */ 8224 if (!flow_idx) { 8225 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8226 MLX5_IPOOL_FOREACH(priv->flows[MLX5_FLOW_TYPE_GEN], idx, flow) 8227 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8228 #endif 8229 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8230 sh->rx_domain, 8231 sh->tx_domain, file); 8232 } 8233 /* dump one */ 8234 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8235 (uintptr_t)(void *)flow_idx); 8236 if (!flow) 8237 return -ENOENT; 8238 8239 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8240 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8241 #endif 8242 handle_idx = flow->dev_handles; 8243 while (handle_idx) { 8244 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8245 handle_idx); 8246 if (!dh) 8247 return -ENOENT; 8248 if (dh->drv_flow) { 8249 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8250 file); 8251 if (ret) 8252 return -ENOENT; 8253 } 8254 handle_idx = dh->next.next; 8255 } 8256 return 0; 8257 } 8258 8259 /** 8260 * Get aged-out flows. 8261 * 8262 * @param[in] dev 8263 * Pointer to the Ethernet device structure. 8264 * @param[in] context 8265 * The address of an array of pointers to the aged-out flows contexts. 8266 * @param[in] nb_countexts 8267 * The length of context array pointers. 8268 * @param[out] error 8269 * Perform verbose error reporting if not NULL. Initialized in case of 8270 * error only. 8271 * 8272 * @return 8273 * how many contexts get in success, otherwise negative errno value. 8274 * if nb_contexts is 0, return the amount of all aged contexts. 8275 * if nb_contexts is not 0 , return the amount of aged flows reported 8276 * in the context array. 8277 */ 8278 int 8279 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8280 uint32_t nb_contexts, struct rte_flow_error *error) 8281 { 8282 const struct mlx5_flow_driver_ops *fops; 8283 struct rte_flow_attr attr = { .transfer = 0 }; 8284 8285 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8286 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8287 return fops->get_aged_flows(dev, contexts, nb_contexts, 8288 error); 8289 } 8290 DRV_LOG(ERR, 8291 "port %u get aged flows is not supported.", 8292 dev->data->port_id); 8293 return -ENOTSUP; 8294 } 8295 8296 /* Wrapper for driver action_validate op callback */ 8297 static int 8298 flow_drv_action_validate(struct rte_eth_dev *dev, 8299 const struct rte_flow_indir_action_conf *conf, 8300 const struct rte_flow_action *action, 8301 const struct mlx5_flow_driver_ops *fops, 8302 struct rte_flow_error *error) 8303 { 8304 static const char err_msg[] = "indirect action validation unsupported"; 8305 8306 if (!fops->action_validate) { 8307 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8308 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8309 NULL, err_msg); 8310 return -rte_errno; 8311 } 8312 return fops->action_validate(dev, conf, action, error); 8313 } 8314 8315 /** 8316 * Destroys the shared action by handle. 8317 * 8318 * @param dev 8319 * Pointer to Ethernet device structure. 8320 * @param[in] handle 8321 * Handle for the indirect action object to be destroyed. 8322 * @param[out] error 8323 * Perform verbose error reporting if not NULL. PMDs initialize this 8324 * structure in case of error only. 8325 * 8326 * @return 8327 * 0 on success, a negative errno value otherwise and rte_errno is set. 8328 * 8329 * @note: wrapper for driver action_create op callback. 8330 */ 8331 static int 8332 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8333 struct rte_flow_action_handle *handle, 8334 struct rte_flow_error *error) 8335 { 8336 static const char err_msg[] = "indirect action destruction unsupported"; 8337 struct rte_flow_attr attr = { .transfer = 0 }; 8338 const struct mlx5_flow_driver_ops *fops = 8339 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8340 8341 if (!fops->action_destroy) { 8342 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8343 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8344 NULL, err_msg); 8345 return -rte_errno; 8346 } 8347 return fops->action_destroy(dev, handle, error); 8348 } 8349 8350 /* Wrapper for driver action_destroy op callback */ 8351 static int 8352 flow_drv_action_update(struct rte_eth_dev *dev, 8353 struct rte_flow_action_handle *handle, 8354 const void *update, 8355 const struct mlx5_flow_driver_ops *fops, 8356 struct rte_flow_error *error) 8357 { 8358 static const char err_msg[] = "indirect action update unsupported"; 8359 8360 if (!fops->action_update) { 8361 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8362 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8363 NULL, err_msg); 8364 return -rte_errno; 8365 } 8366 return fops->action_update(dev, handle, update, error); 8367 } 8368 8369 /* Wrapper for driver action_destroy op callback */ 8370 static int 8371 flow_drv_action_query(struct rte_eth_dev *dev, 8372 const struct rte_flow_action_handle *handle, 8373 void *data, 8374 const struct mlx5_flow_driver_ops *fops, 8375 struct rte_flow_error *error) 8376 { 8377 static const char err_msg[] = "indirect action query unsupported"; 8378 8379 if (!fops->action_query) { 8380 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8381 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8382 NULL, err_msg); 8383 return -rte_errno; 8384 } 8385 return fops->action_query(dev, handle, data, error); 8386 } 8387 8388 /** 8389 * Create indirect action for reuse in multiple flow rules. 8390 * 8391 * @param dev 8392 * Pointer to Ethernet device structure. 8393 * @param conf 8394 * Pointer to indirect action object configuration. 8395 * @param[in] action 8396 * Action configuration for indirect action object creation. 8397 * @param[out] error 8398 * Perform verbose error reporting if not NULL. PMDs initialize this 8399 * structure in case of error only. 8400 * @return 8401 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8402 */ 8403 static struct rte_flow_action_handle * 8404 mlx5_action_handle_create(struct rte_eth_dev *dev, 8405 const struct rte_flow_indir_action_conf *conf, 8406 const struct rte_flow_action *action, 8407 struct rte_flow_error *error) 8408 { 8409 static const char err_msg[] = "indirect action creation unsupported"; 8410 struct rte_flow_attr attr = { .transfer = 0 }; 8411 const struct mlx5_flow_driver_ops *fops = 8412 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8413 8414 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8415 return NULL; 8416 if (!fops->action_create) { 8417 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8418 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8419 NULL, err_msg); 8420 return NULL; 8421 } 8422 return fops->action_create(dev, conf, action, error); 8423 } 8424 8425 /** 8426 * Updates inplace the indirect action configuration pointed by *handle* 8427 * with the configuration provided as *update* argument. 8428 * The update of the indirect action configuration effects all flow rules 8429 * reusing the action via handle. 8430 * 8431 * @param dev 8432 * Pointer to Ethernet device structure. 8433 * @param[in] handle 8434 * Handle for the indirect action to be updated. 8435 * @param[in] update 8436 * Action specification used to modify the action pointed by handle. 8437 * *update* could be of same type with the action pointed by the *handle* 8438 * handle argument, or some other structures like a wrapper, depending on 8439 * the indirect action type. 8440 * @param[out] error 8441 * Perform verbose error reporting if not NULL. PMDs initialize this 8442 * structure in case of error only. 8443 * 8444 * @return 8445 * 0 on success, a negative errno value otherwise and rte_errno is set. 8446 */ 8447 static int 8448 mlx5_action_handle_update(struct rte_eth_dev *dev, 8449 struct rte_flow_action_handle *handle, 8450 const void *update, 8451 struct rte_flow_error *error) 8452 { 8453 struct rte_flow_attr attr = { .transfer = 0 }; 8454 const struct mlx5_flow_driver_ops *fops = 8455 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8456 int ret; 8457 8458 ret = flow_drv_action_validate(dev, NULL, 8459 (const struct rte_flow_action *)update, fops, error); 8460 if (ret) 8461 return ret; 8462 return flow_drv_action_update(dev, handle, update, fops, 8463 error); 8464 } 8465 8466 /** 8467 * Query the indirect action by handle. 8468 * 8469 * This function allows retrieving action-specific data such as counters. 8470 * Data is gathered by special action which may be present/referenced in 8471 * more than one flow rule definition. 8472 * 8473 * see @RTE_FLOW_ACTION_TYPE_COUNT 8474 * 8475 * @param dev 8476 * Pointer to Ethernet device structure. 8477 * @param[in] handle 8478 * Handle for the indirect action to query. 8479 * @param[in, out] data 8480 * Pointer to storage for the associated query data type. 8481 * @param[out] error 8482 * Perform verbose error reporting if not NULL. PMDs initialize this 8483 * structure in case of error only. 8484 * 8485 * @return 8486 * 0 on success, a negative errno value otherwise and rte_errno is set. 8487 */ 8488 static int 8489 mlx5_action_handle_query(struct rte_eth_dev *dev, 8490 const struct rte_flow_action_handle *handle, 8491 void *data, 8492 struct rte_flow_error *error) 8493 { 8494 struct rte_flow_attr attr = { .transfer = 0 }; 8495 const struct mlx5_flow_driver_ops *fops = 8496 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8497 8498 return flow_drv_action_query(dev, handle, data, fops, error); 8499 } 8500 8501 /** 8502 * Destroy all indirect actions (shared RSS). 8503 * 8504 * @param dev 8505 * Pointer to Ethernet device. 8506 * 8507 * @return 8508 * 0 on success, a negative errno value otherwise and rte_errno is set. 8509 */ 8510 int 8511 mlx5_action_handle_flush(struct rte_eth_dev *dev) 8512 { 8513 struct rte_flow_error error; 8514 struct mlx5_priv *priv = dev->data->dev_private; 8515 struct mlx5_shared_action_rss *shared_rss; 8516 int ret = 0; 8517 uint32_t idx; 8518 8519 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 8520 priv->rss_shared_actions, idx, shared_rss, next) { 8521 ret |= mlx5_action_handle_destroy(dev, 8522 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 8523 } 8524 return ret; 8525 } 8526 8527 #ifndef HAVE_MLX5DV_DR 8528 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8529 #else 8530 #define MLX5_DOMAIN_SYNC_FLOW \ 8531 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8532 #endif 8533 8534 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8535 { 8536 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8537 const struct mlx5_flow_driver_ops *fops; 8538 int ret; 8539 struct rte_flow_attr attr = { .transfer = 0 }; 8540 8541 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8542 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8543 if (ret > 0) 8544 ret = -ret; 8545 return ret; 8546 } 8547 8548 const struct mlx5_flow_tunnel * 8549 mlx5_get_tof(const struct rte_flow_item *item, 8550 const struct rte_flow_action *action, 8551 enum mlx5_tof_rule_type *rule_type) 8552 { 8553 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 8554 if (item->type == (typeof(item->type)) 8555 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 8556 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 8557 return flow_items_to_tunnel(item); 8558 } 8559 } 8560 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 8561 if (action->type == (typeof(action->type)) 8562 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 8563 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 8564 return flow_actions_to_tunnel(action); 8565 } 8566 } 8567 return NULL; 8568 } 8569 8570 /** 8571 * tunnel offload functionalilty is defined for DV environment only 8572 */ 8573 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8574 __extension__ 8575 union tunnel_offload_mark { 8576 uint32_t val; 8577 struct { 8578 uint32_t app_reserve:8; 8579 uint32_t table_id:15; 8580 uint32_t transfer:1; 8581 uint32_t _unused_:8; 8582 }; 8583 }; 8584 8585 static bool 8586 mlx5_access_tunnel_offload_db 8587 (struct rte_eth_dev *dev, 8588 bool (*match)(struct rte_eth_dev *, 8589 struct mlx5_flow_tunnel *, const void *), 8590 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8591 void (*miss)(struct rte_eth_dev *, void *), 8592 void *ctx, bool lock_op); 8593 8594 static int 8595 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 8596 struct rte_flow *flow, 8597 const struct rte_flow_attr *attr, 8598 const struct rte_flow_action *app_actions, 8599 uint32_t flow_idx, 8600 const struct mlx5_flow_tunnel *tunnel, 8601 struct tunnel_default_miss_ctx *ctx, 8602 struct rte_flow_error *error) 8603 { 8604 struct mlx5_priv *priv = dev->data->dev_private; 8605 struct mlx5_flow *dev_flow; 8606 struct rte_flow_attr miss_attr = *attr; 8607 const struct rte_flow_item miss_items[2] = { 8608 { 8609 .type = RTE_FLOW_ITEM_TYPE_ETH, 8610 .spec = NULL, 8611 .last = NULL, 8612 .mask = NULL 8613 }, 8614 { 8615 .type = RTE_FLOW_ITEM_TYPE_END, 8616 .spec = NULL, 8617 .last = NULL, 8618 .mask = NULL 8619 } 8620 }; 8621 union tunnel_offload_mark mark_id; 8622 struct rte_flow_action_mark miss_mark; 8623 struct rte_flow_action miss_actions[3] = { 8624 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 8625 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 8626 }; 8627 const struct rte_flow_action_jump *jump_data; 8628 uint32_t i, flow_table = 0; /* prevent compilation warning */ 8629 struct flow_grp_info grp_info = { 8630 .external = 1, 8631 .transfer = attr->transfer, 8632 .fdb_def_rule = !!priv->fdb_def_rule, 8633 .std_tbl_fix = 0, 8634 }; 8635 int ret; 8636 8637 if (!attr->transfer) { 8638 uint32_t q_size; 8639 8640 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 8641 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 8642 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 8643 0, SOCKET_ID_ANY); 8644 if (!ctx->queue) 8645 return rte_flow_error_set 8646 (error, ENOMEM, 8647 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8648 NULL, "invalid default miss RSS"); 8649 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8650 ctx->action_rss.level = 0, 8651 ctx->action_rss.types = priv->rss_conf.rss_hf, 8652 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 8653 ctx->action_rss.queue_num = priv->reta_idx_n, 8654 ctx->action_rss.key = priv->rss_conf.rss_key, 8655 ctx->action_rss.queue = ctx->queue; 8656 if (!priv->reta_idx_n || !priv->rxqs_n) 8657 return rte_flow_error_set 8658 (error, EINVAL, 8659 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8660 NULL, "invalid port configuration"); 8661 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 8662 ctx->action_rss.types = 0; 8663 for (i = 0; i != priv->reta_idx_n; ++i) 8664 ctx->queue[i] = (*priv->reta_idx)[i]; 8665 } else { 8666 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 8667 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 8668 } 8669 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 8670 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 8671 jump_data = app_actions->conf; 8672 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 8673 miss_attr.group = jump_data->group; 8674 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 8675 &flow_table, &grp_info, error); 8676 if (ret) 8677 return rte_flow_error_set(error, EINVAL, 8678 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 8679 NULL, "invalid tunnel id"); 8680 mark_id.app_reserve = 0; 8681 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 8682 mark_id.transfer = !!attr->transfer; 8683 mark_id._unused_ = 0; 8684 miss_mark.id = mark_id.val; 8685 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 8686 miss_items, miss_actions, flow_idx, error); 8687 if (!dev_flow) 8688 return -rte_errno; 8689 dev_flow->flow = flow; 8690 dev_flow->external = true; 8691 dev_flow->tunnel = tunnel; 8692 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 8693 /* Subflow object was created, we must include one in the list. */ 8694 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 8695 dev_flow->handle, next); 8696 DRV_LOG(DEBUG, 8697 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 8698 dev->data->port_id, tunnel->app_tunnel.type, 8699 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 8700 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 8701 miss_actions, error); 8702 if (!ret) 8703 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 8704 error); 8705 8706 return ret; 8707 } 8708 8709 static const struct mlx5_flow_tbl_data_entry * 8710 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 8711 { 8712 struct mlx5_priv *priv = dev->data->dev_private; 8713 struct mlx5_dev_ctx_shared *sh = priv->sh; 8714 struct mlx5_list_entry *he; 8715 union tunnel_offload_mark mbits = { .val = mark }; 8716 union mlx5_flow_tbl_key table_key = { 8717 { 8718 .level = tunnel_id_to_flow_tbl(mbits.table_id), 8719 .id = 0, 8720 .reserved = 0, 8721 .dummy = 0, 8722 .is_fdb = !!mbits.transfer, 8723 .is_egress = 0, 8724 } 8725 }; 8726 struct mlx5_flow_cb_ctx ctx = { 8727 .data = &table_key.v64, 8728 }; 8729 8730 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 8731 return he ? 8732 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 8733 } 8734 8735 static void 8736 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 8737 struct mlx5_list_entry *entry) 8738 { 8739 struct mlx5_dev_ctx_shared *sh = tool_ctx; 8740 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8741 8742 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8743 tunnel_flow_tbl_to_id(tte->flow_table)); 8744 mlx5_free(tte); 8745 } 8746 8747 static int 8748 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 8749 struct mlx5_list_entry *entry, void *cb_ctx) 8750 { 8751 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 8752 union tunnel_tbl_key tbl = { 8753 .val = *(uint64_t *)(ctx->data), 8754 }; 8755 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8756 8757 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 8758 } 8759 8760 static struct mlx5_list_entry * 8761 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 8762 { 8763 struct mlx5_dev_ctx_shared *sh = tool_ctx; 8764 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 8765 struct tunnel_tbl_entry *tte; 8766 union tunnel_tbl_key tbl = { 8767 .val = *(uint64_t *)(ctx->data), 8768 }; 8769 8770 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 8771 sizeof(*tte), 0, 8772 SOCKET_ID_ANY); 8773 if (!tte) 8774 goto err; 8775 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8776 &tte->flow_table); 8777 if (tte->flow_table >= MLX5_MAX_TABLES) { 8778 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 8779 tte->flow_table); 8780 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 8781 tte->flow_table); 8782 goto err; 8783 } else if (!tte->flow_table) { 8784 goto err; 8785 } 8786 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 8787 tte->tunnel_id = tbl.tunnel_id; 8788 tte->group = tbl.group; 8789 return &tte->hash; 8790 err: 8791 if (tte) 8792 mlx5_free(tte); 8793 return NULL; 8794 } 8795 8796 static struct mlx5_list_entry * 8797 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 8798 struct mlx5_list_entry *oentry, 8799 void *cb_ctx __rte_unused) 8800 { 8801 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 8802 0, SOCKET_ID_ANY); 8803 8804 if (!tte) 8805 return NULL; 8806 memcpy(tte, oentry, sizeof(*tte)); 8807 return &tte->hash; 8808 } 8809 8810 static void 8811 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 8812 struct mlx5_list_entry *entry) 8813 { 8814 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 8815 8816 mlx5_free(tte); 8817 } 8818 8819 static uint32_t 8820 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 8821 const struct mlx5_flow_tunnel *tunnel, 8822 uint32_t group, uint32_t *table, 8823 struct rte_flow_error *error) 8824 { 8825 struct mlx5_list_entry *he; 8826 struct tunnel_tbl_entry *tte; 8827 union tunnel_tbl_key key = { 8828 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 8829 .group = group 8830 }; 8831 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8832 struct mlx5_hlist *group_hash; 8833 struct mlx5_flow_cb_ctx ctx = { 8834 .data = &key.val, 8835 }; 8836 8837 group_hash = tunnel ? tunnel->groups : thub->groups; 8838 he = mlx5_hlist_register(group_hash, key.val, &ctx); 8839 if (!he) 8840 return rte_flow_error_set(error, EINVAL, 8841 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8842 NULL, 8843 "tunnel group index not supported"); 8844 tte = container_of(he, typeof(*tte), hash); 8845 *table = tte->flow_table; 8846 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 8847 dev->data->port_id, key.tunnel_id, group, *table); 8848 return 0; 8849 } 8850 8851 static void 8852 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8853 struct mlx5_flow_tunnel *tunnel) 8854 { 8855 struct mlx5_priv *priv = dev->data->dev_private; 8856 struct mlx5_indexed_pool *ipool; 8857 8858 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8859 dev->data->port_id, tunnel->tunnel_id); 8860 LIST_REMOVE(tunnel, chain); 8861 mlx5_hlist_destroy(tunnel->groups); 8862 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8863 mlx5_ipool_free(ipool, tunnel->tunnel_id); 8864 } 8865 8866 static bool 8867 mlx5_access_tunnel_offload_db 8868 (struct rte_eth_dev *dev, 8869 bool (*match)(struct rte_eth_dev *, 8870 struct mlx5_flow_tunnel *, const void *), 8871 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 8872 void (*miss)(struct rte_eth_dev *, void *), 8873 void *ctx, bool lock_op) 8874 { 8875 bool verdict = false; 8876 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8877 struct mlx5_flow_tunnel *tunnel; 8878 8879 rte_spinlock_lock(&thub->sl); 8880 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 8881 verdict = match(dev, tunnel, (const void *)ctx); 8882 if (verdict) 8883 break; 8884 } 8885 if (!lock_op) 8886 rte_spinlock_unlock(&thub->sl); 8887 if (verdict && hit) 8888 hit(dev, tunnel, ctx); 8889 if (!verdict && miss) 8890 miss(dev, ctx); 8891 if (lock_op) 8892 rte_spinlock_unlock(&thub->sl); 8893 8894 return verdict; 8895 } 8896 8897 struct tunnel_db_find_tunnel_id_ctx { 8898 uint32_t tunnel_id; 8899 struct mlx5_flow_tunnel *tunnel; 8900 }; 8901 8902 static bool 8903 find_tunnel_id_match(struct rte_eth_dev *dev, 8904 struct mlx5_flow_tunnel *tunnel, const void *x) 8905 { 8906 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8907 8908 RTE_SET_USED(dev); 8909 return tunnel->tunnel_id == ctx->tunnel_id; 8910 } 8911 8912 static void 8913 find_tunnel_id_hit(struct rte_eth_dev *dev, 8914 struct mlx5_flow_tunnel *tunnel, void *x) 8915 { 8916 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 8917 RTE_SET_USED(dev); 8918 ctx->tunnel = tunnel; 8919 } 8920 8921 static struct mlx5_flow_tunnel * 8922 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 8923 { 8924 struct tunnel_db_find_tunnel_id_ctx ctx = { 8925 .tunnel_id = id, 8926 }; 8927 8928 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 8929 find_tunnel_id_hit, NULL, &ctx, true); 8930 8931 return ctx.tunnel; 8932 } 8933 8934 static struct mlx5_flow_tunnel * 8935 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 8936 const struct rte_flow_tunnel *app_tunnel) 8937 { 8938 struct mlx5_priv *priv = dev->data->dev_private; 8939 struct mlx5_indexed_pool *ipool; 8940 struct mlx5_flow_tunnel *tunnel; 8941 uint32_t id; 8942 8943 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 8944 tunnel = mlx5_ipool_zmalloc(ipool, &id); 8945 if (!tunnel) 8946 return NULL; 8947 if (id >= MLX5_MAX_TUNNELS) { 8948 mlx5_ipool_free(ipool, id); 8949 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 8950 return NULL; 8951 } 8952 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 8953 priv->sh, 8954 mlx5_flow_tunnel_grp2tbl_create_cb, 8955 mlx5_flow_tunnel_grp2tbl_match_cb, 8956 mlx5_flow_tunnel_grp2tbl_remove_cb, 8957 mlx5_flow_tunnel_grp2tbl_clone_cb, 8958 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 8959 if (!tunnel->groups) { 8960 mlx5_ipool_free(ipool, id); 8961 return NULL; 8962 } 8963 /* initiate new PMD tunnel */ 8964 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8965 tunnel->tunnel_id = id; 8966 tunnel->action.type = (typeof(tunnel->action.type)) 8967 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8968 tunnel->action.conf = tunnel; 8969 tunnel->item.type = (typeof(tunnel->item.type)) 8970 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8971 tunnel->item.spec = tunnel; 8972 tunnel->item.last = NULL; 8973 tunnel->item.mask = NULL; 8974 8975 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8976 dev->data->port_id, tunnel->tunnel_id); 8977 8978 return tunnel; 8979 } 8980 8981 struct tunnel_db_get_tunnel_ctx { 8982 const struct rte_flow_tunnel *app_tunnel; 8983 struct mlx5_flow_tunnel *tunnel; 8984 }; 8985 8986 static bool get_tunnel_match(struct rte_eth_dev *dev, 8987 struct mlx5_flow_tunnel *tunnel, const void *x) 8988 { 8989 const struct tunnel_db_get_tunnel_ctx *ctx = x; 8990 8991 RTE_SET_USED(dev); 8992 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 8993 sizeof(*ctx->app_tunnel)); 8994 } 8995 8996 static void get_tunnel_hit(struct rte_eth_dev *dev, 8997 struct mlx5_flow_tunnel *tunnel, void *x) 8998 { 8999 /* called under tunnel spinlock protection */ 9000 struct tunnel_db_get_tunnel_ctx *ctx = x; 9001 9002 RTE_SET_USED(dev); 9003 tunnel->refctn++; 9004 ctx->tunnel = tunnel; 9005 } 9006 9007 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 9008 { 9009 /* called under tunnel spinlock protection */ 9010 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9011 struct tunnel_db_get_tunnel_ctx *ctx = x; 9012 9013 rte_spinlock_unlock(&thub->sl); 9014 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 9015 rte_spinlock_lock(&thub->sl); 9016 if (ctx->tunnel) { 9017 ctx->tunnel->refctn = 1; 9018 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 9019 } 9020 } 9021 9022 9023 static int 9024 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 9025 const struct rte_flow_tunnel *app_tunnel, 9026 struct mlx5_flow_tunnel **tunnel) 9027 { 9028 struct tunnel_db_get_tunnel_ctx ctx = { 9029 .app_tunnel = app_tunnel, 9030 }; 9031 9032 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 9033 get_tunnel_miss, &ctx, true); 9034 *tunnel = ctx.tunnel; 9035 return ctx.tunnel ? 0 : -ENOMEM; 9036 } 9037 9038 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9039 { 9040 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9041 9042 if (!thub) 9043 return; 9044 if (!LIST_EMPTY(&thub->tunnels)) 9045 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9046 mlx5_hlist_destroy(thub->groups); 9047 mlx5_free(thub); 9048 } 9049 9050 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9051 { 9052 int err; 9053 struct mlx5_flow_tunnel_hub *thub; 9054 9055 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9056 0, SOCKET_ID_ANY); 9057 if (!thub) 9058 return -ENOMEM; 9059 LIST_INIT(&thub->tunnels); 9060 rte_spinlock_init(&thub->sl); 9061 thub->groups = mlx5_hlist_create("flow groups", 64, 9062 false, true, sh, 9063 mlx5_flow_tunnel_grp2tbl_create_cb, 9064 mlx5_flow_tunnel_grp2tbl_match_cb, 9065 mlx5_flow_tunnel_grp2tbl_remove_cb, 9066 mlx5_flow_tunnel_grp2tbl_clone_cb, 9067 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9068 if (!thub->groups) { 9069 err = -rte_errno; 9070 goto err; 9071 } 9072 sh->tunnel_hub = thub; 9073 9074 return 0; 9075 9076 err: 9077 if (thub->groups) 9078 mlx5_hlist_destroy(thub->groups); 9079 if (thub) 9080 mlx5_free(thub); 9081 return err; 9082 } 9083 9084 static inline bool 9085 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9086 struct rte_flow_tunnel *tunnel, 9087 const char *err_msg) 9088 { 9089 err_msg = NULL; 9090 if (!is_tunnel_offload_active(dev)) { 9091 err_msg = "tunnel offload was not activated"; 9092 goto out; 9093 } else if (!tunnel) { 9094 err_msg = "no application tunnel"; 9095 goto out; 9096 } 9097 9098 switch (tunnel->type) { 9099 default: 9100 err_msg = "unsupported tunnel type"; 9101 goto out; 9102 case RTE_FLOW_ITEM_TYPE_VXLAN: 9103 break; 9104 } 9105 9106 out: 9107 return !err_msg; 9108 } 9109 9110 static int 9111 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9112 struct rte_flow_tunnel *app_tunnel, 9113 struct rte_flow_action **actions, 9114 uint32_t *num_of_actions, 9115 struct rte_flow_error *error) 9116 { 9117 int ret; 9118 struct mlx5_flow_tunnel *tunnel; 9119 const char *err_msg = NULL; 9120 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 9121 9122 if (!verdict) 9123 return rte_flow_error_set(error, EINVAL, 9124 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9125 err_msg); 9126 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9127 if (ret < 0) { 9128 return rte_flow_error_set(error, ret, 9129 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9130 "failed to initialize pmd tunnel"); 9131 } 9132 *actions = &tunnel->action; 9133 *num_of_actions = 1; 9134 return 0; 9135 } 9136 9137 static int 9138 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9139 struct rte_flow_tunnel *app_tunnel, 9140 struct rte_flow_item **items, 9141 uint32_t *num_of_items, 9142 struct rte_flow_error *error) 9143 { 9144 int ret; 9145 struct mlx5_flow_tunnel *tunnel; 9146 const char *err_msg = NULL; 9147 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 9148 9149 if (!verdict) 9150 return rte_flow_error_set(error, EINVAL, 9151 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9152 err_msg); 9153 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9154 if (ret < 0) { 9155 return rte_flow_error_set(error, ret, 9156 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9157 "failed to initialize pmd tunnel"); 9158 } 9159 *items = &tunnel->item; 9160 *num_of_items = 1; 9161 return 0; 9162 } 9163 9164 struct tunnel_db_element_release_ctx { 9165 struct rte_flow_item *items; 9166 struct rte_flow_action *actions; 9167 uint32_t num_elements; 9168 struct rte_flow_error *error; 9169 int ret; 9170 }; 9171 9172 static bool 9173 tunnel_element_release_match(struct rte_eth_dev *dev, 9174 struct mlx5_flow_tunnel *tunnel, const void *x) 9175 { 9176 const struct tunnel_db_element_release_ctx *ctx = x; 9177 9178 RTE_SET_USED(dev); 9179 if (ctx->num_elements != 1) 9180 return false; 9181 else if (ctx->items) 9182 return ctx->items == &tunnel->item; 9183 else if (ctx->actions) 9184 return ctx->actions == &tunnel->action; 9185 9186 return false; 9187 } 9188 9189 static void 9190 tunnel_element_release_hit(struct rte_eth_dev *dev, 9191 struct mlx5_flow_tunnel *tunnel, void *x) 9192 { 9193 struct tunnel_db_element_release_ctx *ctx = x; 9194 ctx->ret = 0; 9195 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9196 mlx5_flow_tunnel_free(dev, tunnel); 9197 } 9198 9199 static void 9200 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9201 { 9202 struct tunnel_db_element_release_ctx *ctx = x; 9203 RTE_SET_USED(dev); 9204 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9205 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9206 "invalid argument"); 9207 } 9208 9209 static int 9210 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9211 struct rte_flow_item *pmd_items, 9212 uint32_t num_items, struct rte_flow_error *err) 9213 { 9214 struct tunnel_db_element_release_ctx ctx = { 9215 .items = pmd_items, 9216 .actions = NULL, 9217 .num_elements = num_items, 9218 .error = err, 9219 }; 9220 9221 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9222 tunnel_element_release_hit, 9223 tunnel_element_release_miss, &ctx, false); 9224 9225 return ctx.ret; 9226 } 9227 9228 static int 9229 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9230 struct rte_flow_action *pmd_actions, 9231 uint32_t num_actions, struct rte_flow_error *err) 9232 { 9233 struct tunnel_db_element_release_ctx ctx = { 9234 .items = NULL, 9235 .actions = pmd_actions, 9236 .num_elements = num_actions, 9237 .error = err, 9238 }; 9239 9240 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9241 tunnel_element_release_hit, 9242 tunnel_element_release_miss, &ctx, false); 9243 9244 return ctx.ret; 9245 } 9246 9247 static int 9248 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9249 struct rte_mbuf *m, 9250 struct rte_flow_restore_info *info, 9251 struct rte_flow_error *err) 9252 { 9253 uint64_t ol_flags = m->ol_flags; 9254 const struct mlx5_flow_tbl_data_entry *tble; 9255 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 9256 9257 if (!is_tunnel_offload_active(dev)) { 9258 info->flags = 0; 9259 return 0; 9260 } 9261 9262 if ((ol_flags & mask) != mask) 9263 goto err; 9264 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9265 if (!tble) { 9266 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9267 dev->data->port_id, m->hash.fdir.hi); 9268 goto err; 9269 } 9270 MLX5_ASSERT(tble->tunnel); 9271 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9272 info->group_id = tble->group_id; 9273 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9274 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9275 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9276 9277 return 0; 9278 9279 err: 9280 return rte_flow_error_set(err, EINVAL, 9281 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9282 "failed to get restore info"); 9283 } 9284 9285 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9286 static int 9287 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9288 __rte_unused struct rte_flow_tunnel *app_tunnel, 9289 __rte_unused struct rte_flow_action **actions, 9290 __rte_unused uint32_t *num_of_actions, 9291 __rte_unused struct rte_flow_error *error) 9292 { 9293 return -ENOTSUP; 9294 } 9295 9296 static int 9297 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9298 __rte_unused struct rte_flow_tunnel *app_tunnel, 9299 __rte_unused struct rte_flow_item **items, 9300 __rte_unused uint32_t *num_of_items, 9301 __rte_unused struct rte_flow_error *error) 9302 { 9303 return -ENOTSUP; 9304 } 9305 9306 static int 9307 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9308 __rte_unused struct rte_flow_item *pmd_items, 9309 __rte_unused uint32_t num_items, 9310 __rte_unused struct rte_flow_error *err) 9311 { 9312 return -ENOTSUP; 9313 } 9314 9315 static int 9316 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9317 __rte_unused struct rte_flow_action *pmd_action, 9318 __rte_unused uint32_t num_actions, 9319 __rte_unused struct rte_flow_error *err) 9320 { 9321 return -ENOTSUP; 9322 } 9323 9324 static int 9325 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9326 __rte_unused struct rte_mbuf *m, 9327 __rte_unused struct rte_flow_restore_info *i, 9328 __rte_unused struct rte_flow_error *err) 9329 { 9330 return -ENOTSUP; 9331 } 9332 9333 static int 9334 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9335 __rte_unused struct rte_flow *flow, 9336 __rte_unused const struct rte_flow_attr *attr, 9337 __rte_unused const struct rte_flow_action *actions, 9338 __rte_unused uint32_t flow_idx, 9339 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9340 __rte_unused struct tunnel_default_miss_ctx *ctx, 9341 __rte_unused struct rte_flow_error *error) 9342 { 9343 return -ENOTSUP; 9344 } 9345 9346 static struct mlx5_flow_tunnel * 9347 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9348 __rte_unused uint32_t id) 9349 { 9350 return NULL; 9351 } 9352 9353 static void 9354 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9355 __rte_unused struct mlx5_flow_tunnel *tunnel) 9356 { 9357 } 9358 9359 static uint32_t 9360 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 9361 __rte_unused const struct mlx5_flow_tunnel *t, 9362 __rte_unused uint32_t group, 9363 __rte_unused uint32_t *table, 9364 struct rte_flow_error *error) 9365 { 9366 return rte_flow_error_set(error, ENOTSUP, 9367 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9368 "tunnel offload requires DV support"); 9369 } 9370 9371 void 9372 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 9373 __rte_unused uint16_t port_id) 9374 { 9375 } 9376 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9377 9378 static void 9379 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 9380 { 9381 int ret; 9382 struct rte_flow_error error; 9383 9384 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9385 char *item_name; 9386 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 9387 sizeof(item_name), 9388 (void *)(uintptr_t)item->type, &error); 9389 if (ret > 0) 9390 printf("%s ", item_name); 9391 else 9392 printf("%d\n", (int)item->type); 9393 } 9394 printf("END\n"); 9395 } 9396