1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 #include <rte_common.h> 14 #include <rte_ether.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_eal_paging.h> 17 #include <rte_flow.h> 18 #include <rte_cycles.h> 19 #include <rte_flow_driver.h> 20 #include <rte_malloc.h> 21 #include <rte_ip.h> 22 23 #include <mlx5_glue.h> 24 #include <mlx5_devx_cmds.h> 25 #include <mlx5_prm.h> 26 #include <mlx5_malloc.h> 27 28 #include "mlx5_defs.h" 29 #include "mlx5.h" 30 #include "mlx5_flow.h" 31 #include "mlx5_flow_os.h" 32 #include "mlx5_rxtx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 static struct mlx5_flow_tunnel * 37 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 38 static void 39 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 40 static const struct mlx5_flow_tbl_data_entry * 41 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark); 42 static int 43 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 44 const struct rte_flow_tunnel *app_tunnel, 45 struct mlx5_flow_tunnel **tunnel); 46 47 48 /** Device flow drivers. */ 49 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 50 51 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 52 53 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 54 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 55 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 56 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 57 #endif 58 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 59 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 60 }; 61 62 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 63 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 64 (const int []){ \ 65 __VA_ARGS__, 0, \ 66 } 67 68 /** Node object of input graph for mlx5_flow_expand_rss(). */ 69 struct mlx5_flow_expand_node { 70 const int *const next; 71 /**< 72 * List of next node indexes. Index 0 is interpreted as a terminator. 73 */ 74 const enum rte_flow_item_type type; 75 /**< Pattern item type of current node. */ 76 uint64_t rss_types; 77 /**< 78 * RSS types bit-field associated with this node 79 * (see ETH_RSS_* definitions). 80 */ 81 }; 82 83 /** Object returned by mlx5_flow_expand_rss(). */ 84 struct mlx5_flow_expand_rss { 85 uint32_t entries; 86 /**< Number of entries @p patterns and @p priorities. */ 87 struct { 88 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 89 uint32_t priority; /**< Priority offset for each expansion. */ 90 } entry[]; 91 }; 92 93 static enum rte_flow_item_type 94 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 95 { 96 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 97 uint16_t ether_type = 0; 98 uint16_t ether_type_m; 99 uint8_t ip_next_proto = 0; 100 uint8_t ip_next_proto_m; 101 102 if (item == NULL || item->spec == NULL) 103 return ret; 104 switch (item->type) { 105 case RTE_FLOW_ITEM_TYPE_ETH: 106 if (item->mask) 107 ether_type_m = ((const struct rte_flow_item_eth *) 108 (item->mask))->type; 109 else 110 ether_type_m = rte_flow_item_eth_mask.type; 111 if (ether_type_m != RTE_BE16(0xFFFF)) 112 break; 113 ether_type = ((const struct rte_flow_item_eth *) 114 (item->spec))->type; 115 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 116 ret = RTE_FLOW_ITEM_TYPE_IPV4; 117 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 118 ret = RTE_FLOW_ITEM_TYPE_IPV6; 119 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 120 ret = RTE_FLOW_ITEM_TYPE_VLAN; 121 else 122 ret = RTE_FLOW_ITEM_TYPE_END; 123 break; 124 case RTE_FLOW_ITEM_TYPE_VLAN: 125 if (item->mask) 126 ether_type_m = ((const struct rte_flow_item_vlan *) 127 (item->mask))->inner_type; 128 else 129 ether_type_m = rte_flow_item_vlan_mask.inner_type; 130 if (ether_type_m != RTE_BE16(0xFFFF)) 131 break; 132 ether_type = ((const struct rte_flow_item_vlan *) 133 (item->spec))->inner_type; 134 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 135 ret = RTE_FLOW_ITEM_TYPE_IPV4; 136 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 137 ret = RTE_FLOW_ITEM_TYPE_IPV6; 138 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 139 ret = RTE_FLOW_ITEM_TYPE_VLAN; 140 else 141 ret = RTE_FLOW_ITEM_TYPE_END; 142 break; 143 case RTE_FLOW_ITEM_TYPE_IPV4: 144 if (item->mask) 145 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 146 (item->mask))->hdr.next_proto_id; 147 else 148 ip_next_proto_m = 149 rte_flow_item_ipv4_mask.hdr.next_proto_id; 150 if (ip_next_proto_m != 0xFF) 151 break; 152 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 153 (item->spec))->hdr.next_proto_id; 154 if (ip_next_proto == IPPROTO_UDP) 155 ret = RTE_FLOW_ITEM_TYPE_UDP; 156 else if (ip_next_proto == IPPROTO_TCP) 157 ret = RTE_FLOW_ITEM_TYPE_TCP; 158 else if (ip_next_proto == IPPROTO_IP) 159 ret = RTE_FLOW_ITEM_TYPE_IPV4; 160 else if (ip_next_proto == IPPROTO_IPV6) 161 ret = RTE_FLOW_ITEM_TYPE_IPV6; 162 else 163 ret = RTE_FLOW_ITEM_TYPE_END; 164 break; 165 case RTE_FLOW_ITEM_TYPE_IPV6: 166 if (item->mask) 167 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 168 (item->mask))->hdr.proto; 169 else 170 ip_next_proto_m = 171 rte_flow_item_ipv6_mask.hdr.proto; 172 if (ip_next_proto_m != 0xFF) 173 break; 174 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 175 (item->spec))->hdr.proto; 176 if (ip_next_proto == IPPROTO_UDP) 177 ret = RTE_FLOW_ITEM_TYPE_UDP; 178 else if (ip_next_proto == IPPROTO_TCP) 179 ret = RTE_FLOW_ITEM_TYPE_TCP; 180 else if (ip_next_proto == IPPROTO_IP) 181 ret = RTE_FLOW_ITEM_TYPE_IPV4; 182 else if (ip_next_proto == IPPROTO_IPV6) 183 ret = RTE_FLOW_ITEM_TYPE_IPV6; 184 else 185 ret = RTE_FLOW_ITEM_TYPE_END; 186 break; 187 default: 188 ret = RTE_FLOW_ITEM_TYPE_VOID; 189 break; 190 } 191 return ret; 192 } 193 194 /** 195 * Expand RSS flows into several possible flows according to the RSS hash 196 * fields requested and the driver capabilities. 197 * 198 * @param[out] buf 199 * Buffer to store the result expansion. 200 * @param[in] size 201 * Buffer size in bytes. If 0, @p buf can be NULL. 202 * @param[in] pattern 203 * User flow pattern. 204 * @param[in] types 205 * RSS types to expand (see ETH_RSS_* definitions). 206 * @param[in] graph 207 * Input graph to expand @p pattern according to @p types. 208 * @param[in] graph_root_index 209 * Index of root node in @p graph, typically 0. 210 * 211 * @return 212 * A positive value representing the size of @p buf in bytes regardless of 213 * @p size on success, a negative errno value otherwise and rte_errno is 214 * set, the following errors are defined: 215 * 216 * -E2BIG: graph-depth @p graph is too deep. 217 */ 218 static int 219 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 220 const struct rte_flow_item *pattern, uint64_t types, 221 const struct mlx5_flow_expand_node graph[], 222 int graph_root_index) 223 { 224 const int elt_n = 8; 225 const struct rte_flow_item *item; 226 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 227 const int *next_node; 228 const int *stack[elt_n]; 229 int stack_pos = 0; 230 struct rte_flow_item flow_items[elt_n]; 231 unsigned int i; 232 size_t lsize; 233 size_t user_pattern_size = 0; 234 void *addr = NULL; 235 const struct mlx5_flow_expand_node *next = NULL; 236 struct rte_flow_item missed_item; 237 int missed = 0; 238 int elt = 0; 239 const struct rte_flow_item *last_item = NULL; 240 241 memset(&missed_item, 0, sizeof(missed_item)); 242 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 243 elt_n * sizeof(buf->entry[0]); 244 if (lsize <= size) { 245 buf->entry[0].priority = 0; 246 buf->entry[0].pattern = (void *)&buf->entry[elt_n]; 247 buf->entries = 0; 248 addr = buf->entry[0].pattern; 249 } 250 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 251 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 252 last_item = item; 253 for (i = 0; node->next && node->next[i]; ++i) { 254 next = &graph[node->next[i]]; 255 if (next->type == item->type) 256 break; 257 } 258 if (next) 259 node = next; 260 user_pattern_size += sizeof(*item); 261 } 262 user_pattern_size += sizeof(*item); /* Handle END item. */ 263 lsize += user_pattern_size; 264 /* Copy the user pattern in the first entry of the buffer. */ 265 if (lsize <= size) { 266 rte_memcpy(addr, pattern, user_pattern_size); 267 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 268 buf->entries = 1; 269 } 270 /* Start expanding. */ 271 memset(flow_items, 0, sizeof(flow_items)); 272 user_pattern_size -= sizeof(*item); 273 /* 274 * Check if the last valid item has spec set, need complete pattern, 275 * and the pattern can be used for expansion. 276 */ 277 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 278 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 279 /* Item type END indicates expansion is not required. */ 280 return lsize; 281 } 282 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 283 next = NULL; 284 missed = 1; 285 for (i = 0; node->next && node->next[i]; ++i) { 286 next = &graph[node->next[i]]; 287 if (next->type == missed_item.type) { 288 flow_items[0].type = missed_item.type; 289 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 290 break; 291 } 292 next = NULL; 293 } 294 } 295 if (next && missed) { 296 elt = 2; /* missed item + item end. */ 297 node = next; 298 lsize += elt * sizeof(*item) + user_pattern_size; 299 if ((node->rss_types & types) && lsize <= size) { 300 buf->entry[buf->entries].priority = 1; 301 buf->entry[buf->entries].pattern = addr; 302 buf->entries++; 303 rte_memcpy(addr, buf->entry[0].pattern, 304 user_pattern_size); 305 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 306 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 307 addr = (void *)(((uintptr_t)addr) + 308 elt * sizeof(*item)); 309 } 310 } 311 memset(flow_items, 0, sizeof(flow_items)); 312 next_node = node->next; 313 stack[stack_pos] = next_node; 314 node = next_node ? &graph[*next_node] : NULL; 315 while (node) { 316 flow_items[stack_pos].type = node->type; 317 if (node->rss_types & types) { 318 /* 319 * compute the number of items to copy from the 320 * expansion and copy it. 321 * When the stack_pos is 0, there are 1 element in it, 322 * plus the addition END item. 323 */ 324 elt = stack_pos + 2; 325 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 326 lsize += elt * sizeof(*item) + user_pattern_size; 327 if (lsize <= size) { 328 size_t n = elt * sizeof(*item); 329 330 buf->entry[buf->entries].priority = 331 stack_pos + 1 + missed; 332 buf->entry[buf->entries].pattern = addr; 333 buf->entries++; 334 rte_memcpy(addr, buf->entry[0].pattern, 335 user_pattern_size); 336 addr = (void *)(((uintptr_t)addr) + 337 user_pattern_size); 338 rte_memcpy(addr, &missed_item, 339 missed * sizeof(*item)); 340 addr = (void *)(((uintptr_t)addr) + 341 missed * sizeof(*item)); 342 rte_memcpy(addr, flow_items, n); 343 addr = (void *)(((uintptr_t)addr) + n); 344 } 345 } 346 /* Go deeper. */ 347 if (node->next) { 348 next_node = node->next; 349 if (stack_pos++ == elt_n) { 350 rte_errno = E2BIG; 351 return -rte_errno; 352 } 353 stack[stack_pos] = next_node; 354 } else if (*(next_node + 1)) { 355 /* Follow up with the next possibility. */ 356 ++next_node; 357 } else { 358 /* Move to the next path. */ 359 if (stack_pos) 360 next_node = stack[--stack_pos]; 361 next_node++; 362 stack[stack_pos] = next_node; 363 } 364 node = *next_node ? &graph[*next_node] : NULL; 365 }; 366 /* no expanded flows but we have missed item, create one rule for it */ 367 if (buf->entries == 1 && missed != 0) { 368 elt = 2; 369 lsize += elt * sizeof(*item) + user_pattern_size; 370 if (lsize <= size) { 371 buf->entry[buf->entries].priority = 1; 372 buf->entry[buf->entries].pattern = addr; 373 buf->entries++; 374 flow_items[0].type = missed_item.type; 375 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 376 rte_memcpy(addr, buf->entry[0].pattern, 377 user_pattern_size); 378 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 379 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 380 addr = (void *)(((uintptr_t)addr) + 381 elt * sizeof(*item)); 382 } 383 } 384 return lsize; 385 } 386 387 enum mlx5_expansion { 388 MLX5_EXPANSION_ROOT, 389 MLX5_EXPANSION_ROOT_OUTER, 390 MLX5_EXPANSION_ROOT_ETH_VLAN, 391 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 392 MLX5_EXPANSION_OUTER_ETH, 393 MLX5_EXPANSION_OUTER_ETH_VLAN, 394 MLX5_EXPANSION_OUTER_VLAN, 395 MLX5_EXPANSION_OUTER_IPV4, 396 MLX5_EXPANSION_OUTER_IPV4_UDP, 397 MLX5_EXPANSION_OUTER_IPV4_TCP, 398 MLX5_EXPANSION_OUTER_IPV6, 399 MLX5_EXPANSION_OUTER_IPV6_UDP, 400 MLX5_EXPANSION_OUTER_IPV6_TCP, 401 MLX5_EXPANSION_VXLAN, 402 MLX5_EXPANSION_VXLAN_GPE, 403 MLX5_EXPANSION_GRE, 404 MLX5_EXPANSION_MPLS, 405 MLX5_EXPANSION_ETH, 406 MLX5_EXPANSION_ETH_VLAN, 407 MLX5_EXPANSION_VLAN, 408 MLX5_EXPANSION_IPV4, 409 MLX5_EXPANSION_IPV4_UDP, 410 MLX5_EXPANSION_IPV4_TCP, 411 MLX5_EXPANSION_IPV6, 412 MLX5_EXPANSION_IPV6_UDP, 413 MLX5_EXPANSION_IPV6_TCP, 414 }; 415 416 /** Supported expansion of items. */ 417 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 418 [MLX5_EXPANSION_ROOT] = { 419 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 420 MLX5_EXPANSION_IPV4, 421 MLX5_EXPANSION_IPV6), 422 .type = RTE_FLOW_ITEM_TYPE_END, 423 }, 424 [MLX5_EXPANSION_ROOT_OUTER] = { 425 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 426 MLX5_EXPANSION_OUTER_IPV4, 427 MLX5_EXPANSION_OUTER_IPV6), 428 .type = RTE_FLOW_ITEM_TYPE_END, 429 }, 430 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 431 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 432 .type = RTE_FLOW_ITEM_TYPE_END, 433 }, 434 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 435 .next = MLX5_FLOW_EXPAND_RSS_NEXT 436 (MLX5_EXPANSION_OUTER_ETH_VLAN), 437 .type = RTE_FLOW_ITEM_TYPE_END, 438 }, 439 [MLX5_EXPANSION_OUTER_ETH] = { 440 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 441 MLX5_EXPANSION_OUTER_IPV6, 442 MLX5_EXPANSION_MPLS), 443 .type = RTE_FLOW_ITEM_TYPE_ETH, 444 .rss_types = 0, 445 }, 446 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 447 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 448 .type = RTE_FLOW_ITEM_TYPE_ETH, 449 .rss_types = 0, 450 }, 451 [MLX5_EXPANSION_OUTER_VLAN] = { 452 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 453 MLX5_EXPANSION_OUTER_IPV6), 454 .type = RTE_FLOW_ITEM_TYPE_VLAN, 455 }, 456 [MLX5_EXPANSION_OUTER_IPV4] = { 457 .next = MLX5_FLOW_EXPAND_RSS_NEXT 458 (MLX5_EXPANSION_OUTER_IPV4_UDP, 459 MLX5_EXPANSION_OUTER_IPV4_TCP, 460 MLX5_EXPANSION_GRE, 461 MLX5_EXPANSION_IPV4, 462 MLX5_EXPANSION_IPV6), 463 .type = RTE_FLOW_ITEM_TYPE_IPV4, 464 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 465 ETH_RSS_NONFRAG_IPV4_OTHER, 466 }, 467 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 468 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 469 MLX5_EXPANSION_VXLAN_GPE), 470 .type = RTE_FLOW_ITEM_TYPE_UDP, 471 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 472 }, 473 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 474 .type = RTE_FLOW_ITEM_TYPE_TCP, 475 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 476 }, 477 [MLX5_EXPANSION_OUTER_IPV6] = { 478 .next = MLX5_FLOW_EXPAND_RSS_NEXT 479 (MLX5_EXPANSION_OUTER_IPV6_UDP, 480 MLX5_EXPANSION_OUTER_IPV6_TCP, 481 MLX5_EXPANSION_IPV4, 482 MLX5_EXPANSION_IPV6), 483 .type = RTE_FLOW_ITEM_TYPE_IPV6, 484 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 485 ETH_RSS_NONFRAG_IPV6_OTHER, 486 }, 487 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 488 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 489 MLX5_EXPANSION_VXLAN_GPE), 490 .type = RTE_FLOW_ITEM_TYPE_UDP, 491 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 492 }, 493 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 494 .type = RTE_FLOW_ITEM_TYPE_TCP, 495 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 496 }, 497 [MLX5_EXPANSION_VXLAN] = { 498 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 499 MLX5_EXPANSION_IPV4, 500 MLX5_EXPANSION_IPV6), 501 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 502 }, 503 [MLX5_EXPANSION_VXLAN_GPE] = { 504 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 505 MLX5_EXPANSION_IPV4, 506 MLX5_EXPANSION_IPV6), 507 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 508 }, 509 [MLX5_EXPANSION_GRE] = { 510 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 511 .type = RTE_FLOW_ITEM_TYPE_GRE, 512 }, 513 [MLX5_EXPANSION_MPLS] = { 514 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 515 MLX5_EXPANSION_IPV6), 516 .type = RTE_FLOW_ITEM_TYPE_MPLS, 517 }, 518 [MLX5_EXPANSION_ETH] = { 519 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 520 MLX5_EXPANSION_IPV6), 521 .type = RTE_FLOW_ITEM_TYPE_ETH, 522 }, 523 [MLX5_EXPANSION_ETH_VLAN] = { 524 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 525 .type = RTE_FLOW_ITEM_TYPE_ETH, 526 }, 527 [MLX5_EXPANSION_VLAN] = { 528 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 529 MLX5_EXPANSION_IPV6), 530 .type = RTE_FLOW_ITEM_TYPE_VLAN, 531 }, 532 [MLX5_EXPANSION_IPV4] = { 533 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 534 MLX5_EXPANSION_IPV4_TCP), 535 .type = RTE_FLOW_ITEM_TYPE_IPV4, 536 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 537 ETH_RSS_NONFRAG_IPV4_OTHER, 538 }, 539 [MLX5_EXPANSION_IPV4_UDP] = { 540 .type = RTE_FLOW_ITEM_TYPE_UDP, 541 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 542 }, 543 [MLX5_EXPANSION_IPV4_TCP] = { 544 .type = RTE_FLOW_ITEM_TYPE_TCP, 545 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 546 }, 547 [MLX5_EXPANSION_IPV6] = { 548 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 549 MLX5_EXPANSION_IPV6_TCP), 550 .type = RTE_FLOW_ITEM_TYPE_IPV6, 551 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 552 ETH_RSS_NONFRAG_IPV6_OTHER, 553 }, 554 [MLX5_EXPANSION_IPV6_UDP] = { 555 .type = RTE_FLOW_ITEM_TYPE_UDP, 556 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 557 }, 558 [MLX5_EXPANSION_IPV6_TCP] = { 559 .type = RTE_FLOW_ITEM_TYPE_TCP, 560 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 561 }, 562 }; 563 564 static struct rte_flow_shared_action * 565 mlx5_shared_action_create(struct rte_eth_dev *dev, 566 const struct rte_flow_shared_action_conf *conf, 567 const struct rte_flow_action *action, 568 struct rte_flow_error *error); 569 static int mlx5_shared_action_destroy 570 (struct rte_eth_dev *dev, 571 struct rte_flow_shared_action *shared_action, 572 struct rte_flow_error *error); 573 static int mlx5_shared_action_update 574 (struct rte_eth_dev *dev, 575 struct rte_flow_shared_action *shared_action, 576 const struct rte_flow_action *action, 577 struct rte_flow_error *error); 578 static int mlx5_shared_action_query 579 (struct rte_eth_dev *dev, 580 const struct rte_flow_shared_action *action, 581 void *data, 582 struct rte_flow_error *error); 583 static inline bool 584 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 585 struct rte_flow_tunnel *tunnel, 586 const char *err_msg) 587 { 588 err_msg = NULL; 589 if (!is_tunnel_offload_active(dev)) { 590 err_msg = "tunnel offload was not activated"; 591 goto out; 592 } else if (!tunnel) { 593 err_msg = "no application tunnel"; 594 goto out; 595 } 596 597 switch (tunnel->type) { 598 default: 599 err_msg = "unsupported tunnel type"; 600 goto out; 601 case RTE_FLOW_ITEM_TYPE_VXLAN: 602 break; 603 } 604 605 out: 606 return !err_msg; 607 } 608 609 610 static int 611 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 612 struct rte_flow_tunnel *app_tunnel, 613 struct rte_flow_action **actions, 614 uint32_t *num_of_actions, 615 struct rte_flow_error *error) 616 { 617 int ret; 618 struct mlx5_flow_tunnel *tunnel; 619 const char *err_msg = NULL; 620 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 621 622 if (!verdict) 623 return rte_flow_error_set(error, EINVAL, 624 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 625 err_msg); 626 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 627 if (ret < 0) { 628 return rte_flow_error_set(error, ret, 629 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 630 "failed to initialize pmd tunnel"); 631 } 632 *actions = &tunnel->action; 633 *num_of_actions = 1; 634 return 0; 635 } 636 637 static int 638 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 639 struct rte_flow_tunnel *app_tunnel, 640 struct rte_flow_item **items, 641 uint32_t *num_of_items, 642 struct rte_flow_error *error) 643 { 644 int ret; 645 struct mlx5_flow_tunnel *tunnel; 646 const char *err_msg = NULL; 647 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 648 649 if (!verdict) 650 return rte_flow_error_set(error, EINVAL, 651 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 652 err_msg); 653 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 654 if (ret < 0) { 655 return rte_flow_error_set(error, ret, 656 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 657 "failed to initialize pmd tunnel"); 658 } 659 *items = &tunnel->item; 660 *num_of_items = 1; 661 return 0; 662 } 663 664 static int 665 mlx5_flow_item_release(struct rte_eth_dev *dev, 666 struct rte_flow_item *pmd_items, 667 uint32_t num_items, struct rte_flow_error *err) 668 { 669 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 670 struct mlx5_flow_tunnel *tun; 671 672 LIST_FOREACH(tun, &thub->tunnels, chain) { 673 if (&tun->item == pmd_items) 674 break; 675 } 676 if (!tun || num_items != 1) 677 return rte_flow_error_set(err, EINVAL, 678 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 679 "invalid argument"); 680 if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED)) 681 mlx5_flow_tunnel_free(dev, tun); 682 return 0; 683 } 684 685 static int 686 mlx5_flow_action_release(struct rte_eth_dev *dev, 687 struct rte_flow_action *pmd_actions, 688 uint32_t num_actions, struct rte_flow_error *err) 689 { 690 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 691 struct mlx5_flow_tunnel *tun; 692 693 LIST_FOREACH(tun, &thub->tunnels, chain) { 694 if (&tun->action == pmd_actions) 695 break; 696 } 697 if (!tun || num_actions != 1) 698 return rte_flow_error_set(err, EINVAL, 699 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 700 "invalid argument"); 701 if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED)) 702 mlx5_flow_tunnel_free(dev, tun); 703 704 return 0; 705 } 706 707 static int 708 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 709 struct rte_mbuf *m, 710 struct rte_flow_restore_info *info, 711 struct rte_flow_error *err) 712 { 713 uint64_t ol_flags = m->ol_flags; 714 const struct mlx5_flow_tbl_data_entry *tble; 715 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 716 717 if ((ol_flags & mask) != mask) 718 goto err; 719 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 720 if (!tble) { 721 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 722 dev->data->port_id, m->hash.fdir.hi); 723 goto err; 724 } 725 MLX5_ASSERT(tble->tunnel); 726 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 727 info->group_id = tble->group_id; 728 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 729 RTE_FLOW_RESTORE_INFO_GROUP_ID | 730 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 731 732 return 0; 733 734 err: 735 return rte_flow_error_set(err, EINVAL, 736 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 737 "failed to get restore info"); 738 } 739 740 static const struct rte_flow_ops mlx5_flow_ops = { 741 .validate = mlx5_flow_validate, 742 .create = mlx5_flow_create, 743 .destroy = mlx5_flow_destroy, 744 .flush = mlx5_flow_flush, 745 .isolate = mlx5_flow_isolate, 746 .query = mlx5_flow_query, 747 .dev_dump = mlx5_flow_dev_dump, 748 .get_aged_flows = mlx5_flow_get_aged_flows, 749 .shared_action_create = mlx5_shared_action_create, 750 .shared_action_destroy = mlx5_shared_action_destroy, 751 .shared_action_update = mlx5_shared_action_update, 752 .shared_action_query = mlx5_shared_action_query, 753 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 754 .tunnel_match = mlx5_flow_tunnel_match, 755 .tunnel_action_decap_release = mlx5_flow_action_release, 756 .tunnel_item_release = mlx5_flow_item_release, 757 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 758 }; 759 760 /* Convert FDIR request to Generic flow. */ 761 struct mlx5_fdir { 762 struct rte_flow_attr attr; 763 struct rte_flow_item items[4]; 764 struct rte_flow_item_eth l2; 765 struct rte_flow_item_eth l2_mask; 766 union { 767 struct rte_flow_item_ipv4 ipv4; 768 struct rte_flow_item_ipv6 ipv6; 769 } l3; 770 union { 771 struct rte_flow_item_ipv4 ipv4; 772 struct rte_flow_item_ipv6 ipv6; 773 } l3_mask; 774 union { 775 struct rte_flow_item_udp udp; 776 struct rte_flow_item_tcp tcp; 777 } l4; 778 union { 779 struct rte_flow_item_udp udp; 780 struct rte_flow_item_tcp tcp; 781 } l4_mask; 782 struct rte_flow_action actions[2]; 783 struct rte_flow_action_queue queue; 784 }; 785 786 /* Tunnel information. */ 787 struct mlx5_flow_tunnel_info { 788 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 789 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 790 }; 791 792 static struct mlx5_flow_tunnel_info tunnels_info[] = { 793 { 794 .tunnel = MLX5_FLOW_LAYER_VXLAN, 795 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 796 }, 797 { 798 .tunnel = MLX5_FLOW_LAYER_GENEVE, 799 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 800 }, 801 { 802 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 803 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 804 }, 805 { 806 .tunnel = MLX5_FLOW_LAYER_GRE, 807 .ptype = RTE_PTYPE_TUNNEL_GRE, 808 }, 809 { 810 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 811 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 812 }, 813 { 814 .tunnel = MLX5_FLOW_LAYER_MPLS, 815 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 816 }, 817 { 818 .tunnel = MLX5_FLOW_LAYER_NVGRE, 819 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 820 }, 821 { 822 .tunnel = MLX5_FLOW_LAYER_IPIP, 823 .ptype = RTE_PTYPE_TUNNEL_IP, 824 }, 825 { 826 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 827 .ptype = RTE_PTYPE_TUNNEL_IP, 828 }, 829 { 830 .tunnel = MLX5_FLOW_LAYER_GTP, 831 .ptype = RTE_PTYPE_TUNNEL_GTPU, 832 }, 833 }; 834 835 /** 836 * Translate tag ID to register. 837 * 838 * @param[in] dev 839 * Pointer to the Ethernet device structure. 840 * @param[in] feature 841 * The feature that request the register. 842 * @param[in] id 843 * The request register ID. 844 * @param[out] error 845 * Error description in case of any. 846 * 847 * @return 848 * The request register on success, a negative errno 849 * value otherwise and rte_errno is set. 850 */ 851 int 852 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 853 enum mlx5_feature_name feature, 854 uint32_t id, 855 struct rte_flow_error *error) 856 { 857 struct mlx5_priv *priv = dev->data->dev_private; 858 struct mlx5_dev_config *config = &priv->config; 859 enum modify_reg start_reg; 860 bool skip_mtr_reg = false; 861 862 switch (feature) { 863 case MLX5_HAIRPIN_RX: 864 return REG_B; 865 case MLX5_HAIRPIN_TX: 866 return REG_A; 867 case MLX5_METADATA_RX: 868 switch (config->dv_xmeta_en) { 869 case MLX5_XMETA_MODE_LEGACY: 870 return REG_B; 871 case MLX5_XMETA_MODE_META16: 872 return REG_C_0; 873 case MLX5_XMETA_MODE_META32: 874 return REG_C_1; 875 } 876 break; 877 case MLX5_METADATA_TX: 878 return REG_A; 879 case MLX5_METADATA_FDB: 880 switch (config->dv_xmeta_en) { 881 case MLX5_XMETA_MODE_LEGACY: 882 return REG_NON; 883 case MLX5_XMETA_MODE_META16: 884 return REG_C_0; 885 case MLX5_XMETA_MODE_META32: 886 return REG_C_1; 887 } 888 break; 889 case MLX5_FLOW_MARK: 890 switch (config->dv_xmeta_en) { 891 case MLX5_XMETA_MODE_LEGACY: 892 return REG_NON; 893 case MLX5_XMETA_MODE_META16: 894 return REG_C_1; 895 case MLX5_XMETA_MODE_META32: 896 return REG_C_0; 897 } 898 break; 899 case MLX5_MTR_SFX: 900 /* 901 * If meter color and flow match share one register, flow match 902 * should use the meter color register for match. 903 */ 904 if (priv->mtr_reg_share) 905 return priv->mtr_color_reg; 906 else 907 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 908 REG_C_3; 909 case MLX5_MTR_COLOR: 910 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 911 return priv->mtr_color_reg; 912 case MLX5_COPY_MARK: 913 /* 914 * Metadata COPY_MARK register using is in meter suffix sub 915 * flow while with meter. It's safe to share the same register. 916 */ 917 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 918 case MLX5_APP_TAG: 919 /* 920 * If meter is enable, it will engage the register for color 921 * match and flow match. If meter color match is not using the 922 * REG_C_2, need to skip the REG_C_x be used by meter color 923 * match. 924 * If meter is disable, free to use all available registers. 925 */ 926 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 927 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 928 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 929 if (id > (REG_C_7 - start_reg)) 930 return rte_flow_error_set(error, EINVAL, 931 RTE_FLOW_ERROR_TYPE_ITEM, 932 NULL, "invalid tag id"); 933 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 934 return rte_flow_error_set(error, ENOTSUP, 935 RTE_FLOW_ERROR_TYPE_ITEM, 936 NULL, "unsupported tag id"); 937 /* 938 * This case means meter is using the REG_C_x great than 2. 939 * Take care not to conflict with meter color REG_C_x. 940 * If the available index REG_C_y >= REG_C_x, skip the 941 * color register. 942 */ 943 if (skip_mtr_reg && config->flow_mreg_c 944 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 945 if (id >= (REG_C_7 - start_reg)) 946 return rte_flow_error_set(error, EINVAL, 947 RTE_FLOW_ERROR_TYPE_ITEM, 948 NULL, "invalid tag id"); 949 if (config->flow_mreg_c 950 [id + 1 + start_reg - REG_C_0] != REG_NON) 951 return config->flow_mreg_c 952 [id + 1 + start_reg - REG_C_0]; 953 return rte_flow_error_set(error, ENOTSUP, 954 RTE_FLOW_ERROR_TYPE_ITEM, 955 NULL, "unsupported tag id"); 956 } 957 return config->flow_mreg_c[id + start_reg - REG_C_0]; 958 } 959 MLX5_ASSERT(false); 960 return rte_flow_error_set(error, EINVAL, 961 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 962 NULL, "invalid feature name"); 963 } 964 965 /** 966 * Check extensive flow metadata register support. 967 * 968 * @param dev 969 * Pointer to rte_eth_dev structure. 970 * 971 * @return 972 * True if device supports extensive flow metadata register, otherwise false. 973 */ 974 bool 975 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 976 { 977 struct mlx5_priv *priv = dev->data->dev_private; 978 struct mlx5_dev_config *config = &priv->config; 979 980 /* 981 * Having available reg_c can be regarded inclusively as supporting 982 * extensive flow metadata register, which could mean, 983 * - metadata register copy action by modify header. 984 * - 16 modify header actions is supported. 985 * - reg_c's are preserved across different domain (FDB and NIC) on 986 * packet loopback by flow lookup miss. 987 */ 988 return config->flow_mreg_c[2] != REG_NON; 989 } 990 991 /** 992 * Verify the @p item specifications (spec, last, mask) are compatible with the 993 * NIC capabilities. 994 * 995 * @param[in] item 996 * Item specification. 997 * @param[in] mask 998 * @p item->mask or flow default bit-masks. 999 * @param[in] nic_mask 1000 * Bit-masks covering supported fields by the NIC to compare with user mask. 1001 * @param[in] size 1002 * Bit-masks size in bytes. 1003 * @param[in] range_accepted 1004 * True if range of values is accepted for specific fields, false otherwise. 1005 * @param[out] error 1006 * Pointer to error structure. 1007 * 1008 * @return 1009 * 0 on success, a negative errno value otherwise and rte_errno is set. 1010 */ 1011 int 1012 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1013 const uint8_t *mask, 1014 const uint8_t *nic_mask, 1015 unsigned int size, 1016 bool range_accepted, 1017 struct rte_flow_error *error) 1018 { 1019 unsigned int i; 1020 1021 MLX5_ASSERT(nic_mask); 1022 for (i = 0; i < size; ++i) 1023 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1024 return rte_flow_error_set(error, ENOTSUP, 1025 RTE_FLOW_ERROR_TYPE_ITEM, 1026 item, 1027 "mask enables non supported" 1028 " bits"); 1029 if (!item->spec && (item->mask || item->last)) 1030 return rte_flow_error_set(error, EINVAL, 1031 RTE_FLOW_ERROR_TYPE_ITEM, item, 1032 "mask/last without a spec is not" 1033 " supported"); 1034 if (item->spec && item->last && !range_accepted) { 1035 uint8_t spec[size]; 1036 uint8_t last[size]; 1037 unsigned int i; 1038 int ret; 1039 1040 for (i = 0; i < size; ++i) { 1041 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1042 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1043 } 1044 ret = memcmp(spec, last, size); 1045 if (ret != 0) 1046 return rte_flow_error_set(error, EINVAL, 1047 RTE_FLOW_ERROR_TYPE_ITEM, 1048 item, 1049 "range is not valid"); 1050 } 1051 return 0; 1052 } 1053 1054 /** 1055 * Adjust the hash fields according to the @p flow information. 1056 * 1057 * @param[in] dev_flow. 1058 * Pointer to the mlx5_flow. 1059 * @param[in] tunnel 1060 * 1 when the hash field is for a tunnel item. 1061 * @param[in] layer_types 1062 * ETH_RSS_* types. 1063 * @param[in] hash_fields 1064 * Item hash fields. 1065 * 1066 * @return 1067 * The hash fields that should be used. 1068 */ 1069 uint64_t 1070 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1071 int tunnel __rte_unused, uint64_t layer_types, 1072 uint64_t hash_fields) 1073 { 1074 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1075 int rss_request_inner = rss_desc->level >= 2; 1076 1077 /* Check RSS hash level for tunnel. */ 1078 if (tunnel && rss_request_inner) 1079 hash_fields |= IBV_RX_HASH_INNER; 1080 else if (tunnel || rss_request_inner) 1081 return 0; 1082 #endif 1083 /* Check if requested layer matches RSS hash fields. */ 1084 if (!(rss_desc->types & layer_types)) 1085 return 0; 1086 return hash_fields; 1087 } 1088 1089 /** 1090 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1091 * if several tunnel rules are used on this queue, the tunnel ptype will be 1092 * cleared. 1093 * 1094 * @param rxq_ctrl 1095 * Rx queue to update. 1096 */ 1097 static void 1098 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1099 { 1100 unsigned int i; 1101 uint32_t tunnel_ptype = 0; 1102 1103 /* Look up for the ptype to use. */ 1104 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1105 if (!rxq_ctrl->flow_tunnels_n[i]) 1106 continue; 1107 if (!tunnel_ptype) { 1108 tunnel_ptype = tunnels_info[i].ptype; 1109 } else { 1110 tunnel_ptype = 0; 1111 break; 1112 } 1113 } 1114 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1115 } 1116 1117 /** 1118 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1119 * flow. 1120 * 1121 * @param[in] dev 1122 * Pointer to the Ethernet device structure. 1123 * @param[in] dev_handle 1124 * Pointer to device flow handle structure. 1125 */ 1126 static void 1127 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1128 struct mlx5_flow_handle *dev_handle) 1129 { 1130 struct mlx5_priv *priv = dev->data->dev_private; 1131 const int mark = dev_handle->mark; 1132 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1133 struct mlx5_hrxq *hrxq; 1134 unsigned int i; 1135 1136 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1137 return; 1138 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1139 dev_handle->rix_hrxq); 1140 if (!hrxq) 1141 return; 1142 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1143 int idx = hrxq->ind_table->queues[i]; 1144 struct mlx5_rxq_ctrl *rxq_ctrl = 1145 container_of((*priv->rxqs)[idx], 1146 struct mlx5_rxq_ctrl, rxq); 1147 1148 /* 1149 * To support metadata register copy on Tx loopback, 1150 * this must be always enabled (metadata may arive 1151 * from other port - not from local flows only. 1152 */ 1153 if (priv->config.dv_flow_en && 1154 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1155 mlx5_flow_ext_mreg_supported(dev)) { 1156 rxq_ctrl->rxq.mark = 1; 1157 rxq_ctrl->flow_mark_n = 1; 1158 } else if (mark) { 1159 rxq_ctrl->rxq.mark = 1; 1160 rxq_ctrl->flow_mark_n++; 1161 } 1162 if (tunnel) { 1163 unsigned int j; 1164 1165 /* Increase the counter matching the flow. */ 1166 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1167 if ((tunnels_info[j].tunnel & 1168 dev_handle->layers) == 1169 tunnels_info[j].tunnel) { 1170 rxq_ctrl->flow_tunnels_n[j]++; 1171 break; 1172 } 1173 } 1174 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1175 } 1176 } 1177 } 1178 1179 /** 1180 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1181 * 1182 * @param[in] dev 1183 * Pointer to the Ethernet device structure. 1184 * @param[in] flow 1185 * Pointer to flow structure. 1186 */ 1187 static void 1188 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1189 { 1190 struct mlx5_priv *priv = dev->data->dev_private; 1191 uint32_t handle_idx; 1192 struct mlx5_flow_handle *dev_handle; 1193 1194 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1195 handle_idx, dev_handle, next) 1196 flow_drv_rxq_flags_set(dev, dev_handle); 1197 } 1198 1199 /** 1200 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1201 * device flow if no other flow uses it with the same kind of request. 1202 * 1203 * @param dev 1204 * Pointer to Ethernet device. 1205 * @param[in] dev_handle 1206 * Pointer to the device flow handle structure. 1207 */ 1208 static void 1209 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1210 struct mlx5_flow_handle *dev_handle) 1211 { 1212 struct mlx5_priv *priv = dev->data->dev_private; 1213 const int mark = dev_handle->mark; 1214 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1215 struct mlx5_hrxq *hrxq; 1216 unsigned int i; 1217 1218 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1219 return; 1220 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1221 dev_handle->rix_hrxq); 1222 if (!hrxq) 1223 return; 1224 MLX5_ASSERT(dev->data->dev_started); 1225 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1226 int idx = hrxq->ind_table->queues[i]; 1227 struct mlx5_rxq_ctrl *rxq_ctrl = 1228 container_of((*priv->rxqs)[idx], 1229 struct mlx5_rxq_ctrl, rxq); 1230 1231 if (priv->config.dv_flow_en && 1232 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1233 mlx5_flow_ext_mreg_supported(dev)) { 1234 rxq_ctrl->rxq.mark = 1; 1235 rxq_ctrl->flow_mark_n = 1; 1236 } else if (mark) { 1237 rxq_ctrl->flow_mark_n--; 1238 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1239 } 1240 if (tunnel) { 1241 unsigned int j; 1242 1243 /* Decrease the counter matching the flow. */ 1244 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1245 if ((tunnels_info[j].tunnel & 1246 dev_handle->layers) == 1247 tunnels_info[j].tunnel) { 1248 rxq_ctrl->flow_tunnels_n[j]--; 1249 break; 1250 } 1251 } 1252 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1253 } 1254 } 1255 } 1256 1257 /** 1258 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1259 * @p flow if no other flow uses it with the same kind of request. 1260 * 1261 * @param dev 1262 * Pointer to Ethernet device. 1263 * @param[in] flow 1264 * Pointer to the flow. 1265 */ 1266 static void 1267 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1268 { 1269 struct mlx5_priv *priv = dev->data->dev_private; 1270 uint32_t handle_idx; 1271 struct mlx5_flow_handle *dev_handle; 1272 1273 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1274 handle_idx, dev_handle, next) 1275 flow_drv_rxq_flags_trim(dev, dev_handle); 1276 } 1277 1278 /** 1279 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1280 * 1281 * @param dev 1282 * Pointer to Ethernet device. 1283 */ 1284 static void 1285 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1286 { 1287 struct mlx5_priv *priv = dev->data->dev_private; 1288 unsigned int i; 1289 1290 for (i = 0; i != priv->rxqs_n; ++i) { 1291 struct mlx5_rxq_ctrl *rxq_ctrl; 1292 unsigned int j; 1293 1294 if (!(*priv->rxqs)[i]) 1295 continue; 1296 rxq_ctrl = container_of((*priv->rxqs)[i], 1297 struct mlx5_rxq_ctrl, rxq); 1298 rxq_ctrl->flow_mark_n = 0; 1299 rxq_ctrl->rxq.mark = 0; 1300 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1301 rxq_ctrl->flow_tunnels_n[j] = 0; 1302 rxq_ctrl->rxq.tunnel = 0; 1303 } 1304 } 1305 1306 /** 1307 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1308 * 1309 * @param[in] dev 1310 * Pointer to the Ethernet device structure. 1311 */ 1312 void 1313 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1314 { 1315 struct mlx5_priv *priv = dev->data->dev_private; 1316 struct mlx5_rxq_data *data; 1317 unsigned int i; 1318 1319 for (i = 0; i != priv->rxqs_n; ++i) { 1320 if (!(*priv->rxqs)[i]) 1321 continue; 1322 data = (*priv->rxqs)[i]; 1323 if (!rte_flow_dynf_metadata_avail()) { 1324 data->dynf_meta = 0; 1325 data->flow_meta_mask = 0; 1326 data->flow_meta_offset = -1; 1327 } else { 1328 data->dynf_meta = 1; 1329 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1330 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1331 } 1332 } 1333 } 1334 1335 /* 1336 * return a pointer to the desired action in the list of actions. 1337 * 1338 * @param[in] actions 1339 * The list of actions to search the action in. 1340 * @param[in] action 1341 * The action to find. 1342 * 1343 * @return 1344 * Pointer to the action in the list, if found. NULL otherwise. 1345 */ 1346 const struct rte_flow_action * 1347 mlx5_flow_find_action(const struct rte_flow_action *actions, 1348 enum rte_flow_action_type action) 1349 { 1350 if (actions == NULL) 1351 return NULL; 1352 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1353 if (actions->type == action) 1354 return actions; 1355 return NULL; 1356 } 1357 1358 /* 1359 * Validate the flag action. 1360 * 1361 * @param[in] action_flags 1362 * Bit-fields that holds the actions detected until now. 1363 * @param[in] attr 1364 * Attributes of flow that includes this action. 1365 * @param[out] error 1366 * Pointer to error structure. 1367 * 1368 * @return 1369 * 0 on success, a negative errno value otherwise and rte_errno is set. 1370 */ 1371 int 1372 mlx5_flow_validate_action_flag(uint64_t action_flags, 1373 const struct rte_flow_attr *attr, 1374 struct rte_flow_error *error) 1375 { 1376 if (action_flags & MLX5_FLOW_ACTION_MARK) 1377 return rte_flow_error_set(error, EINVAL, 1378 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1379 "can't mark and flag in same flow"); 1380 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1381 return rte_flow_error_set(error, EINVAL, 1382 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1383 "can't have 2 flag" 1384 " actions in same flow"); 1385 if (attr->egress) 1386 return rte_flow_error_set(error, ENOTSUP, 1387 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1388 "flag action not supported for " 1389 "egress"); 1390 return 0; 1391 } 1392 1393 /* 1394 * Validate the mark action. 1395 * 1396 * @param[in] action 1397 * Pointer to the queue action. 1398 * @param[in] action_flags 1399 * Bit-fields that holds the actions detected until now. 1400 * @param[in] attr 1401 * Attributes of flow that includes this action. 1402 * @param[out] error 1403 * Pointer to error structure. 1404 * 1405 * @return 1406 * 0 on success, a negative errno value otherwise and rte_errno is set. 1407 */ 1408 int 1409 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1410 uint64_t action_flags, 1411 const struct rte_flow_attr *attr, 1412 struct rte_flow_error *error) 1413 { 1414 const struct rte_flow_action_mark *mark = action->conf; 1415 1416 if (!mark) 1417 return rte_flow_error_set(error, EINVAL, 1418 RTE_FLOW_ERROR_TYPE_ACTION, 1419 action, 1420 "configuration cannot be null"); 1421 if (mark->id >= MLX5_FLOW_MARK_MAX) 1422 return rte_flow_error_set(error, EINVAL, 1423 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1424 &mark->id, 1425 "mark id must in 0 <= id < " 1426 RTE_STR(MLX5_FLOW_MARK_MAX)); 1427 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1428 return rte_flow_error_set(error, EINVAL, 1429 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1430 "can't flag and mark in same flow"); 1431 if (action_flags & MLX5_FLOW_ACTION_MARK) 1432 return rte_flow_error_set(error, EINVAL, 1433 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1434 "can't have 2 mark actions in same" 1435 " flow"); 1436 if (attr->egress) 1437 return rte_flow_error_set(error, ENOTSUP, 1438 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1439 "mark action not supported for " 1440 "egress"); 1441 return 0; 1442 } 1443 1444 /* 1445 * Validate the drop action. 1446 * 1447 * @param[in] action_flags 1448 * Bit-fields that holds the actions detected until now. 1449 * @param[in] attr 1450 * Attributes of flow that includes this action. 1451 * @param[out] error 1452 * Pointer to error structure. 1453 * 1454 * @return 1455 * 0 on success, a negative errno value otherwise and rte_errno is set. 1456 */ 1457 int 1458 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1459 const struct rte_flow_attr *attr, 1460 struct rte_flow_error *error) 1461 { 1462 if (attr->egress) 1463 return rte_flow_error_set(error, ENOTSUP, 1464 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1465 "drop action not supported for " 1466 "egress"); 1467 return 0; 1468 } 1469 1470 /* 1471 * Validate the queue action. 1472 * 1473 * @param[in] action 1474 * Pointer to the queue action. 1475 * @param[in] action_flags 1476 * Bit-fields that holds the actions detected until now. 1477 * @param[in] dev 1478 * Pointer to the Ethernet device structure. 1479 * @param[in] attr 1480 * Attributes of flow that includes this action. 1481 * @param[out] error 1482 * Pointer to error structure. 1483 * 1484 * @return 1485 * 0 on success, a negative errno value otherwise and rte_errno is set. 1486 */ 1487 int 1488 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1489 uint64_t action_flags, 1490 struct rte_eth_dev *dev, 1491 const struct rte_flow_attr *attr, 1492 struct rte_flow_error *error) 1493 { 1494 struct mlx5_priv *priv = dev->data->dev_private; 1495 const struct rte_flow_action_queue *queue = action->conf; 1496 1497 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1498 return rte_flow_error_set(error, EINVAL, 1499 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1500 "can't have 2 fate actions in" 1501 " same flow"); 1502 if (!priv->rxqs_n) 1503 return rte_flow_error_set(error, EINVAL, 1504 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1505 NULL, "No Rx queues configured"); 1506 if (queue->index >= priv->rxqs_n) 1507 return rte_flow_error_set(error, EINVAL, 1508 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1509 &queue->index, 1510 "queue index out of range"); 1511 if (!(*priv->rxqs)[queue->index]) 1512 return rte_flow_error_set(error, EINVAL, 1513 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1514 &queue->index, 1515 "queue is not configured"); 1516 if (attr->egress) 1517 return rte_flow_error_set(error, ENOTSUP, 1518 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1519 "queue action not supported for " 1520 "egress"); 1521 return 0; 1522 } 1523 1524 /* 1525 * Validate the rss action. 1526 * 1527 * @param[in] dev 1528 * Pointer to the Ethernet device structure. 1529 * @param[in] action 1530 * Pointer to the queue action. 1531 * @param[out] error 1532 * Pointer to error structure. 1533 * 1534 * @return 1535 * 0 on success, a negative errno value otherwise and rte_errno is set. 1536 */ 1537 int 1538 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1539 const struct rte_flow_action *action, 1540 struct rte_flow_error *error) 1541 { 1542 struct mlx5_priv *priv = dev->data->dev_private; 1543 const struct rte_flow_action_rss *rss = action->conf; 1544 unsigned int i; 1545 1546 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1547 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1548 return rte_flow_error_set(error, ENOTSUP, 1549 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1550 &rss->func, 1551 "RSS hash function not supported"); 1552 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1553 if (rss->level > 2) 1554 #else 1555 if (rss->level > 1) 1556 #endif 1557 return rte_flow_error_set(error, ENOTSUP, 1558 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1559 &rss->level, 1560 "tunnel RSS is not supported"); 1561 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1562 if (rss->key_len == 0 && rss->key != NULL) 1563 return rte_flow_error_set(error, ENOTSUP, 1564 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1565 &rss->key_len, 1566 "RSS hash key length 0"); 1567 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1568 return rte_flow_error_set(error, ENOTSUP, 1569 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1570 &rss->key_len, 1571 "RSS hash key too small"); 1572 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1573 return rte_flow_error_set(error, ENOTSUP, 1574 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1575 &rss->key_len, 1576 "RSS hash key too large"); 1577 if (rss->queue_num > priv->config.ind_table_max_size) 1578 return rte_flow_error_set(error, ENOTSUP, 1579 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1580 &rss->queue_num, 1581 "number of queues too large"); 1582 if (rss->types & MLX5_RSS_HF_MASK) 1583 return rte_flow_error_set(error, ENOTSUP, 1584 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1585 &rss->types, 1586 "some RSS protocols are not" 1587 " supported"); 1588 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1589 !(rss->types & ETH_RSS_IP)) 1590 return rte_flow_error_set(error, EINVAL, 1591 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1592 "L3 partial RSS requested but L3 RSS" 1593 " type not specified"); 1594 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1595 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1596 return rte_flow_error_set(error, EINVAL, 1597 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1598 "L4 partial RSS requested but L4 RSS" 1599 " type not specified"); 1600 if (!priv->rxqs_n) 1601 return rte_flow_error_set(error, EINVAL, 1602 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1603 NULL, "No Rx queues configured"); 1604 if (!rss->queue_num) 1605 return rte_flow_error_set(error, EINVAL, 1606 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1607 NULL, "No queues configured"); 1608 for (i = 0; i != rss->queue_num; ++i) { 1609 if (rss->queue[i] >= priv->rxqs_n) 1610 return rte_flow_error_set 1611 (error, EINVAL, 1612 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1613 &rss->queue[i], "queue index out of range"); 1614 if (!(*priv->rxqs)[rss->queue[i]]) 1615 return rte_flow_error_set 1616 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1617 &rss->queue[i], "queue is not configured"); 1618 } 1619 return 0; 1620 } 1621 1622 /* 1623 * Validate the rss action. 1624 * 1625 * @param[in] action 1626 * Pointer to the queue action. 1627 * @param[in] action_flags 1628 * Bit-fields that holds the actions detected until now. 1629 * @param[in] dev 1630 * Pointer to the Ethernet device structure. 1631 * @param[in] attr 1632 * Attributes of flow that includes this action. 1633 * @param[in] item_flags 1634 * Items that were detected. 1635 * @param[out] error 1636 * Pointer to error structure. 1637 * 1638 * @return 1639 * 0 on success, a negative errno value otherwise and rte_errno is set. 1640 */ 1641 int 1642 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1643 uint64_t action_flags, 1644 struct rte_eth_dev *dev, 1645 const struct rte_flow_attr *attr, 1646 uint64_t item_flags, 1647 struct rte_flow_error *error) 1648 { 1649 const struct rte_flow_action_rss *rss = action->conf; 1650 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1651 int ret; 1652 1653 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1654 return rte_flow_error_set(error, EINVAL, 1655 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1656 "can't have 2 fate actions" 1657 " in same flow"); 1658 ret = mlx5_validate_action_rss(dev, action, error); 1659 if (ret) 1660 return ret; 1661 if (attr->egress) 1662 return rte_flow_error_set(error, ENOTSUP, 1663 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1664 "rss action not supported for " 1665 "egress"); 1666 if (rss->level > 1 && !tunnel) 1667 return rte_flow_error_set(error, EINVAL, 1668 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1669 "inner RSS is not supported for " 1670 "non-tunnel flows"); 1671 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1672 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1673 return rte_flow_error_set(error, EINVAL, 1674 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1675 "RSS on eCPRI is not supported now"); 1676 } 1677 return 0; 1678 } 1679 1680 /* 1681 * Validate the default miss action. 1682 * 1683 * @param[in] action_flags 1684 * Bit-fields that holds the actions detected until now. 1685 * @param[out] error 1686 * Pointer to error structure. 1687 * 1688 * @return 1689 * 0 on success, a negative errno value otherwise and rte_errno is set. 1690 */ 1691 int 1692 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1693 const struct rte_flow_attr *attr, 1694 struct rte_flow_error *error) 1695 { 1696 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1697 return rte_flow_error_set(error, EINVAL, 1698 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1699 "can't have 2 fate actions in" 1700 " same flow"); 1701 if (attr->egress) 1702 return rte_flow_error_set(error, ENOTSUP, 1703 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1704 "default miss action not supported " 1705 "for egress"); 1706 if (attr->group) 1707 return rte_flow_error_set(error, ENOTSUP, 1708 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1709 "only group 0 is supported"); 1710 if (attr->transfer) 1711 return rte_flow_error_set(error, ENOTSUP, 1712 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1713 NULL, "transfer is not supported"); 1714 return 0; 1715 } 1716 1717 /* 1718 * Validate the count action. 1719 * 1720 * @param[in] dev 1721 * Pointer to the Ethernet device structure. 1722 * @param[in] attr 1723 * Attributes of flow that includes this action. 1724 * @param[out] error 1725 * Pointer to error structure. 1726 * 1727 * @return 1728 * 0 on success, a negative errno value otherwise and rte_errno is set. 1729 */ 1730 int 1731 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1732 const struct rte_flow_attr *attr, 1733 struct rte_flow_error *error) 1734 { 1735 if (attr->egress) 1736 return rte_flow_error_set(error, ENOTSUP, 1737 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1738 "count action not supported for " 1739 "egress"); 1740 return 0; 1741 } 1742 1743 /** 1744 * Verify the @p attributes will be correctly understood by the NIC and store 1745 * them in the @p flow if everything is correct. 1746 * 1747 * @param[in] dev 1748 * Pointer to the Ethernet device structure. 1749 * @param[in] attributes 1750 * Pointer to flow attributes 1751 * @param[out] error 1752 * Pointer to error structure. 1753 * 1754 * @return 1755 * 0 on success, a negative errno value otherwise and rte_errno is set. 1756 */ 1757 int 1758 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1759 const struct rte_flow_attr *attributes, 1760 struct rte_flow_error *error) 1761 { 1762 struct mlx5_priv *priv = dev->data->dev_private; 1763 uint32_t priority_max = priv->config.flow_prio - 1; 1764 1765 if (attributes->group) 1766 return rte_flow_error_set(error, ENOTSUP, 1767 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1768 NULL, "groups is not supported"); 1769 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1770 attributes->priority >= priority_max) 1771 return rte_flow_error_set(error, ENOTSUP, 1772 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1773 NULL, "priority out of range"); 1774 if (attributes->egress) 1775 return rte_flow_error_set(error, ENOTSUP, 1776 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1777 "egress is not supported"); 1778 if (attributes->transfer && !priv->config.dv_esw_en) 1779 return rte_flow_error_set(error, ENOTSUP, 1780 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1781 NULL, "transfer is not supported"); 1782 if (!attributes->ingress) 1783 return rte_flow_error_set(error, EINVAL, 1784 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1785 NULL, 1786 "ingress attribute is mandatory"); 1787 return 0; 1788 } 1789 1790 /** 1791 * Validate ICMP6 item. 1792 * 1793 * @param[in] item 1794 * Item specification. 1795 * @param[in] item_flags 1796 * Bit-fields that holds the items detected until now. 1797 * @param[in] ext_vlan_sup 1798 * Whether extended VLAN features are supported or not. 1799 * @param[out] error 1800 * Pointer to error structure. 1801 * 1802 * @return 1803 * 0 on success, a negative errno value otherwise and rte_errno is set. 1804 */ 1805 int 1806 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1807 uint64_t item_flags, 1808 uint8_t target_protocol, 1809 struct rte_flow_error *error) 1810 { 1811 const struct rte_flow_item_icmp6 *mask = item->mask; 1812 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1813 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1814 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1815 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1816 MLX5_FLOW_LAYER_OUTER_L4; 1817 int ret; 1818 1819 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1820 return rte_flow_error_set(error, EINVAL, 1821 RTE_FLOW_ERROR_TYPE_ITEM, item, 1822 "protocol filtering not compatible" 1823 " with ICMP6 layer"); 1824 if (!(item_flags & l3m)) 1825 return rte_flow_error_set(error, EINVAL, 1826 RTE_FLOW_ERROR_TYPE_ITEM, item, 1827 "IPv6 is mandatory to filter on" 1828 " ICMP6"); 1829 if (item_flags & l4m) 1830 return rte_flow_error_set(error, EINVAL, 1831 RTE_FLOW_ERROR_TYPE_ITEM, item, 1832 "multiple L4 layers not supported"); 1833 if (!mask) 1834 mask = &rte_flow_item_icmp6_mask; 1835 ret = mlx5_flow_item_acceptable 1836 (item, (const uint8_t *)mask, 1837 (const uint8_t *)&rte_flow_item_icmp6_mask, 1838 sizeof(struct rte_flow_item_icmp6), 1839 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1840 if (ret < 0) 1841 return ret; 1842 return 0; 1843 } 1844 1845 /** 1846 * Validate ICMP item. 1847 * 1848 * @param[in] item 1849 * Item specification. 1850 * @param[in] item_flags 1851 * Bit-fields that holds the items detected until now. 1852 * @param[out] error 1853 * Pointer to error structure. 1854 * 1855 * @return 1856 * 0 on success, a negative errno value otherwise and rte_errno is set. 1857 */ 1858 int 1859 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1860 uint64_t item_flags, 1861 uint8_t target_protocol, 1862 struct rte_flow_error *error) 1863 { 1864 const struct rte_flow_item_icmp *mask = item->mask; 1865 const struct rte_flow_item_icmp nic_mask = { 1866 .hdr.icmp_type = 0xff, 1867 .hdr.icmp_code = 0xff, 1868 .hdr.icmp_ident = RTE_BE16(0xffff), 1869 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1870 }; 1871 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1872 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1873 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1874 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1875 MLX5_FLOW_LAYER_OUTER_L4; 1876 int ret; 1877 1878 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1879 return rte_flow_error_set(error, EINVAL, 1880 RTE_FLOW_ERROR_TYPE_ITEM, item, 1881 "protocol filtering not compatible" 1882 " with ICMP layer"); 1883 if (!(item_flags & l3m)) 1884 return rte_flow_error_set(error, EINVAL, 1885 RTE_FLOW_ERROR_TYPE_ITEM, item, 1886 "IPv4 is mandatory to filter" 1887 " on ICMP"); 1888 if (item_flags & l4m) 1889 return rte_flow_error_set(error, EINVAL, 1890 RTE_FLOW_ERROR_TYPE_ITEM, item, 1891 "multiple L4 layers not supported"); 1892 if (!mask) 1893 mask = &nic_mask; 1894 ret = mlx5_flow_item_acceptable 1895 (item, (const uint8_t *)mask, 1896 (const uint8_t *)&nic_mask, 1897 sizeof(struct rte_flow_item_icmp), 1898 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1899 if (ret < 0) 1900 return ret; 1901 return 0; 1902 } 1903 1904 /** 1905 * Validate Ethernet item. 1906 * 1907 * @param[in] item 1908 * Item specification. 1909 * @param[in] item_flags 1910 * Bit-fields that holds the items detected until now. 1911 * @param[out] error 1912 * Pointer to error structure. 1913 * 1914 * @return 1915 * 0 on success, a negative errno value otherwise and rte_errno is set. 1916 */ 1917 int 1918 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1919 uint64_t item_flags, bool ext_vlan_sup, 1920 struct rte_flow_error *error) 1921 { 1922 const struct rte_flow_item_eth *mask = item->mask; 1923 const struct rte_flow_item_eth nic_mask = { 1924 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1925 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1926 .type = RTE_BE16(0xffff), 1927 .has_vlan = ext_vlan_sup ? 1 : 0, 1928 }; 1929 int ret; 1930 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1931 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1932 MLX5_FLOW_LAYER_OUTER_L2; 1933 1934 if (item_flags & ethm) 1935 return rte_flow_error_set(error, ENOTSUP, 1936 RTE_FLOW_ERROR_TYPE_ITEM, item, 1937 "multiple L2 layers not supported"); 1938 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1939 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1940 return rte_flow_error_set(error, EINVAL, 1941 RTE_FLOW_ERROR_TYPE_ITEM, item, 1942 "L2 layer should not follow " 1943 "L3 layers"); 1944 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1945 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1946 return rte_flow_error_set(error, EINVAL, 1947 RTE_FLOW_ERROR_TYPE_ITEM, item, 1948 "L2 layer should not follow VLAN"); 1949 if (!mask) 1950 mask = &rte_flow_item_eth_mask; 1951 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1952 (const uint8_t *)&nic_mask, 1953 sizeof(struct rte_flow_item_eth), 1954 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1955 return ret; 1956 } 1957 1958 /** 1959 * Validate VLAN item. 1960 * 1961 * @param[in] item 1962 * Item specification. 1963 * @param[in] item_flags 1964 * Bit-fields that holds the items detected until now. 1965 * @param[in] dev 1966 * Ethernet device flow is being created on. 1967 * @param[out] error 1968 * Pointer to error structure. 1969 * 1970 * @return 1971 * 0 on success, a negative errno value otherwise and rte_errno is set. 1972 */ 1973 int 1974 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1975 uint64_t item_flags, 1976 struct rte_eth_dev *dev, 1977 struct rte_flow_error *error) 1978 { 1979 const struct rte_flow_item_vlan *spec = item->spec; 1980 const struct rte_flow_item_vlan *mask = item->mask; 1981 const struct rte_flow_item_vlan nic_mask = { 1982 .tci = RTE_BE16(UINT16_MAX), 1983 .inner_type = RTE_BE16(UINT16_MAX), 1984 }; 1985 uint16_t vlan_tag = 0; 1986 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1987 int ret; 1988 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1989 MLX5_FLOW_LAYER_INNER_L4) : 1990 (MLX5_FLOW_LAYER_OUTER_L3 | 1991 MLX5_FLOW_LAYER_OUTER_L4); 1992 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1993 MLX5_FLOW_LAYER_OUTER_VLAN; 1994 1995 if (item_flags & vlanm) 1996 return rte_flow_error_set(error, EINVAL, 1997 RTE_FLOW_ERROR_TYPE_ITEM, item, 1998 "multiple VLAN layers not supported"); 1999 else if ((item_flags & l34m) != 0) 2000 return rte_flow_error_set(error, EINVAL, 2001 RTE_FLOW_ERROR_TYPE_ITEM, item, 2002 "VLAN cannot follow L3/L4 layer"); 2003 if (!mask) 2004 mask = &rte_flow_item_vlan_mask; 2005 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2006 (const uint8_t *)&nic_mask, 2007 sizeof(struct rte_flow_item_vlan), 2008 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2009 if (ret) 2010 return ret; 2011 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2012 struct mlx5_priv *priv = dev->data->dev_private; 2013 2014 if (priv->vmwa_context) { 2015 /* 2016 * Non-NULL context means we have a virtual machine 2017 * and SR-IOV enabled, we have to create VLAN interface 2018 * to make hypervisor to setup E-Switch vport 2019 * context correctly. We avoid creating the multiple 2020 * VLAN interfaces, so we cannot support VLAN tag mask. 2021 */ 2022 return rte_flow_error_set(error, EINVAL, 2023 RTE_FLOW_ERROR_TYPE_ITEM, 2024 item, 2025 "VLAN tag mask is not" 2026 " supported in virtual" 2027 " environment"); 2028 } 2029 } 2030 if (spec) { 2031 vlan_tag = spec->tci; 2032 vlan_tag &= mask->tci; 2033 } 2034 /* 2035 * From verbs perspective an empty VLAN is equivalent 2036 * to a packet without VLAN layer. 2037 */ 2038 if (!vlan_tag) 2039 return rte_flow_error_set(error, EINVAL, 2040 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2041 item->spec, 2042 "VLAN cannot be empty"); 2043 return 0; 2044 } 2045 2046 /** 2047 * Validate IPV4 item. 2048 * 2049 * @param[in] item 2050 * Item specification. 2051 * @param[in] item_flags 2052 * Bit-fields that holds the items detected until now. 2053 * @param[in] last_item 2054 * Previous validated item in the pattern items. 2055 * @param[in] ether_type 2056 * Type in the ethernet layer header (including dot1q). 2057 * @param[in] acc_mask 2058 * Acceptable mask, if NULL default internal default mask 2059 * will be used to check whether item fields are supported. 2060 * @param[in] range_accepted 2061 * True if range of values is accepted for specific fields, false otherwise. 2062 * @param[out] error 2063 * Pointer to error structure. 2064 * 2065 * @return 2066 * 0 on success, a negative errno value otherwise and rte_errno is set. 2067 */ 2068 int 2069 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2070 uint64_t item_flags, 2071 uint64_t last_item, 2072 uint16_t ether_type, 2073 const struct rte_flow_item_ipv4 *acc_mask, 2074 bool range_accepted, 2075 struct rte_flow_error *error) 2076 { 2077 const struct rte_flow_item_ipv4 *mask = item->mask; 2078 const struct rte_flow_item_ipv4 *spec = item->spec; 2079 const struct rte_flow_item_ipv4 nic_mask = { 2080 .hdr = { 2081 .src_addr = RTE_BE32(0xffffffff), 2082 .dst_addr = RTE_BE32(0xffffffff), 2083 .type_of_service = 0xff, 2084 .next_proto_id = 0xff, 2085 }, 2086 }; 2087 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2088 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2089 MLX5_FLOW_LAYER_OUTER_L3; 2090 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2091 MLX5_FLOW_LAYER_OUTER_L4; 2092 int ret; 2093 uint8_t next_proto = 0xFF; 2094 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2095 MLX5_FLOW_LAYER_OUTER_VLAN | 2096 MLX5_FLOW_LAYER_INNER_VLAN); 2097 2098 if ((last_item & l2_vlan) && ether_type && 2099 ether_type != RTE_ETHER_TYPE_IPV4) 2100 return rte_flow_error_set(error, EINVAL, 2101 RTE_FLOW_ERROR_TYPE_ITEM, item, 2102 "IPv4 cannot follow L2/VLAN layer " 2103 "which ether type is not IPv4"); 2104 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2105 if (mask && spec) 2106 next_proto = mask->hdr.next_proto_id & 2107 spec->hdr.next_proto_id; 2108 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2109 return rte_flow_error_set(error, EINVAL, 2110 RTE_FLOW_ERROR_TYPE_ITEM, 2111 item, 2112 "multiple tunnel " 2113 "not supported"); 2114 } 2115 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2116 return rte_flow_error_set(error, EINVAL, 2117 RTE_FLOW_ERROR_TYPE_ITEM, item, 2118 "wrong tunnel type - IPv6 specified " 2119 "but IPv4 item provided"); 2120 if (item_flags & l3m) 2121 return rte_flow_error_set(error, ENOTSUP, 2122 RTE_FLOW_ERROR_TYPE_ITEM, item, 2123 "multiple L3 layers not supported"); 2124 else if (item_flags & l4m) 2125 return rte_flow_error_set(error, EINVAL, 2126 RTE_FLOW_ERROR_TYPE_ITEM, item, 2127 "L3 cannot follow an L4 layer."); 2128 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2129 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2130 return rte_flow_error_set(error, EINVAL, 2131 RTE_FLOW_ERROR_TYPE_ITEM, item, 2132 "L3 cannot follow an NVGRE layer."); 2133 if (!mask) 2134 mask = &rte_flow_item_ipv4_mask; 2135 else if (mask->hdr.next_proto_id != 0 && 2136 mask->hdr.next_proto_id != 0xff) 2137 return rte_flow_error_set(error, EINVAL, 2138 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2139 "partial mask is not supported" 2140 " for protocol"); 2141 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2142 acc_mask ? (const uint8_t *)acc_mask 2143 : (const uint8_t *)&nic_mask, 2144 sizeof(struct rte_flow_item_ipv4), 2145 range_accepted, error); 2146 if (ret < 0) 2147 return ret; 2148 return 0; 2149 } 2150 2151 /** 2152 * Validate IPV6 item. 2153 * 2154 * @param[in] item 2155 * Item specification. 2156 * @param[in] item_flags 2157 * Bit-fields that holds the items detected until now. 2158 * @param[in] last_item 2159 * Previous validated item in the pattern items. 2160 * @param[in] ether_type 2161 * Type in the ethernet layer header (including dot1q). 2162 * @param[in] acc_mask 2163 * Acceptable mask, if NULL default internal default mask 2164 * will be used to check whether item fields are supported. 2165 * @param[out] error 2166 * Pointer to error structure. 2167 * 2168 * @return 2169 * 0 on success, a negative errno value otherwise and rte_errno is set. 2170 */ 2171 int 2172 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2173 uint64_t item_flags, 2174 uint64_t last_item, 2175 uint16_t ether_type, 2176 const struct rte_flow_item_ipv6 *acc_mask, 2177 struct rte_flow_error *error) 2178 { 2179 const struct rte_flow_item_ipv6 *mask = item->mask; 2180 const struct rte_flow_item_ipv6 *spec = item->spec; 2181 const struct rte_flow_item_ipv6 nic_mask = { 2182 .hdr = { 2183 .src_addr = 2184 "\xff\xff\xff\xff\xff\xff\xff\xff" 2185 "\xff\xff\xff\xff\xff\xff\xff\xff", 2186 .dst_addr = 2187 "\xff\xff\xff\xff\xff\xff\xff\xff" 2188 "\xff\xff\xff\xff\xff\xff\xff\xff", 2189 .vtc_flow = RTE_BE32(0xffffffff), 2190 .proto = 0xff, 2191 }, 2192 }; 2193 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2194 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2195 MLX5_FLOW_LAYER_OUTER_L3; 2196 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2197 MLX5_FLOW_LAYER_OUTER_L4; 2198 int ret; 2199 uint8_t next_proto = 0xFF; 2200 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2201 MLX5_FLOW_LAYER_OUTER_VLAN | 2202 MLX5_FLOW_LAYER_INNER_VLAN); 2203 2204 if ((last_item & l2_vlan) && ether_type && 2205 ether_type != RTE_ETHER_TYPE_IPV6) 2206 return rte_flow_error_set(error, EINVAL, 2207 RTE_FLOW_ERROR_TYPE_ITEM, item, 2208 "IPv6 cannot follow L2/VLAN layer " 2209 "which ether type is not IPv6"); 2210 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2211 next_proto = spec->hdr.proto; 2212 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 2213 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2214 return rte_flow_error_set(error, EINVAL, 2215 RTE_FLOW_ERROR_TYPE_ITEM, 2216 item, 2217 "multiple tunnel " 2218 "not supported"); 2219 } 2220 if (next_proto == IPPROTO_HOPOPTS || 2221 next_proto == IPPROTO_ROUTING || 2222 next_proto == IPPROTO_FRAGMENT || 2223 next_proto == IPPROTO_ESP || 2224 next_proto == IPPROTO_AH || 2225 next_proto == IPPROTO_DSTOPTS) 2226 return rte_flow_error_set(error, EINVAL, 2227 RTE_FLOW_ERROR_TYPE_ITEM, item, 2228 "IPv6 proto (next header) should " 2229 "not be set as extension header"); 2230 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2231 return rte_flow_error_set(error, EINVAL, 2232 RTE_FLOW_ERROR_TYPE_ITEM, item, 2233 "wrong tunnel type - IPv4 specified " 2234 "but IPv6 item provided"); 2235 if (item_flags & l3m) 2236 return rte_flow_error_set(error, ENOTSUP, 2237 RTE_FLOW_ERROR_TYPE_ITEM, item, 2238 "multiple L3 layers not supported"); 2239 else if (item_flags & l4m) 2240 return rte_flow_error_set(error, EINVAL, 2241 RTE_FLOW_ERROR_TYPE_ITEM, item, 2242 "L3 cannot follow an L4 layer."); 2243 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2244 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2245 return rte_flow_error_set(error, EINVAL, 2246 RTE_FLOW_ERROR_TYPE_ITEM, item, 2247 "L3 cannot follow an NVGRE layer."); 2248 if (!mask) 2249 mask = &rte_flow_item_ipv6_mask; 2250 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2251 acc_mask ? (const uint8_t *)acc_mask 2252 : (const uint8_t *)&nic_mask, 2253 sizeof(struct rte_flow_item_ipv6), 2254 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2255 if (ret < 0) 2256 return ret; 2257 return 0; 2258 } 2259 2260 /** 2261 * Validate UDP item. 2262 * 2263 * @param[in] item 2264 * Item specification. 2265 * @param[in] item_flags 2266 * Bit-fields that holds the items detected until now. 2267 * @param[in] target_protocol 2268 * The next protocol in the previous item. 2269 * @param[in] flow_mask 2270 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2271 * @param[out] error 2272 * Pointer to error structure. 2273 * 2274 * @return 2275 * 0 on success, a negative errno value otherwise and rte_errno is set. 2276 */ 2277 int 2278 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2279 uint64_t item_flags, 2280 uint8_t target_protocol, 2281 struct rte_flow_error *error) 2282 { 2283 const struct rte_flow_item_udp *mask = item->mask; 2284 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2285 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2286 MLX5_FLOW_LAYER_OUTER_L3; 2287 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2288 MLX5_FLOW_LAYER_OUTER_L4; 2289 int ret; 2290 2291 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2292 return rte_flow_error_set(error, EINVAL, 2293 RTE_FLOW_ERROR_TYPE_ITEM, item, 2294 "protocol filtering not compatible" 2295 " with UDP layer"); 2296 if (!(item_flags & l3m)) 2297 return rte_flow_error_set(error, EINVAL, 2298 RTE_FLOW_ERROR_TYPE_ITEM, item, 2299 "L3 is mandatory to filter on L4"); 2300 if (item_flags & l4m) 2301 return rte_flow_error_set(error, EINVAL, 2302 RTE_FLOW_ERROR_TYPE_ITEM, item, 2303 "multiple L4 layers not supported"); 2304 if (!mask) 2305 mask = &rte_flow_item_udp_mask; 2306 ret = mlx5_flow_item_acceptable 2307 (item, (const uint8_t *)mask, 2308 (const uint8_t *)&rte_flow_item_udp_mask, 2309 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2310 error); 2311 if (ret < 0) 2312 return ret; 2313 return 0; 2314 } 2315 2316 /** 2317 * Validate TCP item. 2318 * 2319 * @param[in] item 2320 * Item specification. 2321 * @param[in] item_flags 2322 * Bit-fields that holds the items detected until now. 2323 * @param[in] target_protocol 2324 * The next protocol in the previous item. 2325 * @param[out] error 2326 * Pointer to error structure. 2327 * 2328 * @return 2329 * 0 on success, a negative errno value otherwise and rte_errno is set. 2330 */ 2331 int 2332 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2333 uint64_t item_flags, 2334 uint8_t target_protocol, 2335 const struct rte_flow_item_tcp *flow_mask, 2336 struct rte_flow_error *error) 2337 { 2338 const struct rte_flow_item_tcp *mask = item->mask; 2339 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2340 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2341 MLX5_FLOW_LAYER_OUTER_L3; 2342 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2343 MLX5_FLOW_LAYER_OUTER_L4; 2344 int ret; 2345 2346 MLX5_ASSERT(flow_mask); 2347 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2348 return rte_flow_error_set(error, EINVAL, 2349 RTE_FLOW_ERROR_TYPE_ITEM, item, 2350 "protocol filtering not compatible" 2351 " with TCP layer"); 2352 if (!(item_flags & l3m)) 2353 return rte_flow_error_set(error, EINVAL, 2354 RTE_FLOW_ERROR_TYPE_ITEM, item, 2355 "L3 is mandatory to filter on L4"); 2356 if (item_flags & l4m) 2357 return rte_flow_error_set(error, EINVAL, 2358 RTE_FLOW_ERROR_TYPE_ITEM, item, 2359 "multiple L4 layers not supported"); 2360 if (!mask) 2361 mask = &rte_flow_item_tcp_mask; 2362 ret = mlx5_flow_item_acceptable 2363 (item, (const uint8_t *)mask, 2364 (const uint8_t *)flow_mask, 2365 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2366 error); 2367 if (ret < 0) 2368 return ret; 2369 return 0; 2370 } 2371 2372 /** 2373 * Validate VXLAN item. 2374 * 2375 * @param[in] item 2376 * Item specification. 2377 * @param[in] item_flags 2378 * Bit-fields that holds the items detected until now. 2379 * @param[in] target_protocol 2380 * The next protocol in the previous item. 2381 * @param[out] error 2382 * Pointer to error structure. 2383 * 2384 * @return 2385 * 0 on success, a negative errno value otherwise and rte_errno is set. 2386 */ 2387 int 2388 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2389 uint64_t item_flags, 2390 struct rte_flow_error *error) 2391 { 2392 const struct rte_flow_item_vxlan *spec = item->spec; 2393 const struct rte_flow_item_vxlan *mask = item->mask; 2394 int ret; 2395 union vni { 2396 uint32_t vlan_id; 2397 uint8_t vni[4]; 2398 } id = { .vlan_id = 0, }; 2399 2400 2401 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2402 return rte_flow_error_set(error, ENOTSUP, 2403 RTE_FLOW_ERROR_TYPE_ITEM, item, 2404 "multiple tunnel layers not" 2405 " supported"); 2406 /* 2407 * Verify only UDPv4 is present as defined in 2408 * https://tools.ietf.org/html/rfc7348 2409 */ 2410 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2411 return rte_flow_error_set(error, EINVAL, 2412 RTE_FLOW_ERROR_TYPE_ITEM, item, 2413 "no outer UDP layer found"); 2414 if (!mask) 2415 mask = &rte_flow_item_vxlan_mask; 2416 ret = mlx5_flow_item_acceptable 2417 (item, (const uint8_t *)mask, 2418 (const uint8_t *)&rte_flow_item_vxlan_mask, 2419 sizeof(struct rte_flow_item_vxlan), 2420 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2421 if (ret < 0) 2422 return ret; 2423 if (spec) { 2424 memcpy(&id.vni[1], spec->vni, 3); 2425 memcpy(&id.vni[1], mask->vni, 3); 2426 } 2427 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2428 return rte_flow_error_set(error, ENOTSUP, 2429 RTE_FLOW_ERROR_TYPE_ITEM, item, 2430 "VXLAN tunnel must be fully defined"); 2431 return 0; 2432 } 2433 2434 /** 2435 * Validate VXLAN_GPE item. 2436 * 2437 * @param[in] item 2438 * Item specification. 2439 * @param[in] item_flags 2440 * Bit-fields that holds the items detected until now. 2441 * @param[in] priv 2442 * Pointer to the private data structure. 2443 * @param[in] target_protocol 2444 * The next protocol in the previous item. 2445 * @param[out] error 2446 * Pointer to error structure. 2447 * 2448 * @return 2449 * 0 on success, a negative errno value otherwise and rte_errno is set. 2450 */ 2451 int 2452 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2453 uint64_t item_flags, 2454 struct rte_eth_dev *dev, 2455 struct rte_flow_error *error) 2456 { 2457 struct mlx5_priv *priv = dev->data->dev_private; 2458 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2459 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2460 int ret; 2461 union vni { 2462 uint32_t vlan_id; 2463 uint8_t vni[4]; 2464 } id = { .vlan_id = 0, }; 2465 2466 if (!priv->config.l3_vxlan_en) 2467 return rte_flow_error_set(error, ENOTSUP, 2468 RTE_FLOW_ERROR_TYPE_ITEM, item, 2469 "L3 VXLAN is not enabled by device" 2470 " parameter and/or not configured in" 2471 " firmware"); 2472 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2473 return rte_flow_error_set(error, ENOTSUP, 2474 RTE_FLOW_ERROR_TYPE_ITEM, item, 2475 "multiple tunnel layers not" 2476 " supported"); 2477 /* 2478 * Verify only UDPv4 is present as defined in 2479 * https://tools.ietf.org/html/rfc7348 2480 */ 2481 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2482 return rte_flow_error_set(error, EINVAL, 2483 RTE_FLOW_ERROR_TYPE_ITEM, item, 2484 "no outer UDP layer found"); 2485 if (!mask) 2486 mask = &rte_flow_item_vxlan_gpe_mask; 2487 ret = mlx5_flow_item_acceptable 2488 (item, (const uint8_t *)mask, 2489 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2490 sizeof(struct rte_flow_item_vxlan_gpe), 2491 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2492 if (ret < 0) 2493 return ret; 2494 if (spec) { 2495 if (spec->protocol) 2496 return rte_flow_error_set(error, ENOTSUP, 2497 RTE_FLOW_ERROR_TYPE_ITEM, 2498 item, 2499 "VxLAN-GPE protocol" 2500 " not supported"); 2501 memcpy(&id.vni[1], spec->vni, 3); 2502 memcpy(&id.vni[1], mask->vni, 3); 2503 } 2504 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2505 return rte_flow_error_set(error, ENOTSUP, 2506 RTE_FLOW_ERROR_TYPE_ITEM, item, 2507 "VXLAN-GPE tunnel must be fully" 2508 " defined"); 2509 return 0; 2510 } 2511 /** 2512 * Validate GRE Key item. 2513 * 2514 * @param[in] item 2515 * Item specification. 2516 * @param[in] item_flags 2517 * Bit flags to mark detected items. 2518 * @param[in] gre_item 2519 * Pointer to gre_item 2520 * @param[out] error 2521 * Pointer to error structure. 2522 * 2523 * @return 2524 * 0 on success, a negative errno value otherwise and rte_errno is set. 2525 */ 2526 int 2527 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2528 uint64_t item_flags, 2529 const struct rte_flow_item *gre_item, 2530 struct rte_flow_error *error) 2531 { 2532 const rte_be32_t *mask = item->mask; 2533 int ret = 0; 2534 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2535 const struct rte_flow_item_gre *gre_spec; 2536 const struct rte_flow_item_gre *gre_mask; 2537 2538 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2539 return rte_flow_error_set(error, ENOTSUP, 2540 RTE_FLOW_ERROR_TYPE_ITEM, item, 2541 "Multiple GRE key not support"); 2542 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2543 return rte_flow_error_set(error, ENOTSUP, 2544 RTE_FLOW_ERROR_TYPE_ITEM, item, 2545 "No preceding GRE header"); 2546 if (item_flags & MLX5_FLOW_LAYER_INNER) 2547 return rte_flow_error_set(error, ENOTSUP, 2548 RTE_FLOW_ERROR_TYPE_ITEM, item, 2549 "GRE key following a wrong item"); 2550 gre_mask = gre_item->mask; 2551 if (!gre_mask) 2552 gre_mask = &rte_flow_item_gre_mask; 2553 gre_spec = gre_item->spec; 2554 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2555 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2556 return rte_flow_error_set(error, EINVAL, 2557 RTE_FLOW_ERROR_TYPE_ITEM, item, 2558 "Key bit must be on"); 2559 2560 if (!mask) 2561 mask = &gre_key_default_mask; 2562 ret = mlx5_flow_item_acceptable 2563 (item, (const uint8_t *)mask, 2564 (const uint8_t *)&gre_key_default_mask, 2565 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2566 return ret; 2567 } 2568 2569 /** 2570 * Validate GRE item. 2571 * 2572 * @param[in] item 2573 * Item specification. 2574 * @param[in] item_flags 2575 * Bit flags to mark detected items. 2576 * @param[in] target_protocol 2577 * The next protocol in the previous item. 2578 * @param[out] error 2579 * Pointer to error structure. 2580 * 2581 * @return 2582 * 0 on success, a negative errno value otherwise and rte_errno is set. 2583 */ 2584 int 2585 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2586 uint64_t item_flags, 2587 uint8_t target_protocol, 2588 struct rte_flow_error *error) 2589 { 2590 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2591 const struct rte_flow_item_gre *mask = item->mask; 2592 int ret; 2593 const struct rte_flow_item_gre nic_mask = { 2594 .c_rsvd0_ver = RTE_BE16(0xB000), 2595 .protocol = RTE_BE16(UINT16_MAX), 2596 }; 2597 2598 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2599 return rte_flow_error_set(error, EINVAL, 2600 RTE_FLOW_ERROR_TYPE_ITEM, item, 2601 "protocol filtering not compatible" 2602 " with this GRE layer"); 2603 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2604 return rte_flow_error_set(error, ENOTSUP, 2605 RTE_FLOW_ERROR_TYPE_ITEM, item, 2606 "multiple tunnel layers not" 2607 " supported"); 2608 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2609 return rte_flow_error_set(error, ENOTSUP, 2610 RTE_FLOW_ERROR_TYPE_ITEM, item, 2611 "L3 Layer is missing"); 2612 if (!mask) 2613 mask = &rte_flow_item_gre_mask; 2614 ret = mlx5_flow_item_acceptable 2615 (item, (const uint8_t *)mask, 2616 (const uint8_t *)&nic_mask, 2617 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2618 error); 2619 if (ret < 0) 2620 return ret; 2621 #ifndef HAVE_MLX5DV_DR 2622 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2623 if (spec && (spec->protocol & mask->protocol)) 2624 return rte_flow_error_set(error, ENOTSUP, 2625 RTE_FLOW_ERROR_TYPE_ITEM, item, 2626 "without MPLS support the" 2627 " specification cannot be used for" 2628 " filtering"); 2629 #endif 2630 #endif 2631 return 0; 2632 } 2633 2634 /** 2635 * Validate Geneve item. 2636 * 2637 * @param[in] item 2638 * Item specification. 2639 * @param[in] itemFlags 2640 * Bit-fields that holds the items detected until now. 2641 * @param[in] enPriv 2642 * Pointer to the private data structure. 2643 * @param[out] error 2644 * Pointer to error structure. 2645 * 2646 * @return 2647 * 0 on success, a negative errno value otherwise and rte_errno is set. 2648 */ 2649 2650 int 2651 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2652 uint64_t item_flags, 2653 struct rte_eth_dev *dev, 2654 struct rte_flow_error *error) 2655 { 2656 struct mlx5_priv *priv = dev->data->dev_private; 2657 const struct rte_flow_item_geneve *spec = item->spec; 2658 const struct rte_flow_item_geneve *mask = item->mask; 2659 int ret; 2660 uint16_t gbhdr; 2661 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2662 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2663 const struct rte_flow_item_geneve nic_mask = { 2664 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2665 .vni = "\xff\xff\xff", 2666 .protocol = RTE_BE16(UINT16_MAX), 2667 }; 2668 2669 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2670 return rte_flow_error_set(error, ENOTSUP, 2671 RTE_FLOW_ERROR_TYPE_ITEM, item, 2672 "L3 Geneve is not enabled by device" 2673 " parameter and/or not configured in" 2674 " firmware"); 2675 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2676 return rte_flow_error_set(error, ENOTSUP, 2677 RTE_FLOW_ERROR_TYPE_ITEM, item, 2678 "multiple tunnel layers not" 2679 " supported"); 2680 /* 2681 * Verify only UDPv4 is present as defined in 2682 * https://tools.ietf.org/html/rfc7348 2683 */ 2684 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2685 return rte_flow_error_set(error, EINVAL, 2686 RTE_FLOW_ERROR_TYPE_ITEM, item, 2687 "no outer UDP layer found"); 2688 if (!mask) 2689 mask = &rte_flow_item_geneve_mask; 2690 ret = mlx5_flow_item_acceptable 2691 (item, (const uint8_t *)mask, 2692 (const uint8_t *)&nic_mask, 2693 sizeof(struct rte_flow_item_geneve), 2694 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2695 if (ret) 2696 return ret; 2697 if (spec) { 2698 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2699 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2700 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2701 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2702 return rte_flow_error_set(error, ENOTSUP, 2703 RTE_FLOW_ERROR_TYPE_ITEM, 2704 item, 2705 "Geneve protocol unsupported" 2706 " fields are being used"); 2707 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2708 return rte_flow_error_set 2709 (error, ENOTSUP, 2710 RTE_FLOW_ERROR_TYPE_ITEM, 2711 item, 2712 "Unsupported Geneve options length"); 2713 } 2714 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2715 return rte_flow_error_set 2716 (error, ENOTSUP, 2717 RTE_FLOW_ERROR_TYPE_ITEM, item, 2718 "Geneve tunnel must be fully defined"); 2719 return 0; 2720 } 2721 2722 /** 2723 * Validate MPLS item. 2724 * 2725 * @param[in] dev 2726 * Pointer to the rte_eth_dev structure. 2727 * @param[in] item 2728 * Item specification. 2729 * @param[in] item_flags 2730 * Bit-fields that holds the items detected until now. 2731 * @param[in] prev_layer 2732 * The protocol layer indicated in previous item. 2733 * @param[out] error 2734 * Pointer to error structure. 2735 * 2736 * @return 2737 * 0 on success, a negative errno value otherwise and rte_errno is set. 2738 */ 2739 int 2740 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2741 const struct rte_flow_item *item __rte_unused, 2742 uint64_t item_flags __rte_unused, 2743 uint64_t prev_layer __rte_unused, 2744 struct rte_flow_error *error) 2745 { 2746 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2747 const struct rte_flow_item_mpls *mask = item->mask; 2748 struct mlx5_priv *priv = dev->data->dev_private; 2749 int ret; 2750 2751 if (!priv->config.mpls_en) 2752 return rte_flow_error_set(error, ENOTSUP, 2753 RTE_FLOW_ERROR_TYPE_ITEM, item, 2754 "MPLS not supported or" 2755 " disabled in firmware" 2756 " configuration."); 2757 /* MPLS over IP, UDP, GRE is allowed */ 2758 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2759 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2760 MLX5_FLOW_LAYER_GRE))) 2761 return rte_flow_error_set(error, EINVAL, 2762 RTE_FLOW_ERROR_TYPE_ITEM, item, 2763 "protocol filtering not compatible" 2764 " with MPLS layer"); 2765 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2766 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2767 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2768 return rte_flow_error_set(error, ENOTSUP, 2769 RTE_FLOW_ERROR_TYPE_ITEM, item, 2770 "multiple tunnel layers not" 2771 " supported"); 2772 if (!mask) 2773 mask = &rte_flow_item_mpls_mask; 2774 ret = mlx5_flow_item_acceptable 2775 (item, (const uint8_t *)mask, 2776 (const uint8_t *)&rte_flow_item_mpls_mask, 2777 sizeof(struct rte_flow_item_mpls), 2778 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2779 if (ret < 0) 2780 return ret; 2781 return 0; 2782 #else 2783 return rte_flow_error_set(error, ENOTSUP, 2784 RTE_FLOW_ERROR_TYPE_ITEM, item, 2785 "MPLS is not supported by Verbs, please" 2786 " update."); 2787 #endif 2788 } 2789 2790 /** 2791 * Validate NVGRE item. 2792 * 2793 * @param[in] item 2794 * Item specification. 2795 * @param[in] item_flags 2796 * Bit flags to mark detected items. 2797 * @param[in] target_protocol 2798 * The next protocol in the previous item. 2799 * @param[out] error 2800 * Pointer to error structure. 2801 * 2802 * @return 2803 * 0 on success, a negative errno value otherwise and rte_errno is set. 2804 */ 2805 int 2806 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2807 uint64_t item_flags, 2808 uint8_t target_protocol, 2809 struct rte_flow_error *error) 2810 { 2811 const struct rte_flow_item_nvgre *mask = item->mask; 2812 int ret; 2813 2814 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2815 return rte_flow_error_set(error, EINVAL, 2816 RTE_FLOW_ERROR_TYPE_ITEM, item, 2817 "protocol filtering not compatible" 2818 " with this GRE layer"); 2819 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2820 return rte_flow_error_set(error, ENOTSUP, 2821 RTE_FLOW_ERROR_TYPE_ITEM, item, 2822 "multiple tunnel layers not" 2823 " supported"); 2824 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2825 return rte_flow_error_set(error, ENOTSUP, 2826 RTE_FLOW_ERROR_TYPE_ITEM, item, 2827 "L3 Layer is missing"); 2828 if (!mask) 2829 mask = &rte_flow_item_nvgre_mask; 2830 ret = mlx5_flow_item_acceptable 2831 (item, (const uint8_t *)mask, 2832 (const uint8_t *)&rte_flow_item_nvgre_mask, 2833 sizeof(struct rte_flow_item_nvgre), 2834 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2835 if (ret < 0) 2836 return ret; 2837 return 0; 2838 } 2839 2840 /** 2841 * Validate eCPRI item. 2842 * 2843 * @param[in] item 2844 * Item specification. 2845 * @param[in] item_flags 2846 * Bit-fields that holds the items detected until now. 2847 * @param[in] last_item 2848 * Previous validated item in the pattern items. 2849 * @param[in] ether_type 2850 * Type in the ethernet layer header (including dot1q). 2851 * @param[in] acc_mask 2852 * Acceptable mask, if NULL default internal default mask 2853 * will be used to check whether item fields are supported. 2854 * @param[out] error 2855 * Pointer to error structure. 2856 * 2857 * @return 2858 * 0 on success, a negative errno value otherwise and rte_errno is set. 2859 */ 2860 int 2861 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 2862 uint64_t item_flags, 2863 uint64_t last_item, 2864 uint16_t ether_type, 2865 const struct rte_flow_item_ecpri *acc_mask, 2866 struct rte_flow_error *error) 2867 { 2868 const struct rte_flow_item_ecpri *mask = item->mask; 2869 const struct rte_flow_item_ecpri nic_mask = { 2870 .hdr = { 2871 .common = { 2872 .u32 = 2873 RTE_BE32(((const struct rte_ecpri_common_hdr) { 2874 .type = 0xFF, 2875 }).u32), 2876 }, 2877 .dummy[0] = 0xFFFFFFFF, 2878 }, 2879 }; 2880 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 2881 MLX5_FLOW_LAYER_OUTER_VLAN); 2882 struct rte_flow_item_ecpri mask_lo; 2883 2884 if ((last_item & outer_l2_vlan) && ether_type && 2885 ether_type != RTE_ETHER_TYPE_ECPRI) 2886 return rte_flow_error_set(error, EINVAL, 2887 RTE_FLOW_ERROR_TYPE_ITEM, item, 2888 "eCPRI cannot follow L2/VLAN layer " 2889 "which ether type is not 0xAEFE."); 2890 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2891 return rte_flow_error_set(error, EINVAL, 2892 RTE_FLOW_ERROR_TYPE_ITEM, item, 2893 "eCPRI with tunnel is not supported " 2894 "right now."); 2895 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 2896 return rte_flow_error_set(error, ENOTSUP, 2897 RTE_FLOW_ERROR_TYPE_ITEM, item, 2898 "multiple L3 layers not supported"); 2899 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 2900 return rte_flow_error_set(error, EINVAL, 2901 RTE_FLOW_ERROR_TYPE_ITEM, item, 2902 "eCPRI cannot follow a TCP layer."); 2903 /* In specification, eCPRI could be over UDP layer. */ 2904 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 2905 return rte_flow_error_set(error, EINVAL, 2906 RTE_FLOW_ERROR_TYPE_ITEM, item, 2907 "eCPRI over UDP layer is not yet " 2908 "supported right now."); 2909 /* Mask for type field in common header could be zero. */ 2910 if (!mask) 2911 mask = &rte_flow_item_ecpri_mask; 2912 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 2913 /* Input mask is in big-endian format. */ 2914 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 2915 return rte_flow_error_set(error, EINVAL, 2916 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2917 "partial mask is not supported " 2918 "for protocol"); 2919 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 2920 return rte_flow_error_set(error, EINVAL, 2921 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2922 "message header mask must be after " 2923 "a type mask"); 2924 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2925 acc_mask ? (const uint8_t *)acc_mask 2926 : (const uint8_t *)&nic_mask, 2927 sizeof(struct rte_flow_item_ecpri), 2928 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2929 } 2930 2931 /* Allocate unique ID for the split Q/RSS subflows. */ 2932 static uint32_t 2933 flow_qrss_get_id(struct rte_eth_dev *dev) 2934 { 2935 struct mlx5_priv *priv = dev->data->dev_private; 2936 uint32_t qrss_id, ret; 2937 2938 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2939 if (ret) 2940 return 0; 2941 MLX5_ASSERT(qrss_id); 2942 return qrss_id; 2943 } 2944 2945 /* Free unique ID for the split Q/RSS subflows. */ 2946 static void 2947 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2948 { 2949 struct mlx5_priv *priv = dev->data->dev_private; 2950 2951 if (qrss_id) 2952 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2953 } 2954 2955 /** 2956 * Release resource related QUEUE/RSS action split. 2957 * 2958 * @param dev 2959 * Pointer to Ethernet device. 2960 * @param flow 2961 * Flow to release id's from. 2962 */ 2963 static void 2964 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2965 struct rte_flow *flow) 2966 { 2967 struct mlx5_priv *priv = dev->data->dev_private; 2968 uint32_t handle_idx; 2969 struct mlx5_flow_handle *dev_handle; 2970 2971 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2972 handle_idx, dev_handle, next) 2973 if (dev_handle->split_flow_id) 2974 flow_qrss_free_id(dev, dev_handle->split_flow_id); 2975 } 2976 2977 static int 2978 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2979 const struct rte_flow_attr *attr __rte_unused, 2980 const struct rte_flow_item items[] __rte_unused, 2981 const struct rte_flow_action actions[] __rte_unused, 2982 bool external __rte_unused, 2983 int hairpin __rte_unused, 2984 struct rte_flow_error *error) 2985 { 2986 return rte_flow_error_set(error, ENOTSUP, 2987 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2988 } 2989 2990 static struct mlx5_flow * 2991 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2992 const struct rte_flow_attr *attr __rte_unused, 2993 const struct rte_flow_item items[] __rte_unused, 2994 const struct rte_flow_action actions[] __rte_unused, 2995 struct rte_flow_error *error) 2996 { 2997 rte_flow_error_set(error, ENOTSUP, 2998 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2999 return NULL; 3000 } 3001 3002 static int 3003 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3004 struct mlx5_flow *dev_flow __rte_unused, 3005 const struct rte_flow_attr *attr __rte_unused, 3006 const struct rte_flow_item items[] __rte_unused, 3007 const struct rte_flow_action actions[] __rte_unused, 3008 struct rte_flow_error *error) 3009 { 3010 return rte_flow_error_set(error, ENOTSUP, 3011 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3012 } 3013 3014 static int 3015 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3016 struct rte_flow *flow __rte_unused, 3017 struct rte_flow_error *error) 3018 { 3019 return rte_flow_error_set(error, ENOTSUP, 3020 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3021 } 3022 3023 static void 3024 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3025 struct rte_flow *flow __rte_unused) 3026 { 3027 } 3028 3029 static void 3030 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3031 struct rte_flow *flow __rte_unused) 3032 { 3033 } 3034 3035 static int 3036 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3037 struct rte_flow *flow __rte_unused, 3038 const struct rte_flow_action *actions __rte_unused, 3039 void *data __rte_unused, 3040 struct rte_flow_error *error) 3041 { 3042 return rte_flow_error_set(error, ENOTSUP, 3043 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3044 } 3045 3046 static int 3047 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3048 uint32_t domains __rte_unused, 3049 uint32_t flags __rte_unused) 3050 { 3051 return 0; 3052 } 3053 3054 /* Void driver to protect from null pointer reference. */ 3055 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3056 .validate = flow_null_validate, 3057 .prepare = flow_null_prepare, 3058 .translate = flow_null_translate, 3059 .apply = flow_null_apply, 3060 .remove = flow_null_remove, 3061 .destroy = flow_null_destroy, 3062 .query = flow_null_query, 3063 .sync_domain = flow_null_sync_domain, 3064 }; 3065 3066 /** 3067 * Select flow driver type according to flow attributes and device 3068 * configuration. 3069 * 3070 * @param[in] dev 3071 * Pointer to the dev structure. 3072 * @param[in] attr 3073 * Pointer to the flow attributes. 3074 * 3075 * @return 3076 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3077 */ 3078 static enum mlx5_flow_drv_type 3079 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3080 { 3081 struct mlx5_priv *priv = dev->data->dev_private; 3082 /* The OS can determine first a specific flow type (DV, VERBS) */ 3083 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3084 3085 if (type != MLX5_FLOW_TYPE_MAX) 3086 return type; 3087 /* If no OS specific type - continue with DV/VERBS selection */ 3088 if (attr->transfer && priv->config.dv_esw_en) 3089 type = MLX5_FLOW_TYPE_DV; 3090 if (!attr->transfer) 3091 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3092 MLX5_FLOW_TYPE_VERBS; 3093 return type; 3094 } 3095 3096 #define flow_get_drv_ops(type) flow_drv_ops[type] 3097 3098 /** 3099 * Flow driver validation API. This abstracts calling driver specific functions. 3100 * The type of flow driver is determined according to flow attributes. 3101 * 3102 * @param[in] dev 3103 * Pointer to the dev structure. 3104 * @param[in] attr 3105 * Pointer to the flow attributes. 3106 * @param[in] items 3107 * Pointer to the list of items. 3108 * @param[in] actions 3109 * Pointer to the list of actions. 3110 * @param[in] external 3111 * This flow rule is created by request external to PMD. 3112 * @param[in] hairpin 3113 * Number of hairpin TX actions, 0 means classic flow. 3114 * @param[out] error 3115 * Pointer to the error structure. 3116 * 3117 * @return 3118 * 0 on success, a negative errno value otherwise and rte_errno is set. 3119 */ 3120 static inline int 3121 flow_drv_validate(struct rte_eth_dev *dev, 3122 const struct rte_flow_attr *attr, 3123 const struct rte_flow_item items[], 3124 const struct rte_flow_action actions[], 3125 bool external, int hairpin, struct rte_flow_error *error) 3126 { 3127 const struct mlx5_flow_driver_ops *fops; 3128 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3129 3130 fops = flow_get_drv_ops(type); 3131 return fops->validate(dev, attr, items, actions, external, 3132 hairpin, error); 3133 } 3134 3135 /** 3136 * Flow driver preparation API. This abstracts calling driver specific 3137 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3138 * calculates the size of memory required for device flow, allocates the memory, 3139 * initializes the device flow and returns the pointer. 3140 * 3141 * @note 3142 * This function initializes device flow structure such as dv or verbs in 3143 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3144 * rest. For example, adding returning device flow to flow->dev_flow list and 3145 * setting backward reference to the flow should be done out of this function. 3146 * layers field is not filled either. 3147 * 3148 * @param[in] dev 3149 * Pointer to the dev structure. 3150 * @param[in] attr 3151 * Pointer to the flow attributes. 3152 * @param[in] items 3153 * Pointer to the list of items. 3154 * @param[in] actions 3155 * Pointer to the list of actions. 3156 * @param[in] flow_idx 3157 * This memory pool index to the flow. 3158 * @param[out] error 3159 * Pointer to the error structure. 3160 * 3161 * @return 3162 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3163 */ 3164 static inline struct mlx5_flow * 3165 flow_drv_prepare(struct rte_eth_dev *dev, 3166 const struct rte_flow *flow, 3167 const struct rte_flow_attr *attr, 3168 const struct rte_flow_item items[], 3169 const struct rte_flow_action actions[], 3170 uint32_t flow_idx, 3171 struct rte_flow_error *error) 3172 { 3173 const struct mlx5_flow_driver_ops *fops; 3174 enum mlx5_flow_drv_type type = flow->drv_type; 3175 struct mlx5_flow *mlx5_flow = NULL; 3176 3177 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3178 fops = flow_get_drv_ops(type); 3179 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3180 if (mlx5_flow) 3181 mlx5_flow->flow_idx = flow_idx; 3182 return mlx5_flow; 3183 } 3184 3185 /** 3186 * Flow driver translation API. This abstracts calling driver specific 3187 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3188 * translates a generic flow into a driver flow. flow_drv_prepare() must 3189 * precede. 3190 * 3191 * @note 3192 * dev_flow->layers could be filled as a result of parsing during translation 3193 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3194 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3195 * flow->actions could be overwritten even though all the expanded dev_flows 3196 * have the same actions. 3197 * 3198 * @param[in] dev 3199 * Pointer to the rte dev structure. 3200 * @param[in, out] dev_flow 3201 * Pointer to the mlx5 flow. 3202 * @param[in] attr 3203 * Pointer to the flow attributes. 3204 * @param[in] items 3205 * Pointer to the list of items. 3206 * @param[in] actions 3207 * Pointer to the list of actions. 3208 * @param[out] error 3209 * Pointer to the error structure. 3210 * 3211 * @return 3212 * 0 on success, a negative errno value otherwise and rte_errno is set. 3213 */ 3214 static inline int 3215 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3216 const struct rte_flow_attr *attr, 3217 const struct rte_flow_item items[], 3218 const struct rte_flow_action actions[], 3219 struct rte_flow_error *error) 3220 { 3221 const struct mlx5_flow_driver_ops *fops; 3222 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3223 3224 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3225 fops = flow_get_drv_ops(type); 3226 return fops->translate(dev, dev_flow, attr, items, actions, error); 3227 } 3228 3229 /** 3230 * Flow driver apply API. This abstracts calling driver specific functions. 3231 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3232 * translated driver flows on to device. flow_drv_translate() must precede. 3233 * 3234 * @param[in] dev 3235 * Pointer to Ethernet device structure. 3236 * @param[in, out] flow 3237 * Pointer to flow structure. 3238 * @param[out] error 3239 * Pointer to error structure. 3240 * 3241 * @return 3242 * 0 on success, a negative errno value otherwise and rte_errno is set. 3243 */ 3244 static inline int 3245 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3246 struct rte_flow_error *error) 3247 { 3248 const struct mlx5_flow_driver_ops *fops; 3249 enum mlx5_flow_drv_type type = flow->drv_type; 3250 3251 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3252 fops = flow_get_drv_ops(type); 3253 return fops->apply(dev, flow, error); 3254 } 3255 3256 /** 3257 * Flow driver remove API. This abstracts calling driver specific functions. 3258 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3259 * on device. All the resources of the flow should be freed by calling 3260 * flow_drv_destroy(). 3261 * 3262 * @param[in] dev 3263 * Pointer to Ethernet device. 3264 * @param[in, out] flow 3265 * Pointer to flow structure. 3266 */ 3267 static inline void 3268 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 3269 { 3270 const struct mlx5_flow_driver_ops *fops; 3271 enum mlx5_flow_drv_type type = flow->drv_type; 3272 3273 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3274 fops = flow_get_drv_ops(type); 3275 fops->remove(dev, flow); 3276 } 3277 3278 /** 3279 * Flow driver destroy API. This abstracts calling driver specific functions. 3280 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3281 * on device and releases resources of the flow. 3282 * 3283 * @param[in] dev 3284 * Pointer to Ethernet device. 3285 * @param[in, out] flow 3286 * Pointer to flow structure. 3287 */ 3288 static inline void 3289 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3290 { 3291 const struct mlx5_flow_driver_ops *fops; 3292 enum mlx5_flow_drv_type type = flow->drv_type; 3293 3294 flow_mreg_split_qrss_release(dev, flow); 3295 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3296 fops = flow_get_drv_ops(type); 3297 fops->destroy(dev, flow); 3298 } 3299 3300 /** 3301 * Get RSS action from the action list. 3302 * 3303 * @param[in] actions 3304 * Pointer to the list of actions. 3305 * 3306 * @return 3307 * Pointer to the RSS action if exist, else return NULL. 3308 */ 3309 static const struct rte_flow_action_rss* 3310 flow_get_rss_action(const struct rte_flow_action actions[]) 3311 { 3312 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3313 switch (actions->type) { 3314 case RTE_FLOW_ACTION_TYPE_RSS: 3315 return (const struct rte_flow_action_rss *) 3316 actions->conf; 3317 default: 3318 break; 3319 } 3320 } 3321 return NULL; 3322 } 3323 3324 /* maps shared action to translated non shared in some actions array */ 3325 struct mlx5_translated_shared_action { 3326 struct rte_flow_shared_action *action; /**< Shared action */ 3327 int index; /**< Index in related array of rte_flow_action */ 3328 }; 3329 3330 /** 3331 * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related 3332 * non shared action if translation possible. 3333 * This functionality used to run same execution path for both shared & non 3334 * shared actions on flow create. All necessary preparations for shared 3335 * action handling should be preformed on *shared* actions list returned 3336 * from this call. 3337 * 3338 * @param[in] actions 3339 * List of actions to translate. 3340 * @param[out] shared 3341 * List to store translated shared actions. 3342 * @param[in, out] shared_n 3343 * Size of *shared* array. On return should be updated with number of shared 3344 * actions retrieved from the *actions* list. 3345 * @param[out] translated_actions 3346 * List of actions where all shared actions were translated to non shared 3347 * if possible. NULL if no translation took place. 3348 * @param[out] error 3349 * Pointer to the error structure. 3350 * 3351 * @return 3352 * 0 on success, a negative errno value otherwise and rte_errno is set. 3353 */ 3354 static int 3355 flow_shared_actions_translate(const struct rte_flow_action actions[], 3356 struct mlx5_translated_shared_action *shared, 3357 int *shared_n, 3358 struct rte_flow_action **translated_actions, 3359 struct rte_flow_error *error) 3360 { 3361 struct rte_flow_action *translated = NULL; 3362 size_t actions_size; 3363 int n; 3364 int copied_n = 0; 3365 struct mlx5_translated_shared_action *shared_end = NULL; 3366 3367 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3368 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED) 3369 continue; 3370 if (copied_n == *shared_n) { 3371 return rte_flow_error_set 3372 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3373 NULL, "too many shared actions"); 3374 } 3375 rte_memcpy(&shared[copied_n].action, &actions[n].conf, 3376 sizeof(actions[n].conf)); 3377 shared[copied_n].index = n; 3378 copied_n++; 3379 } 3380 n++; 3381 *shared_n = copied_n; 3382 if (!copied_n) 3383 return 0; 3384 actions_size = sizeof(struct rte_flow_action) * n; 3385 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3386 if (!translated) { 3387 rte_errno = ENOMEM; 3388 return -ENOMEM; 3389 } 3390 memcpy(translated, actions, actions_size); 3391 for (shared_end = shared + copied_n; shared < shared_end; shared++) { 3392 const struct rte_flow_shared_action *shared_action; 3393 3394 shared_action = shared->action; 3395 switch (shared_action->type) { 3396 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 3397 translated[shared->index].type = 3398 RTE_FLOW_ACTION_TYPE_RSS; 3399 translated[shared->index].conf = 3400 &shared_action->rss.origin; 3401 break; 3402 default: 3403 mlx5_free(translated); 3404 return rte_flow_error_set 3405 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3406 NULL, "invalid shared action type"); 3407 } 3408 } 3409 *translated_actions = translated; 3410 return 0; 3411 } 3412 3413 /** 3414 * Get Shared RSS action from the action list. 3415 * 3416 * @param[in] shared 3417 * Pointer to the list of actions. 3418 * @param[in] shared_n 3419 * Actions list length. 3420 * 3421 * @return 3422 * Pointer to the MLX5 RSS action if exists, otherwise return NULL. 3423 */ 3424 static struct mlx5_shared_action_rss * 3425 flow_get_shared_rss_action(struct mlx5_translated_shared_action *shared, 3426 int shared_n) 3427 { 3428 struct mlx5_translated_shared_action *shared_end; 3429 3430 for (shared_end = shared + shared_n; shared < shared_end; shared++) { 3431 struct rte_flow_shared_action *shared_action; 3432 3433 shared_action = shared->action; 3434 switch (shared_action->type) { 3435 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 3436 __atomic_add_fetch(&shared_action->refcnt, 1, 3437 __ATOMIC_RELAXED); 3438 return &shared_action->rss; 3439 default: 3440 break; 3441 } 3442 } 3443 return NULL; 3444 } 3445 3446 struct rte_flow_shared_action * 3447 mlx5_flow_get_shared_rss(struct rte_flow *flow) 3448 { 3449 if (flow->shared_rss) 3450 return container_of(flow->shared_rss, 3451 struct rte_flow_shared_action, rss); 3452 else 3453 return NULL; 3454 } 3455 3456 static unsigned int 3457 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3458 { 3459 const struct rte_flow_item *item; 3460 unsigned int has_vlan = 0; 3461 3462 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3463 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3464 has_vlan = 1; 3465 break; 3466 } 3467 } 3468 if (has_vlan) 3469 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3470 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3471 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3472 MLX5_EXPANSION_ROOT_OUTER; 3473 } 3474 3475 /** 3476 * Get layer flags from the prefix flow. 3477 * 3478 * Some flows may be split to several subflows, the prefix subflow gets the 3479 * match items and the suffix sub flow gets the actions. 3480 * Some actions need the user defined match item flags to get the detail for 3481 * the action. 3482 * This function helps the suffix flow to get the item layer flags from prefix 3483 * subflow. 3484 * 3485 * @param[in] dev_flow 3486 * Pointer the created preifx subflow. 3487 * 3488 * @return 3489 * The layers get from prefix subflow. 3490 */ 3491 static inline uint64_t 3492 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3493 { 3494 uint64_t layers = 0; 3495 3496 /* 3497 * Layers bits could be localization, but usually the compiler will 3498 * help to do the optimization work for source code. 3499 * If no decap actions, use the layers directly. 3500 */ 3501 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3502 return dev_flow->handle->layers; 3503 /* Convert L3 layers with decap action. */ 3504 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3505 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3506 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3507 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3508 /* Convert L4 layers with decap action. */ 3509 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3510 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3511 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3512 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3513 return layers; 3514 } 3515 3516 /** 3517 * Get metadata split action information. 3518 * 3519 * @param[in] actions 3520 * Pointer to the list of actions. 3521 * @param[out] qrss 3522 * Pointer to the return pointer. 3523 * @param[out] qrss_type 3524 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3525 * if no QUEUE/RSS is found. 3526 * @param[out] encap_idx 3527 * Pointer to the index of the encap action if exists, otherwise the last 3528 * action index. 3529 * 3530 * @return 3531 * Total number of actions. 3532 */ 3533 static int 3534 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3535 const struct rte_flow_action **qrss, 3536 int *encap_idx) 3537 { 3538 const struct rte_flow_action_raw_encap *raw_encap; 3539 int actions_n = 0; 3540 int raw_decap_idx = -1; 3541 3542 *encap_idx = -1; 3543 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3544 switch (actions->type) { 3545 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3546 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3547 *encap_idx = actions_n; 3548 break; 3549 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3550 raw_decap_idx = actions_n; 3551 break; 3552 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3553 raw_encap = actions->conf; 3554 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3555 *encap_idx = raw_decap_idx != -1 ? 3556 raw_decap_idx : actions_n; 3557 break; 3558 case RTE_FLOW_ACTION_TYPE_QUEUE: 3559 case RTE_FLOW_ACTION_TYPE_RSS: 3560 *qrss = actions; 3561 break; 3562 default: 3563 break; 3564 } 3565 actions_n++; 3566 } 3567 if (*encap_idx == -1) 3568 *encap_idx = actions_n; 3569 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3570 return actions_n + 1; 3571 } 3572 3573 /** 3574 * Check meter action from the action list. 3575 * 3576 * @param[in] actions 3577 * Pointer to the list of actions. 3578 * @param[out] mtr 3579 * Pointer to the meter exist flag. 3580 * 3581 * @return 3582 * Total number of actions. 3583 */ 3584 static int 3585 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 3586 { 3587 int actions_n = 0; 3588 3589 MLX5_ASSERT(mtr); 3590 *mtr = 0; 3591 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3592 switch (actions->type) { 3593 case RTE_FLOW_ACTION_TYPE_METER: 3594 *mtr = 1; 3595 break; 3596 default: 3597 break; 3598 } 3599 actions_n++; 3600 } 3601 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3602 return actions_n + 1; 3603 } 3604 3605 /** 3606 * Check if the flow should be split due to hairpin. 3607 * The reason for the split is that in current HW we can't 3608 * support encap and push-vlan on Rx, so if a flow contains 3609 * these actions we move it to Tx. 3610 * 3611 * @param dev 3612 * Pointer to Ethernet device. 3613 * @param[in] attr 3614 * Flow rule attributes. 3615 * @param[in] actions 3616 * Associated actions (list terminated by the END action). 3617 * 3618 * @return 3619 * > 0 the number of actions and the flow should be split, 3620 * 0 when no split required. 3621 */ 3622 static int 3623 flow_check_hairpin_split(struct rte_eth_dev *dev, 3624 const struct rte_flow_attr *attr, 3625 const struct rte_flow_action actions[]) 3626 { 3627 int queue_action = 0; 3628 int action_n = 0; 3629 int split = 0; 3630 const struct rte_flow_action_queue *queue; 3631 const struct rte_flow_action_rss *rss; 3632 const struct rte_flow_action_raw_encap *raw_encap; 3633 const struct rte_eth_hairpin_conf *conf; 3634 3635 if (!attr->ingress) 3636 return 0; 3637 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3638 switch (actions->type) { 3639 case RTE_FLOW_ACTION_TYPE_QUEUE: 3640 queue = actions->conf; 3641 if (queue == NULL) 3642 return 0; 3643 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3644 if (conf != NULL && !!conf->tx_explicit) 3645 return 0; 3646 queue_action = 1; 3647 action_n++; 3648 break; 3649 case RTE_FLOW_ACTION_TYPE_RSS: 3650 rss = actions->conf; 3651 if (rss == NULL || rss->queue_num == 0) 3652 return 0; 3653 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3654 if (conf != NULL && !!conf->tx_explicit) 3655 return 0; 3656 queue_action = 1; 3657 action_n++; 3658 break; 3659 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3660 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3661 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3662 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3663 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3664 split++; 3665 action_n++; 3666 break; 3667 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3668 raw_encap = actions->conf; 3669 if (raw_encap->size > 3670 (sizeof(struct rte_flow_item_eth) + 3671 sizeof(struct rte_flow_item_ipv4))) 3672 split++; 3673 action_n++; 3674 break; 3675 default: 3676 action_n++; 3677 break; 3678 } 3679 } 3680 if (split && queue_action) 3681 return action_n; 3682 return 0; 3683 } 3684 3685 /* Declare flow create/destroy prototype in advance. */ 3686 static uint32_t 3687 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3688 const struct rte_flow_attr *attr, 3689 const struct rte_flow_item items[], 3690 const struct rte_flow_action actions[], 3691 bool external, struct rte_flow_error *error); 3692 3693 static void 3694 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3695 uint32_t flow_idx); 3696 3697 /** 3698 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3699 * 3700 * As mark_id is unique, if there's already a registered flow for the mark_id, 3701 * return by increasing the reference counter of the resource. Otherwise, create 3702 * the resource (mcp_res) and flow. 3703 * 3704 * Flow looks like, 3705 * - If ingress port is ANY and reg_c[1] is mark_id, 3706 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3707 * 3708 * For default flow (zero mark_id), flow is like, 3709 * - If ingress port is ANY, 3710 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 3711 * 3712 * @param dev 3713 * Pointer to Ethernet device. 3714 * @param mark_id 3715 * ID of MARK action, zero means default flow for META. 3716 * @param[out] error 3717 * Perform verbose error reporting if not NULL. 3718 * 3719 * @return 3720 * Associated resource on success, NULL otherwise and rte_errno is set. 3721 */ 3722 static struct mlx5_flow_mreg_copy_resource * 3723 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 3724 struct rte_flow_error *error) 3725 { 3726 struct mlx5_priv *priv = dev->data->dev_private; 3727 struct rte_flow_attr attr = { 3728 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3729 .ingress = 1, 3730 }; 3731 struct mlx5_rte_flow_item_tag tag_spec = { 3732 .data = mark_id, 3733 }; 3734 struct rte_flow_item items[] = { 3735 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 3736 }; 3737 struct rte_flow_action_mark ftag = { 3738 .id = mark_id, 3739 }; 3740 struct mlx5_flow_action_copy_mreg cp_mreg = { 3741 .dst = REG_B, 3742 .src = REG_NON, 3743 }; 3744 struct rte_flow_action_jump jump = { 3745 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3746 }; 3747 struct rte_flow_action actions[] = { 3748 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 3749 }; 3750 struct mlx5_flow_mreg_copy_resource *mcp_res; 3751 uint32_t idx = 0; 3752 int ret; 3753 3754 /* Fill the register fileds in the flow. */ 3755 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3756 if (ret < 0) 3757 return NULL; 3758 tag_spec.id = ret; 3759 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3760 if (ret < 0) 3761 return NULL; 3762 cp_mreg.src = ret; 3763 /* Check if already registered. */ 3764 MLX5_ASSERT(priv->mreg_cp_tbl); 3765 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 3766 if (mcp_res) { 3767 /* For non-default rule. */ 3768 if (mark_id != MLX5_DEFAULT_COPY_ID) 3769 mcp_res->refcnt++; 3770 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 3771 mcp_res->refcnt == 1); 3772 return mcp_res; 3773 } 3774 /* Provide the full width of FLAG specific value. */ 3775 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3776 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3777 /* Build a new flow. */ 3778 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3779 items[0] = (struct rte_flow_item){ 3780 .type = (enum rte_flow_item_type) 3781 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3782 .spec = &tag_spec, 3783 }; 3784 items[1] = (struct rte_flow_item){ 3785 .type = RTE_FLOW_ITEM_TYPE_END, 3786 }; 3787 actions[0] = (struct rte_flow_action){ 3788 .type = (enum rte_flow_action_type) 3789 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3790 .conf = &ftag, 3791 }; 3792 actions[1] = (struct rte_flow_action){ 3793 .type = (enum rte_flow_action_type) 3794 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3795 .conf = &cp_mreg, 3796 }; 3797 actions[2] = (struct rte_flow_action){ 3798 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3799 .conf = &jump, 3800 }; 3801 actions[3] = (struct rte_flow_action){ 3802 .type = RTE_FLOW_ACTION_TYPE_END, 3803 }; 3804 } else { 3805 /* Default rule, wildcard match. */ 3806 attr.priority = MLX5_FLOW_PRIO_RSVD; 3807 items[0] = (struct rte_flow_item){ 3808 .type = RTE_FLOW_ITEM_TYPE_END, 3809 }; 3810 actions[0] = (struct rte_flow_action){ 3811 .type = (enum rte_flow_action_type) 3812 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3813 .conf = &cp_mreg, 3814 }; 3815 actions[1] = (struct rte_flow_action){ 3816 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3817 .conf = &jump, 3818 }; 3819 actions[2] = (struct rte_flow_action){ 3820 .type = RTE_FLOW_ACTION_TYPE_END, 3821 }; 3822 } 3823 /* Build a new entry. */ 3824 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3825 if (!mcp_res) { 3826 rte_errno = ENOMEM; 3827 return NULL; 3828 } 3829 mcp_res->idx = idx; 3830 /* 3831 * The copy Flows are not included in any list. There 3832 * ones are referenced from other Flows and can not 3833 * be applied, removed, deleted in ardbitrary order 3834 * by list traversing. 3835 */ 3836 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3837 actions, false, error); 3838 if (!mcp_res->rix_flow) 3839 goto error; 3840 mcp_res->refcnt++; 3841 mcp_res->hlist_ent.key = mark_id; 3842 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3843 &mcp_res->hlist_ent); 3844 MLX5_ASSERT(!ret); 3845 if (ret) 3846 goto error; 3847 return mcp_res; 3848 error: 3849 if (mcp_res->rix_flow) 3850 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3851 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3852 return NULL; 3853 } 3854 3855 /** 3856 * Release flow in RX_CP_TBL. 3857 * 3858 * @param dev 3859 * Pointer to Ethernet device. 3860 * @flow 3861 * Parent flow for wich copying is provided. 3862 */ 3863 static void 3864 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3865 struct rte_flow *flow) 3866 { 3867 struct mlx5_flow_mreg_copy_resource *mcp_res; 3868 struct mlx5_priv *priv = dev->data->dev_private; 3869 3870 if (!flow->rix_mreg_copy) 3871 return; 3872 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3873 flow->rix_mreg_copy); 3874 if (!mcp_res || !priv->mreg_cp_tbl) 3875 return; 3876 if (flow->copy_applied) { 3877 MLX5_ASSERT(mcp_res->appcnt); 3878 flow->copy_applied = 0; 3879 --mcp_res->appcnt; 3880 if (!mcp_res->appcnt) { 3881 struct rte_flow *mcp_flow = mlx5_ipool_get 3882 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3883 mcp_res->rix_flow); 3884 3885 if (mcp_flow) 3886 flow_drv_remove(dev, mcp_flow); 3887 } 3888 } 3889 /* 3890 * We do not check availability of metadata registers here, 3891 * because copy resources are not allocated in this case. 3892 */ 3893 if (--mcp_res->refcnt) 3894 return; 3895 MLX5_ASSERT(mcp_res->rix_flow); 3896 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3897 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3898 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3899 flow->rix_mreg_copy = 0; 3900 } 3901 3902 /** 3903 * Start flow in RX_CP_TBL. 3904 * 3905 * @param dev 3906 * Pointer to Ethernet device. 3907 * @flow 3908 * Parent flow for wich copying is provided. 3909 * 3910 * @return 3911 * 0 on success, a negative errno value otherwise and rte_errno is set. 3912 */ 3913 static int 3914 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3915 struct rte_flow *flow) 3916 { 3917 struct mlx5_flow_mreg_copy_resource *mcp_res; 3918 struct mlx5_priv *priv = dev->data->dev_private; 3919 int ret; 3920 3921 if (!flow->rix_mreg_copy || flow->copy_applied) 3922 return 0; 3923 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3924 flow->rix_mreg_copy); 3925 if (!mcp_res) 3926 return 0; 3927 if (!mcp_res->appcnt) { 3928 struct rte_flow *mcp_flow = mlx5_ipool_get 3929 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3930 mcp_res->rix_flow); 3931 3932 if (mcp_flow) { 3933 ret = flow_drv_apply(dev, mcp_flow, NULL); 3934 if (ret) 3935 return ret; 3936 } 3937 } 3938 ++mcp_res->appcnt; 3939 flow->copy_applied = 1; 3940 return 0; 3941 } 3942 3943 /** 3944 * Stop flow in RX_CP_TBL. 3945 * 3946 * @param dev 3947 * Pointer to Ethernet device. 3948 * @flow 3949 * Parent flow for wich copying is provided. 3950 */ 3951 static void 3952 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3953 struct rte_flow *flow) 3954 { 3955 struct mlx5_flow_mreg_copy_resource *mcp_res; 3956 struct mlx5_priv *priv = dev->data->dev_private; 3957 3958 if (!flow->rix_mreg_copy || !flow->copy_applied) 3959 return; 3960 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3961 flow->rix_mreg_copy); 3962 if (!mcp_res) 3963 return; 3964 MLX5_ASSERT(mcp_res->appcnt); 3965 --mcp_res->appcnt; 3966 flow->copy_applied = 0; 3967 if (!mcp_res->appcnt) { 3968 struct rte_flow *mcp_flow = mlx5_ipool_get 3969 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3970 mcp_res->rix_flow); 3971 3972 if (mcp_flow) 3973 flow_drv_remove(dev, mcp_flow); 3974 } 3975 } 3976 3977 /** 3978 * Remove the default copy action from RX_CP_TBL. 3979 * 3980 * @param dev 3981 * Pointer to Ethernet device. 3982 */ 3983 static void 3984 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3985 { 3986 struct mlx5_flow_mreg_copy_resource *mcp_res; 3987 struct mlx5_priv *priv = dev->data->dev_private; 3988 3989 /* Check if default flow is registered. */ 3990 if (!priv->mreg_cp_tbl) 3991 return; 3992 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3993 MLX5_DEFAULT_COPY_ID); 3994 if (!mcp_res) 3995 return; 3996 MLX5_ASSERT(mcp_res->rix_flow); 3997 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3998 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3999 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4000 } 4001 4002 /** 4003 * Add the default copy action in in RX_CP_TBL. 4004 * 4005 * @param dev 4006 * Pointer to Ethernet device. 4007 * @param[out] error 4008 * Perform verbose error reporting if not NULL. 4009 * 4010 * @return 4011 * 0 for success, negative value otherwise and rte_errno is set. 4012 */ 4013 static int 4014 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4015 struct rte_flow_error *error) 4016 { 4017 struct mlx5_priv *priv = dev->data->dev_private; 4018 struct mlx5_flow_mreg_copy_resource *mcp_res; 4019 4020 /* Check whether extensive metadata feature is engaged. */ 4021 if (!priv->config.dv_flow_en || 4022 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4023 !mlx5_flow_ext_mreg_supported(dev) || 4024 !priv->sh->dv_regc0_mask) 4025 return 0; 4026 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 4027 if (!mcp_res) 4028 return -rte_errno; 4029 return 0; 4030 } 4031 4032 /** 4033 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4034 * 4035 * All the flow having Q/RSS action should be split by 4036 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4037 * performs the following, 4038 * - CQE->flow_tag := reg_c[1] (MARK) 4039 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4040 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4041 * but there should be a flow per each MARK ID set by MARK action. 4042 * 4043 * For the aforementioned reason, if there's a MARK action in flow's action 4044 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4045 * the MARK ID to CQE's flow_tag like, 4046 * - If reg_c[1] is mark_id, 4047 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4048 * 4049 * For SET_META action which stores value in reg_c[0], as the destination is 4050 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4051 * MARK ID means the default flow. The default flow looks like, 4052 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4053 * 4054 * @param dev 4055 * Pointer to Ethernet device. 4056 * @param flow 4057 * Pointer to flow structure. 4058 * @param[in] actions 4059 * Pointer to the list of actions. 4060 * @param[out] error 4061 * Perform verbose error reporting if not NULL. 4062 * 4063 * @return 4064 * 0 on success, negative value otherwise and rte_errno is set. 4065 */ 4066 static int 4067 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4068 struct rte_flow *flow, 4069 const struct rte_flow_action *actions, 4070 struct rte_flow_error *error) 4071 { 4072 struct mlx5_priv *priv = dev->data->dev_private; 4073 struct mlx5_dev_config *config = &priv->config; 4074 struct mlx5_flow_mreg_copy_resource *mcp_res; 4075 const struct rte_flow_action_mark *mark; 4076 4077 /* Check whether extensive metadata feature is engaged. */ 4078 if (!config->dv_flow_en || 4079 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4080 !mlx5_flow_ext_mreg_supported(dev) || 4081 !priv->sh->dv_regc0_mask) 4082 return 0; 4083 /* Find MARK action. */ 4084 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4085 switch (actions->type) { 4086 case RTE_FLOW_ACTION_TYPE_FLAG: 4087 mcp_res = flow_mreg_add_copy_action 4088 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4089 if (!mcp_res) 4090 return -rte_errno; 4091 flow->rix_mreg_copy = mcp_res->idx; 4092 if (dev->data->dev_started) { 4093 mcp_res->appcnt++; 4094 flow->copy_applied = 1; 4095 } 4096 return 0; 4097 case RTE_FLOW_ACTION_TYPE_MARK: 4098 mark = (const struct rte_flow_action_mark *) 4099 actions->conf; 4100 mcp_res = 4101 flow_mreg_add_copy_action(dev, mark->id, error); 4102 if (!mcp_res) 4103 return -rte_errno; 4104 flow->rix_mreg_copy = mcp_res->idx; 4105 if (dev->data->dev_started) { 4106 mcp_res->appcnt++; 4107 flow->copy_applied = 1; 4108 } 4109 return 0; 4110 default: 4111 break; 4112 } 4113 } 4114 return 0; 4115 } 4116 4117 #define MLX5_MAX_SPLIT_ACTIONS 24 4118 #define MLX5_MAX_SPLIT_ITEMS 24 4119 4120 /** 4121 * Split the hairpin flow. 4122 * Since HW can't support encap and push-vlan on Rx, we move these 4123 * actions to Tx. 4124 * If the count action is after the encap then we also 4125 * move the count action. in this case the count will also measure 4126 * the outer bytes. 4127 * 4128 * @param dev 4129 * Pointer to Ethernet device. 4130 * @param[in] actions 4131 * Associated actions (list terminated by the END action). 4132 * @param[out] actions_rx 4133 * Rx flow actions. 4134 * @param[out] actions_tx 4135 * Tx flow actions.. 4136 * @param[out] pattern_tx 4137 * The pattern items for the Tx flow. 4138 * @param[out] flow_id 4139 * The flow ID connected to this flow. 4140 * 4141 * @return 4142 * 0 on success. 4143 */ 4144 static int 4145 flow_hairpin_split(struct rte_eth_dev *dev, 4146 const struct rte_flow_action actions[], 4147 struct rte_flow_action actions_rx[], 4148 struct rte_flow_action actions_tx[], 4149 struct rte_flow_item pattern_tx[], 4150 uint32_t *flow_id) 4151 { 4152 struct mlx5_priv *priv = dev->data->dev_private; 4153 const struct rte_flow_action_raw_encap *raw_encap; 4154 const struct rte_flow_action_raw_decap *raw_decap; 4155 struct mlx5_rte_flow_action_set_tag *set_tag; 4156 struct rte_flow_action *tag_action; 4157 struct mlx5_rte_flow_item_tag *tag_item; 4158 struct rte_flow_item *item; 4159 char *addr; 4160 int encap = 0; 4161 4162 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 4163 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4164 switch (actions->type) { 4165 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4166 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4167 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4168 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4169 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4170 rte_memcpy(actions_tx, actions, 4171 sizeof(struct rte_flow_action)); 4172 actions_tx++; 4173 break; 4174 case RTE_FLOW_ACTION_TYPE_COUNT: 4175 if (encap) { 4176 rte_memcpy(actions_tx, actions, 4177 sizeof(struct rte_flow_action)); 4178 actions_tx++; 4179 } else { 4180 rte_memcpy(actions_rx, actions, 4181 sizeof(struct rte_flow_action)); 4182 actions_rx++; 4183 } 4184 break; 4185 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4186 raw_encap = actions->conf; 4187 if (raw_encap->size > 4188 (sizeof(struct rte_flow_item_eth) + 4189 sizeof(struct rte_flow_item_ipv4))) { 4190 memcpy(actions_tx, actions, 4191 sizeof(struct rte_flow_action)); 4192 actions_tx++; 4193 encap = 1; 4194 } else { 4195 rte_memcpy(actions_rx, actions, 4196 sizeof(struct rte_flow_action)); 4197 actions_rx++; 4198 } 4199 break; 4200 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4201 raw_decap = actions->conf; 4202 if (raw_decap->size < 4203 (sizeof(struct rte_flow_item_eth) + 4204 sizeof(struct rte_flow_item_ipv4))) { 4205 memcpy(actions_tx, actions, 4206 sizeof(struct rte_flow_action)); 4207 actions_tx++; 4208 } else { 4209 rte_memcpy(actions_rx, actions, 4210 sizeof(struct rte_flow_action)); 4211 actions_rx++; 4212 } 4213 break; 4214 default: 4215 rte_memcpy(actions_rx, actions, 4216 sizeof(struct rte_flow_action)); 4217 actions_rx++; 4218 break; 4219 } 4220 } 4221 /* Add set meta action and end action for the Rx flow. */ 4222 tag_action = actions_rx; 4223 tag_action->type = (enum rte_flow_action_type) 4224 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4225 actions_rx++; 4226 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4227 actions_rx++; 4228 set_tag = (void *)actions_rx; 4229 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 4230 MLX5_ASSERT(set_tag->id > REG_NON); 4231 set_tag->data = *flow_id; 4232 tag_action->conf = set_tag; 4233 /* Create Tx item list. */ 4234 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4235 addr = (void *)&pattern_tx[2]; 4236 item = pattern_tx; 4237 item->type = (enum rte_flow_item_type) 4238 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4239 tag_item = (void *)addr; 4240 tag_item->data = *flow_id; 4241 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4242 MLX5_ASSERT(set_tag->id > REG_NON); 4243 item->spec = tag_item; 4244 addr += sizeof(struct mlx5_rte_flow_item_tag); 4245 tag_item = (void *)addr; 4246 tag_item->data = UINT32_MAX; 4247 tag_item->id = UINT16_MAX; 4248 item->mask = tag_item; 4249 item->last = NULL; 4250 item++; 4251 item->type = RTE_FLOW_ITEM_TYPE_END; 4252 return 0; 4253 } 4254 4255 __extension__ 4256 union tunnel_offload_mark { 4257 uint32_t val; 4258 struct { 4259 uint32_t app_reserve:8; 4260 uint32_t table_id:15; 4261 uint32_t transfer:1; 4262 uint32_t _unused_:8; 4263 }; 4264 }; 4265 4266 struct tunnel_default_miss_ctx { 4267 uint16_t *queue; 4268 __extension__ 4269 union { 4270 struct rte_flow_action_rss action_rss; 4271 struct rte_flow_action_queue miss_queue; 4272 struct rte_flow_action_jump miss_jump; 4273 uint8_t raw[0]; 4274 }; 4275 }; 4276 4277 static int 4278 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 4279 struct rte_flow *flow, 4280 const struct rte_flow_attr *attr, 4281 const struct rte_flow_action *app_actions, 4282 uint32_t flow_idx, 4283 struct tunnel_default_miss_ctx *ctx, 4284 struct rte_flow_error *error) 4285 { 4286 struct mlx5_priv *priv = dev->data->dev_private; 4287 struct mlx5_flow *dev_flow; 4288 struct rte_flow_attr miss_attr = *attr; 4289 const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf; 4290 const struct rte_flow_item miss_items[2] = { 4291 { 4292 .type = RTE_FLOW_ITEM_TYPE_ETH, 4293 .spec = NULL, 4294 .last = NULL, 4295 .mask = NULL 4296 }, 4297 { 4298 .type = RTE_FLOW_ITEM_TYPE_END, 4299 .spec = NULL, 4300 .last = NULL, 4301 .mask = NULL 4302 } 4303 }; 4304 union tunnel_offload_mark mark_id; 4305 struct rte_flow_action_mark miss_mark; 4306 struct rte_flow_action miss_actions[3] = { 4307 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 4308 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 4309 }; 4310 const struct rte_flow_action_jump *jump_data; 4311 uint32_t i, flow_table = 0; /* prevent compilation warning */ 4312 struct flow_grp_info grp_info = { 4313 .external = 1, 4314 .transfer = attr->transfer, 4315 .fdb_def_rule = !!priv->fdb_def_rule, 4316 .std_tbl_fix = 0, 4317 }; 4318 int ret; 4319 4320 if (!attr->transfer) { 4321 uint32_t q_size; 4322 4323 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 4324 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 4325 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 4326 0, SOCKET_ID_ANY); 4327 if (!ctx->queue) 4328 return rte_flow_error_set 4329 (error, ENOMEM, 4330 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 4331 NULL, "invalid default miss RSS"); 4332 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4333 ctx->action_rss.level = 0, 4334 ctx->action_rss.types = priv->rss_conf.rss_hf, 4335 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 4336 ctx->action_rss.queue_num = priv->reta_idx_n, 4337 ctx->action_rss.key = priv->rss_conf.rss_key, 4338 ctx->action_rss.queue = ctx->queue; 4339 if (!priv->reta_idx_n || !priv->rxqs_n) 4340 return rte_flow_error_set 4341 (error, EINVAL, 4342 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 4343 NULL, "invalid port configuration"); 4344 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 4345 ctx->action_rss.types = 0; 4346 for (i = 0; i != priv->reta_idx_n; ++i) 4347 ctx->queue[i] = (*priv->reta_idx)[i]; 4348 } else { 4349 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 4350 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 4351 } 4352 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 4353 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 4354 jump_data = app_actions->conf; 4355 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 4356 miss_attr.group = jump_data->group; 4357 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 4358 &flow_table, grp_info, error); 4359 if (ret) 4360 return rte_flow_error_set(error, EINVAL, 4361 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 4362 NULL, "invalid tunnel id"); 4363 mark_id.app_reserve = 0; 4364 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 4365 mark_id.transfer = !!attr->transfer; 4366 mark_id._unused_ = 0; 4367 miss_mark.id = mark_id.val; 4368 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 4369 miss_items, miss_actions, flow_idx, error); 4370 if (!dev_flow) 4371 return -rte_errno; 4372 dev_flow->flow = flow; 4373 dev_flow->external = true; 4374 dev_flow->tunnel = tunnel; 4375 /* Subflow object was created, we must include one in the list. */ 4376 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4377 dev_flow->handle, next); 4378 DRV_LOG(DEBUG, 4379 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 4380 dev->data->port_id, tunnel->app_tunnel.type, 4381 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 4382 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 4383 miss_actions, error); 4384 if (!ret) 4385 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 4386 error); 4387 4388 return ret; 4389 } 4390 4391 /** 4392 * The last stage of splitting chain, just creates the subflow 4393 * without any modification. 4394 * 4395 * @param[in] dev 4396 * Pointer to Ethernet device. 4397 * @param[in] flow 4398 * Parent flow structure pointer. 4399 * @param[in, out] sub_flow 4400 * Pointer to return the created subflow, may be NULL. 4401 * @param[in] prefix_layers 4402 * Prefix subflow layers, may be 0. 4403 * @param[in] prefix_mark 4404 * Prefix subflow mark flag, may be 0. 4405 * @param[in] attr 4406 * Flow rule attributes. 4407 * @param[in] items 4408 * Pattern specification (list terminated by the END pattern item). 4409 * @param[in] actions 4410 * Associated actions (list terminated by the END action). 4411 * @param[in] external 4412 * This flow rule is created by request external to PMD. 4413 * @param[in] flow_idx 4414 * This memory pool index to the flow. 4415 * @param[out] error 4416 * Perform verbose error reporting if not NULL. 4417 * @return 4418 * 0 on success, negative value otherwise 4419 */ 4420 static int 4421 flow_create_split_inner(struct rte_eth_dev *dev, 4422 struct rte_flow *flow, 4423 struct mlx5_flow **sub_flow, 4424 uint64_t prefix_layers, 4425 uint32_t prefix_mark, 4426 const struct rte_flow_attr *attr, 4427 const struct rte_flow_item items[], 4428 const struct rte_flow_action actions[], 4429 bool external, uint32_t flow_idx, 4430 struct rte_flow_error *error) 4431 { 4432 struct mlx5_flow *dev_flow; 4433 4434 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4435 flow_idx, error); 4436 if (!dev_flow) 4437 return -rte_errno; 4438 dev_flow->flow = flow; 4439 dev_flow->external = external; 4440 /* Subflow object was created, we must include one in the list. */ 4441 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4442 dev_flow->handle, next); 4443 /* 4444 * If dev_flow is as one of the suffix flow, some actions in suffix 4445 * flow may need some user defined item layer flags, and pass the 4446 * Metadate rxq mark flag to suffix flow as well. 4447 */ 4448 if (prefix_layers) 4449 dev_flow->handle->layers = prefix_layers; 4450 if (prefix_mark) 4451 dev_flow->handle->mark = 1; 4452 if (sub_flow) 4453 *sub_flow = dev_flow; 4454 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4455 } 4456 4457 /** 4458 * Split the meter flow. 4459 * 4460 * As meter flow will split to three sub flow, other than meter 4461 * action, the other actions make sense to only meter accepts 4462 * the packet. If it need to be dropped, no other additional 4463 * actions should be take. 4464 * 4465 * One kind of special action which decapsulates the L3 tunnel 4466 * header will be in the prefix sub flow, as not to take the 4467 * L3 tunnel header into account. 4468 * 4469 * @param dev 4470 * Pointer to Ethernet device. 4471 * @param[in] items 4472 * Pattern specification (list terminated by the END pattern item). 4473 * @param[out] sfx_items 4474 * Suffix flow match items (list terminated by the END pattern item). 4475 * @param[in] actions 4476 * Associated actions (list terminated by the END action). 4477 * @param[out] actions_sfx 4478 * Suffix flow actions. 4479 * @param[out] actions_pre 4480 * Prefix flow actions. 4481 * @param[out] pattern_sfx 4482 * The pattern items for the suffix flow. 4483 * @param[out] tag_sfx 4484 * Pointer to suffix flow tag. 4485 * 4486 * @return 4487 * 0 on success. 4488 */ 4489 static int 4490 flow_meter_split_prep(struct rte_eth_dev *dev, 4491 const struct rte_flow_item items[], 4492 struct rte_flow_item sfx_items[], 4493 const struct rte_flow_action actions[], 4494 struct rte_flow_action actions_sfx[], 4495 struct rte_flow_action actions_pre[]) 4496 { 4497 struct rte_flow_action *tag_action = NULL; 4498 struct rte_flow_item *tag_item; 4499 struct mlx5_rte_flow_action_set_tag *set_tag; 4500 struct rte_flow_error error; 4501 const struct rte_flow_action_raw_encap *raw_encap; 4502 const struct rte_flow_action_raw_decap *raw_decap; 4503 struct mlx5_rte_flow_item_tag *tag_spec; 4504 struct mlx5_rte_flow_item_tag *tag_mask; 4505 uint32_t tag_id; 4506 bool copy_vlan = false; 4507 4508 /* Prepare the actions for prefix and suffix flow. */ 4509 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4510 struct rte_flow_action **action_cur = NULL; 4511 4512 switch (actions->type) { 4513 case RTE_FLOW_ACTION_TYPE_METER: 4514 /* Add the extra tag action first. */ 4515 tag_action = actions_pre; 4516 tag_action->type = (enum rte_flow_action_type) 4517 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4518 actions_pre++; 4519 action_cur = &actions_pre; 4520 break; 4521 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4522 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4523 action_cur = &actions_pre; 4524 break; 4525 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4526 raw_encap = actions->conf; 4527 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4528 action_cur = &actions_pre; 4529 break; 4530 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4531 raw_decap = actions->conf; 4532 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4533 action_cur = &actions_pre; 4534 break; 4535 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4536 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4537 copy_vlan = true; 4538 break; 4539 default: 4540 break; 4541 } 4542 if (!action_cur) 4543 action_cur = &actions_sfx; 4544 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 4545 (*action_cur)++; 4546 } 4547 /* Add end action to the actions. */ 4548 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4549 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4550 actions_pre++; 4551 /* Set the tag. */ 4552 set_tag = (void *)actions_pre; 4553 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4554 /* 4555 * Get the id from the qrss_pool to make qrss share the id with meter. 4556 */ 4557 tag_id = flow_qrss_get_id(dev); 4558 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 4559 assert(tag_action); 4560 tag_action->conf = set_tag; 4561 /* Prepare the suffix subflow items. */ 4562 tag_item = sfx_items++; 4563 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4564 int item_type = items->type; 4565 4566 switch (item_type) { 4567 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4568 memcpy(sfx_items, items, sizeof(*sfx_items)); 4569 sfx_items++; 4570 break; 4571 case RTE_FLOW_ITEM_TYPE_VLAN: 4572 if (copy_vlan) { 4573 memcpy(sfx_items, items, sizeof(*sfx_items)); 4574 /* 4575 * Convert to internal match item, it is used 4576 * for vlan push and set vid. 4577 */ 4578 sfx_items->type = (enum rte_flow_item_type) 4579 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4580 sfx_items++; 4581 } 4582 break; 4583 default: 4584 break; 4585 } 4586 } 4587 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4588 sfx_items++; 4589 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4590 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 4591 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4592 tag_mask = tag_spec + 1; 4593 tag_mask->data = 0xffffff00; 4594 tag_item->type = (enum rte_flow_item_type) 4595 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4596 tag_item->spec = tag_spec; 4597 tag_item->last = NULL; 4598 tag_item->mask = tag_mask; 4599 return tag_id; 4600 } 4601 4602 /** 4603 * Split action list having QUEUE/RSS for metadata register copy. 4604 * 4605 * Once Q/RSS action is detected in user's action list, the flow action 4606 * should be split in order to copy metadata registers, which will happen in 4607 * RX_CP_TBL like, 4608 * - CQE->flow_tag := reg_c[1] (MARK) 4609 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4610 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4611 * This is because the last action of each flow must be a terminal action 4612 * (QUEUE, RSS or DROP). 4613 * 4614 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4615 * stored and kept in the mlx5_flow structure per each sub_flow. 4616 * 4617 * The Q/RSS action is replaced with, 4618 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4619 * And the following JUMP action is added at the end, 4620 * - JUMP, to RX_CP_TBL. 4621 * 4622 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4623 * flow_create_split_metadata() routine. The flow will look like, 4624 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4625 * 4626 * @param dev 4627 * Pointer to Ethernet device. 4628 * @param[out] split_actions 4629 * Pointer to store split actions to jump to CP_TBL. 4630 * @param[in] actions 4631 * Pointer to the list of original flow actions. 4632 * @param[in] qrss 4633 * Pointer to the Q/RSS action. 4634 * @param[in] actions_n 4635 * Number of original actions. 4636 * @param[out] error 4637 * Perform verbose error reporting if not NULL. 4638 * 4639 * @return 4640 * non-zero unique flow_id on success, otherwise 0 and 4641 * error/rte_error are set. 4642 */ 4643 static uint32_t 4644 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4645 struct rte_flow_action *split_actions, 4646 const struct rte_flow_action *actions, 4647 const struct rte_flow_action *qrss, 4648 int actions_n, struct rte_flow_error *error) 4649 { 4650 struct mlx5_rte_flow_action_set_tag *set_tag; 4651 struct rte_flow_action_jump *jump; 4652 const int qrss_idx = qrss - actions; 4653 uint32_t flow_id = 0; 4654 int ret = 0; 4655 4656 /* 4657 * Given actions will be split 4658 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4659 * - Add jump to mreg CP_TBL. 4660 * As a result, there will be one more action. 4661 */ 4662 ++actions_n; 4663 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4664 set_tag = (void *)(split_actions + actions_n); 4665 /* 4666 * If tag action is not set to void(it means we are not the meter 4667 * suffix flow), add the tag action. Since meter suffix flow already 4668 * has the tag added. 4669 */ 4670 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4671 /* 4672 * Allocate the new subflow ID. This one is unique within 4673 * device and not shared with representors. Otherwise, 4674 * we would have to resolve multi-thread access synch 4675 * issue. Each flow on the shared device is appended 4676 * with source vport identifier, so the resulting 4677 * flows will be unique in the shared (by master and 4678 * representors) domain even if they have coinciding 4679 * IDs. 4680 */ 4681 flow_id = flow_qrss_get_id(dev); 4682 if (!flow_id) 4683 return rte_flow_error_set(error, ENOMEM, 4684 RTE_FLOW_ERROR_TYPE_ACTION, 4685 NULL, "can't allocate id " 4686 "for split Q/RSS subflow"); 4687 /* Internal SET_TAG action to set flow ID. */ 4688 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4689 .data = flow_id, 4690 }; 4691 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4692 if (ret < 0) 4693 return ret; 4694 set_tag->id = ret; 4695 /* Construct new actions array. */ 4696 /* Replace QUEUE/RSS action. */ 4697 split_actions[qrss_idx] = (struct rte_flow_action){ 4698 .type = (enum rte_flow_action_type) 4699 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4700 .conf = set_tag, 4701 }; 4702 } 4703 /* JUMP action to jump to mreg copy table (CP_TBL). */ 4704 jump = (void *)(set_tag + 1); 4705 *jump = (struct rte_flow_action_jump){ 4706 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4707 }; 4708 split_actions[actions_n - 2] = (struct rte_flow_action){ 4709 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4710 .conf = jump, 4711 }; 4712 split_actions[actions_n - 1] = (struct rte_flow_action){ 4713 .type = RTE_FLOW_ACTION_TYPE_END, 4714 }; 4715 return flow_id; 4716 } 4717 4718 /** 4719 * Extend the given action list for Tx metadata copy. 4720 * 4721 * Copy the given action list to the ext_actions and add flow metadata register 4722 * copy action in order to copy reg_a set by WQE to reg_c[0]. 4723 * 4724 * @param[out] ext_actions 4725 * Pointer to the extended action list. 4726 * @param[in] actions 4727 * Pointer to the list of actions. 4728 * @param[in] actions_n 4729 * Number of actions in the list. 4730 * @param[out] error 4731 * Perform verbose error reporting if not NULL. 4732 * @param[in] encap_idx 4733 * The encap action inndex. 4734 * 4735 * @return 4736 * 0 on success, negative value otherwise 4737 */ 4738 static int 4739 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 4740 struct rte_flow_action *ext_actions, 4741 const struct rte_flow_action *actions, 4742 int actions_n, struct rte_flow_error *error, 4743 int encap_idx) 4744 { 4745 struct mlx5_flow_action_copy_mreg *cp_mreg = 4746 (struct mlx5_flow_action_copy_mreg *) 4747 (ext_actions + actions_n + 1); 4748 int ret; 4749 4750 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4751 if (ret < 0) 4752 return ret; 4753 cp_mreg->dst = ret; 4754 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 4755 if (ret < 0) 4756 return ret; 4757 cp_mreg->src = ret; 4758 if (encap_idx != 0) 4759 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 4760 if (encap_idx == actions_n - 1) { 4761 ext_actions[actions_n - 1] = (struct rte_flow_action){ 4762 .type = (enum rte_flow_action_type) 4763 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4764 .conf = cp_mreg, 4765 }; 4766 ext_actions[actions_n] = (struct rte_flow_action){ 4767 .type = RTE_FLOW_ACTION_TYPE_END, 4768 }; 4769 } else { 4770 ext_actions[encap_idx] = (struct rte_flow_action){ 4771 .type = (enum rte_flow_action_type) 4772 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4773 .conf = cp_mreg, 4774 }; 4775 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 4776 sizeof(*ext_actions) * (actions_n - encap_idx)); 4777 } 4778 return 0; 4779 } 4780 4781 /** 4782 * Check the match action from the action list. 4783 * 4784 * @param[in] actions 4785 * Pointer to the list of actions. 4786 * @param[in] attr 4787 * Flow rule attributes. 4788 * @param[in] action 4789 * The action to be check if exist. 4790 * @param[out] match_action_pos 4791 * Pointer to the position of the matched action if exists, otherwise is -1. 4792 * @param[out] qrss_action_pos 4793 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 4794 * 4795 * @return 4796 * > 0 the total number of actions. 4797 * 0 if not found match action in action list. 4798 */ 4799 static int 4800 flow_check_match_action(const struct rte_flow_action actions[], 4801 const struct rte_flow_attr *attr, 4802 enum rte_flow_action_type action, 4803 int *match_action_pos, int *qrss_action_pos) 4804 { 4805 const struct rte_flow_action_sample *sample; 4806 int actions_n = 0; 4807 int jump_flag = 0; 4808 uint32_t ratio = 0; 4809 int sub_type = 0; 4810 int flag = 0; 4811 4812 *match_action_pos = -1; 4813 *qrss_action_pos = -1; 4814 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4815 if (actions->type == action) { 4816 flag = 1; 4817 *match_action_pos = actions_n; 4818 } 4819 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE || 4820 actions->type == RTE_FLOW_ACTION_TYPE_RSS) 4821 *qrss_action_pos = actions_n; 4822 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP) 4823 jump_flag = 1; 4824 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) { 4825 sample = actions->conf; 4826 ratio = sample->ratio; 4827 sub_type = ((const struct rte_flow_action *) 4828 (sample->actions))->type; 4829 } 4830 actions_n++; 4831 } 4832 if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) { 4833 if (ratio == 1) { 4834 /* JUMP Action not support for Mirroring; 4835 * Mirroring support multi-destination; 4836 */ 4837 if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END) 4838 flag = 0; 4839 } 4840 } 4841 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4842 return flag ? actions_n + 1 : 0; 4843 } 4844 4845 #define SAMPLE_SUFFIX_ITEM 2 4846 4847 /** 4848 * Split the sample flow. 4849 * 4850 * As sample flow will split to two sub flow, sample flow with 4851 * sample action, the other actions will move to new suffix flow. 4852 * 4853 * Also add unique tag id with tag action in the sample flow, 4854 * the same tag id will be as match in the suffix flow. 4855 * 4856 * @param dev 4857 * Pointer to Ethernet device. 4858 * @param[in] fdb_tx 4859 * FDB egress flow flag. 4860 * @param[out] sfx_items 4861 * Suffix flow match items (list terminated by the END pattern item). 4862 * @param[in] actions 4863 * Associated actions (list terminated by the END action). 4864 * @param[out] actions_sfx 4865 * Suffix flow actions. 4866 * @param[out] actions_pre 4867 * Prefix flow actions. 4868 * @param[in] actions_n 4869 * The total number of actions. 4870 * @param[in] sample_action_pos 4871 * The sample action position. 4872 * @param[in] qrss_action_pos 4873 * The Queue/RSS action position. 4874 * @param[out] error 4875 * Perform verbose error reporting if not NULL. 4876 * 4877 * @return 4878 * 0 on success, or unique flow_id, a negative errno value 4879 * otherwise and rte_errno is set. 4880 */ 4881 static int 4882 flow_sample_split_prep(struct rte_eth_dev *dev, 4883 uint32_t fdb_tx, 4884 struct rte_flow_item sfx_items[], 4885 const struct rte_flow_action actions[], 4886 struct rte_flow_action actions_sfx[], 4887 struct rte_flow_action actions_pre[], 4888 int actions_n, 4889 int sample_action_pos, 4890 int qrss_action_pos, 4891 struct rte_flow_error *error) 4892 { 4893 struct mlx5_rte_flow_action_set_tag *set_tag; 4894 struct mlx5_rte_flow_item_tag *tag_spec; 4895 struct mlx5_rte_flow_item_tag *tag_mask; 4896 uint32_t tag_id = 0; 4897 int index; 4898 int ret; 4899 4900 if (sample_action_pos < 0) 4901 return rte_flow_error_set(error, EINVAL, 4902 RTE_FLOW_ERROR_TYPE_ACTION, 4903 NULL, "invalid position of sample " 4904 "action in list"); 4905 if (!fdb_tx) { 4906 /* Prepare the prefix tag action. */ 4907 set_tag = (void *)(actions_pre + actions_n + 1); 4908 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 4909 if (ret < 0) 4910 return ret; 4911 set_tag->id = ret; 4912 tag_id = flow_qrss_get_id(dev); 4913 set_tag->data = tag_id; 4914 /* Prepare the suffix subflow items. */ 4915 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 4916 tag_spec->data = tag_id; 4917 tag_spec->id = set_tag->id; 4918 tag_mask = tag_spec + 1; 4919 tag_mask->data = UINT32_MAX; 4920 sfx_items[0] = (struct rte_flow_item){ 4921 .type = (enum rte_flow_item_type) 4922 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4923 .spec = tag_spec, 4924 .last = NULL, 4925 .mask = tag_mask, 4926 }; 4927 sfx_items[1] = (struct rte_flow_item){ 4928 .type = (enum rte_flow_item_type) 4929 RTE_FLOW_ITEM_TYPE_END, 4930 }; 4931 } 4932 /* Prepare the actions for prefix and suffix flow. */ 4933 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 4934 index = qrss_action_pos; 4935 /* Put the preceding the Queue/RSS action into prefix flow. */ 4936 if (index != 0) 4937 memcpy(actions_pre, actions, 4938 sizeof(struct rte_flow_action) * index); 4939 /* Put others preceding the sample action into prefix flow. */ 4940 if (sample_action_pos > index + 1) 4941 memcpy(actions_pre + index, actions + index + 1, 4942 sizeof(struct rte_flow_action) * 4943 (sample_action_pos - index - 1)); 4944 index = sample_action_pos - 1; 4945 /* Put Queue/RSS action into Suffix flow. */ 4946 memcpy(actions_sfx, actions + qrss_action_pos, 4947 sizeof(struct rte_flow_action)); 4948 actions_sfx++; 4949 } else { 4950 index = sample_action_pos; 4951 if (index != 0) 4952 memcpy(actions_pre, actions, 4953 sizeof(struct rte_flow_action) * index); 4954 } 4955 /* Add the extra tag action for NIC-RX and E-Switch ingress. */ 4956 if (!fdb_tx) { 4957 actions_pre[index++] = 4958 (struct rte_flow_action){ 4959 .type = (enum rte_flow_action_type) 4960 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4961 .conf = set_tag, 4962 }; 4963 } 4964 memcpy(actions_pre + index, actions + sample_action_pos, 4965 sizeof(struct rte_flow_action)); 4966 index += 1; 4967 actions_pre[index] = (struct rte_flow_action){ 4968 .type = (enum rte_flow_action_type) 4969 RTE_FLOW_ACTION_TYPE_END, 4970 }; 4971 /* Put the actions after sample into Suffix flow. */ 4972 memcpy(actions_sfx, actions + sample_action_pos + 1, 4973 sizeof(struct rte_flow_action) * 4974 (actions_n - sample_action_pos - 1)); 4975 return tag_id; 4976 } 4977 4978 /** 4979 * The splitting for metadata feature. 4980 * 4981 * - Q/RSS action on NIC Rx should be split in order to pass by 4982 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4983 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4984 * 4985 * - All the actions on NIC Tx should have a mreg copy action to 4986 * copy reg_a from WQE to reg_c[0]. 4987 * 4988 * @param dev 4989 * Pointer to Ethernet device. 4990 * @param[in] flow 4991 * Parent flow structure pointer. 4992 * @param[in] prefix_layers 4993 * Prefix flow layer flags. 4994 * @param[in] prefix_mark 4995 * Prefix subflow mark flag, may be 0. 4996 * @param[in] attr 4997 * Flow rule attributes. 4998 * @param[in] items 4999 * Pattern specification (list terminated by the END pattern item). 5000 * @param[in] actions 5001 * Associated actions (list terminated by the END action). 5002 * @param[in] external 5003 * This flow rule is created by request external to PMD. 5004 * @param[in] flow_idx 5005 * This memory pool index to the flow. 5006 * @param[out] error 5007 * Perform verbose error reporting if not NULL. 5008 * @return 5009 * 0 on success, negative value otherwise 5010 */ 5011 static int 5012 flow_create_split_metadata(struct rte_eth_dev *dev, 5013 struct rte_flow *flow, 5014 uint64_t prefix_layers, 5015 uint32_t prefix_mark, 5016 const struct rte_flow_attr *attr, 5017 const struct rte_flow_item items[], 5018 const struct rte_flow_action actions[], 5019 bool external, uint32_t flow_idx, 5020 struct rte_flow_error *error) 5021 { 5022 struct mlx5_priv *priv = dev->data->dev_private; 5023 struct mlx5_dev_config *config = &priv->config; 5024 const struct rte_flow_action *qrss = NULL; 5025 struct rte_flow_action *ext_actions = NULL; 5026 struct mlx5_flow *dev_flow = NULL; 5027 uint32_t qrss_id = 0; 5028 int mtr_sfx = 0; 5029 size_t act_size; 5030 int actions_n; 5031 int encap_idx; 5032 int ret; 5033 5034 /* Check whether extensive metadata feature is engaged. */ 5035 if (!config->dv_flow_en || 5036 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5037 !mlx5_flow_ext_mreg_supported(dev)) 5038 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 5039 prefix_mark, attr, items, 5040 actions, external, flow_idx, 5041 error); 5042 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5043 &encap_idx); 5044 if (qrss) { 5045 /* Exclude hairpin flows from splitting. */ 5046 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5047 const struct rte_flow_action_queue *queue; 5048 5049 queue = qrss->conf; 5050 if (mlx5_rxq_get_type(dev, queue->index) == 5051 MLX5_RXQ_TYPE_HAIRPIN) 5052 qrss = NULL; 5053 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5054 const struct rte_flow_action_rss *rss; 5055 5056 rss = qrss->conf; 5057 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5058 MLX5_RXQ_TYPE_HAIRPIN) 5059 qrss = NULL; 5060 } 5061 } 5062 if (qrss) { 5063 /* Check if it is in meter suffix table. */ 5064 mtr_sfx = attr->group == (attr->transfer ? 5065 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 5066 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 5067 /* 5068 * Q/RSS action on NIC Rx should be split in order to pass by 5069 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5070 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5071 */ 5072 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5073 sizeof(struct rte_flow_action_set_tag) + 5074 sizeof(struct rte_flow_action_jump); 5075 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5076 SOCKET_ID_ANY); 5077 if (!ext_actions) 5078 return rte_flow_error_set(error, ENOMEM, 5079 RTE_FLOW_ERROR_TYPE_ACTION, 5080 NULL, "no memory to split " 5081 "metadata flow"); 5082 /* 5083 * If we are the suffix flow of meter, tag already exist. 5084 * Set the tag action to void. 5085 */ 5086 if (mtr_sfx) 5087 ext_actions[qrss - actions].type = 5088 RTE_FLOW_ACTION_TYPE_VOID; 5089 else 5090 ext_actions[qrss - actions].type = 5091 (enum rte_flow_action_type) 5092 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5093 /* 5094 * Create the new actions list with removed Q/RSS action 5095 * and appended set tag and jump to register copy table 5096 * (RX_CP_TBL). We should preallocate unique tag ID here 5097 * in advance, because it is needed for set tag action. 5098 */ 5099 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5100 qrss, actions_n, error); 5101 if (!mtr_sfx && !qrss_id) { 5102 ret = -rte_errno; 5103 goto exit; 5104 } 5105 } else if (attr->egress && !attr->transfer) { 5106 /* 5107 * All the actions on NIC Tx should have a metadata register 5108 * copy action to copy reg_a from WQE to reg_c[meta] 5109 */ 5110 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5111 sizeof(struct mlx5_flow_action_copy_mreg); 5112 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5113 SOCKET_ID_ANY); 5114 if (!ext_actions) 5115 return rte_flow_error_set(error, ENOMEM, 5116 RTE_FLOW_ERROR_TYPE_ACTION, 5117 NULL, "no memory to split " 5118 "metadata flow"); 5119 /* Create the action list appended with copy register. */ 5120 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5121 actions_n, error, encap_idx); 5122 if (ret < 0) 5123 goto exit; 5124 } 5125 /* Add the unmodified original or prefix subflow. */ 5126 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, 5127 prefix_mark, attr, 5128 items, ext_actions ? ext_actions : 5129 actions, external, flow_idx, error); 5130 if (ret < 0) 5131 goto exit; 5132 MLX5_ASSERT(dev_flow); 5133 if (qrss) { 5134 const struct rte_flow_attr q_attr = { 5135 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5136 .ingress = 1, 5137 }; 5138 /* Internal PMD action to set register. */ 5139 struct mlx5_rte_flow_item_tag q_tag_spec = { 5140 .data = qrss_id, 5141 .id = REG_NON, 5142 }; 5143 struct rte_flow_item q_items[] = { 5144 { 5145 .type = (enum rte_flow_item_type) 5146 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5147 .spec = &q_tag_spec, 5148 .last = NULL, 5149 .mask = NULL, 5150 }, 5151 { 5152 .type = RTE_FLOW_ITEM_TYPE_END, 5153 }, 5154 }; 5155 struct rte_flow_action q_actions[] = { 5156 { 5157 .type = qrss->type, 5158 .conf = qrss->conf, 5159 }, 5160 { 5161 .type = RTE_FLOW_ACTION_TYPE_END, 5162 }, 5163 }; 5164 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5165 5166 /* 5167 * Configure the tag item only if there is no meter subflow. 5168 * Since tag is already marked in the meter suffix subflow 5169 * we can just use the meter suffix items as is. 5170 */ 5171 if (qrss_id) { 5172 /* Not meter subflow. */ 5173 MLX5_ASSERT(!mtr_sfx); 5174 /* 5175 * Put unique id in prefix flow due to it is destroyed 5176 * after suffix flow and id will be freed after there 5177 * is no actual flows with this id and identifier 5178 * reallocation becomes possible (for example, for 5179 * other flows in other threads). 5180 */ 5181 dev_flow->handle->split_flow_id = qrss_id; 5182 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5183 error); 5184 if (ret < 0) 5185 goto exit; 5186 q_tag_spec.id = ret; 5187 } 5188 dev_flow = NULL; 5189 /* Add suffix subflow to execute Q/RSS. */ 5190 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 0, 5191 &q_attr, mtr_sfx ? items : 5192 q_items, q_actions, 5193 external, flow_idx, error); 5194 if (ret < 0) 5195 goto exit; 5196 /* qrss ID should be freed if failed. */ 5197 qrss_id = 0; 5198 MLX5_ASSERT(dev_flow); 5199 } 5200 5201 exit: 5202 /* 5203 * We do not destroy the partially created sub_flows in case of error. 5204 * These ones are included into parent flow list and will be destroyed 5205 * by flow_drv_destroy. 5206 */ 5207 flow_qrss_free_id(dev, qrss_id); 5208 mlx5_free(ext_actions); 5209 return ret; 5210 } 5211 5212 /** 5213 * The splitting for meter feature. 5214 * 5215 * - The meter flow will be split to two flows as prefix and 5216 * suffix flow. The packets make sense only it pass the prefix 5217 * meter action. 5218 * 5219 * - Reg_C_5 is used for the packet to match betweend prefix and 5220 * suffix flow. 5221 * 5222 * @param dev 5223 * Pointer to Ethernet device. 5224 * @param[in] flow 5225 * Parent flow structure pointer. 5226 * @param[in] prefix_layers 5227 * Prefix subflow layers, may be 0. 5228 * @param[in] prefix_mark 5229 * Prefix subflow mark flag, may be 0. 5230 * @param[in] attr 5231 * Flow rule attributes. 5232 * @param[in] items 5233 * Pattern specification (list terminated by the END pattern item). 5234 * @param[in] actions 5235 * Associated actions (list terminated by the END action). 5236 * @param[in] external 5237 * This flow rule is created by request external to PMD. 5238 * @param[in] flow_idx 5239 * This memory pool index to the flow. 5240 * @param[out] error 5241 * Perform verbose error reporting if not NULL. 5242 * @return 5243 * 0 on success, negative value otherwise 5244 */ 5245 static int 5246 flow_create_split_meter(struct rte_eth_dev *dev, 5247 struct rte_flow *flow, 5248 uint64_t prefix_layers, 5249 uint32_t prefix_mark, 5250 const struct rte_flow_attr *attr, 5251 const struct rte_flow_item items[], 5252 const struct rte_flow_action actions[], 5253 bool external, uint32_t flow_idx, 5254 struct rte_flow_error *error) 5255 { 5256 struct mlx5_priv *priv = dev->data->dev_private; 5257 struct rte_flow_action *sfx_actions = NULL; 5258 struct rte_flow_action *pre_actions = NULL; 5259 struct rte_flow_item *sfx_items = NULL; 5260 struct mlx5_flow *dev_flow = NULL; 5261 struct rte_flow_attr sfx_attr = *attr; 5262 uint32_t mtr = 0; 5263 uint32_t mtr_tag_id = 0; 5264 size_t act_size; 5265 size_t item_size; 5266 int actions_n = 0; 5267 int ret; 5268 5269 if (priv->mtr_en) 5270 actions_n = flow_check_meter_action(actions, &mtr); 5271 if (mtr) { 5272 /* The five prefix actions: meter, decap, encap, tag, end. */ 5273 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 5274 sizeof(struct mlx5_rte_flow_action_set_tag); 5275 /* tag, vlan, port id, end. */ 5276 #define METER_SUFFIX_ITEM 4 5277 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5278 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5279 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5280 0, SOCKET_ID_ANY); 5281 if (!sfx_actions) 5282 return rte_flow_error_set(error, ENOMEM, 5283 RTE_FLOW_ERROR_TYPE_ACTION, 5284 NULL, "no memory to split " 5285 "meter flow"); 5286 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5287 act_size); 5288 pre_actions = sfx_actions + actions_n; 5289 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 5290 actions, sfx_actions, 5291 pre_actions); 5292 if (!mtr_tag_id) { 5293 ret = -rte_errno; 5294 goto exit; 5295 } 5296 /* Add the prefix subflow. */ 5297 ret = flow_create_split_inner(dev, flow, &dev_flow, 5298 prefix_layers, 0, 5299 attr, items, 5300 pre_actions, external, 5301 flow_idx, error); 5302 if (ret) { 5303 ret = -rte_errno; 5304 goto exit; 5305 } 5306 dev_flow->handle->split_flow_id = mtr_tag_id; 5307 /* Setting the sfx group atrr. */ 5308 sfx_attr.group = sfx_attr.transfer ? 5309 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 5310 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 5311 } 5312 /* Add the prefix subflow. */ 5313 ret = flow_create_split_metadata(dev, flow, dev_flow ? 5314 flow_get_prefix_layer_flags(dev_flow) : 5315 prefix_layers, dev_flow ? 5316 dev_flow->handle->mark : prefix_mark, 5317 &sfx_attr, sfx_items ? 5318 sfx_items : items, 5319 sfx_actions ? sfx_actions : actions, 5320 external, flow_idx, error); 5321 exit: 5322 if (sfx_actions) 5323 mlx5_free(sfx_actions); 5324 return ret; 5325 } 5326 5327 /** 5328 * The splitting for sample feature. 5329 * 5330 * Once Sample action is detected in the action list, the flow actions should 5331 * be split into prefix sub flow and suffix sub flow. 5332 * 5333 * The original items remain in the prefix sub flow, all actions preceding the 5334 * sample action and the sample action itself will be copied to the prefix 5335 * sub flow, the actions following the sample action will be copied to the 5336 * suffix sub flow, Queue action always be located in the suffix sub flow. 5337 * 5338 * In order to make the packet from prefix sub flow matches with suffix sub 5339 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5340 * flow uses tag item with the unique flow id. 5341 * 5342 * @param dev 5343 * Pointer to Ethernet device. 5344 * @param[in] flow 5345 * Parent flow structure pointer. 5346 * @param[in] attr 5347 * Flow rule attributes. 5348 * @param[in] items 5349 * Pattern specification (list terminated by the END pattern item). 5350 * @param[in] actions 5351 * Associated actions (list terminated by the END action). 5352 * @param[in] external 5353 * This flow rule is created by request external to PMD. 5354 * @param[in] flow_idx 5355 * This memory pool index to the flow. 5356 * @param[out] error 5357 * Perform verbose error reporting if not NULL. 5358 * @return 5359 * 0 on success, negative value otherwise 5360 */ 5361 static int 5362 flow_create_split_sample(struct rte_eth_dev *dev, 5363 struct rte_flow *flow, 5364 const struct rte_flow_attr *attr, 5365 const struct rte_flow_item items[], 5366 const struct rte_flow_action actions[], 5367 bool external, uint32_t flow_idx, 5368 struct rte_flow_error *error) 5369 { 5370 struct mlx5_priv *priv = dev->data->dev_private; 5371 struct rte_flow_action *sfx_actions = NULL; 5372 struct rte_flow_action *pre_actions = NULL; 5373 struct rte_flow_item *sfx_items = NULL; 5374 struct mlx5_flow *dev_flow = NULL; 5375 struct rte_flow_attr sfx_attr = *attr; 5376 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5377 struct mlx5_flow_dv_sample_resource *sample_res; 5378 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5379 struct mlx5_flow_tbl_resource *sfx_tbl; 5380 union mlx5_flow_tbl_key sfx_table_key; 5381 #endif 5382 size_t act_size; 5383 size_t item_size; 5384 uint32_t fdb_tx = 0; 5385 int32_t tag_id = 0; 5386 int actions_n = 0; 5387 int sample_action_pos; 5388 int qrss_action_pos; 5389 int ret = 0; 5390 5391 if (priv->sampler_en) 5392 actions_n = flow_check_match_action(actions, attr, 5393 RTE_FLOW_ACTION_TYPE_SAMPLE, 5394 &sample_action_pos, &qrss_action_pos); 5395 if (actions_n) { 5396 /* The prefix actions must includes sample, tag, end. */ 5397 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5398 + sizeof(struct mlx5_rte_flow_action_set_tag); 5399 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5400 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5401 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5402 item_size), 0, SOCKET_ID_ANY); 5403 if (!sfx_actions) 5404 return rte_flow_error_set(error, ENOMEM, 5405 RTE_FLOW_ERROR_TYPE_ACTION, 5406 NULL, "no memory to split " 5407 "sample flow"); 5408 /* The representor_id is -1 for uplink. */ 5409 fdb_tx = (attr->transfer && priv->representor_id != -1); 5410 if (!fdb_tx) 5411 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5412 + act_size); 5413 pre_actions = sfx_actions + actions_n; 5414 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items, 5415 actions, sfx_actions, 5416 pre_actions, actions_n, 5417 sample_action_pos, 5418 qrss_action_pos, error); 5419 if (tag_id < 0 || (!fdb_tx && !tag_id)) { 5420 ret = -rte_errno; 5421 goto exit; 5422 } 5423 /* Add the prefix subflow. */ 5424 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, 0, attr, 5425 items, pre_actions, external, 5426 flow_idx, error); 5427 if (ret) { 5428 ret = -rte_errno; 5429 goto exit; 5430 } 5431 dev_flow->handle->split_flow_id = tag_id; 5432 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5433 /* Set the sfx group attr. */ 5434 sample_res = (struct mlx5_flow_dv_sample_resource *) 5435 dev_flow->dv.sample_res; 5436 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5437 sample_res->normal_path_tbl; 5438 sfx_tbl_data = container_of(sfx_tbl, 5439 struct mlx5_flow_tbl_data_entry, tbl); 5440 sfx_table_key.v64 = sfx_tbl_data->entry.key; 5441 sfx_attr.group = sfx_attr.transfer ? 5442 (sfx_table_key.table_id - 1) : 5443 sfx_table_key.table_id; 5444 #endif 5445 } 5446 /* Add the suffix subflow. */ 5447 ret = flow_create_split_meter(dev, flow, dev_flow ? 5448 flow_get_prefix_layer_flags(dev_flow) : 0, 5449 dev_flow ? dev_flow->handle->mark : 0, 5450 &sfx_attr, sfx_items ? sfx_items : items, 5451 sfx_actions ? sfx_actions : actions, 5452 external, flow_idx, error); 5453 exit: 5454 if (sfx_actions) 5455 mlx5_free(sfx_actions); 5456 return ret; 5457 } 5458 5459 /** 5460 * Split the flow to subflow set. The splitters might be linked 5461 * in the chain, like this: 5462 * flow_create_split_outer() calls: 5463 * flow_create_split_meter() calls: 5464 * flow_create_split_metadata(meter_subflow_0) calls: 5465 * flow_create_split_inner(metadata_subflow_0) 5466 * flow_create_split_inner(metadata_subflow_1) 5467 * flow_create_split_inner(metadata_subflow_2) 5468 * flow_create_split_metadata(meter_subflow_1) calls: 5469 * flow_create_split_inner(metadata_subflow_0) 5470 * flow_create_split_inner(metadata_subflow_1) 5471 * flow_create_split_inner(metadata_subflow_2) 5472 * 5473 * This provide flexible way to add new levels of flow splitting. 5474 * The all of successfully created subflows are included to the 5475 * parent flow dev_flow list. 5476 * 5477 * @param dev 5478 * Pointer to Ethernet device. 5479 * @param[in] flow 5480 * Parent flow structure pointer. 5481 * @param[in] attr 5482 * Flow rule attributes. 5483 * @param[in] items 5484 * Pattern specification (list terminated by the END pattern item). 5485 * @param[in] actions 5486 * Associated actions (list terminated by the END action). 5487 * @param[in] external 5488 * This flow rule is created by request external to PMD. 5489 * @param[in] flow_idx 5490 * This memory pool index to the flow. 5491 * @param[out] error 5492 * Perform verbose error reporting if not NULL. 5493 * @return 5494 * 0 on success, negative value otherwise 5495 */ 5496 static int 5497 flow_create_split_outer(struct rte_eth_dev *dev, 5498 struct rte_flow *flow, 5499 const struct rte_flow_attr *attr, 5500 const struct rte_flow_item items[], 5501 const struct rte_flow_action actions[], 5502 bool external, uint32_t flow_idx, 5503 struct rte_flow_error *error) 5504 { 5505 int ret; 5506 5507 ret = flow_create_split_sample(dev, flow, attr, items, 5508 actions, external, flow_idx, error); 5509 MLX5_ASSERT(ret <= 0); 5510 return ret; 5511 } 5512 5513 static struct mlx5_flow_tunnel * 5514 flow_tunnel_from_rule(struct rte_eth_dev *dev, 5515 const struct rte_flow_attr *attr, 5516 const struct rte_flow_item items[], 5517 const struct rte_flow_action actions[]) 5518 { 5519 struct mlx5_flow_tunnel *tunnel; 5520 5521 #pragma GCC diagnostic push 5522 #pragma GCC diagnostic ignored "-Wcast-qual" 5523 if (is_flow_tunnel_match_rule(dev, attr, items, actions)) 5524 tunnel = (struct mlx5_flow_tunnel *)items[0].spec; 5525 else if (is_flow_tunnel_steer_rule(dev, attr, items, actions)) 5526 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf; 5527 else 5528 tunnel = NULL; 5529 #pragma GCC diagnostic pop 5530 5531 return tunnel; 5532 } 5533 5534 /** 5535 * Create a flow and add it to @p list. 5536 * 5537 * @param dev 5538 * Pointer to Ethernet device. 5539 * @param list 5540 * Pointer to a TAILQ flow list. If this parameter NULL, 5541 * no list insertion occurred, flow is just created, 5542 * this is caller's responsibility to track the 5543 * created flow. 5544 * @param[in] attr 5545 * Flow rule attributes. 5546 * @param[in] items 5547 * Pattern specification (list terminated by the END pattern item). 5548 * @param[in] actions 5549 * Associated actions (list terminated by the END action). 5550 * @param[in] external 5551 * This flow rule is created by request external to PMD. 5552 * @param[out] error 5553 * Perform verbose error reporting if not NULL. 5554 * 5555 * @return 5556 * A flow index on success, 0 otherwise and rte_errno is set. 5557 */ 5558 static uint32_t 5559 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 5560 const struct rte_flow_attr *attr, 5561 const struct rte_flow_item items[], 5562 const struct rte_flow_action original_actions[], 5563 bool external, struct rte_flow_error *error) 5564 { 5565 struct mlx5_priv *priv = dev->data->dev_private; 5566 struct rte_flow *flow = NULL; 5567 struct mlx5_flow *dev_flow; 5568 const struct rte_flow_action_rss *rss; 5569 struct mlx5_translated_shared_action 5570 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5571 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5572 union { 5573 struct mlx5_flow_expand_rss buf; 5574 uint8_t buffer[2048]; 5575 } expand_buffer; 5576 union { 5577 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5578 uint8_t buffer[2048]; 5579 } actions_rx; 5580 union { 5581 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5582 uint8_t buffer[2048]; 5583 } actions_hairpin_tx; 5584 union { 5585 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 5586 uint8_t buffer[2048]; 5587 } items_tx; 5588 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 5589 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 5590 priv->rss_desc)[!!priv->flow_idx]; 5591 const struct rte_flow_action *p_actions_rx; 5592 uint32_t i; 5593 uint32_t idx = 0; 5594 int hairpin_flow; 5595 uint32_t hairpin_id = 0; 5596 struct rte_flow_attr attr_tx = { .priority = 0 }; 5597 struct rte_flow_attr attr_factor = {0}; 5598 const struct rte_flow_action *actions; 5599 struct rte_flow_action *translated_actions = NULL; 5600 struct mlx5_flow_tunnel *tunnel; 5601 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 5602 int ret = flow_shared_actions_translate(original_actions, 5603 shared_actions, 5604 &shared_actions_n, 5605 &translated_actions, error); 5606 5607 if (ret < 0) { 5608 MLX5_ASSERT(translated_actions == NULL); 5609 return 0; 5610 } 5611 actions = translated_actions ? translated_actions : original_actions; 5612 memcpy((void *)&attr_factor, (const void *)attr, sizeof(*attr)); 5613 p_actions_rx = actions; 5614 hairpin_flow = flow_check_hairpin_split(dev, &attr_factor, actions); 5615 ret = flow_drv_validate(dev, &attr_factor, items, p_actions_rx, 5616 external, hairpin_flow, error); 5617 if (ret < 0) 5618 goto error_before_hairpin_split; 5619 if (hairpin_flow > 0) { 5620 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 5621 rte_errno = EINVAL; 5622 goto error_before_hairpin_split; 5623 } 5624 flow_hairpin_split(dev, actions, actions_rx.actions, 5625 actions_hairpin_tx.actions, items_tx.items, 5626 &hairpin_id); 5627 p_actions_rx = actions_rx.actions; 5628 } 5629 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 5630 if (!flow) { 5631 rte_errno = ENOMEM; 5632 goto error_before_flow; 5633 } 5634 flow->drv_type = flow_get_drv_type(dev, &attr_factor); 5635 if (hairpin_id != 0) 5636 flow->hairpin_flow_id = hairpin_id; 5637 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 5638 flow->drv_type < MLX5_FLOW_TYPE_MAX); 5639 memset(rss_desc, 0, sizeof(*rss_desc)); 5640 rss = flow_get_rss_action(p_actions_rx); 5641 if (rss) { 5642 /* 5643 * The following information is required by 5644 * mlx5_flow_hashfields_adjust() in advance. 5645 */ 5646 rss_desc->level = rss->level; 5647 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 5648 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 5649 } 5650 flow->dev_handles = 0; 5651 if (rss && rss->types) { 5652 unsigned int graph_root; 5653 5654 graph_root = find_graph_root(items, rss->level); 5655 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 5656 items, rss->types, 5657 mlx5_support_expansion, graph_root); 5658 MLX5_ASSERT(ret > 0 && 5659 (unsigned int)ret < sizeof(expand_buffer.buffer)); 5660 } else { 5661 buf->entries = 1; 5662 buf->entry[0].pattern = (void *)(uintptr_t)items; 5663 } 5664 flow->shared_rss = flow_get_shared_rss_action(shared_actions, 5665 shared_actions_n); 5666 /* 5667 * Record the start index when there is a nested call. All sub-flows 5668 * need to be translated before another calling. 5669 * No need to use ping-pong buffer to save memory here. 5670 */ 5671 if (priv->flow_idx) { 5672 MLX5_ASSERT(!priv->flow_nested_idx); 5673 priv->flow_nested_idx = priv->flow_idx; 5674 } 5675 for (i = 0; i < buf->entries; ++i) { 5676 /* 5677 * The splitter may create multiple dev_flows, 5678 * depending on configuration. In the simplest 5679 * case it just creates unmodified original flow. 5680 */ 5681 ret = flow_create_split_outer(dev, flow, &attr_factor, 5682 buf->entry[i].pattern, 5683 p_actions_rx, external, idx, 5684 error); 5685 if (ret < 0) 5686 goto error; 5687 if (is_flow_tunnel_steer_rule(dev, attr, 5688 buf->entry[i].pattern, 5689 p_actions_rx)) { 5690 ret = flow_tunnel_add_default_miss(dev, flow, attr, 5691 p_actions_rx, 5692 idx, 5693 &default_miss_ctx, 5694 error); 5695 if (ret < 0) { 5696 mlx5_free(default_miss_ctx.queue); 5697 goto error; 5698 } 5699 } 5700 } 5701 /* Create the tx flow. */ 5702 if (hairpin_flow) { 5703 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 5704 attr_tx.ingress = 0; 5705 attr_tx.egress = 1; 5706 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 5707 actions_hairpin_tx.actions, 5708 idx, error); 5709 if (!dev_flow) 5710 goto error; 5711 dev_flow->flow = flow; 5712 dev_flow->external = 0; 5713 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5714 dev_flow->handle, next); 5715 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 5716 items_tx.items, 5717 actions_hairpin_tx.actions, error); 5718 if (ret < 0) 5719 goto error; 5720 } 5721 /* 5722 * Update the metadata register copy table. If extensive 5723 * metadata feature is enabled and registers are supported 5724 * we might create the extra rte_flow for each unique 5725 * MARK/FLAG action ID. 5726 * 5727 * The table is updated for ingress Flows only, because 5728 * the egress Flows belong to the different device and 5729 * copy table should be updated in peer NIC Rx domain. 5730 */ 5731 if (attr_factor.ingress && 5732 (external || attr_factor.group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 5733 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 5734 if (ret) 5735 goto error; 5736 } 5737 /* 5738 * If the flow is external (from application) OR device is started, then 5739 * the flow will be applied immediately. 5740 */ 5741 if (external || dev->data->dev_started) { 5742 ret = flow_drv_apply(dev, flow, error); 5743 if (ret < 0) 5744 goto error; 5745 } 5746 if (list) 5747 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 5748 flow, next); 5749 flow_rxq_flags_set(dev, flow); 5750 rte_free(translated_actions); 5751 /* Nested flow creation index recovery. */ 5752 priv->flow_idx = priv->flow_nested_idx; 5753 if (priv->flow_nested_idx) 5754 priv->flow_nested_idx = 0; 5755 tunnel = flow_tunnel_from_rule(dev, attr, items, actions); 5756 if (tunnel) { 5757 flow->tunnel = 1; 5758 flow->tunnel_id = tunnel->tunnel_id; 5759 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 5760 mlx5_free(default_miss_ctx.queue); 5761 } 5762 return idx; 5763 error: 5764 MLX5_ASSERT(flow); 5765 ret = rte_errno; /* Save rte_errno before cleanup. */ 5766 flow_mreg_del_copy_action(dev, flow); 5767 flow_drv_destroy(dev, flow); 5768 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 5769 rte_errno = ret; /* Restore rte_errno. */ 5770 error_before_flow: 5771 ret = rte_errno; 5772 if (hairpin_id) 5773 mlx5_flow_id_release(priv->sh->flow_id_pool, 5774 hairpin_id); 5775 rte_errno = ret; 5776 priv->flow_idx = priv->flow_nested_idx; 5777 if (priv->flow_nested_idx) 5778 priv->flow_nested_idx = 0; 5779 error_before_hairpin_split: 5780 rte_free(translated_actions); 5781 return 0; 5782 } 5783 5784 /** 5785 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 5786 * incoming packets to table 1. 5787 * 5788 * Other flow rules, requested for group n, will be created in 5789 * e-switch table n+1. 5790 * Jump action to e-switch group n will be created to group n+1. 5791 * 5792 * Used when working in switchdev mode, to utilise advantages of table 1 5793 * and above. 5794 * 5795 * @param dev 5796 * Pointer to Ethernet device. 5797 * 5798 * @return 5799 * Pointer to flow on success, NULL otherwise and rte_errno is set. 5800 */ 5801 struct rte_flow * 5802 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 5803 { 5804 const struct rte_flow_attr attr = { 5805 .group = 0, 5806 .priority = 0, 5807 .ingress = 1, 5808 .egress = 0, 5809 .transfer = 1, 5810 }; 5811 const struct rte_flow_item pattern = { 5812 .type = RTE_FLOW_ITEM_TYPE_END, 5813 }; 5814 struct rte_flow_action_jump jump = { 5815 .group = 1, 5816 }; 5817 const struct rte_flow_action actions[] = { 5818 { 5819 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5820 .conf = &jump, 5821 }, 5822 { 5823 .type = RTE_FLOW_ACTION_TYPE_END, 5824 }, 5825 }; 5826 struct mlx5_priv *priv = dev->data->dev_private; 5827 struct rte_flow_error error; 5828 5829 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 5830 &attr, &pattern, 5831 actions, false, &error); 5832 } 5833 5834 /** 5835 * Validate a flow supported by the NIC. 5836 * 5837 * @see rte_flow_validate() 5838 * @see rte_flow_ops 5839 */ 5840 int 5841 mlx5_flow_validate(struct rte_eth_dev *dev, 5842 const struct rte_flow_attr *attr, 5843 const struct rte_flow_item items[], 5844 const struct rte_flow_action original_actions[], 5845 struct rte_flow_error *error) 5846 { 5847 int hairpin_flow; 5848 struct mlx5_translated_shared_action 5849 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5850 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5851 const struct rte_flow_action *actions; 5852 struct rte_flow_action *translated_actions = NULL; 5853 int ret = flow_shared_actions_translate(original_actions, 5854 shared_actions, 5855 &shared_actions_n, 5856 &translated_actions, error); 5857 5858 if (ret) 5859 return ret; 5860 actions = translated_actions ? translated_actions : original_actions; 5861 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5862 ret = flow_drv_validate(dev, attr, items, actions, 5863 true, hairpin_flow, error); 5864 rte_free(translated_actions); 5865 return ret; 5866 } 5867 5868 /** 5869 * Create a flow. 5870 * 5871 * @see rte_flow_create() 5872 * @see rte_flow_ops 5873 */ 5874 struct rte_flow * 5875 mlx5_flow_create(struct rte_eth_dev *dev, 5876 const struct rte_flow_attr *attr, 5877 const struct rte_flow_item items[], 5878 const struct rte_flow_action actions[], 5879 struct rte_flow_error *error) 5880 { 5881 struct mlx5_priv *priv = dev->data->dev_private; 5882 5883 /* 5884 * If the device is not started yet, it is not allowed to created a 5885 * flow from application. PMD default flows and traffic control flows 5886 * are not affected. 5887 */ 5888 if (unlikely(!dev->data->dev_started)) { 5889 DRV_LOG(DEBUG, "port %u is not started when " 5890 "inserting a flow", dev->data->port_id); 5891 rte_flow_error_set(error, ENODEV, 5892 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5893 NULL, 5894 "port not started"); 5895 return NULL; 5896 } 5897 5898 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 5899 attr, items, actions, true, error); 5900 } 5901 5902 /** 5903 * Destroy a flow in a list. 5904 * 5905 * @param dev 5906 * Pointer to Ethernet device. 5907 * @param list 5908 * Pointer to the Indexed flow list. If this parameter NULL, 5909 * there is no flow removal from the list. Be noted that as 5910 * flow is add to the indexed list, memory of the indexed 5911 * list points to maybe changed as flow destroyed. 5912 * @param[in] flow_idx 5913 * Index of flow to destroy. 5914 */ 5915 static void 5916 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 5917 uint32_t flow_idx) 5918 { 5919 struct mlx5_priv *priv = dev->data->dev_private; 5920 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5921 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5922 [MLX5_IPOOL_RTE_FLOW], flow_idx); 5923 5924 if (!flow) 5925 return; 5926 /* 5927 * Update RX queue flags only if port is started, otherwise it is 5928 * already clean. 5929 */ 5930 if (dev->data->dev_started) 5931 flow_rxq_flags_trim(dev, flow); 5932 if (flow->hairpin_flow_id) 5933 mlx5_flow_id_release(priv->sh->flow_id_pool, 5934 flow->hairpin_flow_id); 5935 flow_drv_destroy(dev, flow); 5936 if (list) 5937 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 5938 flow_idx, flow, next); 5939 flow_mreg_del_copy_action(dev, flow); 5940 if (flow->fdir) { 5941 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5942 if (priv_fdir_flow->rix_flow == flow_idx) 5943 break; 5944 } 5945 if (priv_fdir_flow) { 5946 LIST_REMOVE(priv_fdir_flow, next); 5947 mlx5_free(priv_fdir_flow->fdir); 5948 mlx5_free(priv_fdir_flow); 5949 } 5950 } 5951 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5952 if (flow->tunnel) { 5953 struct mlx5_flow_tunnel *tunnel; 5954 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 5955 RTE_VERIFY(tunnel); 5956 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 5957 mlx5_flow_tunnel_free(dev, tunnel); 5958 } 5959 } 5960 5961 /** 5962 * Destroy all flows. 5963 * 5964 * @param dev 5965 * Pointer to Ethernet device. 5966 * @param list 5967 * Pointer to the Indexed flow list. 5968 * @param active 5969 * If flushing is called avtively. 5970 */ 5971 void 5972 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 5973 { 5974 uint32_t num_flushed = 0; 5975 5976 while (*list) { 5977 flow_list_destroy(dev, list, *list); 5978 num_flushed++; 5979 } 5980 if (active) { 5981 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 5982 dev->data->port_id, num_flushed); 5983 } 5984 } 5985 5986 /** 5987 * Remove all flows. 5988 * 5989 * @param dev 5990 * Pointer to Ethernet device. 5991 * @param list 5992 * Pointer to the Indexed flow list. 5993 */ 5994 void 5995 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list) 5996 { 5997 struct mlx5_priv *priv = dev->data->dev_private; 5998 struct rte_flow *flow = NULL; 5999 uint32_t idx; 6000 6001 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 6002 flow, next) { 6003 flow_drv_remove(dev, flow); 6004 flow_mreg_stop_copy_action(dev, flow); 6005 } 6006 flow_mreg_del_default_copy_action(dev); 6007 flow_rxq_flags_clear(dev); 6008 } 6009 6010 /** 6011 * Add all flows. 6012 * 6013 * @param dev 6014 * Pointer to Ethernet device. 6015 * @param list 6016 * Pointer to the Indexed flow list. 6017 * 6018 * @return 6019 * 0 on success, a negative errno value otherwise and rte_errno is set. 6020 */ 6021 int 6022 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list) 6023 { 6024 struct mlx5_priv *priv = dev->data->dev_private; 6025 struct rte_flow *flow = NULL; 6026 struct rte_flow_error error; 6027 uint32_t idx; 6028 int ret = 0; 6029 6030 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6031 ret = flow_mreg_add_default_copy_action(dev, &error); 6032 if (ret < 0) 6033 return -rte_errno; 6034 /* Apply Flows created by application. */ 6035 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 6036 flow, next) { 6037 ret = flow_mreg_start_copy_action(dev, flow); 6038 if (ret < 0) 6039 goto error; 6040 ret = flow_drv_apply(dev, flow, &error); 6041 if (ret < 0) 6042 goto error; 6043 flow_rxq_flags_set(dev, flow); 6044 } 6045 return 0; 6046 error: 6047 ret = rte_errno; /* Save rte_errno before cleanup. */ 6048 mlx5_flow_stop(dev, list); 6049 rte_errno = ret; /* Restore rte_errno. */ 6050 return -rte_errno; 6051 } 6052 6053 /** 6054 * Stop all default actions for flows. 6055 * 6056 * @param dev 6057 * Pointer to Ethernet device. 6058 */ 6059 void 6060 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6061 { 6062 flow_mreg_del_default_copy_action(dev); 6063 flow_rxq_flags_clear(dev); 6064 } 6065 6066 /** 6067 * Start all default actions for flows. 6068 * 6069 * @param dev 6070 * Pointer to Ethernet device. 6071 * @return 6072 * 0 on success, a negative errno value otherwise and rte_errno is set. 6073 */ 6074 int 6075 mlx5_flow_start_default(struct rte_eth_dev *dev) 6076 { 6077 struct rte_flow_error error; 6078 6079 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6080 return flow_mreg_add_default_copy_action(dev, &error); 6081 } 6082 6083 /** 6084 * Allocate intermediate resources for flow creation. 6085 * 6086 * @param dev 6087 * Pointer to Ethernet device. 6088 */ 6089 void 6090 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) 6091 { 6092 struct mlx5_priv *priv = dev->data->dev_private; 6093 6094 if (!priv->inter_flows) { 6095 priv->inter_flows = mlx5_malloc(MLX5_MEM_ZERO, 6096 MLX5_NUM_MAX_DEV_FLOWS * 6097 sizeof(struct mlx5_flow) + 6098 (sizeof(struct mlx5_flow_rss_desc) + 6099 sizeof(uint16_t) * UINT16_MAX) * 2, 0, 6100 SOCKET_ID_ANY); 6101 if (!priv->inter_flows) { 6102 DRV_LOG(ERR, "can't allocate intermediate memory."); 6103 return; 6104 } 6105 } 6106 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows) 6107 [MLX5_NUM_MAX_DEV_FLOWS]; 6108 /* Reset the index. */ 6109 priv->flow_idx = 0; 6110 priv->flow_nested_idx = 0; 6111 } 6112 6113 /** 6114 * Free intermediate resources for flows. 6115 * 6116 * @param dev 6117 * Pointer to Ethernet device. 6118 */ 6119 void 6120 mlx5_flow_free_intermediate(struct rte_eth_dev *dev) 6121 { 6122 struct mlx5_priv *priv = dev->data->dev_private; 6123 6124 mlx5_free(priv->inter_flows); 6125 priv->inter_flows = NULL; 6126 } 6127 6128 /** 6129 * Verify the flow list is empty 6130 * 6131 * @param dev 6132 * Pointer to Ethernet device. 6133 * 6134 * @return the number of flows not released. 6135 */ 6136 int 6137 mlx5_flow_verify(struct rte_eth_dev *dev) 6138 { 6139 struct mlx5_priv *priv = dev->data->dev_private; 6140 struct rte_flow *flow; 6141 uint32_t idx; 6142 int ret = 0; 6143 6144 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 6145 flow, next) { 6146 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6147 dev->data->port_id, (void *)flow); 6148 ++ret; 6149 } 6150 return ret; 6151 } 6152 6153 /** 6154 * Enable default hairpin egress flow. 6155 * 6156 * @param dev 6157 * Pointer to Ethernet device. 6158 * @param queue 6159 * The queue index. 6160 * 6161 * @return 6162 * 0 on success, a negative errno value otherwise and rte_errno is set. 6163 */ 6164 int 6165 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6166 uint32_t queue) 6167 { 6168 struct mlx5_priv *priv = dev->data->dev_private; 6169 const struct rte_flow_attr attr = { 6170 .egress = 1, 6171 .priority = 0, 6172 }; 6173 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6174 .queue = queue, 6175 }; 6176 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6177 .queue = UINT32_MAX, 6178 }; 6179 struct rte_flow_item items[] = { 6180 { 6181 .type = (enum rte_flow_item_type) 6182 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6183 .spec = &queue_spec, 6184 .last = NULL, 6185 .mask = &queue_mask, 6186 }, 6187 { 6188 .type = RTE_FLOW_ITEM_TYPE_END, 6189 }, 6190 }; 6191 struct rte_flow_action_jump jump = { 6192 .group = MLX5_HAIRPIN_TX_TABLE, 6193 }; 6194 struct rte_flow_action actions[2]; 6195 uint32_t flow_idx; 6196 struct rte_flow_error error; 6197 6198 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6199 actions[0].conf = &jump; 6200 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6201 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6202 &attr, items, actions, false, &error); 6203 if (!flow_idx) { 6204 DRV_LOG(DEBUG, 6205 "Failed to create ctrl flow: rte_errno(%d)," 6206 " type(%d), message(%s)", 6207 rte_errno, error.type, 6208 error.message ? error.message : " (no stated reason)"); 6209 return -rte_errno; 6210 } 6211 return 0; 6212 } 6213 6214 /** 6215 * Enable a control flow configured from the control plane. 6216 * 6217 * @param dev 6218 * Pointer to Ethernet device. 6219 * @param eth_spec 6220 * An Ethernet flow spec to apply. 6221 * @param eth_mask 6222 * An Ethernet flow mask to apply. 6223 * @param vlan_spec 6224 * A VLAN flow spec to apply. 6225 * @param vlan_mask 6226 * A VLAN flow mask to apply. 6227 * 6228 * @return 6229 * 0 on success, a negative errno value otherwise and rte_errno is set. 6230 */ 6231 int 6232 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6233 struct rte_flow_item_eth *eth_spec, 6234 struct rte_flow_item_eth *eth_mask, 6235 struct rte_flow_item_vlan *vlan_spec, 6236 struct rte_flow_item_vlan *vlan_mask) 6237 { 6238 struct mlx5_priv *priv = dev->data->dev_private; 6239 const struct rte_flow_attr attr = { 6240 .ingress = 1, 6241 .priority = MLX5_FLOW_PRIO_RSVD, 6242 }; 6243 struct rte_flow_item items[] = { 6244 { 6245 .type = RTE_FLOW_ITEM_TYPE_ETH, 6246 .spec = eth_spec, 6247 .last = NULL, 6248 .mask = eth_mask, 6249 }, 6250 { 6251 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6252 RTE_FLOW_ITEM_TYPE_END, 6253 .spec = vlan_spec, 6254 .last = NULL, 6255 .mask = vlan_mask, 6256 }, 6257 { 6258 .type = RTE_FLOW_ITEM_TYPE_END, 6259 }, 6260 }; 6261 uint16_t queue[priv->reta_idx_n]; 6262 struct rte_flow_action_rss action_rss = { 6263 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6264 .level = 0, 6265 .types = priv->rss_conf.rss_hf, 6266 .key_len = priv->rss_conf.rss_key_len, 6267 .queue_num = priv->reta_idx_n, 6268 .key = priv->rss_conf.rss_key, 6269 .queue = queue, 6270 }; 6271 struct rte_flow_action actions[] = { 6272 { 6273 .type = RTE_FLOW_ACTION_TYPE_RSS, 6274 .conf = &action_rss, 6275 }, 6276 { 6277 .type = RTE_FLOW_ACTION_TYPE_END, 6278 }, 6279 }; 6280 uint32_t flow_idx; 6281 struct rte_flow_error error; 6282 unsigned int i; 6283 6284 if (!priv->reta_idx_n || !priv->rxqs_n) { 6285 return 0; 6286 } 6287 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6288 action_rss.types = 0; 6289 for (i = 0; i != priv->reta_idx_n; ++i) 6290 queue[i] = (*priv->reta_idx)[i]; 6291 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6292 &attr, items, actions, false, &error); 6293 if (!flow_idx) 6294 return -rte_errno; 6295 return 0; 6296 } 6297 6298 /** 6299 * Enable a flow control configured from the control plane. 6300 * 6301 * @param dev 6302 * Pointer to Ethernet device. 6303 * @param eth_spec 6304 * An Ethernet flow spec to apply. 6305 * @param eth_mask 6306 * An Ethernet flow mask to apply. 6307 * 6308 * @return 6309 * 0 on success, a negative errno value otherwise and rte_errno is set. 6310 */ 6311 int 6312 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6313 struct rte_flow_item_eth *eth_spec, 6314 struct rte_flow_item_eth *eth_mask) 6315 { 6316 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6317 } 6318 6319 /** 6320 * Create default miss flow rule matching lacp traffic 6321 * 6322 * @param dev 6323 * Pointer to Ethernet device. 6324 * @param eth_spec 6325 * An Ethernet flow spec to apply. 6326 * 6327 * @return 6328 * 0 on success, a negative errno value otherwise and rte_errno is set. 6329 */ 6330 int 6331 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6332 { 6333 struct mlx5_priv *priv = dev->data->dev_private; 6334 /* 6335 * The LACP matching is done by only using ether type since using 6336 * a multicast dst mac causes kernel to give low priority to this flow. 6337 */ 6338 static const struct rte_flow_item_eth lacp_spec = { 6339 .type = RTE_BE16(0x8809), 6340 }; 6341 static const struct rte_flow_item_eth lacp_mask = { 6342 .type = 0xffff, 6343 }; 6344 const struct rte_flow_attr attr = { 6345 .ingress = 1, 6346 }; 6347 struct rte_flow_item items[] = { 6348 { 6349 .type = RTE_FLOW_ITEM_TYPE_ETH, 6350 .spec = &lacp_spec, 6351 .mask = &lacp_mask, 6352 }, 6353 { 6354 .type = RTE_FLOW_ITEM_TYPE_END, 6355 }, 6356 }; 6357 struct rte_flow_action actions[] = { 6358 { 6359 .type = (enum rte_flow_action_type) 6360 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6361 }, 6362 { 6363 .type = RTE_FLOW_ACTION_TYPE_END, 6364 }, 6365 }; 6366 struct rte_flow_error error; 6367 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6368 &attr, items, actions, false, &error); 6369 6370 if (!flow_idx) 6371 return -rte_errno; 6372 return 0; 6373 } 6374 6375 /** 6376 * Destroy a flow. 6377 * 6378 * @see rte_flow_destroy() 6379 * @see rte_flow_ops 6380 */ 6381 int 6382 mlx5_flow_destroy(struct rte_eth_dev *dev, 6383 struct rte_flow *flow, 6384 struct rte_flow_error *error __rte_unused) 6385 { 6386 struct mlx5_priv *priv = dev->data->dev_private; 6387 6388 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6389 return 0; 6390 } 6391 6392 /** 6393 * Destroy all flows. 6394 * 6395 * @see rte_flow_flush() 6396 * @see rte_flow_ops 6397 */ 6398 int 6399 mlx5_flow_flush(struct rte_eth_dev *dev, 6400 struct rte_flow_error *error __rte_unused) 6401 { 6402 struct mlx5_priv *priv = dev->data->dev_private; 6403 6404 mlx5_flow_list_flush(dev, &priv->flows, false); 6405 return 0; 6406 } 6407 6408 /** 6409 * Isolated mode. 6410 * 6411 * @see rte_flow_isolate() 6412 * @see rte_flow_ops 6413 */ 6414 int 6415 mlx5_flow_isolate(struct rte_eth_dev *dev, 6416 int enable, 6417 struct rte_flow_error *error) 6418 { 6419 struct mlx5_priv *priv = dev->data->dev_private; 6420 6421 if (dev->data->dev_started) { 6422 rte_flow_error_set(error, EBUSY, 6423 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6424 NULL, 6425 "port must be stopped first"); 6426 return -rte_errno; 6427 } 6428 priv->isolated = !!enable; 6429 if (enable) 6430 dev->dev_ops = &mlx5_os_dev_ops_isolate; 6431 else 6432 dev->dev_ops = &mlx5_os_dev_ops; 6433 6434 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 6435 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 6436 6437 return 0; 6438 } 6439 6440 /** 6441 * Query a flow. 6442 * 6443 * @see rte_flow_query() 6444 * @see rte_flow_ops 6445 */ 6446 static int 6447 flow_drv_query(struct rte_eth_dev *dev, 6448 uint32_t flow_idx, 6449 const struct rte_flow_action *actions, 6450 void *data, 6451 struct rte_flow_error *error) 6452 { 6453 struct mlx5_priv *priv = dev->data->dev_private; 6454 const struct mlx5_flow_driver_ops *fops; 6455 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6456 [MLX5_IPOOL_RTE_FLOW], 6457 flow_idx); 6458 enum mlx5_flow_drv_type ftype; 6459 6460 if (!flow) { 6461 return rte_flow_error_set(error, ENOENT, 6462 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6463 NULL, 6464 "invalid flow handle"); 6465 } 6466 ftype = flow->drv_type; 6467 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 6468 fops = flow_get_drv_ops(ftype); 6469 6470 return fops->query(dev, flow, actions, data, error); 6471 } 6472 6473 /** 6474 * Query a flow. 6475 * 6476 * @see rte_flow_query() 6477 * @see rte_flow_ops 6478 */ 6479 int 6480 mlx5_flow_query(struct rte_eth_dev *dev, 6481 struct rte_flow *flow, 6482 const struct rte_flow_action *actions, 6483 void *data, 6484 struct rte_flow_error *error) 6485 { 6486 int ret; 6487 6488 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 6489 error); 6490 if (ret < 0) 6491 return ret; 6492 return 0; 6493 } 6494 6495 /** 6496 * Convert a flow director filter to a generic flow. 6497 * 6498 * @param dev 6499 * Pointer to Ethernet device. 6500 * @param fdir_filter 6501 * Flow director filter to add. 6502 * @param attributes 6503 * Generic flow parameters structure. 6504 * 6505 * @return 6506 * 0 on success, a negative errno value otherwise and rte_errno is set. 6507 */ 6508 static int 6509 flow_fdir_filter_convert(struct rte_eth_dev *dev, 6510 const struct rte_eth_fdir_filter *fdir_filter, 6511 struct mlx5_fdir *attributes) 6512 { 6513 struct mlx5_priv *priv = dev->data->dev_private; 6514 const struct rte_eth_fdir_input *input = &fdir_filter->input; 6515 const struct rte_eth_fdir_masks *mask = 6516 &dev->data->dev_conf.fdir_conf.mask; 6517 6518 /* Validate queue number. */ 6519 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 6520 DRV_LOG(ERR, "port %u invalid queue number %d", 6521 dev->data->port_id, fdir_filter->action.rx_queue); 6522 rte_errno = EINVAL; 6523 return -rte_errno; 6524 } 6525 attributes->attr.ingress = 1; 6526 attributes->items[0] = (struct rte_flow_item) { 6527 .type = RTE_FLOW_ITEM_TYPE_ETH, 6528 .spec = &attributes->l2, 6529 .mask = &attributes->l2_mask, 6530 }; 6531 switch (fdir_filter->action.behavior) { 6532 case RTE_ETH_FDIR_ACCEPT: 6533 attributes->actions[0] = (struct rte_flow_action){ 6534 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 6535 .conf = &attributes->queue, 6536 }; 6537 break; 6538 case RTE_ETH_FDIR_REJECT: 6539 attributes->actions[0] = (struct rte_flow_action){ 6540 .type = RTE_FLOW_ACTION_TYPE_DROP, 6541 }; 6542 break; 6543 default: 6544 DRV_LOG(ERR, "port %u invalid behavior %d", 6545 dev->data->port_id, 6546 fdir_filter->action.behavior); 6547 rte_errno = ENOTSUP; 6548 return -rte_errno; 6549 } 6550 attributes->queue.index = fdir_filter->action.rx_queue; 6551 /* Handle L3. */ 6552 switch (fdir_filter->input.flow_type) { 6553 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 6554 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 6555 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 6556 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 6557 .src_addr = input->flow.ip4_flow.src_ip, 6558 .dst_addr = input->flow.ip4_flow.dst_ip, 6559 .time_to_live = input->flow.ip4_flow.ttl, 6560 .type_of_service = input->flow.ip4_flow.tos, 6561 }; 6562 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 6563 .src_addr = mask->ipv4_mask.src_ip, 6564 .dst_addr = mask->ipv4_mask.dst_ip, 6565 .time_to_live = mask->ipv4_mask.ttl, 6566 .type_of_service = mask->ipv4_mask.tos, 6567 .next_proto_id = mask->ipv4_mask.proto, 6568 }; 6569 attributes->items[1] = (struct rte_flow_item){ 6570 .type = RTE_FLOW_ITEM_TYPE_IPV4, 6571 .spec = &attributes->l3, 6572 .mask = &attributes->l3_mask, 6573 }; 6574 break; 6575 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 6576 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 6577 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 6578 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 6579 .hop_limits = input->flow.ipv6_flow.hop_limits, 6580 .proto = input->flow.ipv6_flow.proto, 6581 }; 6582 6583 memcpy(attributes->l3.ipv6.hdr.src_addr, 6584 input->flow.ipv6_flow.src_ip, 6585 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 6586 memcpy(attributes->l3.ipv6.hdr.dst_addr, 6587 input->flow.ipv6_flow.dst_ip, 6588 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 6589 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 6590 mask->ipv6_mask.src_ip, 6591 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 6592 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 6593 mask->ipv6_mask.dst_ip, 6594 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 6595 attributes->items[1] = (struct rte_flow_item){ 6596 .type = RTE_FLOW_ITEM_TYPE_IPV6, 6597 .spec = &attributes->l3, 6598 .mask = &attributes->l3_mask, 6599 }; 6600 break; 6601 default: 6602 DRV_LOG(ERR, "port %u invalid flow type%d", 6603 dev->data->port_id, fdir_filter->input.flow_type); 6604 rte_errno = ENOTSUP; 6605 return -rte_errno; 6606 } 6607 /* Handle L4. */ 6608 switch (fdir_filter->input.flow_type) { 6609 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 6610 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 6611 .src_port = input->flow.udp4_flow.src_port, 6612 .dst_port = input->flow.udp4_flow.dst_port, 6613 }; 6614 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 6615 .src_port = mask->src_port_mask, 6616 .dst_port = mask->dst_port_mask, 6617 }; 6618 attributes->items[2] = (struct rte_flow_item){ 6619 .type = RTE_FLOW_ITEM_TYPE_UDP, 6620 .spec = &attributes->l4, 6621 .mask = &attributes->l4_mask, 6622 }; 6623 break; 6624 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 6625 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 6626 .src_port = input->flow.tcp4_flow.src_port, 6627 .dst_port = input->flow.tcp4_flow.dst_port, 6628 }; 6629 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 6630 .src_port = mask->src_port_mask, 6631 .dst_port = mask->dst_port_mask, 6632 }; 6633 attributes->items[2] = (struct rte_flow_item){ 6634 .type = RTE_FLOW_ITEM_TYPE_TCP, 6635 .spec = &attributes->l4, 6636 .mask = &attributes->l4_mask, 6637 }; 6638 break; 6639 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 6640 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 6641 .src_port = input->flow.udp6_flow.src_port, 6642 .dst_port = input->flow.udp6_flow.dst_port, 6643 }; 6644 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 6645 .src_port = mask->src_port_mask, 6646 .dst_port = mask->dst_port_mask, 6647 }; 6648 attributes->items[2] = (struct rte_flow_item){ 6649 .type = RTE_FLOW_ITEM_TYPE_UDP, 6650 .spec = &attributes->l4, 6651 .mask = &attributes->l4_mask, 6652 }; 6653 break; 6654 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 6655 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 6656 .src_port = input->flow.tcp6_flow.src_port, 6657 .dst_port = input->flow.tcp6_flow.dst_port, 6658 }; 6659 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 6660 .src_port = mask->src_port_mask, 6661 .dst_port = mask->dst_port_mask, 6662 }; 6663 attributes->items[2] = (struct rte_flow_item){ 6664 .type = RTE_FLOW_ITEM_TYPE_TCP, 6665 .spec = &attributes->l4, 6666 .mask = &attributes->l4_mask, 6667 }; 6668 break; 6669 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 6670 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 6671 break; 6672 default: 6673 DRV_LOG(ERR, "port %u invalid flow type%d", 6674 dev->data->port_id, fdir_filter->input.flow_type); 6675 rte_errno = ENOTSUP; 6676 return -rte_errno; 6677 } 6678 return 0; 6679 } 6680 6681 #define FLOW_FDIR_CMP(f1, f2, fld) \ 6682 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 6683 6684 /** 6685 * Compare two FDIR flows. If items and actions are identical, the two flows are 6686 * regarded as same. 6687 * 6688 * @param dev 6689 * Pointer to Ethernet device. 6690 * @param f1 6691 * FDIR flow to compare. 6692 * @param f2 6693 * FDIR flow to compare. 6694 * 6695 * @return 6696 * Zero on match, 1 otherwise. 6697 */ 6698 static int 6699 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 6700 { 6701 if (FLOW_FDIR_CMP(f1, f2, attr) || 6702 FLOW_FDIR_CMP(f1, f2, l2) || 6703 FLOW_FDIR_CMP(f1, f2, l2_mask) || 6704 FLOW_FDIR_CMP(f1, f2, l3) || 6705 FLOW_FDIR_CMP(f1, f2, l3_mask) || 6706 FLOW_FDIR_CMP(f1, f2, l4) || 6707 FLOW_FDIR_CMP(f1, f2, l4_mask) || 6708 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 6709 return 1; 6710 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 6711 FLOW_FDIR_CMP(f1, f2, queue)) 6712 return 1; 6713 return 0; 6714 } 6715 6716 /** 6717 * Search device flow list to find out a matched FDIR flow. 6718 * 6719 * @param dev 6720 * Pointer to Ethernet device. 6721 * @param fdir_flow 6722 * FDIR flow to lookup. 6723 * 6724 * @return 6725 * Index of flow if found, 0 otherwise. 6726 */ 6727 static uint32_t 6728 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 6729 { 6730 struct mlx5_priv *priv = dev->data->dev_private; 6731 uint32_t flow_idx = 0; 6732 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6733 6734 MLX5_ASSERT(fdir_flow); 6735 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 6736 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { 6737 DRV_LOG(DEBUG, "port %u found FDIR flow %u", 6738 dev->data->port_id, flow_idx); 6739 flow_idx = priv_fdir_flow->rix_flow; 6740 break; 6741 } 6742 } 6743 return flow_idx; 6744 } 6745 6746 /** 6747 * Add new flow director filter and store it in list. 6748 * 6749 * @param dev 6750 * Pointer to Ethernet device. 6751 * @param fdir_filter 6752 * Flow director filter to add. 6753 * 6754 * @return 6755 * 0 on success, a negative errno value otherwise and rte_errno is set. 6756 */ 6757 static int 6758 flow_fdir_filter_add(struct rte_eth_dev *dev, 6759 const struct rte_eth_fdir_filter *fdir_filter) 6760 { 6761 struct mlx5_priv *priv = dev->data->dev_private; 6762 struct mlx5_fdir *fdir_flow; 6763 struct rte_flow *flow; 6764 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6765 uint32_t flow_idx; 6766 int ret; 6767 6768 fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*fdir_flow), 0, 6769 SOCKET_ID_ANY); 6770 if (!fdir_flow) { 6771 rte_errno = ENOMEM; 6772 return -rte_errno; 6773 } 6774 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 6775 if (ret) 6776 goto error; 6777 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); 6778 if (flow_idx) { 6779 rte_errno = EEXIST; 6780 goto error; 6781 } 6782 priv_fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, 6783 sizeof(struct mlx5_fdir_flow), 6784 0, SOCKET_ID_ANY); 6785 if (!priv_fdir_flow) { 6786 rte_errno = ENOMEM; 6787 goto error; 6788 } 6789 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 6790 fdir_flow->items, fdir_flow->actions, true, 6791 NULL); 6792 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 6793 if (!flow) 6794 goto error; 6795 flow->fdir = 1; 6796 priv_fdir_flow->fdir = fdir_flow; 6797 priv_fdir_flow->rix_flow = flow_idx; 6798 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); 6799 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 6800 dev->data->port_id, (void *)flow); 6801 return 0; 6802 error: 6803 mlx5_free(priv_fdir_flow); 6804 mlx5_free(fdir_flow); 6805 return -rte_errno; 6806 } 6807 6808 /** 6809 * Delete specific filter. 6810 * 6811 * @param dev 6812 * Pointer to Ethernet device. 6813 * @param fdir_filter 6814 * Filter to be deleted. 6815 * 6816 * @return 6817 * 0 on success, a negative errno value otherwise and rte_errno is set. 6818 */ 6819 static int 6820 flow_fdir_filter_delete(struct rte_eth_dev *dev, 6821 const struct rte_eth_fdir_filter *fdir_filter) 6822 { 6823 struct mlx5_priv *priv = dev->data->dev_private; 6824 uint32_t flow_idx; 6825 struct mlx5_fdir fdir_flow = { 6826 .attr.group = 0, 6827 }; 6828 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6829 int ret; 6830 6831 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 6832 if (ret) 6833 return -rte_errno; 6834 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 6835 /* Find the fdir in priv list */ 6836 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) 6837 break; 6838 } 6839 if (!priv_fdir_flow) 6840 return 0; 6841 LIST_REMOVE(priv_fdir_flow, next); 6842 flow_idx = priv_fdir_flow->rix_flow; 6843 flow_list_destroy(dev, &priv->flows, flow_idx); 6844 mlx5_free(priv_fdir_flow->fdir); 6845 mlx5_free(priv_fdir_flow); 6846 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", 6847 dev->data->port_id, flow_idx); 6848 return 0; 6849 } 6850 6851 /** 6852 * Update queue for specific filter. 6853 * 6854 * @param dev 6855 * Pointer to Ethernet device. 6856 * @param fdir_filter 6857 * Filter to be updated. 6858 * 6859 * @return 6860 * 0 on success, a negative errno value otherwise and rte_errno is set. 6861 */ 6862 static int 6863 flow_fdir_filter_update(struct rte_eth_dev *dev, 6864 const struct rte_eth_fdir_filter *fdir_filter) 6865 { 6866 int ret; 6867 6868 ret = flow_fdir_filter_delete(dev, fdir_filter); 6869 if (ret) 6870 return ret; 6871 return flow_fdir_filter_add(dev, fdir_filter); 6872 } 6873 6874 /** 6875 * Flush all filters. 6876 * 6877 * @param dev 6878 * Pointer to Ethernet device. 6879 */ 6880 static void 6881 flow_fdir_filter_flush(struct rte_eth_dev *dev) 6882 { 6883 struct mlx5_priv *priv = dev->data->dev_private; 6884 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6885 6886 while (!LIST_EMPTY(&priv->fdir_flows)) { 6887 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); 6888 LIST_REMOVE(priv_fdir_flow, next); 6889 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); 6890 mlx5_free(priv_fdir_flow->fdir); 6891 mlx5_free(priv_fdir_flow); 6892 } 6893 } 6894 6895 /** 6896 * Get flow director information. 6897 * 6898 * @param dev 6899 * Pointer to Ethernet device. 6900 * @param[out] fdir_info 6901 * Resulting flow director information. 6902 */ 6903 static void 6904 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 6905 { 6906 struct rte_eth_fdir_masks *mask = 6907 &dev->data->dev_conf.fdir_conf.mask; 6908 6909 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 6910 fdir_info->guarant_spc = 0; 6911 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 6912 fdir_info->max_flexpayload = 0; 6913 fdir_info->flow_types_mask[0] = 0; 6914 fdir_info->flex_payload_unit = 0; 6915 fdir_info->max_flex_payload_segment_num = 0; 6916 fdir_info->flex_payload_limit = 0; 6917 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 6918 } 6919 6920 /** 6921 * Deal with flow director operations. 6922 * 6923 * @param dev 6924 * Pointer to Ethernet device. 6925 * @param filter_op 6926 * Operation to perform. 6927 * @param arg 6928 * Pointer to operation-specific structure. 6929 * 6930 * @return 6931 * 0 on success, a negative errno value otherwise and rte_errno is set. 6932 */ 6933 static int 6934 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 6935 void *arg) 6936 { 6937 enum rte_fdir_mode fdir_mode = 6938 dev->data->dev_conf.fdir_conf.mode; 6939 6940 if (filter_op == RTE_ETH_FILTER_NOP) 6941 return 0; 6942 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 6943 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 6944 DRV_LOG(ERR, "port %u flow director mode %d not supported", 6945 dev->data->port_id, fdir_mode); 6946 rte_errno = EINVAL; 6947 return -rte_errno; 6948 } 6949 switch (filter_op) { 6950 case RTE_ETH_FILTER_ADD: 6951 return flow_fdir_filter_add(dev, arg); 6952 case RTE_ETH_FILTER_UPDATE: 6953 return flow_fdir_filter_update(dev, arg); 6954 case RTE_ETH_FILTER_DELETE: 6955 return flow_fdir_filter_delete(dev, arg); 6956 case RTE_ETH_FILTER_FLUSH: 6957 flow_fdir_filter_flush(dev); 6958 break; 6959 case RTE_ETH_FILTER_INFO: 6960 flow_fdir_info_get(dev, arg); 6961 break; 6962 default: 6963 DRV_LOG(DEBUG, "port %u unknown operation %u", 6964 dev->data->port_id, filter_op); 6965 rte_errno = EINVAL; 6966 return -rte_errno; 6967 } 6968 return 0; 6969 } 6970 6971 /** 6972 * Manage filter operations. 6973 * 6974 * @param dev 6975 * Pointer to Ethernet device structure. 6976 * @param filter_type 6977 * Filter type. 6978 * @param filter_op 6979 * Operation to perform. 6980 * @param arg 6981 * Pointer to operation-specific structure. 6982 * 6983 * @return 6984 * 0 on success, a negative errno value otherwise and rte_errno is set. 6985 */ 6986 int 6987 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 6988 enum rte_filter_type filter_type, 6989 enum rte_filter_op filter_op, 6990 void *arg) 6991 { 6992 switch (filter_type) { 6993 case RTE_ETH_FILTER_GENERIC: 6994 if (filter_op != RTE_ETH_FILTER_GET) { 6995 rte_errno = EINVAL; 6996 return -rte_errno; 6997 } 6998 *(const void **)arg = &mlx5_flow_ops; 6999 return 0; 7000 case RTE_ETH_FILTER_FDIR: 7001 return flow_fdir_ctrl_func(dev, filter_op, arg); 7002 default: 7003 DRV_LOG(ERR, "port %u filter type (%d) not supported", 7004 dev->data->port_id, filter_type); 7005 rte_errno = ENOTSUP; 7006 return -rte_errno; 7007 } 7008 return 0; 7009 } 7010 7011 /** 7012 * Create the needed meter and suffix tables. 7013 * 7014 * @param[in] dev 7015 * Pointer to Ethernet device. 7016 * @param[in] fm 7017 * Pointer to the flow meter. 7018 * 7019 * @return 7020 * Pointer to table set on success, NULL otherwise. 7021 */ 7022 struct mlx5_meter_domains_infos * 7023 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7024 const struct mlx5_flow_meter *fm) 7025 { 7026 const struct mlx5_flow_driver_ops *fops; 7027 7028 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7029 return fops->create_mtr_tbls(dev, fm); 7030 } 7031 7032 /** 7033 * Destroy the meter table set. 7034 * 7035 * @param[in] dev 7036 * Pointer to Ethernet device. 7037 * @param[in] tbl 7038 * Pointer to the meter table set. 7039 * 7040 * @return 7041 * 0 on success. 7042 */ 7043 int 7044 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7045 struct mlx5_meter_domains_infos *tbls) 7046 { 7047 const struct mlx5_flow_driver_ops *fops; 7048 7049 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7050 return fops->destroy_mtr_tbls(dev, tbls); 7051 } 7052 7053 /** 7054 * Create policer rules. 7055 * 7056 * @param[in] dev 7057 * Pointer to Ethernet device. 7058 * @param[in] fm 7059 * Pointer to flow meter structure. 7060 * @param[in] attr 7061 * Pointer to flow attributes. 7062 * 7063 * @return 7064 * 0 on success, -1 otherwise. 7065 */ 7066 int 7067 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 7068 struct mlx5_flow_meter *fm, 7069 const struct rte_flow_attr *attr) 7070 { 7071 const struct mlx5_flow_driver_ops *fops; 7072 7073 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7074 return fops->create_policer_rules(dev, fm, attr); 7075 } 7076 7077 /** 7078 * Destroy policer rules. 7079 * 7080 * @param[in] fm 7081 * Pointer to flow meter structure. 7082 * @param[in] attr 7083 * Pointer to flow attributes. 7084 * 7085 * @return 7086 * 0 on success, -1 otherwise. 7087 */ 7088 int 7089 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 7090 struct mlx5_flow_meter *fm, 7091 const struct rte_flow_attr *attr) 7092 { 7093 const struct mlx5_flow_driver_ops *fops; 7094 7095 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7096 return fops->destroy_policer_rules(dev, fm, attr); 7097 } 7098 7099 /** 7100 * Allocate a counter. 7101 * 7102 * @param[in] dev 7103 * Pointer to Ethernet device structure. 7104 * 7105 * @return 7106 * Index to allocated counter on success, 0 otherwise. 7107 */ 7108 uint32_t 7109 mlx5_counter_alloc(struct rte_eth_dev *dev) 7110 { 7111 const struct mlx5_flow_driver_ops *fops; 7112 struct rte_flow_attr attr = { .transfer = 0 }; 7113 7114 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7115 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7116 return fops->counter_alloc(dev); 7117 } 7118 DRV_LOG(ERR, 7119 "port %u counter allocate is not supported.", 7120 dev->data->port_id); 7121 return 0; 7122 } 7123 7124 /** 7125 * Free a counter. 7126 * 7127 * @param[in] dev 7128 * Pointer to Ethernet device structure. 7129 * @param[in] cnt 7130 * Index to counter to be free. 7131 */ 7132 void 7133 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7134 { 7135 const struct mlx5_flow_driver_ops *fops; 7136 struct rte_flow_attr attr = { .transfer = 0 }; 7137 7138 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7139 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7140 fops->counter_free(dev, cnt); 7141 return; 7142 } 7143 DRV_LOG(ERR, 7144 "port %u counter free is not supported.", 7145 dev->data->port_id); 7146 } 7147 7148 /** 7149 * Query counter statistics. 7150 * 7151 * @param[in] dev 7152 * Pointer to Ethernet device structure. 7153 * @param[in] cnt 7154 * Index to counter to query. 7155 * @param[in] clear 7156 * Set to clear counter statistics. 7157 * @param[out] pkts 7158 * The counter hits packets number to save. 7159 * @param[out] bytes 7160 * The counter hits bytes number to save. 7161 * 7162 * @return 7163 * 0 on success, a negative errno value otherwise. 7164 */ 7165 int 7166 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7167 bool clear, uint64_t *pkts, uint64_t *bytes) 7168 { 7169 const struct mlx5_flow_driver_ops *fops; 7170 struct rte_flow_attr attr = { .transfer = 0 }; 7171 7172 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7173 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7174 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7175 } 7176 DRV_LOG(ERR, 7177 "port %u counter query is not supported.", 7178 dev->data->port_id); 7179 return -ENOTSUP; 7180 } 7181 7182 /** 7183 * Allocate a new memory for the counter values wrapped by all the needed 7184 * management. 7185 * 7186 * @param[in] sh 7187 * Pointer to mlx5_dev_ctx_shared object. 7188 * 7189 * @return 7190 * 0 on success, a negative errno value otherwise. 7191 */ 7192 static int 7193 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7194 { 7195 struct mlx5_devx_mkey_attr mkey_attr; 7196 struct mlx5_counter_stats_mem_mng *mem_mng; 7197 volatile struct flow_counter_stats *raw_data; 7198 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7199 int size = (sizeof(struct flow_counter_stats) * 7200 MLX5_COUNTERS_PER_POOL + 7201 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7202 sizeof(struct mlx5_counter_stats_mem_mng); 7203 size_t pgsize = rte_mem_page_size(); 7204 uint8_t *mem; 7205 int i; 7206 7207 if (pgsize == (size_t)-1) { 7208 DRV_LOG(ERR, "Failed to get mem page size"); 7209 rte_errno = ENOMEM; 7210 return -ENOMEM; 7211 } 7212 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7213 if (!mem) { 7214 rte_errno = ENOMEM; 7215 return -ENOMEM; 7216 } 7217 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7218 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7219 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 7220 IBV_ACCESS_LOCAL_WRITE); 7221 if (!mem_mng->umem) { 7222 rte_errno = errno; 7223 mlx5_free(mem); 7224 return -rte_errno; 7225 } 7226 mkey_attr.addr = (uintptr_t)mem; 7227 mkey_attr.size = size; 7228 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7229 mkey_attr.pd = sh->pdn; 7230 mkey_attr.log_entity_size = 0; 7231 mkey_attr.pg_access = 0; 7232 mkey_attr.klm_array = NULL; 7233 mkey_attr.klm_num = 0; 7234 mkey_attr.relaxed_ordering = sh->cmng.relaxed_ordering; 7235 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7236 if (!mem_mng->dm) { 7237 mlx5_glue->devx_umem_dereg(mem_mng->umem); 7238 rte_errno = errno; 7239 mlx5_free(mem); 7240 return -rte_errno; 7241 } 7242 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7243 raw_data = (volatile struct flow_counter_stats *)mem; 7244 for (i = 0; i < raws_n; ++i) { 7245 mem_mng->raws[i].mem_mng = mem_mng; 7246 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7247 } 7248 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7249 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7250 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7251 next); 7252 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7253 sh->cmng.mem_mng = mem_mng; 7254 return 0; 7255 } 7256 7257 /** 7258 * Set the statistic memory to the new counter pool. 7259 * 7260 * @param[in] sh 7261 * Pointer to mlx5_dev_ctx_shared object. 7262 * @param[in] pool 7263 * Pointer to the pool to set the statistic memory. 7264 * 7265 * @return 7266 * 0 on success, a negative errno value otherwise. 7267 */ 7268 static int 7269 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7270 struct mlx5_flow_counter_pool *pool) 7271 { 7272 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7273 /* Resize statistic memory once used out. */ 7274 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7275 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7276 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7277 return -1; 7278 } 7279 rte_spinlock_lock(&pool->sl); 7280 pool->raw = cmng->mem_mng->raws + pool->index % 7281 MLX5_CNT_CONTAINER_RESIZE; 7282 rte_spinlock_unlock(&pool->sl); 7283 pool->raw_hw = NULL; 7284 return 0; 7285 } 7286 7287 #define MLX5_POOL_QUERY_FREQ_US 1000000 7288 7289 /** 7290 * Set the periodic procedure for triggering asynchronous batch queries for all 7291 * the counter pools. 7292 * 7293 * @param[in] sh 7294 * Pointer to mlx5_dev_ctx_shared object. 7295 */ 7296 void 7297 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7298 { 7299 uint32_t pools_n, us; 7300 7301 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7302 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7303 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7304 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7305 sh->cmng.query_thread_on = 0; 7306 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7307 } else { 7308 sh->cmng.query_thread_on = 1; 7309 } 7310 } 7311 7312 /** 7313 * The periodic procedure for triggering asynchronous batch queries for all the 7314 * counter pools. This function is probably called by the host thread. 7315 * 7316 * @param[in] arg 7317 * The parameter for the alarm process. 7318 */ 7319 void 7320 mlx5_flow_query_alarm(void *arg) 7321 { 7322 struct mlx5_dev_ctx_shared *sh = arg; 7323 int ret; 7324 uint16_t pool_index = sh->cmng.pool_index; 7325 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7326 struct mlx5_flow_counter_pool *pool; 7327 uint16_t n_valid; 7328 7329 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7330 goto set_alarm; 7331 rte_spinlock_lock(&cmng->pool_update_sl); 7332 pool = cmng->pools[pool_index]; 7333 n_valid = cmng->n_valid; 7334 rte_spinlock_unlock(&cmng->pool_update_sl); 7335 /* Set the statistic memory to the new created pool. */ 7336 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7337 goto set_alarm; 7338 if (pool->raw_hw) 7339 /* There is a pool query in progress. */ 7340 goto set_alarm; 7341 pool->raw_hw = 7342 LIST_FIRST(&sh->cmng.free_stat_raws); 7343 if (!pool->raw_hw) 7344 /* No free counter statistics raw memory. */ 7345 goto set_alarm; 7346 /* 7347 * Identify the counters released between query trigger and query 7348 * handle more efficiently. The counter released in this gap period 7349 * should wait for a new round of query as the new arrived packets 7350 * will not be taken into account. 7351 */ 7352 pool->query_gen++; 7353 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7354 MLX5_COUNTERS_PER_POOL, 7355 NULL, NULL, 7356 pool->raw_hw->mem_mng->dm->id, 7357 (void *)(uintptr_t) 7358 pool->raw_hw->data, 7359 sh->devx_comp, 7360 (uint64_t)(uintptr_t)pool); 7361 if (ret) { 7362 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7363 " %d", pool->min_dcs->id); 7364 pool->raw_hw = NULL; 7365 goto set_alarm; 7366 } 7367 LIST_REMOVE(pool->raw_hw, next); 7368 sh->cmng.pending_queries++; 7369 pool_index++; 7370 if (pool_index >= n_valid) 7371 pool_index = 0; 7372 set_alarm: 7373 sh->cmng.pool_index = pool_index; 7374 mlx5_set_query_alarm(sh); 7375 } 7376 7377 /** 7378 * Check and callback event for new aged flow in the counter pool 7379 * 7380 * @param[in] sh 7381 * Pointer to mlx5_dev_ctx_shared object. 7382 * @param[in] pool 7383 * Pointer to Current counter pool. 7384 */ 7385 static void 7386 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7387 struct mlx5_flow_counter_pool *pool) 7388 { 7389 struct mlx5_priv *priv; 7390 struct mlx5_flow_counter *cnt; 7391 struct mlx5_age_info *age_info; 7392 struct mlx5_age_param *age_param; 7393 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7394 struct mlx5_counter_stats_raw *prev = pool->raw; 7395 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7396 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7397 uint16_t expected = AGE_CANDIDATE; 7398 uint32_t i; 7399 7400 pool->time_of_last_age_check = curr_time; 7401 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7402 cnt = MLX5_POOL_GET_CNT(pool, i); 7403 age_param = MLX5_CNT_TO_AGE(cnt); 7404 if (__atomic_load_n(&age_param->state, 7405 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7406 continue; 7407 if (cur->data[i].hits != prev->data[i].hits) { 7408 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7409 __ATOMIC_RELAXED); 7410 continue; 7411 } 7412 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7413 time_delta, 7414 __ATOMIC_RELAXED) <= age_param->timeout) 7415 continue; 7416 /** 7417 * Hold the lock first, or if between the 7418 * state AGE_TMOUT and tailq operation the 7419 * release happened, the release procedure 7420 * may delete a non-existent tailq node. 7421 */ 7422 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7423 age_info = GET_PORT_AGE_INFO(priv); 7424 rte_spinlock_lock(&age_info->aged_sl); 7425 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7426 AGE_TMOUT, false, 7427 __ATOMIC_RELAXED, 7428 __ATOMIC_RELAXED)) { 7429 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7430 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7431 } 7432 rte_spinlock_unlock(&age_info->aged_sl); 7433 } 7434 for (i = 0; i < sh->max_port; i++) { 7435 age_info = &sh->port[i].age_info; 7436 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 7437 continue; 7438 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) 7439 rte_eth_dev_callback_process 7440 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 7441 RTE_ETH_EVENT_FLOW_AGED, NULL); 7442 age_info->flags = 0; 7443 } 7444 } 7445 7446 /** 7447 * Handler for the HW respond about ready values from an asynchronous batch 7448 * query. This function is probably called by the host thread. 7449 * 7450 * @param[in] sh 7451 * The pointer to the shared device context. 7452 * @param[in] async_id 7453 * The Devx async ID. 7454 * @param[in] status 7455 * The status of the completion. 7456 */ 7457 void 7458 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7459 uint64_t async_id, int status) 7460 { 7461 struct mlx5_flow_counter_pool *pool = 7462 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7463 struct mlx5_counter_stats_raw *raw_to_free; 7464 uint8_t query_gen = pool->query_gen ^ 1; 7465 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7466 enum mlx5_counter_type cnt_type = 7467 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7468 MLX5_COUNTER_TYPE_ORIGIN; 7469 7470 if (unlikely(status)) { 7471 raw_to_free = pool->raw_hw; 7472 } else { 7473 raw_to_free = pool->raw; 7474 if (pool->is_aged) 7475 mlx5_flow_aging_check(sh, pool); 7476 rte_spinlock_lock(&pool->sl); 7477 pool->raw = pool->raw_hw; 7478 rte_spinlock_unlock(&pool->sl); 7479 /* Be sure the new raw counters data is updated in memory. */ 7480 rte_io_wmb(); 7481 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7482 rte_spinlock_lock(&cmng->csl[cnt_type]); 7483 TAILQ_CONCAT(&cmng->counters[cnt_type], 7484 &pool->counters[query_gen], next); 7485 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7486 } 7487 } 7488 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7489 pool->raw_hw = NULL; 7490 sh->cmng.pending_queries--; 7491 } 7492 7493 static const struct mlx5_flow_tbl_data_entry * 7494 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 7495 { 7496 struct mlx5_priv *priv = dev->data->dev_private; 7497 struct mlx5_dev_ctx_shared *sh = priv->sh; 7498 struct mlx5_hlist_entry *he; 7499 union tunnel_offload_mark mbits = { .val = mark }; 7500 union mlx5_flow_tbl_key table_key = { 7501 { 7502 .table_id = tunnel_id_to_flow_tbl(mbits.table_id), 7503 .reserved = 0, 7504 .domain = !!mbits.transfer, 7505 .direction = 0, 7506 } 7507 }; 7508 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64); 7509 return he ? 7510 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 7511 } 7512 7513 static uint32_t 7514 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 7515 const struct mlx5_flow_tunnel *tunnel, 7516 uint32_t group, uint32_t *table, 7517 struct rte_flow_error *error) 7518 { 7519 struct mlx5_hlist_entry *he; 7520 struct tunnel_tbl_entry *tte; 7521 union tunnel_tbl_key key = { 7522 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 7523 .group = group 7524 }; 7525 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7526 struct mlx5_hlist *group_hash; 7527 7528 group_hash = tunnel ? tunnel->groups : thub->groups; 7529 he = mlx5_hlist_lookup(group_hash, key.val); 7530 if (!he) { 7531 int ret; 7532 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 7533 sizeof(*tte), 0, 7534 SOCKET_ID_ANY); 7535 if (!tte) 7536 goto err; 7537 tte->hash.key = key.val; 7538 ret = mlx5_flow_id_get(thub->table_ids, &tte->flow_table); 7539 if (ret) { 7540 mlx5_free(tte); 7541 goto err; 7542 } 7543 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 7544 mlx5_hlist_insert(group_hash, &tte->hash); 7545 } else { 7546 tte = container_of(he, typeof(*tte), hash); 7547 } 7548 *table = tte->flow_table; 7549 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 7550 dev->data->port_id, key.tunnel_id, group, *table); 7551 return 0; 7552 7553 err: 7554 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7555 NULL, "tunnel group index not supported"); 7556 } 7557 7558 static int 7559 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7560 struct flow_grp_info grp_info, struct rte_flow_error *error) 7561 { 7562 if (grp_info.transfer && grp_info.external && grp_info.fdb_def_rule) { 7563 if (group == UINT32_MAX) 7564 return rte_flow_error_set 7565 (error, EINVAL, 7566 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7567 NULL, 7568 "group index not supported"); 7569 *table = group + 1; 7570 } else { 7571 *table = group; 7572 } 7573 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7574 return 0; 7575 } 7576 7577 /** 7578 * Translate the rte_flow group index to HW table value. 7579 * 7580 * If tunnel offload is disabled, all group ids converted to flow table 7581 * id using the standard method. 7582 * If tunnel offload is enabled, group id can be converted using the 7583 * standard or tunnel conversion method. Group conversion method 7584 * selection depends on flags in `grp_info` parameter: 7585 * - Internal (grp_info.external == 0) groups conversion uses the 7586 * standard method. 7587 * - Group ids in JUMP action converted with the tunnel conversion. 7588 * - Group id in rule attribute conversion depends on a rule type and 7589 * group id value: 7590 * ** non zero group attributes converted with the tunnel method 7591 * ** zero group attribute in non-tunnel rule is converted using the 7592 * standard method - there's only one root table 7593 * ** zero group attribute in steer tunnel rule is converted with the 7594 * standard method - single root table 7595 * ** zero group attribute in match tunnel rule is a special OvS 7596 * case: that value is used for portability reasons. That group 7597 * id is converted with the tunnel conversion method. 7598 * 7599 * @param[in] dev 7600 * Port device 7601 * @param[in] tunnel 7602 * PMD tunnel offload object 7603 * @param[in] group 7604 * rte_flow group index value. 7605 * @param[out] table 7606 * HW table value. 7607 * @param[in] grp_info 7608 * flags used for conversion 7609 * @param[out] error 7610 * Pointer to error structure. 7611 * 7612 * @return 7613 * 0 on success, a negative errno value otherwise and rte_errno is set. 7614 */ 7615 int 7616 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7617 const struct mlx5_flow_tunnel *tunnel, 7618 uint32_t group, uint32_t *table, 7619 struct flow_grp_info grp_info, 7620 struct rte_flow_error *error) 7621 { 7622 int ret; 7623 bool standard_translation; 7624 7625 if (grp_info.external && group < MLX5_MAX_TABLES_EXTERNAL) 7626 group *= MLX5_FLOW_TABLE_FACTOR; 7627 if (is_tunnel_offload_active(dev)) { 7628 standard_translation = !grp_info.external || 7629 grp_info.std_tbl_fix; 7630 } else { 7631 standard_translation = true; 7632 } 7633 DRV_LOG(DEBUG, 7634 "port %u group=%#x transfer=%d external=%d fdb_def_rule=%d translate=%s", 7635 dev->data->port_id, group, grp_info.transfer, 7636 grp_info.external, grp_info.fdb_def_rule, 7637 standard_translation ? "STANDARD" : "TUNNEL"); 7638 if (standard_translation) 7639 ret = flow_group_to_table(dev->data->port_id, group, table, 7640 grp_info, error); 7641 else 7642 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7643 table, error); 7644 7645 return ret; 7646 } 7647 7648 /** 7649 * Discover availability of metadata reg_c's. 7650 * 7651 * Iteratively use test flows to check availability. 7652 * 7653 * @param[in] dev 7654 * Pointer to the Ethernet device structure. 7655 * 7656 * @return 7657 * 0 on success, a negative errno value otherwise and rte_errno is set. 7658 */ 7659 int 7660 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7661 { 7662 struct mlx5_priv *priv = dev->data->dev_private; 7663 struct mlx5_dev_config *config = &priv->config; 7664 enum modify_reg idx; 7665 int n = 0; 7666 7667 /* reg_c[0] and reg_c[1] are reserved. */ 7668 config->flow_mreg_c[n++] = REG_C_0; 7669 config->flow_mreg_c[n++] = REG_C_1; 7670 /* Discover availability of other reg_c's. */ 7671 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7672 struct rte_flow_attr attr = { 7673 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7674 .priority = MLX5_FLOW_PRIO_RSVD, 7675 .ingress = 1, 7676 }; 7677 struct rte_flow_item items[] = { 7678 [0] = { 7679 .type = RTE_FLOW_ITEM_TYPE_END, 7680 }, 7681 }; 7682 struct rte_flow_action actions[] = { 7683 [0] = { 7684 .type = (enum rte_flow_action_type) 7685 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7686 .conf = &(struct mlx5_flow_action_copy_mreg){ 7687 .src = REG_C_1, 7688 .dst = idx, 7689 }, 7690 }, 7691 [1] = { 7692 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7693 .conf = &(struct rte_flow_action_jump){ 7694 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7695 }, 7696 }, 7697 [2] = { 7698 .type = RTE_FLOW_ACTION_TYPE_END, 7699 }, 7700 }; 7701 uint32_t flow_idx; 7702 struct rte_flow *flow; 7703 struct rte_flow_error error; 7704 7705 if (!config->dv_flow_en) 7706 break; 7707 /* Create internal flow, validation skips copy action. */ 7708 flow_idx = flow_list_create(dev, NULL, &attr, items, 7709 actions, false, &error); 7710 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 7711 flow_idx); 7712 if (!flow) 7713 continue; 7714 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 7715 config->flow_mreg_c[n++] = idx; 7716 flow_list_destroy(dev, NULL, flow_idx); 7717 } 7718 for (; n < MLX5_MREG_C_NUM; ++n) 7719 config->flow_mreg_c[n] = REG_NON; 7720 return 0; 7721 } 7722 7723 /** 7724 * Dump flow raw hw data to file 7725 * 7726 * @param[in] dev 7727 * The pointer to Ethernet device. 7728 * @param[in] file 7729 * A pointer to a file for output. 7730 * @param[out] error 7731 * Perform verbose error reporting if not NULL. PMDs initialize this 7732 * structure in case of error only. 7733 * @return 7734 * 0 on success, a nagative value otherwise. 7735 */ 7736 int 7737 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 7738 FILE *file, 7739 struct rte_flow_error *error __rte_unused) 7740 { 7741 struct mlx5_priv *priv = dev->data->dev_private; 7742 struct mlx5_dev_ctx_shared *sh = priv->sh; 7743 7744 if (!priv->config.dv_flow_en) { 7745 if (fputs("device dv flow disabled\n", file) <= 0) 7746 return -errno; 7747 return -ENOTSUP; 7748 } 7749 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 7750 sh->tx_domain, file); 7751 } 7752 7753 /** 7754 * Get aged-out flows. 7755 * 7756 * @param[in] dev 7757 * Pointer to the Ethernet device structure. 7758 * @param[in] context 7759 * The address of an array of pointers to the aged-out flows contexts. 7760 * @param[in] nb_countexts 7761 * The length of context array pointers. 7762 * @param[out] error 7763 * Perform verbose error reporting if not NULL. Initialized in case of 7764 * error only. 7765 * 7766 * @return 7767 * how many contexts get in success, otherwise negative errno value. 7768 * if nb_contexts is 0, return the amount of all aged contexts. 7769 * if nb_contexts is not 0 , return the amount of aged flows reported 7770 * in the context array. 7771 */ 7772 int 7773 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 7774 uint32_t nb_contexts, struct rte_flow_error *error) 7775 { 7776 const struct mlx5_flow_driver_ops *fops; 7777 struct rte_flow_attr attr = { .transfer = 0 }; 7778 7779 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7780 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7781 return fops->get_aged_flows(dev, contexts, nb_contexts, 7782 error); 7783 } 7784 DRV_LOG(ERR, 7785 "port %u get aged flows is not supported.", 7786 dev->data->port_id); 7787 return -ENOTSUP; 7788 } 7789 7790 /* Wrapper for driver action_validate op callback */ 7791 static int 7792 flow_drv_action_validate(struct rte_eth_dev *dev, 7793 const struct rte_flow_shared_action_conf *conf, 7794 const struct rte_flow_action *action, 7795 const struct mlx5_flow_driver_ops *fops, 7796 struct rte_flow_error *error) 7797 { 7798 static const char err_msg[] = "shared action validation unsupported"; 7799 7800 if (!fops->action_validate) { 7801 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7802 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7803 NULL, err_msg); 7804 return -rte_errno; 7805 } 7806 return fops->action_validate(dev, conf, action, error); 7807 } 7808 7809 /** 7810 * Destroys the shared action by handle. 7811 * 7812 * @param dev 7813 * Pointer to Ethernet device structure. 7814 * @param[in] action 7815 * Handle for the shared action to be destroyed. 7816 * @param[out] error 7817 * Perform verbose error reporting if not NULL. PMDs initialize this 7818 * structure in case of error only. 7819 * 7820 * @return 7821 * 0 on success, a negative errno value otherwise and rte_errno is set. 7822 * 7823 * @note: wrapper for driver action_create op callback. 7824 */ 7825 static int 7826 mlx5_shared_action_destroy(struct rte_eth_dev *dev, 7827 struct rte_flow_shared_action *action, 7828 struct rte_flow_error *error) 7829 { 7830 static const char err_msg[] = "shared action destruction unsupported"; 7831 struct rte_flow_attr attr = { .transfer = 0 }; 7832 const struct mlx5_flow_driver_ops *fops = 7833 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7834 7835 if (!fops->action_destroy) { 7836 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7837 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7838 NULL, err_msg); 7839 return -rte_errno; 7840 } 7841 return fops->action_destroy(dev, action, error); 7842 } 7843 7844 /* Wrapper for driver action_destroy op callback */ 7845 static int 7846 flow_drv_action_update(struct rte_eth_dev *dev, 7847 struct rte_flow_shared_action *action, 7848 const void *action_conf, 7849 const struct mlx5_flow_driver_ops *fops, 7850 struct rte_flow_error *error) 7851 { 7852 static const char err_msg[] = "shared action update unsupported"; 7853 7854 if (!fops->action_update) { 7855 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7856 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7857 NULL, err_msg); 7858 return -rte_errno; 7859 } 7860 return fops->action_update(dev, action, action_conf, error); 7861 } 7862 7863 /** 7864 * Create shared action for reuse in multiple flow rules. 7865 * 7866 * @param dev 7867 * Pointer to Ethernet device structure. 7868 * @param[in] action 7869 * Action configuration for shared action creation. 7870 * @param[out] error 7871 * Perform verbose error reporting if not NULL. PMDs initialize this 7872 * structure in case of error only. 7873 * @return 7874 * A valid handle in case of success, NULL otherwise and rte_errno is set. 7875 */ 7876 static struct rte_flow_shared_action * 7877 mlx5_shared_action_create(struct rte_eth_dev *dev, 7878 const struct rte_flow_shared_action_conf *conf, 7879 const struct rte_flow_action *action, 7880 struct rte_flow_error *error) 7881 { 7882 static const char err_msg[] = "shared action creation unsupported"; 7883 struct rte_flow_attr attr = { .transfer = 0 }; 7884 const struct mlx5_flow_driver_ops *fops = 7885 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7886 7887 if (flow_drv_action_validate(dev, conf, action, fops, error)) 7888 return NULL; 7889 if (!fops->action_create) { 7890 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7891 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7892 NULL, err_msg); 7893 return NULL; 7894 } 7895 return fops->action_create(dev, conf, action, error); 7896 } 7897 7898 /** 7899 * Updates inplace the shared action configuration pointed by *action* handle 7900 * with the configuration provided as *action* argument. 7901 * The update of the shared action configuration effects all flow rules reusing 7902 * the action via handle. 7903 * 7904 * @param dev 7905 * Pointer to Ethernet device structure. 7906 * @param[in] shared_action 7907 * Handle for the shared action to be updated. 7908 * @param[in] action 7909 * Action specification used to modify the action pointed by handle. 7910 * *action* should be of same type with the action pointed by the *action* 7911 * handle argument, otherwise considered as invalid. 7912 * @param[out] error 7913 * Perform verbose error reporting if not NULL. PMDs initialize this 7914 * structure in case of error only. 7915 * 7916 * @return 7917 * 0 on success, a negative errno value otherwise and rte_errno is set. 7918 */ 7919 static int 7920 mlx5_shared_action_update(struct rte_eth_dev *dev, 7921 struct rte_flow_shared_action *shared_action, 7922 const struct rte_flow_action *action, 7923 struct rte_flow_error *error) 7924 { 7925 struct rte_flow_attr attr = { .transfer = 0 }; 7926 const struct mlx5_flow_driver_ops *fops = 7927 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7928 int ret; 7929 7930 switch (shared_action->type) { 7931 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 7932 if (action->type != RTE_FLOW_ACTION_TYPE_RSS) { 7933 return rte_flow_error_set(error, EINVAL, 7934 RTE_FLOW_ERROR_TYPE_ACTION, 7935 NULL, 7936 "update action type invalid"); 7937 } 7938 ret = flow_drv_action_validate(dev, NULL, action, fops, error); 7939 if (ret) 7940 return ret; 7941 return flow_drv_action_update(dev, shared_action, action->conf, 7942 fops, error); 7943 default: 7944 return rte_flow_error_set(error, ENOTSUP, 7945 RTE_FLOW_ERROR_TYPE_ACTION, 7946 NULL, 7947 "action type not supported"); 7948 } 7949 } 7950 7951 /** 7952 * Query the shared action by handle. 7953 * 7954 * This function allows retrieving action-specific data such as counters. 7955 * Data is gathered by special action which may be present/referenced in 7956 * more than one flow rule definition. 7957 * 7958 * \see RTE_FLOW_ACTION_TYPE_COUNT 7959 * 7960 * @param dev 7961 * Pointer to Ethernet device structure. 7962 * @param[in] action 7963 * Handle for the shared action to query. 7964 * @param[in, out] data 7965 * Pointer to storage for the associated query data type. 7966 * @param[out] error 7967 * Perform verbose error reporting if not NULL. PMDs initialize this 7968 * structure in case of error only. 7969 * 7970 * @return 7971 * 0 on success, a negative errno value otherwise and rte_errno is set. 7972 */ 7973 static int 7974 mlx5_shared_action_query(struct rte_eth_dev *dev, 7975 const struct rte_flow_shared_action *action, 7976 void *data, 7977 struct rte_flow_error *error) 7978 { 7979 (void)dev; 7980 switch (action->type) { 7981 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 7982 __atomic_load(&action->refcnt, (uint32_t *)data, 7983 __ATOMIC_RELAXED); 7984 return 0; 7985 default: 7986 return rte_flow_error_set(error, ENOTSUP, 7987 RTE_FLOW_ERROR_TYPE_ACTION, 7988 NULL, 7989 "action type not supported"); 7990 } 7991 } 7992 7993 /** 7994 * Destroy all shared actions. 7995 * 7996 * @param dev 7997 * Pointer to Ethernet device. 7998 * 7999 * @return 8000 * 0 on success, a negative errno value otherwise and rte_errno is set. 8001 */ 8002 int 8003 mlx5_shared_action_flush(struct rte_eth_dev *dev) 8004 { 8005 struct rte_flow_error error; 8006 struct mlx5_priv *priv = dev->data->dev_private; 8007 struct rte_flow_shared_action *action; 8008 int ret = 0; 8009 8010 while (!LIST_EMPTY(&priv->shared_actions)) { 8011 action = LIST_FIRST(&priv->shared_actions); 8012 ret = mlx5_shared_action_destroy(dev, action, &error); 8013 } 8014 return ret; 8015 } 8016 8017 static void 8018 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 8019 struct mlx5_flow_tunnel *tunnel) 8020 { 8021 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8022 struct mlx5_flow_id_pool *id_pool = thub->tunnel_ids; 8023 8024 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 8025 dev->data->port_id, tunnel->tunnel_id); 8026 RTE_VERIFY(!__atomic_load_n(&tunnel->refctn, __ATOMIC_RELAXED)); 8027 LIST_REMOVE(tunnel, chain); 8028 mlx5_flow_id_release(id_pool, tunnel->tunnel_id); 8029 mlx5_hlist_destroy(tunnel->groups, NULL, NULL); 8030 mlx5_free(tunnel); 8031 } 8032 8033 static struct mlx5_flow_tunnel * 8034 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 8035 { 8036 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8037 struct mlx5_flow_tunnel *tun; 8038 8039 LIST_FOREACH(tun, &thub->tunnels, chain) { 8040 if (tun->tunnel_id == id) 8041 break; 8042 } 8043 8044 return tun; 8045 } 8046 8047 static struct mlx5_flow_tunnel * 8048 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 8049 const struct rte_flow_tunnel *app_tunnel) 8050 { 8051 int ret; 8052 struct mlx5_flow_tunnel *tunnel; 8053 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8054 struct mlx5_flow_id_pool *id_pool = thub->tunnel_ids; 8055 uint32_t id; 8056 8057 ret = mlx5_flow_id_get(id_pool, &id); 8058 if (ret) 8059 return NULL; 8060 /** 8061 * mlx5 flow tunnel is an auxlilary data structure 8062 * It's not part of IO. No need to allocate it from 8063 * huge pages pools dedicated for IO 8064 */ 8065 tunnel = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*tunnel), 8066 0, SOCKET_ID_ANY); 8067 if (!tunnel) { 8068 mlx5_flow_id_pool_release(id_pool); 8069 return NULL; 8070 } 8071 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024); 8072 if (!tunnel->groups) { 8073 mlx5_flow_id_pool_release(id_pool); 8074 mlx5_free(tunnel); 8075 return NULL; 8076 } 8077 /* initiate new PMD tunnel */ 8078 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8079 tunnel->tunnel_id = id; 8080 tunnel->action.type = (typeof(tunnel->action.type)) 8081 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8082 tunnel->action.conf = tunnel; 8083 tunnel->item.type = (typeof(tunnel->item.type)) 8084 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8085 tunnel->item.spec = tunnel; 8086 tunnel->item.last = NULL; 8087 tunnel->item.mask = NULL; 8088 8089 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8090 dev->data->port_id, tunnel->tunnel_id); 8091 8092 return tunnel; 8093 } 8094 8095 static int 8096 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 8097 const struct rte_flow_tunnel *app_tunnel, 8098 struct mlx5_flow_tunnel **tunnel) 8099 { 8100 int ret; 8101 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8102 struct mlx5_flow_tunnel *tun; 8103 8104 LIST_FOREACH(tun, &thub->tunnels, chain) { 8105 if (!memcmp(app_tunnel, &tun->app_tunnel, 8106 sizeof(*app_tunnel))) { 8107 *tunnel = tun; 8108 ret = 0; 8109 break; 8110 } 8111 } 8112 if (!tun) { 8113 tun = mlx5_flow_tunnel_allocate(dev, app_tunnel); 8114 if (tun) { 8115 LIST_INSERT_HEAD(&thub->tunnels, tun, chain); 8116 *tunnel = tun; 8117 } else { 8118 ret = -ENOMEM; 8119 } 8120 } 8121 if (tun) 8122 __atomic_add_fetch(&tun->refctn, 1, __ATOMIC_RELAXED); 8123 8124 return ret; 8125 } 8126 8127 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 8128 { 8129 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 8130 8131 if (!thub) 8132 return; 8133 if (!LIST_EMPTY(&thub->tunnels)) 8134 DRV_LOG(WARNING, "port %u tunnels present\n", port_id); 8135 mlx5_flow_id_pool_release(thub->tunnel_ids); 8136 mlx5_flow_id_pool_release(thub->table_ids); 8137 mlx5_hlist_destroy(thub->groups, NULL, NULL); 8138 mlx5_free(thub); 8139 } 8140 8141 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 8142 { 8143 int err; 8144 struct mlx5_flow_tunnel_hub *thub; 8145 8146 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 8147 0, SOCKET_ID_ANY); 8148 if (!thub) 8149 return -ENOMEM; 8150 LIST_INIT(&thub->tunnels); 8151 thub->tunnel_ids = mlx5_flow_id_pool_alloc(MLX5_MAX_TUNNELS); 8152 if (!thub->tunnel_ids) { 8153 err = -rte_errno; 8154 goto err; 8155 } 8156 thub->table_ids = mlx5_flow_id_pool_alloc(MLX5_MAX_TABLES); 8157 if (!thub->table_ids) { 8158 err = -rte_errno; 8159 goto err; 8160 } 8161 thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES); 8162 if (!thub->groups) { 8163 err = -rte_errno; 8164 goto err; 8165 } 8166 sh->tunnel_hub = thub; 8167 8168 return 0; 8169 8170 err: 8171 if (thub->groups) 8172 mlx5_hlist_destroy(thub->groups, NULL, NULL); 8173 if (thub->table_ids) 8174 mlx5_flow_id_pool_release(thub->table_ids); 8175 if (thub->tunnel_ids) 8176 mlx5_flow_id_pool_release(thub->tunnel_ids); 8177 if (thub) 8178 mlx5_free(thub); 8179 return err; 8180 } 8181 8182 #ifndef HAVE_MLX5DV_DR 8183 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8184 #else 8185 #define MLX5_DOMAIN_SYNC_FLOW \ 8186 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8187 #endif 8188 8189 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8190 { 8191 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8192 const struct mlx5_flow_driver_ops *fops; 8193 int ret; 8194 struct rte_flow_attr attr = { .transfer = 0 }; 8195 8196 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8197 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8198 if (ret > 0) 8199 ret = -ret; 8200 return ret; 8201 } 8202