1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 #include <rte_common.h> 14 #include <rte_ether.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_eal_paging.h> 17 #include <rte_flow.h> 18 #include <rte_cycles.h> 19 #include <rte_flow_driver.h> 20 #include <rte_malloc.h> 21 #include <rte_ip.h> 22 23 #include <mlx5_glue.h> 24 #include <mlx5_devx_cmds.h> 25 #include <mlx5_prm.h> 26 #include <mlx5_malloc.h> 27 28 #include "mlx5_defs.h" 29 #include "mlx5.h" 30 #include "mlx5_flow.h" 31 #include "mlx5_flow_os.h" 32 #include "mlx5_rxtx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 static struct mlx5_flow_tunnel * 37 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 38 static void 39 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 40 static const struct mlx5_flow_tbl_data_entry * 41 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark); 42 static int 43 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 44 const struct rte_flow_tunnel *app_tunnel, 45 struct mlx5_flow_tunnel **tunnel); 46 47 48 /** Device flow drivers. */ 49 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 50 51 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 52 53 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 54 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 55 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 56 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 57 #endif 58 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 59 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 60 }; 61 62 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 63 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 64 (const int []){ \ 65 __VA_ARGS__, 0, \ 66 } 67 68 /** Node object of input graph for mlx5_flow_expand_rss(). */ 69 struct mlx5_flow_expand_node { 70 const int *const next; 71 /**< 72 * List of next node indexes. Index 0 is interpreted as a terminator. 73 */ 74 const enum rte_flow_item_type type; 75 /**< Pattern item type of current node. */ 76 uint64_t rss_types; 77 /**< 78 * RSS types bit-field associated with this node 79 * (see ETH_RSS_* definitions). 80 */ 81 }; 82 83 /** Object returned by mlx5_flow_expand_rss(). */ 84 struct mlx5_flow_expand_rss { 85 uint32_t entries; 86 /**< Number of entries @p patterns and @p priorities. */ 87 struct { 88 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 89 uint32_t priority; /**< Priority offset for each expansion. */ 90 } entry[]; 91 }; 92 93 static enum rte_flow_item_type 94 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 95 { 96 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 97 uint16_t ether_type = 0; 98 uint16_t ether_type_m; 99 uint8_t ip_next_proto = 0; 100 uint8_t ip_next_proto_m; 101 102 if (item == NULL || item->spec == NULL) 103 return ret; 104 switch (item->type) { 105 case RTE_FLOW_ITEM_TYPE_ETH: 106 if (item->mask) 107 ether_type_m = ((const struct rte_flow_item_eth *) 108 (item->mask))->type; 109 else 110 ether_type_m = rte_flow_item_eth_mask.type; 111 if (ether_type_m != RTE_BE16(0xFFFF)) 112 break; 113 ether_type = ((const struct rte_flow_item_eth *) 114 (item->spec))->type; 115 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 116 ret = RTE_FLOW_ITEM_TYPE_IPV4; 117 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 118 ret = RTE_FLOW_ITEM_TYPE_IPV6; 119 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 120 ret = RTE_FLOW_ITEM_TYPE_VLAN; 121 else 122 ret = RTE_FLOW_ITEM_TYPE_END; 123 break; 124 case RTE_FLOW_ITEM_TYPE_VLAN: 125 if (item->mask) 126 ether_type_m = ((const struct rte_flow_item_vlan *) 127 (item->mask))->inner_type; 128 else 129 ether_type_m = rte_flow_item_vlan_mask.inner_type; 130 if (ether_type_m != RTE_BE16(0xFFFF)) 131 break; 132 ether_type = ((const struct rte_flow_item_vlan *) 133 (item->spec))->inner_type; 134 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 135 ret = RTE_FLOW_ITEM_TYPE_IPV4; 136 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 137 ret = RTE_FLOW_ITEM_TYPE_IPV6; 138 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 139 ret = RTE_FLOW_ITEM_TYPE_VLAN; 140 else 141 ret = RTE_FLOW_ITEM_TYPE_END; 142 break; 143 case RTE_FLOW_ITEM_TYPE_IPV4: 144 if (item->mask) 145 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 146 (item->mask))->hdr.next_proto_id; 147 else 148 ip_next_proto_m = 149 rte_flow_item_ipv4_mask.hdr.next_proto_id; 150 if (ip_next_proto_m != 0xFF) 151 break; 152 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 153 (item->spec))->hdr.next_proto_id; 154 if (ip_next_proto == IPPROTO_UDP) 155 ret = RTE_FLOW_ITEM_TYPE_UDP; 156 else if (ip_next_proto == IPPROTO_TCP) 157 ret = RTE_FLOW_ITEM_TYPE_TCP; 158 else if (ip_next_proto == IPPROTO_IP) 159 ret = RTE_FLOW_ITEM_TYPE_IPV4; 160 else if (ip_next_proto == IPPROTO_IPV6) 161 ret = RTE_FLOW_ITEM_TYPE_IPV6; 162 else 163 ret = RTE_FLOW_ITEM_TYPE_END; 164 break; 165 case RTE_FLOW_ITEM_TYPE_IPV6: 166 if (item->mask) 167 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 168 (item->mask))->hdr.proto; 169 else 170 ip_next_proto_m = 171 rte_flow_item_ipv6_mask.hdr.proto; 172 if (ip_next_proto_m != 0xFF) 173 break; 174 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 175 (item->spec))->hdr.proto; 176 if (ip_next_proto == IPPROTO_UDP) 177 ret = RTE_FLOW_ITEM_TYPE_UDP; 178 else if (ip_next_proto == IPPROTO_TCP) 179 ret = RTE_FLOW_ITEM_TYPE_TCP; 180 else if (ip_next_proto == IPPROTO_IP) 181 ret = RTE_FLOW_ITEM_TYPE_IPV4; 182 else if (ip_next_proto == IPPROTO_IPV6) 183 ret = RTE_FLOW_ITEM_TYPE_IPV6; 184 else 185 ret = RTE_FLOW_ITEM_TYPE_END; 186 break; 187 default: 188 ret = RTE_FLOW_ITEM_TYPE_VOID; 189 break; 190 } 191 return ret; 192 } 193 194 /** 195 * Expand RSS flows into several possible flows according to the RSS hash 196 * fields requested and the driver capabilities. 197 * 198 * @param[out] buf 199 * Buffer to store the result expansion. 200 * @param[in] size 201 * Buffer size in bytes. If 0, @p buf can be NULL. 202 * @param[in] pattern 203 * User flow pattern. 204 * @param[in] types 205 * RSS types to expand (see ETH_RSS_* definitions). 206 * @param[in] graph 207 * Input graph to expand @p pattern according to @p types. 208 * @param[in] graph_root_index 209 * Index of root node in @p graph, typically 0. 210 * 211 * @return 212 * A positive value representing the size of @p buf in bytes regardless of 213 * @p size on success, a negative errno value otherwise and rte_errno is 214 * set, the following errors are defined: 215 * 216 * -E2BIG: graph-depth @p graph is too deep. 217 */ 218 static int 219 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 220 const struct rte_flow_item *pattern, uint64_t types, 221 const struct mlx5_flow_expand_node graph[], 222 int graph_root_index) 223 { 224 const int elt_n = 8; 225 const struct rte_flow_item *item; 226 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 227 const int *next_node; 228 const int *stack[elt_n]; 229 int stack_pos = 0; 230 struct rte_flow_item flow_items[elt_n]; 231 unsigned int i; 232 size_t lsize; 233 size_t user_pattern_size = 0; 234 void *addr = NULL; 235 const struct mlx5_flow_expand_node *next = NULL; 236 struct rte_flow_item missed_item; 237 int missed = 0; 238 int elt = 0; 239 const struct rte_flow_item *last_item = NULL; 240 241 memset(&missed_item, 0, sizeof(missed_item)); 242 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 243 elt_n * sizeof(buf->entry[0]); 244 if (lsize <= size) { 245 buf->entry[0].priority = 0; 246 buf->entry[0].pattern = (void *)&buf->entry[elt_n]; 247 buf->entries = 0; 248 addr = buf->entry[0].pattern; 249 } 250 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 251 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 252 last_item = item; 253 for (i = 0; node->next && node->next[i]; ++i) { 254 next = &graph[node->next[i]]; 255 if (next->type == item->type) 256 break; 257 } 258 if (next) 259 node = next; 260 user_pattern_size += sizeof(*item); 261 } 262 user_pattern_size += sizeof(*item); /* Handle END item. */ 263 lsize += user_pattern_size; 264 /* Copy the user pattern in the first entry of the buffer. */ 265 if (lsize <= size) { 266 rte_memcpy(addr, pattern, user_pattern_size); 267 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 268 buf->entries = 1; 269 } 270 /* Start expanding. */ 271 memset(flow_items, 0, sizeof(flow_items)); 272 user_pattern_size -= sizeof(*item); 273 /* 274 * Check if the last valid item has spec set, need complete pattern, 275 * and the pattern can be used for expansion. 276 */ 277 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 278 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 279 /* Item type END indicates expansion is not required. */ 280 return lsize; 281 } 282 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 283 next = NULL; 284 missed = 1; 285 for (i = 0; node->next && node->next[i]; ++i) { 286 next = &graph[node->next[i]]; 287 if (next->type == missed_item.type) { 288 flow_items[0].type = missed_item.type; 289 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 290 break; 291 } 292 next = NULL; 293 } 294 } 295 if (next && missed) { 296 elt = 2; /* missed item + item end. */ 297 node = next; 298 lsize += elt * sizeof(*item) + user_pattern_size; 299 if ((node->rss_types & types) && lsize <= size) { 300 buf->entry[buf->entries].priority = 1; 301 buf->entry[buf->entries].pattern = addr; 302 buf->entries++; 303 rte_memcpy(addr, buf->entry[0].pattern, 304 user_pattern_size); 305 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 306 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 307 addr = (void *)(((uintptr_t)addr) + 308 elt * sizeof(*item)); 309 } 310 } 311 memset(flow_items, 0, sizeof(flow_items)); 312 next_node = node->next; 313 stack[stack_pos] = next_node; 314 node = next_node ? &graph[*next_node] : NULL; 315 while (node) { 316 flow_items[stack_pos].type = node->type; 317 if (node->rss_types & types) { 318 /* 319 * compute the number of items to copy from the 320 * expansion and copy it. 321 * When the stack_pos is 0, there are 1 element in it, 322 * plus the addition END item. 323 */ 324 elt = stack_pos + 2; 325 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 326 lsize += elt * sizeof(*item) + user_pattern_size; 327 if (lsize <= size) { 328 size_t n = elt * sizeof(*item); 329 330 buf->entry[buf->entries].priority = 331 stack_pos + 1 + missed; 332 buf->entry[buf->entries].pattern = addr; 333 buf->entries++; 334 rte_memcpy(addr, buf->entry[0].pattern, 335 user_pattern_size); 336 addr = (void *)(((uintptr_t)addr) + 337 user_pattern_size); 338 rte_memcpy(addr, &missed_item, 339 missed * sizeof(*item)); 340 addr = (void *)(((uintptr_t)addr) + 341 missed * sizeof(*item)); 342 rte_memcpy(addr, flow_items, n); 343 addr = (void *)(((uintptr_t)addr) + n); 344 } 345 } 346 /* Go deeper. */ 347 if (node->next) { 348 next_node = node->next; 349 if (stack_pos++ == elt_n) { 350 rte_errno = E2BIG; 351 return -rte_errno; 352 } 353 stack[stack_pos] = next_node; 354 } else if (*(next_node + 1)) { 355 /* Follow up with the next possibility. */ 356 ++next_node; 357 } else { 358 /* Move to the next path. */ 359 if (stack_pos) 360 next_node = stack[--stack_pos]; 361 next_node++; 362 stack[stack_pos] = next_node; 363 } 364 node = *next_node ? &graph[*next_node] : NULL; 365 }; 366 /* no expanded flows but we have missed item, create one rule for it */ 367 if (buf->entries == 1 && missed != 0) { 368 elt = 2; 369 lsize += elt * sizeof(*item) + user_pattern_size; 370 if (lsize <= size) { 371 buf->entry[buf->entries].priority = 1; 372 buf->entry[buf->entries].pattern = addr; 373 buf->entries++; 374 flow_items[0].type = missed_item.type; 375 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 376 rte_memcpy(addr, buf->entry[0].pattern, 377 user_pattern_size); 378 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 379 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 380 addr = (void *)(((uintptr_t)addr) + 381 elt * sizeof(*item)); 382 } 383 } 384 return lsize; 385 } 386 387 enum mlx5_expansion { 388 MLX5_EXPANSION_ROOT, 389 MLX5_EXPANSION_ROOT_OUTER, 390 MLX5_EXPANSION_ROOT_ETH_VLAN, 391 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 392 MLX5_EXPANSION_OUTER_ETH, 393 MLX5_EXPANSION_OUTER_ETH_VLAN, 394 MLX5_EXPANSION_OUTER_VLAN, 395 MLX5_EXPANSION_OUTER_IPV4, 396 MLX5_EXPANSION_OUTER_IPV4_UDP, 397 MLX5_EXPANSION_OUTER_IPV4_TCP, 398 MLX5_EXPANSION_OUTER_IPV6, 399 MLX5_EXPANSION_OUTER_IPV6_UDP, 400 MLX5_EXPANSION_OUTER_IPV6_TCP, 401 MLX5_EXPANSION_VXLAN, 402 MLX5_EXPANSION_VXLAN_GPE, 403 MLX5_EXPANSION_GRE, 404 MLX5_EXPANSION_MPLS, 405 MLX5_EXPANSION_ETH, 406 MLX5_EXPANSION_ETH_VLAN, 407 MLX5_EXPANSION_VLAN, 408 MLX5_EXPANSION_IPV4, 409 MLX5_EXPANSION_IPV4_UDP, 410 MLX5_EXPANSION_IPV4_TCP, 411 MLX5_EXPANSION_IPV6, 412 MLX5_EXPANSION_IPV6_UDP, 413 MLX5_EXPANSION_IPV6_TCP, 414 }; 415 416 /** Supported expansion of items. */ 417 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 418 [MLX5_EXPANSION_ROOT] = { 419 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 420 MLX5_EXPANSION_IPV4, 421 MLX5_EXPANSION_IPV6), 422 .type = RTE_FLOW_ITEM_TYPE_END, 423 }, 424 [MLX5_EXPANSION_ROOT_OUTER] = { 425 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 426 MLX5_EXPANSION_OUTER_IPV4, 427 MLX5_EXPANSION_OUTER_IPV6), 428 .type = RTE_FLOW_ITEM_TYPE_END, 429 }, 430 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 431 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 432 .type = RTE_FLOW_ITEM_TYPE_END, 433 }, 434 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 435 .next = MLX5_FLOW_EXPAND_RSS_NEXT 436 (MLX5_EXPANSION_OUTER_ETH_VLAN), 437 .type = RTE_FLOW_ITEM_TYPE_END, 438 }, 439 [MLX5_EXPANSION_OUTER_ETH] = { 440 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 441 MLX5_EXPANSION_OUTER_IPV6, 442 MLX5_EXPANSION_MPLS), 443 .type = RTE_FLOW_ITEM_TYPE_ETH, 444 .rss_types = 0, 445 }, 446 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 447 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 448 .type = RTE_FLOW_ITEM_TYPE_ETH, 449 .rss_types = 0, 450 }, 451 [MLX5_EXPANSION_OUTER_VLAN] = { 452 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 453 MLX5_EXPANSION_OUTER_IPV6), 454 .type = RTE_FLOW_ITEM_TYPE_VLAN, 455 }, 456 [MLX5_EXPANSION_OUTER_IPV4] = { 457 .next = MLX5_FLOW_EXPAND_RSS_NEXT 458 (MLX5_EXPANSION_OUTER_IPV4_UDP, 459 MLX5_EXPANSION_OUTER_IPV4_TCP, 460 MLX5_EXPANSION_GRE, 461 MLX5_EXPANSION_IPV4, 462 MLX5_EXPANSION_IPV6), 463 .type = RTE_FLOW_ITEM_TYPE_IPV4, 464 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 465 ETH_RSS_NONFRAG_IPV4_OTHER, 466 }, 467 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 468 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 469 MLX5_EXPANSION_VXLAN_GPE), 470 .type = RTE_FLOW_ITEM_TYPE_UDP, 471 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 472 }, 473 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 474 .type = RTE_FLOW_ITEM_TYPE_TCP, 475 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 476 }, 477 [MLX5_EXPANSION_OUTER_IPV6] = { 478 .next = MLX5_FLOW_EXPAND_RSS_NEXT 479 (MLX5_EXPANSION_OUTER_IPV6_UDP, 480 MLX5_EXPANSION_OUTER_IPV6_TCP, 481 MLX5_EXPANSION_IPV4, 482 MLX5_EXPANSION_IPV6), 483 .type = RTE_FLOW_ITEM_TYPE_IPV6, 484 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 485 ETH_RSS_NONFRAG_IPV6_OTHER, 486 }, 487 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 488 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 489 MLX5_EXPANSION_VXLAN_GPE), 490 .type = RTE_FLOW_ITEM_TYPE_UDP, 491 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 492 }, 493 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 494 .type = RTE_FLOW_ITEM_TYPE_TCP, 495 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 496 }, 497 [MLX5_EXPANSION_VXLAN] = { 498 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 499 MLX5_EXPANSION_IPV4, 500 MLX5_EXPANSION_IPV6), 501 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 502 }, 503 [MLX5_EXPANSION_VXLAN_GPE] = { 504 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 505 MLX5_EXPANSION_IPV4, 506 MLX5_EXPANSION_IPV6), 507 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 508 }, 509 [MLX5_EXPANSION_GRE] = { 510 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 511 .type = RTE_FLOW_ITEM_TYPE_GRE, 512 }, 513 [MLX5_EXPANSION_MPLS] = { 514 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 515 MLX5_EXPANSION_IPV6), 516 .type = RTE_FLOW_ITEM_TYPE_MPLS, 517 }, 518 [MLX5_EXPANSION_ETH] = { 519 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 520 MLX5_EXPANSION_IPV6), 521 .type = RTE_FLOW_ITEM_TYPE_ETH, 522 }, 523 [MLX5_EXPANSION_ETH_VLAN] = { 524 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 525 .type = RTE_FLOW_ITEM_TYPE_ETH, 526 }, 527 [MLX5_EXPANSION_VLAN] = { 528 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 529 MLX5_EXPANSION_IPV6), 530 .type = RTE_FLOW_ITEM_TYPE_VLAN, 531 }, 532 [MLX5_EXPANSION_IPV4] = { 533 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 534 MLX5_EXPANSION_IPV4_TCP), 535 .type = RTE_FLOW_ITEM_TYPE_IPV4, 536 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 537 ETH_RSS_NONFRAG_IPV4_OTHER, 538 }, 539 [MLX5_EXPANSION_IPV4_UDP] = { 540 .type = RTE_FLOW_ITEM_TYPE_UDP, 541 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 542 }, 543 [MLX5_EXPANSION_IPV4_TCP] = { 544 .type = RTE_FLOW_ITEM_TYPE_TCP, 545 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 546 }, 547 [MLX5_EXPANSION_IPV6] = { 548 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 549 MLX5_EXPANSION_IPV6_TCP), 550 .type = RTE_FLOW_ITEM_TYPE_IPV6, 551 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 552 ETH_RSS_NONFRAG_IPV6_OTHER, 553 }, 554 [MLX5_EXPANSION_IPV6_UDP] = { 555 .type = RTE_FLOW_ITEM_TYPE_UDP, 556 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 557 }, 558 [MLX5_EXPANSION_IPV6_TCP] = { 559 .type = RTE_FLOW_ITEM_TYPE_TCP, 560 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 561 }, 562 }; 563 564 static struct rte_flow_shared_action * 565 mlx5_shared_action_create(struct rte_eth_dev *dev, 566 const struct rte_flow_shared_action_conf *conf, 567 const struct rte_flow_action *action, 568 struct rte_flow_error *error); 569 static int mlx5_shared_action_destroy 570 (struct rte_eth_dev *dev, 571 struct rte_flow_shared_action *shared_action, 572 struct rte_flow_error *error); 573 static int mlx5_shared_action_update 574 (struct rte_eth_dev *dev, 575 struct rte_flow_shared_action *shared_action, 576 const struct rte_flow_action *action, 577 struct rte_flow_error *error); 578 static int mlx5_shared_action_query 579 (struct rte_eth_dev *dev, 580 const struct rte_flow_shared_action *action, 581 void *data, 582 struct rte_flow_error *error); 583 static inline bool 584 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 585 struct rte_flow_tunnel *tunnel, 586 const char *err_msg) 587 { 588 err_msg = NULL; 589 if (!is_tunnel_offload_active(dev)) { 590 err_msg = "tunnel offload was not activated"; 591 goto out; 592 } else if (!tunnel) { 593 err_msg = "no application tunnel"; 594 goto out; 595 } 596 597 switch (tunnel->type) { 598 default: 599 err_msg = "unsupported tunnel type"; 600 goto out; 601 case RTE_FLOW_ITEM_TYPE_VXLAN: 602 break; 603 } 604 605 out: 606 return !err_msg; 607 } 608 609 610 static int 611 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 612 struct rte_flow_tunnel *app_tunnel, 613 struct rte_flow_action **actions, 614 uint32_t *num_of_actions, 615 struct rte_flow_error *error) 616 { 617 int ret; 618 struct mlx5_flow_tunnel *tunnel; 619 const char *err_msg = NULL; 620 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 621 622 if (!verdict) 623 return rte_flow_error_set(error, EINVAL, 624 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 625 err_msg); 626 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 627 if (ret < 0) { 628 return rte_flow_error_set(error, ret, 629 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 630 "failed to initialize pmd tunnel"); 631 } 632 *actions = &tunnel->action; 633 *num_of_actions = 1; 634 return 0; 635 } 636 637 static int 638 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 639 struct rte_flow_tunnel *app_tunnel, 640 struct rte_flow_item **items, 641 uint32_t *num_of_items, 642 struct rte_flow_error *error) 643 { 644 int ret; 645 struct mlx5_flow_tunnel *tunnel; 646 const char *err_msg = NULL; 647 bool verdict = mlx5_flow_tunnel_validate(dev, app_tunnel, err_msg); 648 649 if (!verdict) 650 return rte_flow_error_set(error, EINVAL, 651 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 652 err_msg); 653 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 654 if (ret < 0) { 655 return rte_flow_error_set(error, ret, 656 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 657 "failed to initialize pmd tunnel"); 658 } 659 *items = &tunnel->item; 660 *num_of_items = 1; 661 return 0; 662 } 663 664 static int 665 mlx5_flow_item_release(struct rte_eth_dev *dev, 666 struct rte_flow_item *pmd_items, 667 uint32_t num_items, struct rte_flow_error *err) 668 { 669 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 670 struct mlx5_flow_tunnel *tun; 671 672 rte_spinlock_lock(&thub->sl); 673 LIST_FOREACH(tun, &thub->tunnels, chain) { 674 if (&tun->item == pmd_items) { 675 LIST_REMOVE(tun, chain); 676 break; 677 } 678 } 679 rte_spinlock_unlock(&thub->sl); 680 if (!tun || num_items != 1) 681 return rte_flow_error_set(err, EINVAL, 682 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 683 "invalid argument"); 684 if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED)) 685 mlx5_flow_tunnel_free(dev, tun); 686 return 0; 687 } 688 689 static int 690 mlx5_flow_action_release(struct rte_eth_dev *dev, 691 struct rte_flow_action *pmd_actions, 692 uint32_t num_actions, struct rte_flow_error *err) 693 { 694 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 695 struct mlx5_flow_tunnel *tun; 696 697 rte_spinlock_lock(&thub->sl); 698 LIST_FOREACH(tun, &thub->tunnels, chain) { 699 if (&tun->action == pmd_actions) { 700 LIST_REMOVE(tun, chain); 701 break; 702 } 703 } 704 rte_spinlock_unlock(&thub->sl); 705 if (!tun || num_actions != 1) 706 return rte_flow_error_set(err, EINVAL, 707 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 708 "invalid argument"); 709 if (!__atomic_sub_fetch(&tun->refctn, 1, __ATOMIC_RELAXED)) 710 mlx5_flow_tunnel_free(dev, tun); 711 712 return 0; 713 } 714 715 static int 716 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 717 struct rte_mbuf *m, 718 struct rte_flow_restore_info *info, 719 struct rte_flow_error *err) 720 { 721 uint64_t ol_flags = m->ol_flags; 722 const struct mlx5_flow_tbl_data_entry *tble; 723 const uint64_t mask = PKT_RX_FDIR | PKT_RX_FDIR_ID; 724 725 if ((ol_flags & mask) != mask) 726 goto err; 727 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 728 if (!tble) { 729 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 730 dev->data->port_id, m->hash.fdir.hi); 731 goto err; 732 } 733 MLX5_ASSERT(tble->tunnel); 734 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 735 info->group_id = tble->group_id; 736 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 737 RTE_FLOW_RESTORE_INFO_GROUP_ID | 738 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 739 740 return 0; 741 742 err: 743 return rte_flow_error_set(err, EINVAL, 744 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 745 "failed to get restore info"); 746 } 747 748 static const struct rte_flow_ops mlx5_flow_ops = { 749 .validate = mlx5_flow_validate, 750 .create = mlx5_flow_create, 751 .destroy = mlx5_flow_destroy, 752 .flush = mlx5_flow_flush, 753 .isolate = mlx5_flow_isolate, 754 .query = mlx5_flow_query, 755 .dev_dump = mlx5_flow_dev_dump, 756 .get_aged_flows = mlx5_flow_get_aged_flows, 757 .shared_action_create = mlx5_shared_action_create, 758 .shared_action_destroy = mlx5_shared_action_destroy, 759 .shared_action_update = mlx5_shared_action_update, 760 .shared_action_query = mlx5_shared_action_query, 761 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 762 .tunnel_match = mlx5_flow_tunnel_match, 763 .tunnel_action_decap_release = mlx5_flow_action_release, 764 .tunnel_item_release = mlx5_flow_item_release, 765 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 766 }; 767 768 /* Convert FDIR request to Generic flow. */ 769 struct mlx5_fdir { 770 struct rte_flow_attr attr; 771 struct rte_flow_item items[4]; 772 struct rte_flow_item_eth l2; 773 struct rte_flow_item_eth l2_mask; 774 union { 775 struct rte_flow_item_ipv4 ipv4; 776 struct rte_flow_item_ipv6 ipv6; 777 } l3; 778 union { 779 struct rte_flow_item_ipv4 ipv4; 780 struct rte_flow_item_ipv6 ipv6; 781 } l3_mask; 782 union { 783 struct rte_flow_item_udp udp; 784 struct rte_flow_item_tcp tcp; 785 } l4; 786 union { 787 struct rte_flow_item_udp udp; 788 struct rte_flow_item_tcp tcp; 789 } l4_mask; 790 struct rte_flow_action actions[2]; 791 struct rte_flow_action_queue queue; 792 }; 793 794 /* Tunnel information. */ 795 struct mlx5_flow_tunnel_info { 796 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 797 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 798 }; 799 800 static struct mlx5_flow_tunnel_info tunnels_info[] = { 801 { 802 .tunnel = MLX5_FLOW_LAYER_VXLAN, 803 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 804 }, 805 { 806 .tunnel = MLX5_FLOW_LAYER_GENEVE, 807 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 808 }, 809 { 810 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 811 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 812 }, 813 { 814 .tunnel = MLX5_FLOW_LAYER_GRE, 815 .ptype = RTE_PTYPE_TUNNEL_GRE, 816 }, 817 { 818 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 819 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 820 }, 821 { 822 .tunnel = MLX5_FLOW_LAYER_MPLS, 823 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 824 }, 825 { 826 .tunnel = MLX5_FLOW_LAYER_NVGRE, 827 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 828 }, 829 { 830 .tunnel = MLX5_FLOW_LAYER_IPIP, 831 .ptype = RTE_PTYPE_TUNNEL_IP, 832 }, 833 { 834 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 835 .ptype = RTE_PTYPE_TUNNEL_IP, 836 }, 837 { 838 .tunnel = MLX5_FLOW_LAYER_GTP, 839 .ptype = RTE_PTYPE_TUNNEL_GTPU, 840 }, 841 }; 842 843 /* Key of thread specific flow workspace data. */ 844 static pthread_key_t key_workspace; 845 846 /* Thread specific flow workspace data once initialization data. */ 847 static pthread_once_t key_workspace_init; 848 849 850 /** 851 * Translate tag ID to register. 852 * 853 * @param[in] dev 854 * Pointer to the Ethernet device structure. 855 * @param[in] feature 856 * The feature that request the register. 857 * @param[in] id 858 * The request register ID. 859 * @param[out] error 860 * Error description in case of any. 861 * 862 * @return 863 * The request register on success, a negative errno 864 * value otherwise and rte_errno is set. 865 */ 866 int 867 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 868 enum mlx5_feature_name feature, 869 uint32_t id, 870 struct rte_flow_error *error) 871 { 872 struct mlx5_priv *priv = dev->data->dev_private; 873 struct mlx5_dev_config *config = &priv->config; 874 enum modify_reg start_reg; 875 bool skip_mtr_reg = false; 876 877 switch (feature) { 878 case MLX5_HAIRPIN_RX: 879 return REG_B; 880 case MLX5_HAIRPIN_TX: 881 return REG_A; 882 case MLX5_METADATA_RX: 883 switch (config->dv_xmeta_en) { 884 case MLX5_XMETA_MODE_LEGACY: 885 return REG_B; 886 case MLX5_XMETA_MODE_META16: 887 return REG_C_0; 888 case MLX5_XMETA_MODE_META32: 889 return REG_C_1; 890 } 891 break; 892 case MLX5_METADATA_TX: 893 return REG_A; 894 case MLX5_METADATA_FDB: 895 switch (config->dv_xmeta_en) { 896 case MLX5_XMETA_MODE_LEGACY: 897 return REG_NON; 898 case MLX5_XMETA_MODE_META16: 899 return REG_C_0; 900 case MLX5_XMETA_MODE_META32: 901 return REG_C_1; 902 } 903 break; 904 case MLX5_FLOW_MARK: 905 switch (config->dv_xmeta_en) { 906 case MLX5_XMETA_MODE_LEGACY: 907 return REG_NON; 908 case MLX5_XMETA_MODE_META16: 909 return REG_C_1; 910 case MLX5_XMETA_MODE_META32: 911 return REG_C_0; 912 } 913 break; 914 case MLX5_MTR_SFX: 915 /* 916 * If meter color and flow match share one register, flow match 917 * should use the meter color register for match. 918 */ 919 if (priv->mtr_reg_share) 920 return priv->mtr_color_reg; 921 else 922 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 923 REG_C_3; 924 case MLX5_MTR_COLOR: 925 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 926 return priv->mtr_color_reg; 927 case MLX5_COPY_MARK: 928 /* 929 * Metadata COPY_MARK register using is in meter suffix sub 930 * flow while with meter. It's safe to share the same register. 931 */ 932 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 933 case MLX5_APP_TAG: 934 /* 935 * If meter is enable, it will engage the register for color 936 * match and flow match. If meter color match is not using the 937 * REG_C_2, need to skip the REG_C_x be used by meter color 938 * match. 939 * If meter is disable, free to use all available registers. 940 */ 941 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 942 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 943 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 944 if (id > (REG_C_7 - start_reg)) 945 return rte_flow_error_set(error, EINVAL, 946 RTE_FLOW_ERROR_TYPE_ITEM, 947 NULL, "invalid tag id"); 948 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 949 return rte_flow_error_set(error, ENOTSUP, 950 RTE_FLOW_ERROR_TYPE_ITEM, 951 NULL, "unsupported tag id"); 952 /* 953 * This case means meter is using the REG_C_x great than 2. 954 * Take care not to conflict with meter color REG_C_x. 955 * If the available index REG_C_y >= REG_C_x, skip the 956 * color register. 957 */ 958 if (skip_mtr_reg && config->flow_mreg_c 959 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 960 if (id >= (REG_C_7 - start_reg)) 961 return rte_flow_error_set(error, EINVAL, 962 RTE_FLOW_ERROR_TYPE_ITEM, 963 NULL, "invalid tag id"); 964 if (config->flow_mreg_c 965 [id + 1 + start_reg - REG_C_0] != REG_NON) 966 return config->flow_mreg_c 967 [id + 1 + start_reg - REG_C_0]; 968 return rte_flow_error_set(error, ENOTSUP, 969 RTE_FLOW_ERROR_TYPE_ITEM, 970 NULL, "unsupported tag id"); 971 } 972 return config->flow_mreg_c[id + start_reg - REG_C_0]; 973 } 974 MLX5_ASSERT(false); 975 return rte_flow_error_set(error, EINVAL, 976 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 977 NULL, "invalid feature name"); 978 } 979 980 /** 981 * Check extensive flow metadata register support. 982 * 983 * @param dev 984 * Pointer to rte_eth_dev structure. 985 * 986 * @return 987 * True if device supports extensive flow metadata register, otherwise false. 988 */ 989 bool 990 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 991 { 992 struct mlx5_priv *priv = dev->data->dev_private; 993 struct mlx5_dev_config *config = &priv->config; 994 995 /* 996 * Having available reg_c can be regarded inclusively as supporting 997 * extensive flow metadata register, which could mean, 998 * - metadata register copy action by modify header. 999 * - 16 modify header actions is supported. 1000 * - reg_c's are preserved across different domain (FDB and NIC) on 1001 * packet loopback by flow lookup miss. 1002 */ 1003 return config->flow_mreg_c[2] != REG_NON; 1004 } 1005 1006 /** 1007 * Verify the @p item specifications (spec, last, mask) are compatible with the 1008 * NIC capabilities. 1009 * 1010 * @param[in] item 1011 * Item specification. 1012 * @param[in] mask 1013 * @p item->mask or flow default bit-masks. 1014 * @param[in] nic_mask 1015 * Bit-masks covering supported fields by the NIC to compare with user mask. 1016 * @param[in] size 1017 * Bit-masks size in bytes. 1018 * @param[in] range_accepted 1019 * True if range of values is accepted for specific fields, false otherwise. 1020 * @param[out] error 1021 * Pointer to error structure. 1022 * 1023 * @return 1024 * 0 on success, a negative errno value otherwise and rte_errno is set. 1025 */ 1026 int 1027 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1028 const uint8_t *mask, 1029 const uint8_t *nic_mask, 1030 unsigned int size, 1031 bool range_accepted, 1032 struct rte_flow_error *error) 1033 { 1034 unsigned int i; 1035 1036 MLX5_ASSERT(nic_mask); 1037 for (i = 0; i < size; ++i) 1038 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1039 return rte_flow_error_set(error, ENOTSUP, 1040 RTE_FLOW_ERROR_TYPE_ITEM, 1041 item, 1042 "mask enables non supported" 1043 " bits"); 1044 if (!item->spec && (item->mask || item->last)) 1045 return rte_flow_error_set(error, EINVAL, 1046 RTE_FLOW_ERROR_TYPE_ITEM, item, 1047 "mask/last without a spec is not" 1048 " supported"); 1049 if (item->spec && item->last && !range_accepted) { 1050 uint8_t spec[size]; 1051 uint8_t last[size]; 1052 unsigned int i; 1053 int ret; 1054 1055 for (i = 0; i < size; ++i) { 1056 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1057 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1058 } 1059 ret = memcmp(spec, last, size); 1060 if (ret != 0) 1061 return rte_flow_error_set(error, EINVAL, 1062 RTE_FLOW_ERROR_TYPE_ITEM, 1063 item, 1064 "range is not valid"); 1065 } 1066 return 0; 1067 } 1068 1069 /** 1070 * Adjust the hash fields according to the @p flow information. 1071 * 1072 * @param[in] dev_flow. 1073 * Pointer to the mlx5_flow. 1074 * @param[in] tunnel 1075 * 1 when the hash field is for a tunnel item. 1076 * @param[in] layer_types 1077 * ETH_RSS_* types. 1078 * @param[in] hash_fields 1079 * Item hash fields. 1080 * 1081 * @return 1082 * The hash fields that should be used. 1083 */ 1084 uint64_t 1085 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1086 int tunnel __rte_unused, uint64_t layer_types, 1087 uint64_t hash_fields) 1088 { 1089 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1090 int rss_request_inner = rss_desc->level >= 2; 1091 1092 /* Check RSS hash level for tunnel. */ 1093 if (tunnel && rss_request_inner) 1094 hash_fields |= IBV_RX_HASH_INNER; 1095 else if (tunnel || rss_request_inner) 1096 return 0; 1097 #endif 1098 /* Check if requested layer matches RSS hash fields. */ 1099 if (!(rss_desc->types & layer_types)) 1100 return 0; 1101 return hash_fields; 1102 } 1103 1104 /** 1105 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1106 * if several tunnel rules are used on this queue, the tunnel ptype will be 1107 * cleared. 1108 * 1109 * @param rxq_ctrl 1110 * Rx queue to update. 1111 */ 1112 static void 1113 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1114 { 1115 unsigned int i; 1116 uint32_t tunnel_ptype = 0; 1117 1118 /* Look up for the ptype to use. */ 1119 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1120 if (!rxq_ctrl->flow_tunnels_n[i]) 1121 continue; 1122 if (!tunnel_ptype) { 1123 tunnel_ptype = tunnels_info[i].ptype; 1124 } else { 1125 tunnel_ptype = 0; 1126 break; 1127 } 1128 } 1129 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1130 } 1131 1132 /** 1133 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 1134 * flow. 1135 * 1136 * @param[in] dev 1137 * Pointer to the Ethernet device structure. 1138 * @param[in] dev_handle 1139 * Pointer to device flow handle structure. 1140 */ 1141 static void 1142 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1143 struct mlx5_flow_handle *dev_handle) 1144 { 1145 struct mlx5_priv *priv = dev->data->dev_private; 1146 const int mark = dev_handle->mark; 1147 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1148 struct mlx5_hrxq *hrxq; 1149 unsigned int i; 1150 1151 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1152 return; 1153 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1154 dev_handle->rix_hrxq); 1155 if (!hrxq) 1156 return; 1157 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1158 int idx = hrxq->ind_table->queues[i]; 1159 struct mlx5_rxq_ctrl *rxq_ctrl = 1160 container_of((*priv->rxqs)[idx], 1161 struct mlx5_rxq_ctrl, rxq); 1162 1163 /* 1164 * To support metadata register copy on Tx loopback, 1165 * this must be always enabled (metadata may arive 1166 * from other port - not from local flows only. 1167 */ 1168 if (priv->config.dv_flow_en && 1169 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1170 mlx5_flow_ext_mreg_supported(dev)) { 1171 rxq_ctrl->rxq.mark = 1; 1172 rxq_ctrl->flow_mark_n = 1; 1173 } else if (mark) { 1174 rxq_ctrl->rxq.mark = 1; 1175 rxq_ctrl->flow_mark_n++; 1176 } 1177 if (tunnel) { 1178 unsigned int j; 1179 1180 /* Increase the counter matching the flow. */ 1181 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1182 if ((tunnels_info[j].tunnel & 1183 dev_handle->layers) == 1184 tunnels_info[j].tunnel) { 1185 rxq_ctrl->flow_tunnels_n[j]++; 1186 break; 1187 } 1188 } 1189 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1190 } 1191 } 1192 } 1193 1194 /** 1195 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1196 * 1197 * @param[in] dev 1198 * Pointer to the Ethernet device structure. 1199 * @param[in] flow 1200 * Pointer to flow structure. 1201 */ 1202 static void 1203 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1204 { 1205 struct mlx5_priv *priv = dev->data->dev_private; 1206 uint32_t handle_idx; 1207 struct mlx5_flow_handle *dev_handle; 1208 1209 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1210 handle_idx, dev_handle, next) 1211 flow_drv_rxq_flags_set(dev, dev_handle); 1212 } 1213 1214 /** 1215 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1216 * device flow if no other flow uses it with the same kind of request. 1217 * 1218 * @param dev 1219 * Pointer to Ethernet device. 1220 * @param[in] dev_handle 1221 * Pointer to the device flow handle structure. 1222 */ 1223 static void 1224 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1225 struct mlx5_flow_handle *dev_handle) 1226 { 1227 struct mlx5_priv *priv = dev->data->dev_private; 1228 const int mark = dev_handle->mark; 1229 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1230 struct mlx5_hrxq *hrxq; 1231 unsigned int i; 1232 1233 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1234 return; 1235 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1236 dev_handle->rix_hrxq); 1237 if (!hrxq) 1238 return; 1239 MLX5_ASSERT(dev->data->dev_started); 1240 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1241 int idx = hrxq->ind_table->queues[i]; 1242 struct mlx5_rxq_ctrl *rxq_ctrl = 1243 container_of((*priv->rxqs)[idx], 1244 struct mlx5_rxq_ctrl, rxq); 1245 1246 if (priv->config.dv_flow_en && 1247 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1248 mlx5_flow_ext_mreg_supported(dev)) { 1249 rxq_ctrl->rxq.mark = 1; 1250 rxq_ctrl->flow_mark_n = 1; 1251 } else if (mark) { 1252 rxq_ctrl->flow_mark_n--; 1253 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1254 } 1255 if (tunnel) { 1256 unsigned int j; 1257 1258 /* Decrease the counter matching the flow. */ 1259 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1260 if ((tunnels_info[j].tunnel & 1261 dev_handle->layers) == 1262 tunnels_info[j].tunnel) { 1263 rxq_ctrl->flow_tunnels_n[j]--; 1264 break; 1265 } 1266 } 1267 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1268 } 1269 } 1270 } 1271 1272 /** 1273 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1274 * @p flow if no other flow uses it with the same kind of request. 1275 * 1276 * @param dev 1277 * Pointer to Ethernet device. 1278 * @param[in] flow 1279 * Pointer to the flow. 1280 */ 1281 static void 1282 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1283 { 1284 struct mlx5_priv *priv = dev->data->dev_private; 1285 uint32_t handle_idx; 1286 struct mlx5_flow_handle *dev_handle; 1287 1288 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1289 handle_idx, dev_handle, next) 1290 flow_drv_rxq_flags_trim(dev, dev_handle); 1291 } 1292 1293 /** 1294 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1295 * 1296 * @param dev 1297 * Pointer to Ethernet device. 1298 */ 1299 static void 1300 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1301 { 1302 struct mlx5_priv *priv = dev->data->dev_private; 1303 unsigned int i; 1304 1305 for (i = 0; i != priv->rxqs_n; ++i) { 1306 struct mlx5_rxq_ctrl *rxq_ctrl; 1307 unsigned int j; 1308 1309 if (!(*priv->rxqs)[i]) 1310 continue; 1311 rxq_ctrl = container_of((*priv->rxqs)[i], 1312 struct mlx5_rxq_ctrl, rxq); 1313 rxq_ctrl->flow_mark_n = 0; 1314 rxq_ctrl->rxq.mark = 0; 1315 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1316 rxq_ctrl->flow_tunnels_n[j] = 0; 1317 rxq_ctrl->rxq.tunnel = 0; 1318 } 1319 } 1320 1321 /** 1322 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1323 * 1324 * @param[in] dev 1325 * Pointer to the Ethernet device structure. 1326 */ 1327 void 1328 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1329 { 1330 struct mlx5_priv *priv = dev->data->dev_private; 1331 struct mlx5_rxq_data *data; 1332 unsigned int i; 1333 1334 for (i = 0; i != priv->rxqs_n; ++i) { 1335 if (!(*priv->rxqs)[i]) 1336 continue; 1337 data = (*priv->rxqs)[i]; 1338 if (!rte_flow_dynf_metadata_avail()) { 1339 data->dynf_meta = 0; 1340 data->flow_meta_mask = 0; 1341 data->flow_meta_offset = -1; 1342 } else { 1343 data->dynf_meta = 1; 1344 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1345 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1346 } 1347 } 1348 } 1349 1350 /* 1351 * return a pointer to the desired action in the list of actions. 1352 * 1353 * @param[in] actions 1354 * The list of actions to search the action in. 1355 * @param[in] action 1356 * The action to find. 1357 * 1358 * @return 1359 * Pointer to the action in the list, if found. NULL otherwise. 1360 */ 1361 const struct rte_flow_action * 1362 mlx5_flow_find_action(const struct rte_flow_action *actions, 1363 enum rte_flow_action_type action) 1364 { 1365 if (actions == NULL) 1366 return NULL; 1367 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1368 if (actions->type == action) 1369 return actions; 1370 return NULL; 1371 } 1372 1373 /* 1374 * Validate the flag action. 1375 * 1376 * @param[in] action_flags 1377 * Bit-fields that holds the actions detected until now. 1378 * @param[in] attr 1379 * Attributes of flow that includes this action. 1380 * @param[out] error 1381 * Pointer to error structure. 1382 * 1383 * @return 1384 * 0 on success, a negative errno value otherwise and rte_errno is set. 1385 */ 1386 int 1387 mlx5_flow_validate_action_flag(uint64_t action_flags, 1388 const struct rte_flow_attr *attr, 1389 struct rte_flow_error *error) 1390 { 1391 if (action_flags & MLX5_FLOW_ACTION_MARK) 1392 return rte_flow_error_set(error, EINVAL, 1393 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1394 "can't mark and flag in same flow"); 1395 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1396 return rte_flow_error_set(error, EINVAL, 1397 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1398 "can't have 2 flag" 1399 " actions in same flow"); 1400 if (attr->egress) 1401 return rte_flow_error_set(error, ENOTSUP, 1402 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1403 "flag action not supported for " 1404 "egress"); 1405 return 0; 1406 } 1407 1408 /* 1409 * Validate the mark action. 1410 * 1411 * @param[in] action 1412 * Pointer to the queue action. 1413 * @param[in] action_flags 1414 * Bit-fields that holds the actions detected until now. 1415 * @param[in] attr 1416 * Attributes of flow that includes this action. 1417 * @param[out] error 1418 * Pointer to error structure. 1419 * 1420 * @return 1421 * 0 on success, a negative errno value otherwise and rte_errno is set. 1422 */ 1423 int 1424 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1425 uint64_t action_flags, 1426 const struct rte_flow_attr *attr, 1427 struct rte_flow_error *error) 1428 { 1429 const struct rte_flow_action_mark *mark = action->conf; 1430 1431 if (!mark) 1432 return rte_flow_error_set(error, EINVAL, 1433 RTE_FLOW_ERROR_TYPE_ACTION, 1434 action, 1435 "configuration cannot be null"); 1436 if (mark->id >= MLX5_FLOW_MARK_MAX) 1437 return rte_flow_error_set(error, EINVAL, 1438 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1439 &mark->id, 1440 "mark id must in 0 <= id < " 1441 RTE_STR(MLX5_FLOW_MARK_MAX)); 1442 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1443 return rte_flow_error_set(error, EINVAL, 1444 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1445 "can't flag and mark in same flow"); 1446 if (action_flags & MLX5_FLOW_ACTION_MARK) 1447 return rte_flow_error_set(error, EINVAL, 1448 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1449 "can't have 2 mark actions in same" 1450 " flow"); 1451 if (attr->egress) 1452 return rte_flow_error_set(error, ENOTSUP, 1453 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1454 "mark action not supported for " 1455 "egress"); 1456 return 0; 1457 } 1458 1459 /* 1460 * Validate the drop action. 1461 * 1462 * @param[in] action_flags 1463 * Bit-fields that holds the actions detected until now. 1464 * @param[in] attr 1465 * Attributes of flow that includes this action. 1466 * @param[out] error 1467 * Pointer to error structure. 1468 * 1469 * @return 1470 * 0 on success, a negative errno value otherwise and rte_errno is set. 1471 */ 1472 int 1473 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1474 const struct rte_flow_attr *attr, 1475 struct rte_flow_error *error) 1476 { 1477 if (attr->egress) 1478 return rte_flow_error_set(error, ENOTSUP, 1479 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1480 "drop action not supported for " 1481 "egress"); 1482 return 0; 1483 } 1484 1485 /* 1486 * Validate the queue action. 1487 * 1488 * @param[in] action 1489 * Pointer to the queue action. 1490 * @param[in] action_flags 1491 * Bit-fields that holds the actions detected until now. 1492 * @param[in] dev 1493 * Pointer to the Ethernet device structure. 1494 * @param[in] attr 1495 * Attributes of flow that includes this action. 1496 * @param[out] error 1497 * Pointer to error structure. 1498 * 1499 * @return 1500 * 0 on success, a negative errno value otherwise and rte_errno is set. 1501 */ 1502 int 1503 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1504 uint64_t action_flags, 1505 struct rte_eth_dev *dev, 1506 const struct rte_flow_attr *attr, 1507 struct rte_flow_error *error) 1508 { 1509 struct mlx5_priv *priv = dev->data->dev_private; 1510 const struct rte_flow_action_queue *queue = action->conf; 1511 1512 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1513 return rte_flow_error_set(error, EINVAL, 1514 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1515 "can't have 2 fate actions in" 1516 " same flow"); 1517 if (!priv->rxqs_n) 1518 return rte_flow_error_set(error, EINVAL, 1519 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1520 NULL, "No Rx queues configured"); 1521 if (queue->index >= priv->rxqs_n) 1522 return rte_flow_error_set(error, EINVAL, 1523 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1524 &queue->index, 1525 "queue index out of range"); 1526 if (!(*priv->rxqs)[queue->index]) 1527 return rte_flow_error_set(error, EINVAL, 1528 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1529 &queue->index, 1530 "queue is not configured"); 1531 if (attr->egress) 1532 return rte_flow_error_set(error, ENOTSUP, 1533 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1534 "queue action not supported for " 1535 "egress"); 1536 return 0; 1537 } 1538 1539 /* 1540 * Validate the rss action. 1541 * 1542 * @param[in] dev 1543 * Pointer to the Ethernet device structure. 1544 * @param[in] action 1545 * Pointer to the queue action. 1546 * @param[out] error 1547 * Pointer to error structure. 1548 * 1549 * @return 1550 * 0 on success, a negative errno value otherwise and rte_errno is set. 1551 */ 1552 int 1553 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1554 const struct rte_flow_action *action, 1555 struct rte_flow_error *error) 1556 { 1557 struct mlx5_priv *priv = dev->data->dev_private; 1558 const struct rte_flow_action_rss *rss = action->conf; 1559 unsigned int i; 1560 1561 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1562 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1563 return rte_flow_error_set(error, ENOTSUP, 1564 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1565 &rss->func, 1566 "RSS hash function not supported"); 1567 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1568 if (rss->level > 2) 1569 #else 1570 if (rss->level > 1) 1571 #endif 1572 return rte_flow_error_set(error, ENOTSUP, 1573 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1574 &rss->level, 1575 "tunnel RSS is not supported"); 1576 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1577 if (rss->key_len == 0 && rss->key != NULL) 1578 return rte_flow_error_set(error, ENOTSUP, 1579 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1580 &rss->key_len, 1581 "RSS hash key length 0"); 1582 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1583 return rte_flow_error_set(error, ENOTSUP, 1584 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1585 &rss->key_len, 1586 "RSS hash key too small"); 1587 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1588 return rte_flow_error_set(error, ENOTSUP, 1589 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1590 &rss->key_len, 1591 "RSS hash key too large"); 1592 if (rss->queue_num > priv->config.ind_table_max_size) 1593 return rte_flow_error_set(error, ENOTSUP, 1594 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1595 &rss->queue_num, 1596 "number of queues too large"); 1597 if (rss->types & MLX5_RSS_HF_MASK) 1598 return rte_flow_error_set(error, ENOTSUP, 1599 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1600 &rss->types, 1601 "some RSS protocols are not" 1602 " supported"); 1603 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1604 !(rss->types & ETH_RSS_IP)) 1605 return rte_flow_error_set(error, EINVAL, 1606 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1607 "L3 partial RSS requested but L3 RSS" 1608 " type not specified"); 1609 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1610 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1611 return rte_flow_error_set(error, EINVAL, 1612 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1613 "L4 partial RSS requested but L4 RSS" 1614 " type not specified"); 1615 if (!priv->rxqs_n) 1616 return rte_flow_error_set(error, EINVAL, 1617 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1618 NULL, "No Rx queues configured"); 1619 if (!rss->queue_num) 1620 return rte_flow_error_set(error, EINVAL, 1621 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1622 NULL, "No queues configured"); 1623 for (i = 0; i != rss->queue_num; ++i) { 1624 if (rss->queue[i] >= priv->rxqs_n) 1625 return rte_flow_error_set 1626 (error, EINVAL, 1627 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1628 &rss->queue[i], "queue index out of range"); 1629 if (!(*priv->rxqs)[rss->queue[i]]) 1630 return rte_flow_error_set 1631 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1632 &rss->queue[i], "queue is not configured"); 1633 } 1634 return 0; 1635 } 1636 1637 /* 1638 * Validate the rss action. 1639 * 1640 * @param[in] action 1641 * Pointer to the queue action. 1642 * @param[in] action_flags 1643 * Bit-fields that holds the actions detected until now. 1644 * @param[in] dev 1645 * Pointer to the Ethernet device structure. 1646 * @param[in] attr 1647 * Attributes of flow that includes this action. 1648 * @param[in] item_flags 1649 * Items that were detected. 1650 * @param[out] error 1651 * Pointer to error structure. 1652 * 1653 * @return 1654 * 0 on success, a negative errno value otherwise and rte_errno is set. 1655 */ 1656 int 1657 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1658 uint64_t action_flags, 1659 struct rte_eth_dev *dev, 1660 const struct rte_flow_attr *attr, 1661 uint64_t item_flags, 1662 struct rte_flow_error *error) 1663 { 1664 const struct rte_flow_action_rss *rss = action->conf; 1665 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1666 int ret; 1667 1668 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1669 return rte_flow_error_set(error, EINVAL, 1670 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1671 "can't have 2 fate actions" 1672 " in same flow"); 1673 ret = mlx5_validate_action_rss(dev, action, error); 1674 if (ret) 1675 return ret; 1676 if (attr->egress) 1677 return rte_flow_error_set(error, ENOTSUP, 1678 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1679 "rss action not supported for " 1680 "egress"); 1681 if (rss->level > 1 && !tunnel) 1682 return rte_flow_error_set(error, EINVAL, 1683 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1684 "inner RSS is not supported for " 1685 "non-tunnel flows"); 1686 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1687 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1688 return rte_flow_error_set(error, EINVAL, 1689 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1690 "RSS on eCPRI is not supported now"); 1691 } 1692 return 0; 1693 } 1694 1695 /* 1696 * Validate the default miss action. 1697 * 1698 * @param[in] action_flags 1699 * Bit-fields that holds the actions detected until now. 1700 * @param[out] error 1701 * Pointer to error structure. 1702 * 1703 * @return 1704 * 0 on success, a negative errno value otherwise and rte_errno is set. 1705 */ 1706 int 1707 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1708 const struct rte_flow_attr *attr, 1709 struct rte_flow_error *error) 1710 { 1711 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1712 return rte_flow_error_set(error, EINVAL, 1713 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1714 "can't have 2 fate actions in" 1715 " same flow"); 1716 if (attr->egress) 1717 return rte_flow_error_set(error, ENOTSUP, 1718 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1719 "default miss action not supported " 1720 "for egress"); 1721 if (attr->group) 1722 return rte_flow_error_set(error, ENOTSUP, 1723 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1724 "only group 0 is supported"); 1725 if (attr->transfer) 1726 return rte_flow_error_set(error, ENOTSUP, 1727 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1728 NULL, "transfer is not supported"); 1729 return 0; 1730 } 1731 1732 /* 1733 * Validate the count action. 1734 * 1735 * @param[in] dev 1736 * Pointer to the Ethernet device structure. 1737 * @param[in] attr 1738 * Attributes of flow that includes this action. 1739 * @param[out] error 1740 * Pointer to error structure. 1741 * 1742 * @return 1743 * 0 on success, a negative errno value otherwise and rte_errno is set. 1744 */ 1745 int 1746 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1747 const struct rte_flow_attr *attr, 1748 struct rte_flow_error *error) 1749 { 1750 if (attr->egress) 1751 return rte_flow_error_set(error, ENOTSUP, 1752 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1753 "count action not supported for " 1754 "egress"); 1755 return 0; 1756 } 1757 1758 /** 1759 * Verify the @p attributes will be correctly understood by the NIC and store 1760 * them in the @p flow if everything is correct. 1761 * 1762 * @param[in] dev 1763 * Pointer to the Ethernet device structure. 1764 * @param[in] attributes 1765 * Pointer to flow attributes 1766 * @param[out] error 1767 * Pointer to error structure. 1768 * 1769 * @return 1770 * 0 on success, a negative errno value otherwise and rte_errno is set. 1771 */ 1772 int 1773 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1774 const struct rte_flow_attr *attributes, 1775 struct rte_flow_error *error) 1776 { 1777 struct mlx5_priv *priv = dev->data->dev_private; 1778 uint32_t priority_max = priv->config.flow_prio - 1; 1779 1780 if (attributes->group) 1781 return rte_flow_error_set(error, ENOTSUP, 1782 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1783 NULL, "groups is not supported"); 1784 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1785 attributes->priority >= priority_max) 1786 return rte_flow_error_set(error, ENOTSUP, 1787 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1788 NULL, "priority out of range"); 1789 if (attributes->egress) 1790 return rte_flow_error_set(error, ENOTSUP, 1791 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1792 "egress is not supported"); 1793 if (attributes->transfer && !priv->config.dv_esw_en) 1794 return rte_flow_error_set(error, ENOTSUP, 1795 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1796 NULL, "transfer is not supported"); 1797 if (!attributes->ingress) 1798 return rte_flow_error_set(error, EINVAL, 1799 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1800 NULL, 1801 "ingress attribute is mandatory"); 1802 return 0; 1803 } 1804 1805 /** 1806 * Validate ICMP6 item. 1807 * 1808 * @param[in] item 1809 * Item specification. 1810 * @param[in] item_flags 1811 * Bit-fields that holds the items detected until now. 1812 * @param[in] ext_vlan_sup 1813 * Whether extended VLAN features are supported or not. 1814 * @param[out] error 1815 * Pointer to error structure. 1816 * 1817 * @return 1818 * 0 on success, a negative errno value otherwise and rte_errno is set. 1819 */ 1820 int 1821 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1822 uint64_t item_flags, 1823 uint8_t target_protocol, 1824 struct rte_flow_error *error) 1825 { 1826 const struct rte_flow_item_icmp6 *mask = item->mask; 1827 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1828 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1829 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1830 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1831 MLX5_FLOW_LAYER_OUTER_L4; 1832 int ret; 1833 1834 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1835 return rte_flow_error_set(error, EINVAL, 1836 RTE_FLOW_ERROR_TYPE_ITEM, item, 1837 "protocol filtering not compatible" 1838 " with ICMP6 layer"); 1839 if (!(item_flags & l3m)) 1840 return rte_flow_error_set(error, EINVAL, 1841 RTE_FLOW_ERROR_TYPE_ITEM, item, 1842 "IPv6 is mandatory to filter on" 1843 " ICMP6"); 1844 if (item_flags & l4m) 1845 return rte_flow_error_set(error, EINVAL, 1846 RTE_FLOW_ERROR_TYPE_ITEM, item, 1847 "multiple L4 layers not supported"); 1848 if (!mask) 1849 mask = &rte_flow_item_icmp6_mask; 1850 ret = mlx5_flow_item_acceptable 1851 (item, (const uint8_t *)mask, 1852 (const uint8_t *)&rte_flow_item_icmp6_mask, 1853 sizeof(struct rte_flow_item_icmp6), 1854 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1855 if (ret < 0) 1856 return ret; 1857 return 0; 1858 } 1859 1860 /** 1861 * Validate ICMP item. 1862 * 1863 * @param[in] item 1864 * Item specification. 1865 * @param[in] item_flags 1866 * Bit-fields that holds the items detected until now. 1867 * @param[out] error 1868 * Pointer to error structure. 1869 * 1870 * @return 1871 * 0 on success, a negative errno value otherwise and rte_errno is set. 1872 */ 1873 int 1874 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1875 uint64_t item_flags, 1876 uint8_t target_protocol, 1877 struct rte_flow_error *error) 1878 { 1879 const struct rte_flow_item_icmp *mask = item->mask; 1880 const struct rte_flow_item_icmp nic_mask = { 1881 .hdr.icmp_type = 0xff, 1882 .hdr.icmp_code = 0xff, 1883 .hdr.icmp_ident = RTE_BE16(0xffff), 1884 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1885 }; 1886 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1887 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1888 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1889 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1890 MLX5_FLOW_LAYER_OUTER_L4; 1891 int ret; 1892 1893 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1894 return rte_flow_error_set(error, EINVAL, 1895 RTE_FLOW_ERROR_TYPE_ITEM, item, 1896 "protocol filtering not compatible" 1897 " with ICMP layer"); 1898 if (!(item_flags & l3m)) 1899 return rte_flow_error_set(error, EINVAL, 1900 RTE_FLOW_ERROR_TYPE_ITEM, item, 1901 "IPv4 is mandatory to filter" 1902 " on ICMP"); 1903 if (item_flags & l4m) 1904 return rte_flow_error_set(error, EINVAL, 1905 RTE_FLOW_ERROR_TYPE_ITEM, item, 1906 "multiple L4 layers not supported"); 1907 if (!mask) 1908 mask = &nic_mask; 1909 ret = mlx5_flow_item_acceptable 1910 (item, (const uint8_t *)mask, 1911 (const uint8_t *)&nic_mask, 1912 sizeof(struct rte_flow_item_icmp), 1913 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1914 if (ret < 0) 1915 return ret; 1916 return 0; 1917 } 1918 1919 /** 1920 * Validate Ethernet item. 1921 * 1922 * @param[in] item 1923 * Item specification. 1924 * @param[in] item_flags 1925 * Bit-fields that holds the items detected until now. 1926 * @param[out] error 1927 * Pointer to error structure. 1928 * 1929 * @return 1930 * 0 on success, a negative errno value otherwise and rte_errno is set. 1931 */ 1932 int 1933 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1934 uint64_t item_flags, bool ext_vlan_sup, 1935 struct rte_flow_error *error) 1936 { 1937 const struct rte_flow_item_eth *mask = item->mask; 1938 const struct rte_flow_item_eth nic_mask = { 1939 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1940 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1941 .type = RTE_BE16(0xffff), 1942 .has_vlan = ext_vlan_sup ? 1 : 0, 1943 }; 1944 int ret; 1945 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1946 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1947 MLX5_FLOW_LAYER_OUTER_L2; 1948 1949 if (item_flags & ethm) 1950 return rte_flow_error_set(error, ENOTSUP, 1951 RTE_FLOW_ERROR_TYPE_ITEM, item, 1952 "multiple L2 layers not supported"); 1953 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1954 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1955 return rte_flow_error_set(error, EINVAL, 1956 RTE_FLOW_ERROR_TYPE_ITEM, item, 1957 "L2 layer should not follow " 1958 "L3 layers"); 1959 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1960 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1961 return rte_flow_error_set(error, EINVAL, 1962 RTE_FLOW_ERROR_TYPE_ITEM, item, 1963 "L2 layer should not follow VLAN"); 1964 if (!mask) 1965 mask = &rte_flow_item_eth_mask; 1966 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1967 (const uint8_t *)&nic_mask, 1968 sizeof(struct rte_flow_item_eth), 1969 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1970 return ret; 1971 } 1972 1973 /** 1974 * Validate VLAN item. 1975 * 1976 * @param[in] item 1977 * Item specification. 1978 * @param[in] item_flags 1979 * Bit-fields that holds the items detected until now. 1980 * @param[in] dev 1981 * Ethernet device flow is being created on. 1982 * @param[out] error 1983 * Pointer to error structure. 1984 * 1985 * @return 1986 * 0 on success, a negative errno value otherwise and rte_errno is set. 1987 */ 1988 int 1989 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1990 uint64_t item_flags, 1991 struct rte_eth_dev *dev, 1992 struct rte_flow_error *error) 1993 { 1994 const struct rte_flow_item_vlan *spec = item->spec; 1995 const struct rte_flow_item_vlan *mask = item->mask; 1996 const struct rte_flow_item_vlan nic_mask = { 1997 .tci = RTE_BE16(UINT16_MAX), 1998 .inner_type = RTE_BE16(UINT16_MAX), 1999 }; 2000 uint16_t vlan_tag = 0; 2001 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2002 int ret; 2003 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2004 MLX5_FLOW_LAYER_INNER_L4) : 2005 (MLX5_FLOW_LAYER_OUTER_L3 | 2006 MLX5_FLOW_LAYER_OUTER_L4); 2007 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2008 MLX5_FLOW_LAYER_OUTER_VLAN; 2009 2010 if (item_flags & vlanm) 2011 return rte_flow_error_set(error, EINVAL, 2012 RTE_FLOW_ERROR_TYPE_ITEM, item, 2013 "multiple VLAN layers not supported"); 2014 else if ((item_flags & l34m) != 0) 2015 return rte_flow_error_set(error, EINVAL, 2016 RTE_FLOW_ERROR_TYPE_ITEM, item, 2017 "VLAN cannot follow L3/L4 layer"); 2018 if (!mask) 2019 mask = &rte_flow_item_vlan_mask; 2020 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2021 (const uint8_t *)&nic_mask, 2022 sizeof(struct rte_flow_item_vlan), 2023 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2024 if (ret) 2025 return ret; 2026 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2027 struct mlx5_priv *priv = dev->data->dev_private; 2028 2029 if (priv->vmwa_context) { 2030 /* 2031 * Non-NULL context means we have a virtual machine 2032 * and SR-IOV enabled, we have to create VLAN interface 2033 * to make hypervisor to setup E-Switch vport 2034 * context correctly. We avoid creating the multiple 2035 * VLAN interfaces, so we cannot support VLAN tag mask. 2036 */ 2037 return rte_flow_error_set(error, EINVAL, 2038 RTE_FLOW_ERROR_TYPE_ITEM, 2039 item, 2040 "VLAN tag mask is not" 2041 " supported in virtual" 2042 " environment"); 2043 } 2044 } 2045 if (spec) { 2046 vlan_tag = spec->tci; 2047 vlan_tag &= mask->tci; 2048 } 2049 /* 2050 * From verbs perspective an empty VLAN is equivalent 2051 * to a packet without VLAN layer. 2052 */ 2053 if (!vlan_tag) 2054 return rte_flow_error_set(error, EINVAL, 2055 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2056 item->spec, 2057 "VLAN cannot be empty"); 2058 return 0; 2059 } 2060 2061 /** 2062 * Validate IPV4 item. 2063 * 2064 * @param[in] item 2065 * Item specification. 2066 * @param[in] item_flags 2067 * Bit-fields that holds the items detected until now. 2068 * @param[in] last_item 2069 * Previous validated item in the pattern items. 2070 * @param[in] ether_type 2071 * Type in the ethernet layer header (including dot1q). 2072 * @param[in] acc_mask 2073 * Acceptable mask, if NULL default internal default mask 2074 * will be used to check whether item fields are supported. 2075 * @param[in] range_accepted 2076 * True if range of values is accepted for specific fields, false otherwise. 2077 * @param[out] error 2078 * Pointer to error structure. 2079 * 2080 * @return 2081 * 0 on success, a negative errno value otherwise and rte_errno is set. 2082 */ 2083 int 2084 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2085 uint64_t item_flags, 2086 uint64_t last_item, 2087 uint16_t ether_type, 2088 const struct rte_flow_item_ipv4 *acc_mask, 2089 bool range_accepted, 2090 struct rte_flow_error *error) 2091 { 2092 const struct rte_flow_item_ipv4 *mask = item->mask; 2093 const struct rte_flow_item_ipv4 *spec = item->spec; 2094 const struct rte_flow_item_ipv4 nic_mask = { 2095 .hdr = { 2096 .src_addr = RTE_BE32(0xffffffff), 2097 .dst_addr = RTE_BE32(0xffffffff), 2098 .type_of_service = 0xff, 2099 .next_proto_id = 0xff, 2100 }, 2101 }; 2102 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2103 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2104 MLX5_FLOW_LAYER_OUTER_L3; 2105 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2106 MLX5_FLOW_LAYER_OUTER_L4; 2107 int ret; 2108 uint8_t next_proto = 0xFF; 2109 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2110 MLX5_FLOW_LAYER_OUTER_VLAN | 2111 MLX5_FLOW_LAYER_INNER_VLAN); 2112 2113 if ((last_item & l2_vlan) && ether_type && 2114 ether_type != RTE_ETHER_TYPE_IPV4) 2115 return rte_flow_error_set(error, EINVAL, 2116 RTE_FLOW_ERROR_TYPE_ITEM, item, 2117 "IPv4 cannot follow L2/VLAN layer " 2118 "which ether type is not IPv4"); 2119 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2120 if (mask && spec) 2121 next_proto = mask->hdr.next_proto_id & 2122 spec->hdr.next_proto_id; 2123 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2124 return rte_flow_error_set(error, EINVAL, 2125 RTE_FLOW_ERROR_TYPE_ITEM, 2126 item, 2127 "multiple tunnel " 2128 "not supported"); 2129 } 2130 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2131 return rte_flow_error_set(error, EINVAL, 2132 RTE_FLOW_ERROR_TYPE_ITEM, item, 2133 "wrong tunnel type - IPv6 specified " 2134 "but IPv4 item provided"); 2135 if (item_flags & l3m) 2136 return rte_flow_error_set(error, ENOTSUP, 2137 RTE_FLOW_ERROR_TYPE_ITEM, item, 2138 "multiple L3 layers not supported"); 2139 else if (item_flags & l4m) 2140 return rte_flow_error_set(error, EINVAL, 2141 RTE_FLOW_ERROR_TYPE_ITEM, item, 2142 "L3 cannot follow an L4 layer."); 2143 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2144 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2145 return rte_flow_error_set(error, EINVAL, 2146 RTE_FLOW_ERROR_TYPE_ITEM, item, 2147 "L3 cannot follow an NVGRE layer."); 2148 if (!mask) 2149 mask = &rte_flow_item_ipv4_mask; 2150 else if (mask->hdr.next_proto_id != 0 && 2151 mask->hdr.next_proto_id != 0xff) 2152 return rte_flow_error_set(error, EINVAL, 2153 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2154 "partial mask is not supported" 2155 " for protocol"); 2156 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2157 acc_mask ? (const uint8_t *)acc_mask 2158 : (const uint8_t *)&nic_mask, 2159 sizeof(struct rte_flow_item_ipv4), 2160 range_accepted, error); 2161 if (ret < 0) 2162 return ret; 2163 return 0; 2164 } 2165 2166 /** 2167 * Validate IPV6 item. 2168 * 2169 * @param[in] item 2170 * Item specification. 2171 * @param[in] item_flags 2172 * Bit-fields that holds the items detected until now. 2173 * @param[in] last_item 2174 * Previous validated item in the pattern items. 2175 * @param[in] ether_type 2176 * Type in the ethernet layer header (including dot1q). 2177 * @param[in] acc_mask 2178 * Acceptable mask, if NULL default internal default mask 2179 * will be used to check whether item fields are supported. 2180 * @param[out] error 2181 * Pointer to error structure. 2182 * 2183 * @return 2184 * 0 on success, a negative errno value otherwise and rte_errno is set. 2185 */ 2186 int 2187 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2188 uint64_t item_flags, 2189 uint64_t last_item, 2190 uint16_t ether_type, 2191 const struct rte_flow_item_ipv6 *acc_mask, 2192 struct rte_flow_error *error) 2193 { 2194 const struct rte_flow_item_ipv6 *mask = item->mask; 2195 const struct rte_flow_item_ipv6 *spec = item->spec; 2196 const struct rte_flow_item_ipv6 nic_mask = { 2197 .hdr = { 2198 .src_addr = 2199 "\xff\xff\xff\xff\xff\xff\xff\xff" 2200 "\xff\xff\xff\xff\xff\xff\xff\xff", 2201 .dst_addr = 2202 "\xff\xff\xff\xff\xff\xff\xff\xff" 2203 "\xff\xff\xff\xff\xff\xff\xff\xff", 2204 .vtc_flow = RTE_BE32(0xffffffff), 2205 .proto = 0xff, 2206 }, 2207 }; 2208 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2209 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2210 MLX5_FLOW_LAYER_OUTER_L3; 2211 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2212 MLX5_FLOW_LAYER_OUTER_L4; 2213 int ret; 2214 uint8_t next_proto = 0xFF; 2215 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2216 MLX5_FLOW_LAYER_OUTER_VLAN | 2217 MLX5_FLOW_LAYER_INNER_VLAN); 2218 2219 if ((last_item & l2_vlan) && ether_type && 2220 ether_type != RTE_ETHER_TYPE_IPV6) 2221 return rte_flow_error_set(error, EINVAL, 2222 RTE_FLOW_ERROR_TYPE_ITEM, item, 2223 "IPv6 cannot follow L2/VLAN layer " 2224 "which ether type is not IPv6"); 2225 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2226 next_proto = spec->hdr.proto; 2227 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 2228 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2229 return rte_flow_error_set(error, EINVAL, 2230 RTE_FLOW_ERROR_TYPE_ITEM, 2231 item, 2232 "multiple tunnel " 2233 "not supported"); 2234 } 2235 if (next_proto == IPPROTO_HOPOPTS || 2236 next_proto == IPPROTO_ROUTING || 2237 next_proto == IPPROTO_FRAGMENT || 2238 next_proto == IPPROTO_ESP || 2239 next_proto == IPPROTO_AH || 2240 next_proto == IPPROTO_DSTOPTS) 2241 return rte_flow_error_set(error, EINVAL, 2242 RTE_FLOW_ERROR_TYPE_ITEM, item, 2243 "IPv6 proto (next header) should " 2244 "not be set as extension header"); 2245 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2246 return rte_flow_error_set(error, EINVAL, 2247 RTE_FLOW_ERROR_TYPE_ITEM, item, 2248 "wrong tunnel type - IPv4 specified " 2249 "but IPv6 item provided"); 2250 if (item_flags & l3m) 2251 return rte_flow_error_set(error, ENOTSUP, 2252 RTE_FLOW_ERROR_TYPE_ITEM, item, 2253 "multiple L3 layers not supported"); 2254 else if (item_flags & l4m) 2255 return rte_flow_error_set(error, EINVAL, 2256 RTE_FLOW_ERROR_TYPE_ITEM, item, 2257 "L3 cannot follow an L4 layer."); 2258 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2259 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2260 return rte_flow_error_set(error, EINVAL, 2261 RTE_FLOW_ERROR_TYPE_ITEM, item, 2262 "L3 cannot follow an NVGRE layer."); 2263 if (!mask) 2264 mask = &rte_flow_item_ipv6_mask; 2265 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2266 acc_mask ? (const uint8_t *)acc_mask 2267 : (const uint8_t *)&nic_mask, 2268 sizeof(struct rte_flow_item_ipv6), 2269 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2270 if (ret < 0) 2271 return ret; 2272 return 0; 2273 } 2274 2275 /** 2276 * Validate UDP item. 2277 * 2278 * @param[in] item 2279 * Item specification. 2280 * @param[in] item_flags 2281 * Bit-fields that holds the items detected until now. 2282 * @param[in] target_protocol 2283 * The next protocol in the previous item. 2284 * @param[in] flow_mask 2285 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2286 * @param[out] error 2287 * Pointer to error structure. 2288 * 2289 * @return 2290 * 0 on success, a negative errno value otherwise and rte_errno is set. 2291 */ 2292 int 2293 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2294 uint64_t item_flags, 2295 uint8_t target_protocol, 2296 struct rte_flow_error *error) 2297 { 2298 const struct rte_flow_item_udp *mask = item->mask; 2299 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2300 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2301 MLX5_FLOW_LAYER_OUTER_L3; 2302 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2303 MLX5_FLOW_LAYER_OUTER_L4; 2304 int ret; 2305 2306 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2307 return rte_flow_error_set(error, EINVAL, 2308 RTE_FLOW_ERROR_TYPE_ITEM, item, 2309 "protocol filtering not compatible" 2310 " with UDP layer"); 2311 if (!(item_flags & l3m)) 2312 return rte_flow_error_set(error, EINVAL, 2313 RTE_FLOW_ERROR_TYPE_ITEM, item, 2314 "L3 is mandatory to filter on L4"); 2315 if (item_flags & l4m) 2316 return rte_flow_error_set(error, EINVAL, 2317 RTE_FLOW_ERROR_TYPE_ITEM, item, 2318 "multiple L4 layers not supported"); 2319 if (!mask) 2320 mask = &rte_flow_item_udp_mask; 2321 ret = mlx5_flow_item_acceptable 2322 (item, (const uint8_t *)mask, 2323 (const uint8_t *)&rte_flow_item_udp_mask, 2324 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2325 error); 2326 if (ret < 0) 2327 return ret; 2328 return 0; 2329 } 2330 2331 /** 2332 * Validate TCP item. 2333 * 2334 * @param[in] item 2335 * Item specification. 2336 * @param[in] item_flags 2337 * Bit-fields that holds the items detected until now. 2338 * @param[in] target_protocol 2339 * The next protocol in the previous item. 2340 * @param[out] error 2341 * Pointer to error structure. 2342 * 2343 * @return 2344 * 0 on success, a negative errno value otherwise and rte_errno is set. 2345 */ 2346 int 2347 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2348 uint64_t item_flags, 2349 uint8_t target_protocol, 2350 const struct rte_flow_item_tcp *flow_mask, 2351 struct rte_flow_error *error) 2352 { 2353 const struct rte_flow_item_tcp *mask = item->mask; 2354 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2355 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2356 MLX5_FLOW_LAYER_OUTER_L3; 2357 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2358 MLX5_FLOW_LAYER_OUTER_L4; 2359 int ret; 2360 2361 MLX5_ASSERT(flow_mask); 2362 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2363 return rte_flow_error_set(error, EINVAL, 2364 RTE_FLOW_ERROR_TYPE_ITEM, item, 2365 "protocol filtering not compatible" 2366 " with TCP layer"); 2367 if (!(item_flags & l3m)) 2368 return rte_flow_error_set(error, EINVAL, 2369 RTE_FLOW_ERROR_TYPE_ITEM, item, 2370 "L3 is mandatory to filter on L4"); 2371 if (item_flags & l4m) 2372 return rte_flow_error_set(error, EINVAL, 2373 RTE_FLOW_ERROR_TYPE_ITEM, item, 2374 "multiple L4 layers not supported"); 2375 if (!mask) 2376 mask = &rte_flow_item_tcp_mask; 2377 ret = mlx5_flow_item_acceptable 2378 (item, (const uint8_t *)mask, 2379 (const uint8_t *)flow_mask, 2380 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2381 error); 2382 if (ret < 0) 2383 return ret; 2384 return 0; 2385 } 2386 2387 /** 2388 * Validate VXLAN item. 2389 * 2390 * @param[in] item 2391 * Item specification. 2392 * @param[in] item_flags 2393 * Bit-fields that holds the items detected until now. 2394 * @param[in] target_protocol 2395 * The next protocol in the previous item. 2396 * @param[out] error 2397 * Pointer to error structure. 2398 * 2399 * @return 2400 * 0 on success, a negative errno value otherwise and rte_errno is set. 2401 */ 2402 int 2403 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2404 uint64_t item_flags, 2405 struct rte_flow_error *error) 2406 { 2407 const struct rte_flow_item_vxlan *spec = item->spec; 2408 const struct rte_flow_item_vxlan *mask = item->mask; 2409 int ret; 2410 union vni { 2411 uint32_t vlan_id; 2412 uint8_t vni[4]; 2413 } id = { .vlan_id = 0, }; 2414 2415 2416 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2417 return rte_flow_error_set(error, ENOTSUP, 2418 RTE_FLOW_ERROR_TYPE_ITEM, item, 2419 "multiple tunnel layers not" 2420 " supported"); 2421 /* 2422 * Verify only UDPv4 is present as defined in 2423 * https://tools.ietf.org/html/rfc7348 2424 */ 2425 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2426 return rte_flow_error_set(error, EINVAL, 2427 RTE_FLOW_ERROR_TYPE_ITEM, item, 2428 "no outer UDP layer found"); 2429 if (!mask) 2430 mask = &rte_flow_item_vxlan_mask; 2431 ret = mlx5_flow_item_acceptable 2432 (item, (const uint8_t *)mask, 2433 (const uint8_t *)&rte_flow_item_vxlan_mask, 2434 sizeof(struct rte_flow_item_vxlan), 2435 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2436 if (ret < 0) 2437 return ret; 2438 if (spec) { 2439 memcpy(&id.vni[1], spec->vni, 3); 2440 memcpy(&id.vni[1], mask->vni, 3); 2441 } 2442 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2443 return rte_flow_error_set(error, ENOTSUP, 2444 RTE_FLOW_ERROR_TYPE_ITEM, item, 2445 "VXLAN tunnel must be fully defined"); 2446 return 0; 2447 } 2448 2449 /** 2450 * Validate VXLAN_GPE item. 2451 * 2452 * @param[in] item 2453 * Item specification. 2454 * @param[in] item_flags 2455 * Bit-fields that holds the items detected until now. 2456 * @param[in] priv 2457 * Pointer to the private data structure. 2458 * @param[in] target_protocol 2459 * The next protocol in the previous item. 2460 * @param[out] error 2461 * Pointer to error structure. 2462 * 2463 * @return 2464 * 0 on success, a negative errno value otherwise and rte_errno is set. 2465 */ 2466 int 2467 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2468 uint64_t item_flags, 2469 struct rte_eth_dev *dev, 2470 struct rte_flow_error *error) 2471 { 2472 struct mlx5_priv *priv = dev->data->dev_private; 2473 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2474 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2475 int ret; 2476 union vni { 2477 uint32_t vlan_id; 2478 uint8_t vni[4]; 2479 } id = { .vlan_id = 0, }; 2480 2481 if (!priv->config.l3_vxlan_en) 2482 return rte_flow_error_set(error, ENOTSUP, 2483 RTE_FLOW_ERROR_TYPE_ITEM, item, 2484 "L3 VXLAN is not enabled by device" 2485 " parameter and/or not configured in" 2486 " firmware"); 2487 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2488 return rte_flow_error_set(error, ENOTSUP, 2489 RTE_FLOW_ERROR_TYPE_ITEM, item, 2490 "multiple tunnel layers not" 2491 " supported"); 2492 /* 2493 * Verify only UDPv4 is present as defined in 2494 * https://tools.ietf.org/html/rfc7348 2495 */ 2496 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2497 return rte_flow_error_set(error, EINVAL, 2498 RTE_FLOW_ERROR_TYPE_ITEM, item, 2499 "no outer UDP layer found"); 2500 if (!mask) 2501 mask = &rte_flow_item_vxlan_gpe_mask; 2502 ret = mlx5_flow_item_acceptable 2503 (item, (const uint8_t *)mask, 2504 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2505 sizeof(struct rte_flow_item_vxlan_gpe), 2506 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2507 if (ret < 0) 2508 return ret; 2509 if (spec) { 2510 if (spec->protocol) 2511 return rte_flow_error_set(error, ENOTSUP, 2512 RTE_FLOW_ERROR_TYPE_ITEM, 2513 item, 2514 "VxLAN-GPE protocol" 2515 " not supported"); 2516 memcpy(&id.vni[1], spec->vni, 3); 2517 memcpy(&id.vni[1], mask->vni, 3); 2518 } 2519 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2520 return rte_flow_error_set(error, ENOTSUP, 2521 RTE_FLOW_ERROR_TYPE_ITEM, item, 2522 "VXLAN-GPE tunnel must be fully" 2523 " defined"); 2524 return 0; 2525 } 2526 /** 2527 * Validate GRE Key item. 2528 * 2529 * @param[in] item 2530 * Item specification. 2531 * @param[in] item_flags 2532 * Bit flags to mark detected items. 2533 * @param[in] gre_item 2534 * Pointer to gre_item 2535 * @param[out] error 2536 * Pointer to error structure. 2537 * 2538 * @return 2539 * 0 on success, a negative errno value otherwise and rte_errno is set. 2540 */ 2541 int 2542 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2543 uint64_t item_flags, 2544 const struct rte_flow_item *gre_item, 2545 struct rte_flow_error *error) 2546 { 2547 const rte_be32_t *mask = item->mask; 2548 int ret = 0; 2549 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2550 const struct rte_flow_item_gre *gre_spec; 2551 const struct rte_flow_item_gre *gre_mask; 2552 2553 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2554 return rte_flow_error_set(error, ENOTSUP, 2555 RTE_FLOW_ERROR_TYPE_ITEM, item, 2556 "Multiple GRE key not support"); 2557 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2558 return rte_flow_error_set(error, ENOTSUP, 2559 RTE_FLOW_ERROR_TYPE_ITEM, item, 2560 "No preceding GRE header"); 2561 if (item_flags & MLX5_FLOW_LAYER_INNER) 2562 return rte_flow_error_set(error, ENOTSUP, 2563 RTE_FLOW_ERROR_TYPE_ITEM, item, 2564 "GRE key following a wrong item"); 2565 gre_mask = gre_item->mask; 2566 if (!gre_mask) 2567 gre_mask = &rte_flow_item_gre_mask; 2568 gre_spec = gre_item->spec; 2569 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2570 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2571 return rte_flow_error_set(error, EINVAL, 2572 RTE_FLOW_ERROR_TYPE_ITEM, item, 2573 "Key bit must be on"); 2574 2575 if (!mask) 2576 mask = &gre_key_default_mask; 2577 ret = mlx5_flow_item_acceptable 2578 (item, (const uint8_t *)mask, 2579 (const uint8_t *)&gre_key_default_mask, 2580 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2581 return ret; 2582 } 2583 2584 /** 2585 * Validate GRE item. 2586 * 2587 * @param[in] item 2588 * Item specification. 2589 * @param[in] item_flags 2590 * Bit flags to mark detected items. 2591 * @param[in] target_protocol 2592 * The next protocol in the previous item. 2593 * @param[out] error 2594 * Pointer to error structure. 2595 * 2596 * @return 2597 * 0 on success, a negative errno value otherwise and rte_errno is set. 2598 */ 2599 int 2600 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2601 uint64_t item_flags, 2602 uint8_t target_protocol, 2603 struct rte_flow_error *error) 2604 { 2605 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2606 const struct rte_flow_item_gre *mask = item->mask; 2607 int ret; 2608 const struct rte_flow_item_gre nic_mask = { 2609 .c_rsvd0_ver = RTE_BE16(0xB000), 2610 .protocol = RTE_BE16(UINT16_MAX), 2611 }; 2612 2613 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2614 return rte_flow_error_set(error, EINVAL, 2615 RTE_FLOW_ERROR_TYPE_ITEM, item, 2616 "protocol filtering not compatible" 2617 " with this GRE layer"); 2618 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2619 return rte_flow_error_set(error, ENOTSUP, 2620 RTE_FLOW_ERROR_TYPE_ITEM, item, 2621 "multiple tunnel layers not" 2622 " supported"); 2623 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2624 return rte_flow_error_set(error, ENOTSUP, 2625 RTE_FLOW_ERROR_TYPE_ITEM, item, 2626 "L3 Layer is missing"); 2627 if (!mask) 2628 mask = &rte_flow_item_gre_mask; 2629 ret = mlx5_flow_item_acceptable 2630 (item, (const uint8_t *)mask, 2631 (const uint8_t *)&nic_mask, 2632 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2633 error); 2634 if (ret < 0) 2635 return ret; 2636 #ifndef HAVE_MLX5DV_DR 2637 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2638 if (spec && (spec->protocol & mask->protocol)) 2639 return rte_flow_error_set(error, ENOTSUP, 2640 RTE_FLOW_ERROR_TYPE_ITEM, item, 2641 "without MPLS support the" 2642 " specification cannot be used for" 2643 " filtering"); 2644 #endif 2645 #endif 2646 return 0; 2647 } 2648 2649 /** 2650 * Validate Geneve item. 2651 * 2652 * @param[in] item 2653 * Item specification. 2654 * @param[in] itemFlags 2655 * Bit-fields that holds the items detected until now. 2656 * @param[in] enPriv 2657 * Pointer to the private data structure. 2658 * @param[out] error 2659 * Pointer to error structure. 2660 * 2661 * @return 2662 * 0 on success, a negative errno value otherwise and rte_errno is set. 2663 */ 2664 2665 int 2666 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2667 uint64_t item_flags, 2668 struct rte_eth_dev *dev, 2669 struct rte_flow_error *error) 2670 { 2671 struct mlx5_priv *priv = dev->data->dev_private; 2672 const struct rte_flow_item_geneve *spec = item->spec; 2673 const struct rte_flow_item_geneve *mask = item->mask; 2674 int ret; 2675 uint16_t gbhdr; 2676 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2677 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2678 const struct rte_flow_item_geneve nic_mask = { 2679 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2680 .vni = "\xff\xff\xff", 2681 .protocol = RTE_BE16(UINT16_MAX), 2682 }; 2683 2684 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2685 return rte_flow_error_set(error, ENOTSUP, 2686 RTE_FLOW_ERROR_TYPE_ITEM, item, 2687 "L3 Geneve is not enabled by device" 2688 " parameter and/or not configured in" 2689 " firmware"); 2690 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2691 return rte_flow_error_set(error, ENOTSUP, 2692 RTE_FLOW_ERROR_TYPE_ITEM, item, 2693 "multiple tunnel layers not" 2694 " supported"); 2695 /* 2696 * Verify only UDPv4 is present as defined in 2697 * https://tools.ietf.org/html/rfc7348 2698 */ 2699 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2700 return rte_flow_error_set(error, EINVAL, 2701 RTE_FLOW_ERROR_TYPE_ITEM, item, 2702 "no outer UDP layer found"); 2703 if (!mask) 2704 mask = &rte_flow_item_geneve_mask; 2705 ret = mlx5_flow_item_acceptable 2706 (item, (const uint8_t *)mask, 2707 (const uint8_t *)&nic_mask, 2708 sizeof(struct rte_flow_item_geneve), 2709 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2710 if (ret) 2711 return ret; 2712 if (spec) { 2713 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2714 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2715 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2716 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2717 return rte_flow_error_set(error, ENOTSUP, 2718 RTE_FLOW_ERROR_TYPE_ITEM, 2719 item, 2720 "Geneve protocol unsupported" 2721 " fields are being used"); 2722 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2723 return rte_flow_error_set 2724 (error, ENOTSUP, 2725 RTE_FLOW_ERROR_TYPE_ITEM, 2726 item, 2727 "Unsupported Geneve options length"); 2728 } 2729 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2730 return rte_flow_error_set 2731 (error, ENOTSUP, 2732 RTE_FLOW_ERROR_TYPE_ITEM, item, 2733 "Geneve tunnel must be fully defined"); 2734 return 0; 2735 } 2736 2737 /** 2738 * Validate MPLS item. 2739 * 2740 * @param[in] dev 2741 * Pointer to the rte_eth_dev structure. 2742 * @param[in] item 2743 * Item specification. 2744 * @param[in] item_flags 2745 * Bit-fields that holds the items detected until now. 2746 * @param[in] prev_layer 2747 * The protocol layer indicated in previous item. 2748 * @param[out] error 2749 * Pointer to error structure. 2750 * 2751 * @return 2752 * 0 on success, a negative errno value otherwise and rte_errno is set. 2753 */ 2754 int 2755 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2756 const struct rte_flow_item *item __rte_unused, 2757 uint64_t item_flags __rte_unused, 2758 uint64_t prev_layer __rte_unused, 2759 struct rte_flow_error *error) 2760 { 2761 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2762 const struct rte_flow_item_mpls *mask = item->mask; 2763 struct mlx5_priv *priv = dev->data->dev_private; 2764 int ret; 2765 2766 if (!priv->config.mpls_en) 2767 return rte_flow_error_set(error, ENOTSUP, 2768 RTE_FLOW_ERROR_TYPE_ITEM, item, 2769 "MPLS not supported or" 2770 " disabled in firmware" 2771 " configuration."); 2772 /* MPLS over IP, UDP, GRE is allowed */ 2773 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2774 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2775 MLX5_FLOW_LAYER_GRE))) 2776 return rte_flow_error_set(error, EINVAL, 2777 RTE_FLOW_ERROR_TYPE_ITEM, item, 2778 "protocol filtering not compatible" 2779 " with MPLS layer"); 2780 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2781 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2782 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2783 return rte_flow_error_set(error, ENOTSUP, 2784 RTE_FLOW_ERROR_TYPE_ITEM, item, 2785 "multiple tunnel layers not" 2786 " supported"); 2787 if (!mask) 2788 mask = &rte_flow_item_mpls_mask; 2789 ret = mlx5_flow_item_acceptable 2790 (item, (const uint8_t *)mask, 2791 (const uint8_t *)&rte_flow_item_mpls_mask, 2792 sizeof(struct rte_flow_item_mpls), 2793 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2794 if (ret < 0) 2795 return ret; 2796 return 0; 2797 #else 2798 return rte_flow_error_set(error, ENOTSUP, 2799 RTE_FLOW_ERROR_TYPE_ITEM, item, 2800 "MPLS is not supported by Verbs, please" 2801 " update."); 2802 #endif 2803 } 2804 2805 /** 2806 * Validate NVGRE item. 2807 * 2808 * @param[in] item 2809 * Item specification. 2810 * @param[in] item_flags 2811 * Bit flags to mark detected items. 2812 * @param[in] target_protocol 2813 * The next protocol in the previous item. 2814 * @param[out] error 2815 * Pointer to error structure. 2816 * 2817 * @return 2818 * 0 on success, a negative errno value otherwise and rte_errno is set. 2819 */ 2820 int 2821 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2822 uint64_t item_flags, 2823 uint8_t target_protocol, 2824 struct rte_flow_error *error) 2825 { 2826 const struct rte_flow_item_nvgre *mask = item->mask; 2827 int ret; 2828 2829 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2830 return rte_flow_error_set(error, EINVAL, 2831 RTE_FLOW_ERROR_TYPE_ITEM, item, 2832 "protocol filtering not compatible" 2833 " with this GRE layer"); 2834 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2835 return rte_flow_error_set(error, ENOTSUP, 2836 RTE_FLOW_ERROR_TYPE_ITEM, item, 2837 "multiple tunnel layers not" 2838 " supported"); 2839 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2840 return rte_flow_error_set(error, ENOTSUP, 2841 RTE_FLOW_ERROR_TYPE_ITEM, item, 2842 "L3 Layer is missing"); 2843 if (!mask) 2844 mask = &rte_flow_item_nvgre_mask; 2845 ret = mlx5_flow_item_acceptable 2846 (item, (const uint8_t *)mask, 2847 (const uint8_t *)&rte_flow_item_nvgre_mask, 2848 sizeof(struct rte_flow_item_nvgre), 2849 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2850 if (ret < 0) 2851 return ret; 2852 return 0; 2853 } 2854 2855 /** 2856 * Validate eCPRI item. 2857 * 2858 * @param[in] item 2859 * Item specification. 2860 * @param[in] item_flags 2861 * Bit-fields that holds the items detected until now. 2862 * @param[in] last_item 2863 * Previous validated item in the pattern items. 2864 * @param[in] ether_type 2865 * Type in the ethernet layer header (including dot1q). 2866 * @param[in] acc_mask 2867 * Acceptable mask, if NULL default internal default mask 2868 * will be used to check whether item fields are supported. 2869 * @param[out] error 2870 * Pointer to error structure. 2871 * 2872 * @return 2873 * 0 on success, a negative errno value otherwise and rte_errno is set. 2874 */ 2875 int 2876 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 2877 uint64_t item_flags, 2878 uint64_t last_item, 2879 uint16_t ether_type, 2880 const struct rte_flow_item_ecpri *acc_mask, 2881 struct rte_flow_error *error) 2882 { 2883 const struct rte_flow_item_ecpri *mask = item->mask; 2884 const struct rte_flow_item_ecpri nic_mask = { 2885 .hdr = { 2886 .common = { 2887 .u32 = 2888 RTE_BE32(((const struct rte_ecpri_common_hdr) { 2889 .type = 0xFF, 2890 }).u32), 2891 }, 2892 .dummy[0] = 0xFFFFFFFF, 2893 }, 2894 }; 2895 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 2896 MLX5_FLOW_LAYER_OUTER_VLAN); 2897 struct rte_flow_item_ecpri mask_lo; 2898 2899 if ((last_item & outer_l2_vlan) && ether_type && 2900 ether_type != RTE_ETHER_TYPE_ECPRI) 2901 return rte_flow_error_set(error, EINVAL, 2902 RTE_FLOW_ERROR_TYPE_ITEM, item, 2903 "eCPRI cannot follow L2/VLAN layer " 2904 "which ether type is not 0xAEFE."); 2905 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2906 return rte_flow_error_set(error, EINVAL, 2907 RTE_FLOW_ERROR_TYPE_ITEM, item, 2908 "eCPRI with tunnel is not supported " 2909 "right now."); 2910 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 2911 return rte_flow_error_set(error, ENOTSUP, 2912 RTE_FLOW_ERROR_TYPE_ITEM, item, 2913 "multiple L3 layers not supported"); 2914 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 2915 return rte_flow_error_set(error, EINVAL, 2916 RTE_FLOW_ERROR_TYPE_ITEM, item, 2917 "eCPRI cannot follow a TCP layer."); 2918 /* In specification, eCPRI could be over UDP layer. */ 2919 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 2920 return rte_flow_error_set(error, EINVAL, 2921 RTE_FLOW_ERROR_TYPE_ITEM, item, 2922 "eCPRI over UDP layer is not yet " 2923 "supported right now."); 2924 /* Mask for type field in common header could be zero. */ 2925 if (!mask) 2926 mask = &rte_flow_item_ecpri_mask; 2927 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 2928 /* Input mask is in big-endian format. */ 2929 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 2930 return rte_flow_error_set(error, EINVAL, 2931 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2932 "partial mask is not supported " 2933 "for protocol"); 2934 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 2935 return rte_flow_error_set(error, EINVAL, 2936 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2937 "message header mask must be after " 2938 "a type mask"); 2939 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2940 acc_mask ? (const uint8_t *)acc_mask 2941 : (const uint8_t *)&nic_mask, 2942 sizeof(struct rte_flow_item_ecpri), 2943 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2944 } 2945 2946 /** 2947 * Release resource related QUEUE/RSS action split. 2948 * 2949 * @param dev 2950 * Pointer to Ethernet device. 2951 * @param flow 2952 * Flow to release id's from. 2953 */ 2954 static void 2955 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2956 struct rte_flow *flow) 2957 { 2958 struct mlx5_priv *priv = dev->data->dev_private; 2959 uint32_t handle_idx; 2960 struct mlx5_flow_handle *dev_handle; 2961 2962 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2963 handle_idx, dev_handle, next) 2964 if (dev_handle->split_flow_id) 2965 mlx5_ipool_free(priv->sh->ipool 2966 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 2967 dev_handle->split_flow_id); 2968 } 2969 2970 static int 2971 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2972 const struct rte_flow_attr *attr __rte_unused, 2973 const struct rte_flow_item items[] __rte_unused, 2974 const struct rte_flow_action actions[] __rte_unused, 2975 bool external __rte_unused, 2976 int hairpin __rte_unused, 2977 struct rte_flow_error *error) 2978 { 2979 return rte_flow_error_set(error, ENOTSUP, 2980 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2981 } 2982 2983 static struct mlx5_flow * 2984 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2985 const struct rte_flow_attr *attr __rte_unused, 2986 const struct rte_flow_item items[] __rte_unused, 2987 const struct rte_flow_action actions[] __rte_unused, 2988 struct rte_flow_error *error) 2989 { 2990 rte_flow_error_set(error, ENOTSUP, 2991 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2992 return NULL; 2993 } 2994 2995 static int 2996 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2997 struct mlx5_flow *dev_flow __rte_unused, 2998 const struct rte_flow_attr *attr __rte_unused, 2999 const struct rte_flow_item items[] __rte_unused, 3000 const struct rte_flow_action actions[] __rte_unused, 3001 struct rte_flow_error *error) 3002 { 3003 return rte_flow_error_set(error, ENOTSUP, 3004 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3005 } 3006 3007 static int 3008 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3009 struct rte_flow *flow __rte_unused, 3010 struct rte_flow_error *error) 3011 { 3012 return rte_flow_error_set(error, ENOTSUP, 3013 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3014 } 3015 3016 static void 3017 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3018 struct rte_flow *flow __rte_unused) 3019 { 3020 } 3021 3022 static void 3023 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3024 struct rte_flow *flow __rte_unused) 3025 { 3026 } 3027 3028 static int 3029 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3030 struct rte_flow *flow __rte_unused, 3031 const struct rte_flow_action *actions __rte_unused, 3032 void *data __rte_unused, 3033 struct rte_flow_error *error) 3034 { 3035 return rte_flow_error_set(error, ENOTSUP, 3036 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3037 } 3038 3039 static int 3040 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3041 uint32_t domains __rte_unused, 3042 uint32_t flags __rte_unused) 3043 { 3044 return 0; 3045 } 3046 3047 /* Void driver to protect from null pointer reference. */ 3048 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3049 .validate = flow_null_validate, 3050 .prepare = flow_null_prepare, 3051 .translate = flow_null_translate, 3052 .apply = flow_null_apply, 3053 .remove = flow_null_remove, 3054 .destroy = flow_null_destroy, 3055 .query = flow_null_query, 3056 .sync_domain = flow_null_sync_domain, 3057 }; 3058 3059 /** 3060 * Select flow driver type according to flow attributes and device 3061 * configuration. 3062 * 3063 * @param[in] dev 3064 * Pointer to the dev structure. 3065 * @param[in] attr 3066 * Pointer to the flow attributes. 3067 * 3068 * @return 3069 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3070 */ 3071 static enum mlx5_flow_drv_type 3072 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3073 { 3074 struct mlx5_priv *priv = dev->data->dev_private; 3075 /* The OS can determine first a specific flow type (DV, VERBS) */ 3076 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3077 3078 if (type != MLX5_FLOW_TYPE_MAX) 3079 return type; 3080 /* If no OS specific type - continue with DV/VERBS selection */ 3081 if (attr->transfer && priv->config.dv_esw_en) 3082 type = MLX5_FLOW_TYPE_DV; 3083 if (!attr->transfer) 3084 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3085 MLX5_FLOW_TYPE_VERBS; 3086 return type; 3087 } 3088 3089 #define flow_get_drv_ops(type) flow_drv_ops[type] 3090 3091 /** 3092 * Flow driver validation API. This abstracts calling driver specific functions. 3093 * The type of flow driver is determined according to flow attributes. 3094 * 3095 * @param[in] dev 3096 * Pointer to the dev structure. 3097 * @param[in] attr 3098 * Pointer to the flow attributes. 3099 * @param[in] items 3100 * Pointer to the list of items. 3101 * @param[in] actions 3102 * Pointer to the list of actions. 3103 * @param[in] external 3104 * This flow rule is created by request external to PMD. 3105 * @param[in] hairpin 3106 * Number of hairpin TX actions, 0 means classic flow. 3107 * @param[out] error 3108 * Pointer to the error structure. 3109 * 3110 * @return 3111 * 0 on success, a negative errno value otherwise and rte_errno is set. 3112 */ 3113 static inline int 3114 flow_drv_validate(struct rte_eth_dev *dev, 3115 const struct rte_flow_attr *attr, 3116 const struct rte_flow_item items[], 3117 const struct rte_flow_action actions[], 3118 bool external, int hairpin, struct rte_flow_error *error) 3119 { 3120 const struct mlx5_flow_driver_ops *fops; 3121 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3122 3123 fops = flow_get_drv_ops(type); 3124 return fops->validate(dev, attr, items, actions, external, 3125 hairpin, error); 3126 } 3127 3128 /** 3129 * Flow driver preparation API. This abstracts calling driver specific 3130 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3131 * calculates the size of memory required for device flow, allocates the memory, 3132 * initializes the device flow and returns the pointer. 3133 * 3134 * @note 3135 * This function initializes device flow structure such as dv or verbs in 3136 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3137 * rest. For example, adding returning device flow to flow->dev_flow list and 3138 * setting backward reference to the flow should be done out of this function. 3139 * layers field is not filled either. 3140 * 3141 * @param[in] dev 3142 * Pointer to the dev structure. 3143 * @param[in] attr 3144 * Pointer to the flow attributes. 3145 * @param[in] items 3146 * Pointer to the list of items. 3147 * @param[in] actions 3148 * Pointer to the list of actions. 3149 * @param[in] flow_idx 3150 * This memory pool index to the flow. 3151 * @param[out] error 3152 * Pointer to the error structure. 3153 * 3154 * @return 3155 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3156 */ 3157 static inline struct mlx5_flow * 3158 flow_drv_prepare(struct rte_eth_dev *dev, 3159 const struct rte_flow *flow, 3160 const struct rte_flow_attr *attr, 3161 const struct rte_flow_item items[], 3162 const struct rte_flow_action actions[], 3163 uint32_t flow_idx, 3164 struct rte_flow_error *error) 3165 { 3166 const struct mlx5_flow_driver_ops *fops; 3167 enum mlx5_flow_drv_type type = flow->drv_type; 3168 struct mlx5_flow *mlx5_flow = NULL; 3169 3170 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3171 fops = flow_get_drv_ops(type); 3172 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3173 if (mlx5_flow) 3174 mlx5_flow->flow_idx = flow_idx; 3175 return mlx5_flow; 3176 } 3177 3178 /** 3179 * Flow driver translation API. This abstracts calling driver specific 3180 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3181 * translates a generic flow into a driver flow. flow_drv_prepare() must 3182 * precede. 3183 * 3184 * @note 3185 * dev_flow->layers could be filled as a result of parsing during translation 3186 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3187 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3188 * flow->actions could be overwritten even though all the expanded dev_flows 3189 * have the same actions. 3190 * 3191 * @param[in] dev 3192 * Pointer to the rte dev structure. 3193 * @param[in, out] dev_flow 3194 * Pointer to the mlx5 flow. 3195 * @param[in] attr 3196 * Pointer to the flow attributes. 3197 * @param[in] items 3198 * Pointer to the list of items. 3199 * @param[in] actions 3200 * Pointer to the list of actions. 3201 * @param[out] error 3202 * Pointer to the error structure. 3203 * 3204 * @return 3205 * 0 on success, a negative errno value otherwise and rte_errno is set. 3206 */ 3207 static inline int 3208 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3209 const struct rte_flow_attr *attr, 3210 const struct rte_flow_item items[], 3211 const struct rte_flow_action actions[], 3212 struct rte_flow_error *error) 3213 { 3214 const struct mlx5_flow_driver_ops *fops; 3215 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3216 3217 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3218 fops = flow_get_drv_ops(type); 3219 return fops->translate(dev, dev_flow, attr, items, actions, error); 3220 } 3221 3222 /** 3223 * Flow driver apply API. This abstracts calling driver specific functions. 3224 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3225 * translated driver flows on to device. flow_drv_translate() must precede. 3226 * 3227 * @param[in] dev 3228 * Pointer to Ethernet device structure. 3229 * @param[in, out] flow 3230 * Pointer to flow structure. 3231 * @param[out] error 3232 * Pointer to error structure. 3233 * 3234 * @return 3235 * 0 on success, a negative errno value otherwise and rte_errno is set. 3236 */ 3237 static inline int 3238 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3239 struct rte_flow_error *error) 3240 { 3241 const struct mlx5_flow_driver_ops *fops; 3242 enum mlx5_flow_drv_type type = flow->drv_type; 3243 3244 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3245 fops = flow_get_drv_ops(type); 3246 return fops->apply(dev, flow, error); 3247 } 3248 3249 /** 3250 * Flow driver destroy API. This abstracts calling driver specific functions. 3251 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3252 * on device and releases resources of the flow. 3253 * 3254 * @param[in] dev 3255 * Pointer to Ethernet device. 3256 * @param[in, out] flow 3257 * Pointer to flow structure. 3258 */ 3259 static inline void 3260 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3261 { 3262 const struct mlx5_flow_driver_ops *fops; 3263 enum mlx5_flow_drv_type type = flow->drv_type; 3264 3265 flow_mreg_split_qrss_release(dev, flow); 3266 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3267 fops = flow_get_drv_ops(type); 3268 fops->destroy(dev, flow); 3269 } 3270 3271 /** 3272 * Get RSS action from the action list. 3273 * 3274 * @param[in] actions 3275 * Pointer to the list of actions. 3276 * 3277 * @return 3278 * Pointer to the RSS action if exist, else return NULL. 3279 */ 3280 static const struct rte_flow_action_rss* 3281 flow_get_rss_action(const struct rte_flow_action actions[]) 3282 { 3283 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3284 switch (actions->type) { 3285 case RTE_FLOW_ACTION_TYPE_RSS: 3286 return (const struct rte_flow_action_rss *) 3287 actions->conf; 3288 default: 3289 break; 3290 } 3291 } 3292 return NULL; 3293 } 3294 3295 /* maps shared action to translated non shared in some actions array */ 3296 struct mlx5_translated_shared_action { 3297 struct rte_flow_shared_action *action; /**< Shared action */ 3298 int index; /**< Index in related array of rte_flow_action */ 3299 }; 3300 3301 /** 3302 * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related 3303 * non shared action if translation possible. 3304 * This functionality used to run same execution path for both shared & non 3305 * shared actions on flow create. All necessary preparations for shared 3306 * action handling should be preformed on *shared* actions list returned 3307 * from this call. 3308 * 3309 * @param[in] actions 3310 * List of actions to translate. 3311 * @param[out] shared 3312 * List to store translated shared actions. 3313 * @param[in, out] shared_n 3314 * Size of *shared* array. On return should be updated with number of shared 3315 * actions retrieved from the *actions* list. 3316 * @param[out] translated_actions 3317 * List of actions where all shared actions were translated to non shared 3318 * if possible. NULL if no translation took place. 3319 * @param[out] error 3320 * Pointer to the error structure. 3321 * 3322 * @return 3323 * 0 on success, a negative errno value otherwise and rte_errno is set. 3324 */ 3325 static int 3326 flow_shared_actions_translate(const struct rte_flow_action actions[], 3327 struct mlx5_translated_shared_action *shared, 3328 int *shared_n, 3329 struct rte_flow_action **translated_actions, 3330 struct rte_flow_error *error) 3331 { 3332 struct rte_flow_action *translated = NULL; 3333 size_t actions_size; 3334 int n; 3335 int copied_n = 0; 3336 struct mlx5_translated_shared_action *shared_end = NULL; 3337 3338 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3339 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED) 3340 continue; 3341 if (copied_n == *shared_n) { 3342 return rte_flow_error_set 3343 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3344 NULL, "too many shared actions"); 3345 } 3346 rte_memcpy(&shared[copied_n].action, &actions[n].conf, 3347 sizeof(actions[n].conf)); 3348 shared[copied_n].index = n; 3349 copied_n++; 3350 } 3351 n++; 3352 *shared_n = copied_n; 3353 if (!copied_n) 3354 return 0; 3355 actions_size = sizeof(struct rte_flow_action) * n; 3356 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3357 if (!translated) { 3358 rte_errno = ENOMEM; 3359 return -ENOMEM; 3360 } 3361 memcpy(translated, actions, actions_size); 3362 for (shared_end = shared + copied_n; shared < shared_end; shared++) { 3363 const struct rte_flow_shared_action *shared_action; 3364 3365 shared_action = shared->action; 3366 switch (shared_action->type) { 3367 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 3368 translated[shared->index].type = 3369 RTE_FLOW_ACTION_TYPE_RSS; 3370 translated[shared->index].conf = 3371 &shared_action->rss.origin; 3372 break; 3373 default: 3374 mlx5_free(translated); 3375 return rte_flow_error_set 3376 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3377 NULL, "invalid shared action type"); 3378 } 3379 } 3380 *translated_actions = translated; 3381 return 0; 3382 } 3383 3384 /** 3385 * Get Shared RSS action from the action list. 3386 * 3387 * @param[in] shared 3388 * Pointer to the list of actions. 3389 * @param[in] shared_n 3390 * Actions list length. 3391 * 3392 * @return 3393 * Pointer to the MLX5 RSS action if exists, otherwise return NULL. 3394 */ 3395 static struct mlx5_shared_action_rss * 3396 flow_get_shared_rss_action(struct mlx5_translated_shared_action *shared, 3397 int shared_n) 3398 { 3399 struct mlx5_translated_shared_action *shared_end; 3400 3401 for (shared_end = shared + shared_n; shared < shared_end; shared++) { 3402 struct rte_flow_shared_action *shared_action; 3403 3404 shared_action = shared->action; 3405 switch (shared_action->type) { 3406 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 3407 __atomic_add_fetch(&shared_action->refcnt, 1, 3408 __ATOMIC_RELAXED); 3409 return &shared_action->rss; 3410 default: 3411 break; 3412 } 3413 } 3414 return NULL; 3415 } 3416 3417 struct rte_flow_shared_action * 3418 mlx5_flow_get_shared_rss(struct rte_flow *flow) 3419 { 3420 if (flow->shared_rss) 3421 return container_of(flow->shared_rss, 3422 struct rte_flow_shared_action, rss); 3423 else 3424 return NULL; 3425 } 3426 3427 static unsigned int 3428 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3429 { 3430 const struct rte_flow_item *item; 3431 unsigned int has_vlan = 0; 3432 3433 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3434 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3435 has_vlan = 1; 3436 break; 3437 } 3438 } 3439 if (has_vlan) 3440 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3441 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3442 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3443 MLX5_EXPANSION_ROOT_OUTER; 3444 } 3445 3446 /** 3447 * Get layer flags from the prefix flow. 3448 * 3449 * Some flows may be split to several subflows, the prefix subflow gets the 3450 * match items and the suffix sub flow gets the actions. 3451 * Some actions need the user defined match item flags to get the detail for 3452 * the action. 3453 * This function helps the suffix flow to get the item layer flags from prefix 3454 * subflow. 3455 * 3456 * @param[in] dev_flow 3457 * Pointer the created preifx subflow. 3458 * 3459 * @return 3460 * The layers get from prefix subflow. 3461 */ 3462 static inline uint64_t 3463 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3464 { 3465 uint64_t layers = 0; 3466 3467 /* 3468 * Layers bits could be localization, but usually the compiler will 3469 * help to do the optimization work for source code. 3470 * If no decap actions, use the layers directly. 3471 */ 3472 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3473 return dev_flow->handle->layers; 3474 /* Convert L3 layers with decap action. */ 3475 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3476 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3477 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3478 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3479 /* Convert L4 layers with decap action. */ 3480 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3481 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3482 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3483 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3484 return layers; 3485 } 3486 3487 /** 3488 * Get metadata split action information. 3489 * 3490 * @param[in] actions 3491 * Pointer to the list of actions. 3492 * @param[out] qrss 3493 * Pointer to the return pointer. 3494 * @param[out] qrss_type 3495 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3496 * if no QUEUE/RSS is found. 3497 * @param[out] encap_idx 3498 * Pointer to the index of the encap action if exists, otherwise the last 3499 * action index. 3500 * 3501 * @return 3502 * Total number of actions. 3503 */ 3504 static int 3505 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3506 const struct rte_flow_action **qrss, 3507 int *encap_idx) 3508 { 3509 const struct rte_flow_action_raw_encap *raw_encap; 3510 int actions_n = 0; 3511 int raw_decap_idx = -1; 3512 3513 *encap_idx = -1; 3514 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3515 switch (actions->type) { 3516 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3517 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3518 *encap_idx = actions_n; 3519 break; 3520 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3521 raw_decap_idx = actions_n; 3522 break; 3523 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3524 raw_encap = actions->conf; 3525 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3526 *encap_idx = raw_decap_idx != -1 ? 3527 raw_decap_idx : actions_n; 3528 break; 3529 case RTE_FLOW_ACTION_TYPE_QUEUE: 3530 case RTE_FLOW_ACTION_TYPE_RSS: 3531 *qrss = actions; 3532 break; 3533 default: 3534 break; 3535 } 3536 actions_n++; 3537 } 3538 if (*encap_idx == -1) 3539 *encap_idx = actions_n; 3540 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3541 return actions_n + 1; 3542 } 3543 3544 /** 3545 * Check meter action from the action list. 3546 * 3547 * @param[in] actions 3548 * Pointer to the list of actions. 3549 * @param[out] mtr 3550 * Pointer to the meter exist flag. 3551 * 3552 * @return 3553 * Total number of actions. 3554 */ 3555 static int 3556 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 3557 { 3558 int actions_n = 0; 3559 3560 MLX5_ASSERT(mtr); 3561 *mtr = 0; 3562 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3563 switch (actions->type) { 3564 case RTE_FLOW_ACTION_TYPE_METER: 3565 *mtr = 1; 3566 break; 3567 default: 3568 break; 3569 } 3570 actions_n++; 3571 } 3572 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3573 return actions_n + 1; 3574 } 3575 3576 /** 3577 * Check if the flow should be split due to hairpin. 3578 * The reason for the split is that in current HW we can't 3579 * support encap and push-vlan on Rx, so if a flow contains 3580 * these actions we move it to Tx. 3581 * 3582 * @param dev 3583 * Pointer to Ethernet device. 3584 * @param[in] attr 3585 * Flow rule attributes. 3586 * @param[in] actions 3587 * Associated actions (list terminated by the END action). 3588 * 3589 * @return 3590 * > 0 the number of actions and the flow should be split, 3591 * 0 when no split required. 3592 */ 3593 static int 3594 flow_check_hairpin_split(struct rte_eth_dev *dev, 3595 const struct rte_flow_attr *attr, 3596 const struct rte_flow_action actions[]) 3597 { 3598 int queue_action = 0; 3599 int action_n = 0; 3600 int split = 0; 3601 const struct rte_flow_action_queue *queue; 3602 const struct rte_flow_action_rss *rss; 3603 const struct rte_flow_action_raw_encap *raw_encap; 3604 const struct rte_eth_hairpin_conf *conf; 3605 3606 if (!attr->ingress) 3607 return 0; 3608 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3609 switch (actions->type) { 3610 case RTE_FLOW_ACTION_TYPE_QUEUE: 3611 queue = actions->conf; 3612 if (queue == NULL) 3613 return 0; 3614 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 3615 if (conf != NULL && !!conf->tx_explicit) 3616 return 0; 3617 queue_action = 1; 3618 action_n++; 3619 break; 3620 case RTE_FLOW_ACTION_TYPE_RSS: 3621 rss = actions->conf; 3622 if (rss == NULL || rss->queue_num == 0) 3623 return 0; 3624 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 3625 if (conf != NULL && !!conf->tx_explicit) 3626 return 0; 3627 queue_action = 1; 3628 action_n++; 3629 break; 3630 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3631 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3632 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3633 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3634 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3635 split++; 3636 action_n++; 3637 break; 3638 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3639 raw_encap = actions->conf; 3640 if (raw_encap->size > 3641 (sizeof(struct rte_flow_item_eth) + 3642 sizeof(struct rte_flow_item_ipv4))) 3643 split++; 3644 action_n++; 3645 break; 3646 default: 3647 action_n++; 3648 break; 3649 } 3650 } 3651 if (split && queue_action) 3652 return action_n; 3653 return 0; 3654 } 3655 3656 /* Declare flow create/destroy prototype in advance. */ 3657 static uint32_t 3658 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3659 const struct rte_flow_attr *attr, 3660 const struct rte_flow_item items[], 3661 const struct rte_flow_action actions[], 3662 bool external, struct rte_flow_error *error); 3663 3664 static void 3665 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3666 uint32_t flow_idx); 3667 3668 struct mlx5_hlist_entry * 3669 flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key, 3670 void *cb_ctx) 3671 { 3672 struct rte_eth_dev *dev = list->ctx; 3673 struct mlx5_priv *priv = dev->data->dev_private; 3674 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 3675 struct mlx5_flow_mreg_copy_resource *mcp_res; 3676 struct rte_flow_error *error = ctx->error; 3677 uint32_t idx = 0; 3678 int ret; 3679 uint32_t mark_id = key; 3680 struct rte_flow_attr attr = { 3681 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3682 .ingress = 1, 3683 }; 3684 struct mlx5_rte_flow_item_tag tag_spec = { 3685 .data = mark_id, 3686 }; 3687 struct rte_flow_item items[] = { 3688 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 3689 }; 3690 struct rte_flow_action_mark ftag = { 3691 .id = mark_id, 3692 }; 3693 struct mlx5_flow_action_copy_mreg cp_mreg = { 3694 .dst = REG_B, 3695 .src = REG_NON, 3696 }; 3697 struct rte_flow_action_jump jump = { 3698 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3699 }; 3700 struct rte_flow_action actions[] = { 3701 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 3702 }; 3703 3704 /* Fill the register fileds in the flow. */ 3705 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3706 if (ret < 0) 3707 return NULL; 3708 tag_spec.id = ret; 3709 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3710 if (ret < 0) 3711 return NULL; 3712 cp_mreg.src = ret; 3713 /* Provide the full width of FLAG specific value. */ 3714 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3715 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3716 /* Build a new flow. */ 3717 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3718 items[0] = (struct rte_flow_item){ 3719 .type = (enum rte_flow_item_type) 3720 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3721 .spec = &tag_spec, 3722 }; 3723 items[1] = (struct rte_flow_item){ 3724 .type = RTE_FLOW_ITEM_TYPE_END, 3725 }; 3726 actions[0] = (struct rte_flow_action){ 3727 .type = (enum rte_flow_action_type) 3728 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3729 .conf = &ftag, 3730 }; 3731 actions[1] = (struct rte_flow_action){ 3732 .type = (enum rte_flow_action_type) 3733 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3734 .conf = &cp_mreg, 3735 }; 3736 actions[2] = (struct rte_flow_action){ 3737 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3738 .conf = &jump, 3739 }; 3740 actions[3] = (struct rte_flow_action){ 3741 .type = RTE_FLOW_ACTION_TYPE_END, 3742 }; 3743 } else { 3744 /* Default rule, wildcard match. */ 3745 attr.priority = MLX5_FLOW_PRIO_RSVD; 3746 items[0] = (struct rte_flow_item){ 3747 .type = RTE_FLOW_ITEM_TYPE_END, 3748 }; 3749 actions[0] = (struct rte_flow_action){ 3750 .type = (enum rte_flow_action_type) 3751 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3752 .conf = &cp_mreg, 3753 }; 3754 actions[1] = (struct rte_flow_action){ 3755 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3756 .conf = &jump, 3757 }; 3758 actions[2] = (struct rte_flow_action){ 3759 .type = RTE_FLOW_ACTION_TYPE_END, 3760 }; 3761 } 3762 /* Build a new entry. */ 3763 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3764 if (!mcp_res) { 3765 rte_errno = ENOMEM; 3766 return NULL; 3767 } 3768 mcp_res->idx = idx; 3769 /* 3770 * The copy Flows are not included in any list. There 3771 * ones are referenced from other Flows and can not 3772 * be applied, removed, deleted in ardbitrary order 3773 * by list traversing. 3774 */ 3775 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3776 actions, false, error); 3777 if (!mcp_res->rix_flow) { 3778 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 3779 return NULL; 3780 } 3781 return &mcp_res->hlist_ent; 3782 } 3783 3784 /** 3785 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3786 * 3787 * As mark_id is unique, if there's already a registered flow for the mark_id, 3788 * return by increasing the reference counter of the resource. Otherwise, create 3789 * the resource (mcp_res) and flow. 3790 * 3791 * Flow looks like, 3792 * - If ingress port is ANY and reg_c[1] is mark_id, 3793 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3794 * 3795 * For default flow (zero mark_id), flow is like, 3796 * - If ingress port is ANY, 3797 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 3798 * 3799 * @param dev 3800 * Pointer to Ethernet device. 3801 * @param mark_id 3802 * ID of MARK action, zero means default flow for META. 3803 * @param[out] error 3804 * Perform verbose error reporting if not NULL. 3805 * 3806 * @return 3807 * Associated resource on success, NULL otherwise and rte_errno is set. 3808 */ 3809 static struct mlx5_flow_mreg_copy_resource * 3810 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 3811 struct rte_flow_error *error) 3812 { 3813 struct mlx5_priv *priv = dev->data->dev_private; 3814 struct mlx5_hlist_entry *entry; 3815 struct mlx5_flow_cb_ctx ctx = { 3816 .dev = dev, 3817 .error = error, 3818 }; 3819 3820 /* Check if already registered. */ 3821 MLX5_ASSERT(priv->mreg_cp_tbl); 3822 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 3823 if (!entry) 3824 return NULL; 3825 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 3826 hlist_ent); 3827 } 3828 3829 void 3830 flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry) 3831 { 3832 struct mlx5_flow_mreg_copy_resource *mcp_res = 3833 container_of(entry, typeof(*mcp_res), hlist_ent); 3834 struct rte_eth_dev *dev = list->ctx; 3835 struct mlx5_priv *priv = dev->data->dev_private; 3836 3837 MLX5_ASSERT(mcp_res->rix_flow); 3838 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3839 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3840 } 3841 3842 /** 3843 * Release flow in RX_CP_TBL. 3844 * 3845 * @param dev 3846 * Pointer to Ethernet device. 3847 * @flow 3848 * Parent flow for wich copying is provided. 3849 */ 3850 static void 3851 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3852 struct rte_flow *flow) 3853 { 3854 struct mlx5_flow_mreg_copy_resource *mcp_res; 3855 struct mlx5_priv *priv = dev->data->dev_private; 3856 3857 if (!flow->rix_mreg_copy) 3858 return; 3859 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3860 flow->rix_mreg_copy); 3861 if (!mcp_res || !priv->mreg_cp_tbl) 3862 return; 3863 MLX5_ASSERT(mcp_res->rix_flow); 3864 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3865 flow->rix_mreg_copy = 0; 3866 } 3867 3868 /** 3869 * Remove the default copy action from RX_CP_TBL. 3870 * 3871 * This functions is called in the mlx5_dev_start(). No thread safe 3872 * is guaranteed. 3873 * 3874 * @param dev 3875 * Pointer to Ethernet device. 3876 */ 3877 static void 3878 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3879 { 3880 struct mlx5_hlist_entry *entry; 3881 struct mlx5_priv *priv = dev->data->dev_private; 3882 3883 /* Check if default flow is registered. */ 3884 if (!priv->mreg_cp_tbl) 3885 return; 3886 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, 3887 MLX5_DEFAULT_COPY_ID, NULL); 3888 if (!entry) 3889 return; 3890 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 3891 } 3892 3893 /** 3894 * Add the default copy action in in RX_CP_TBL. 3895 * 3896 * This functions is called in the mlx5_dev_start(). No thread safe 3897 * is guaranteed. 3898 * 3899 * @param dev 3900 * Pointer to Ethernet device. 3901 * @param[out] error 3902 * Perform verbose error reporting if not NULL. 3903 * 3904 * @return 3905 * 0 for success, negative value otherwise and rte_errno is set. 3906 */ 3907 static int 3908 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3909 struct rte_flow_error *error) 3910 { 3911 struct mlx5_priv *priv = dev->data->dev_private; 3912 struct mlx5_flow_mreg_copy_resource *mcp_res; 3913 3914 /* Check whether extensive metadata feature is engaged. */ 3915 if (!priv->config.dv_flow_en || 3916 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3917 !mlx5_flow_ext_mreg_supported(dev) || 3918 !priv->sh->dv_regc0_mask) 3919 return 0; 3920 /* 3921 * Add default mreg copy flow may be called multiple time, but 3922 * only be called once in stop. Avoid register it twice. 3923 */ 3924 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL)) 3925 return 0; 3926 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3927 if (!mcp_res) 3928 return -rte_errno; 3929 return 0; 3930 } 3931 3932 /** 3933 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3934 * 3935 * All the flow having Q/RSS action should be split by 3936 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3937 * performs the following, 3938 * - CQE->flow_tag := reg_c[1] (MARK) 3939 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3940 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3941 * but there should be a flow per each MARK ID set by MARK action. 3942 * 3943 * For the aforementioned reason, if there's a MARK action in flow's action 3944 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3945 * the MARK ID to CQE's flow_tag like, 3946 * - If reg_c[1] is mark_id, 3947 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3948 * 3949 * For SET_META action which stores value in reg_c[0], as the destination is 3950 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3951 * MARK ID means the default flow. The default flow looks like, 3952 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3953 * 3954 * @param dev 3955 * Pointer to Ethernet device. 3956 * @param flow 3957 * Pointer to flow structure. 3958 * @param[in] actions 3959 * Pointer to the list of actions. 3960 * @param[out] error 3961 * Perform verbose error reporting if not NULL. 3962 * 3963 * @return 3964 * 0 on success, negative value otherwise and rte_errno is set. 3965 */ 3966 static int 3967 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3968 struct rte_flow *flow, 3969 const struct rte_flow_action *actions, 3970 struct rte_flow_error *error) 3971 { 3972 struct mlx5_priv *priv = dev->data->dev_private; 3973 struct mlx5_dev_config *config = &priv->config; 3974 struct mlx5_flow_mreg_copy_resource *mcp_res; 3975 const struct rte_flow_action_mark *mark; 3976 3977 /* Check whether extensive metadata feature is engaged. */ 3978 if (!config->dv_flow_en || 3979 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3980 !mlx5_flow_ext_mreg_supported(dev) || 3981 !priv->sh->dv_regc0_mask) 3982 return 0; 3983 /* Find MARK action. */ 3984 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3985 switch (actions->type) { 3986 case RTE_FLOW_ACTION_TYPE_FLAG: 3987 mcp_res = flow_mreg_add_copy_action 3988 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3989 if (!mcp_res) 3990 return -rte_errno; 3991 flow->rix_mreg_copy = mcp_res->idx; 3992 return 0; 3993 case RTE_FLOW_ACTION_TYPE_MARK: 3994 mark = (const struct rte_flow_action_mark *) 3995 actions->conf; 3996 mcp_res = 3997 flow_mreg_add_copy_action(dev, mark->id, error); 3998 if (!mcp_res) 3999 return -rte_errno; 4000 flow->rix_mreg_copy = mcp_res->idx; 4001 return 0; 4002 default: 4003 break; 4004 } 4005 } 4006 return 0; 4007 } 4008 4009 #define MLX5_MAX_SPLIT_ACTIONS 24 4010 #define MLX5_MAX_SPLIT_ITEMS 24 4011 4012 /** 4013 * Split the hairpin flow. 4014 * Since HW can't support encap and push-vlan on Rx, we move these 4015 * actions to Tx. 4016 * If the count action is after the encap then we also 4017 * move the count action. in this case the count will also measure 4018 * the outer bytes. 4019 * 4020 * @param dev 4021 * Pointer to Ethernet device. 4022 * @param[in] actions 4023 * Associated actions (list terminated by the END action). 4024 * @param[out] actions_rx 4025 * Rx flow actions. 4026 * @param[out] actions_tx 4027 * Tx flow actions.. 4028 * @param[out] pattern_tx 4029 * The pattern items for the Tx flow. 4030 * @param[out] flow_id 4031 * The flow ID connected to this flow. 4032 * 4033 * @return 4034 * 0 on success. 4035 */ 4036 static int 4037 flow_hairpin_split(struct rte_eth_dev *dev, 4038 const struct rte_flow_action actions[], 4039 struct rte_flow_action actions_rx[], 4040 struct rte_flow_action actions_tx[], 4041 struct rte_flow_item pattern_tx[], 4042 uint32_t flow_id) 4043 { 4044 const struct rte_flow_action_raw_encap *raw_encap; 4045 const struct rte_flow_action_raw_decap *raw_decap; 4046 struct mlx5_rte_flow_action_set_tag *set_tag; 4047 struct rte_flow_action *tag_action; 4048 struct mlx5_rte_flow_item_tag *tag_item; 4049 struct rte_flow_item *item; 4050 char *addr; 4051 int encap = 0; 4052 4053 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4054 switch (actions->type) { 4055 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4056 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4057 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4058 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4059 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4060 rte_memcpy(actions_tx, actions, 4061 sizeof(struct rte_flow_action)); 4062 actions_tx++; 4063 break; 4064 case RTE_FLOW_ACTION_TYPE_COUNT: 4065 if (encap) { 4066 rte_memcpy(actions_tx, actions, 4067 sizeof(struct rte_flow_action)); 4068 actions_tx++; 4069 } else { 4070 rte_memcpy(actions_rx, actions, 4071 sizeof(struct rte_flow_action)); 4072 actions_rx++; 4073 } 4074 break; 4075 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4076 raw_encap = actions->conf; 4077 if (raw_encap->size > 4078 (sizeof(struct rte_flow_item_eth) + 4079 sizeof(struct rte_flow_item_ipv4))) { 4080 memcpy(actions_tx, actions, 4081 sizeof(struct rte_flow_action)); 4082 actions_tx++; 4083 encap = 1; 4084 } else { 4085 rte_memcpy(actions_rx, actions, 4086 sizeof(struct rte_flow_action)); 4087 actions_rx++; 4088 } 4089 break; 4090 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4091 raw_decap = actions->conf; 4092 if (raw_decap->size < 4093 (sizeof(struct rte_flow_item_eth) + 4094 sizeof(struct rte_flow_item_ipv4))) { 4095 memcpy(actions_tx, actions, 4096 sizeof(struct rte_flow_action)); 4097 actions_tx++; 4098 } else { 4099 rte_memcpy(actions_rx, actions, 4100 sizeof(struct rte_flow_action)); 4101 actions_rx++; 4102 } 4103 break; 4104 default: 4105 rte_memcpy(actions_rx, actions, 4106 sizeof(struct rte_flow_action)); 4107 actions_rx++; 4108 break; 4109 } 4110 } 4111 /* Add set meta action and end action for the Rx flow. */ 4112 tag_action = actions_rx; 4113 tag_action->type = (enum rte_flow_action_type) 4114 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4115 actions_rx++; 4116 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4117 actions_rx++; 4118 set_tag = (void *)actions_rx; 4119 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 4120 MLX5_ASSERT(set_tag->id > REG_NON); 4121 set_tag->data = flow_id; 4122 tag_action->conf = set_tag; 4123 /* Create Tx item list. */ 4124 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4125 addr = (void *)&pattern_tx[2]; 4126 item = pattern_tx; 4127 item->type = (enum rte_flow_item_type) 4128 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4129 tag_item = (void *)addr; 4130 tag_item->data = flow_id; 4131 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4132 MLX5_ASSERT(set_tag->id > REG_NON); 4133 item->spec = tag_item; 4134 addr += sizeof(struct mlx5_rte_flow_item_tag); 4135 tag_item = (void *)addr; 4136 tag_item->data = UINT32_MAX; 4137 tag_item->id = UINT16_MAX; 4138 item->mask = tag_item; 4139 item->last = NULL; 4140 item++; 4141 item->type = RTE_FLOW_ITEM_TYPE_END; 4142 return 0; 4143 } 4144 4145 __extension__ 4146 union tunnel_offload_mark { 4147 uint32_t val; 4148 struct { 4149 uint32_t app_reserve:8; 4150 uint32_t table_id:15; 4151 uint32_t transfer:1; 4152 uint32_t _unused_:8; 4153 }; 4154 }; 4155 4156 struct tunnel_default_miss_ctx { 4157 uint16_t *queue; 4158 __extension__ 4159 union { 4160 struct rte_flow_action_rss action_rss; 4161 struct rte_flow_action_queue miss_queue; 4162 struct rte_flow_action_jump miss_jump; 4163 uint8_t raw[0]; 4164 }; 4165 }; 4166 4167 static int 4168 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 4169 struct rte_flow *flow, 4170 const struct rte_flow_attr *attr, 4171 const struct rte_flow_action *app_actions, 4172 uint32_t flow_idx, 4173 struct tunnel_default_miss_ctx *ctx, 4174 struct rte_flow_error *error) 4175 { 4176 struct mlx5_priv *priv = dev->data->dev_private; 4177 struct mlx5_flow *dev_flow; 4178 struct rte_flow_attr miss_attr = *attr; 4179 const struct mlx5_flow_tunnel *tunnel = app_actions[0].conf; 4180 const struct rte_flow_item miss_items[2] = { 4181 { 4182 .type = RTE_FLOW_ITEM_TYPE_ETH, 4183 .spec = NULL, 4184 .last = NULL, 4185 .mask = NULL 4186 }, 4187 { 4188 .type = RTE_FLOW_ITEM_TYPE_END, 4189 .spec = NULL, 4190 .last = NULL, 4191 .mask = NULL 4192 } 4193 }; 4194 union tunnel_offload_mark mark_id; 4195 struct rte_flow_action_mark miss_mark; 4196 struct rte_flow_action miss_actions[3] = { 4197 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 4198 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 4199 }; 4200 const struct rte_flow_action_jump *jump_data; 4201 uint32_t i, flow_table = 0; /* prevent compilation warning */ 4202 struct flow_grp_info grp_info = { 4203 .external = 1, 4204 .transfer = attr->transfer, 4205 .fdb_def_rule = !!priv->fdb_def_rule, 4206 .std_tbl_fix = 0, 4207 }; 4208 int ret; 4209 4210 if (!attr->transfer) { 4211 uint32_t q_size; 4212 4213 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 4214 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 4215 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 4216 0, SOCKET_ID_ANY); 4217 if (!ctx->queue) 4218 return rte_flow_error_set 4219 (error, ENOMEM, 4220 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 4221 NULL, "invalid default miss RSS"); 4222 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4223 ctx->action_rss.level = 0, 4224 ctx->action_rss.types = priv->rss_conf.rss_hf, 4225 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 4226 ctx->action_rss.queue_num = priv->reta_idx_n, 4227 ctx->action_rss.key = priv->rss_conf.rss_key, 4228 ctx->action_rss.queue = ctx->queue; 4229 if (!priv->reta_idx_n || !priv->rxqs_n) 4230 return rte_flow_error_set 4231 (error, EINVAL, 4232 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 4233 NULL, "invalid port configuration"); 4234 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 4235 ctx->action_rss.types = 0; 4236 for (i = 0; i != priv->reta_idx_n; ++i) 4237 ctx->queue[i] = (*priv->reta_idx)[i]; 4238 } else { 4239 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 4240 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 4241 } 4242 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 4243 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 4244 jump_data = app_actions->conf; 4245 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 4246 miss_attr.group = jump_data->group; 4247 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 4248 &flow_table, grp_info, error); 4249 if (ret) 4250 return rte_flow_error_set(error, EINVAL, 4251 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 4252 NULL, "invalid tunnel id"); 4253 mark_id.app_reserve = 0; 4254 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 4255 mark_id.transfer = !!attr->transfer; 4256 mark_id._unused_ = 0; 4257 miss_mark.id = mark_id.val; 4258 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 4259 miss_items, miss_actions, flow_idx, error); 4260 if (!dev_flow) 4261 return -rte_errno; 4262 dev_flow->flow = flow; 4263 dev_flow->external = true; 4264 dev_flow->tunnel = tunnel; 4265 /* Subflow object was created, we must include one in the list. */ 4266 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4267 dev_flow->handle, next); 4268 DRV_LOG(DEBUG, 4269 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 4270 dev->data->port_id, tunnel->app_tunnel.type, 4271 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 4272 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 4273 miss_actions, error); 4274 if (!ret) 4275 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 4276 error); 4277 4278 return ret; 4279 } 4280 4281 /** 4282 * The last stage of splitting chain, just creates the subflow 4283 * without any modification. 4284 * 4285 * @param[in] dev 4286 * Pointer to Ethernet device. 4287 * @param[in] flow 4288 * Parent flow structure pointer. 4289 * @param[in, out] sub_flow 4290 * Pointer to return the created subflow, may be NULL. 4291 * @param[in] prefix_layers 4292 * Prefix subflow layers, may be 0. 4293 * @param[in] prefix_mark 4294 * Prefix subflow mark flag, may be 0. 4295 * @param[in] attr 4296 * Flow rule attributes. 4297 * @param[in] items 4298 * Pattern specification (list terminated by the END pattern item). 4299 * @param[in] actions 4300 * Associated actions (list terminated by the END action). 4301 * @param[in] external 4302 * This flow rule is created by request external to PMD. 4303 * @param[in] flow_idx 4304 * This memory pool index to the flow. 4305 * @param[out] error 4306 * Perform verbose error reporting if not NULL. 4307 * @return 4308 * 0 on success, negative value otherwise 4309 */ 4310 static int 4311 flow_create_split_inner(struct rte_eth_dev *dev, 4312 struct rte_flow *flow, 4313 struct mlx5_flow **sub_flow, 4314 uint64_t prefix_layers, 4315 uint32_t prefix_mark, 4316 const struct rte_flow_attr *attr, 4317 const struct rte_flow_item items[], 4318 const struct rte_flow_action actions[], 4319 bool external, uint32_t flow_idx, 4320 struct rte_flow_error *error) 4321 { 4322 struct mlx5_flow *dev_flow; 4323 4324 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4325 flow_idx, error); 4326 if (!dev_flow) 4327 return -rte_errno; 4328 dev_flow->flow = flow; 4329 dev_flow->external = external; 4330 /* Subflow object was created, we must include one in the list. */ 4331 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4332 dev_flow->handle, next); 4333 /* 4334 * If dev_flow is as one of the suffix flow, some actions in suffix 4335 * flow may need some user defined item layer flags, and pass the 4336 * Metadate rxq mark flag to suffix flow as well. 4337 */ 4338 if (prefix_layers) 4339 dev_flow->handle->layers = prefix_layers; 4340 if (prefix_mark) 4341 dev_flow->handle->mark = 1; 4342 if (sub_flow) 4343 *sub_flow = dev_flow; 4344 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4345 } 4346 4347 /** 4348 * Split the meter flow. 4349 * 4350 * As meter flow will split to three sub flow, other than meter 4351 * action, the other actions make sense to only meter accepts 4352 * the packet. If it need to be dropped, no other additional 4353 * actions should be take. 4354 * 4355 * One kind of special action which decapsulates the L3 tunnel 4356 * header will be in the prefix sub flow, as not to take the 4357 * L3 tunnel header into account. 4358 * 4359 * @param dev 4360 * Pointer to Ethernet device. 4361 * @param[in] items 4362 * Pattern specification (list terminated by the END pattern item). 4363 * @param[out] sfx_items 4364 * Suffix flow match items (list terminated by the END pattern item). 4365 * @param[in] actions 4366 * Associated actions (list terminated by the END action). 4367 * @param[out] actions_sfx 4368 * Suffix flow actions. 4369 * @param[out] actions_pre 4370 * Prefix flow actions. 4371 * @param[out] pattern_sfx 4372 * The pattern items for the suffix flow. 4373 * @param[out] tag_sfx 4374 * Pointer to suffix flow tag. 4375 * 4376 * @return 4377 * 0 on success. 4378 */ 4379 static int 4380 flow_meter_split_prep(struct rte_eth_dev *dev, 4381 const struct rte_flow_item items[], 4382 struct rte_flow_item sfx_items[], 4383 const struct rte_flow_action actions[], 4384 struct rte_flow_action actions_sfx[], 4385 struct rte_flow_action actions_pre[]) 4386 { 4387 struct mlx5_priv *priv = dev->data->dev_private; 4388 struct rte_flow_action *tag_action = NULL; 4389 struct rte_flow_item *tag_item; 4390 struct mlx5_rte_flow_action_set_tag *set_tag; 4391 struct rte_flow_error error; 4392 const struct rte_flow_action_raw_encap *raw_encap; 4393 const struct rte_flow_action_raw_decap *raw_decap; 4394 struct mlx5_rte_flow_item_tag *tag_spec; 4395 struct mlx5_rte_flow_item_tag *tag_mask; 4396 uint32_t tag_id = 0; 4397 bool copy_vlan = false; 4398 4399 /* Prepare the actions for prefix and suffix flow. */ 4400 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4401 struct rte_flow_action **action_cur = NULL; 4402 4403 switch (actions->type) { 4404 case RTE_FLOW_ACTION_TYPE_METER: 4405 /* Add the extra tag action first. */ 4406 tag_action = actions_pre; 4407 tag_action->type = (enum rte_flow_action_type) 4408 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4409 actions_pre++; 4410 action_cur = &actions_pre; 4411 break; 4412 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4413 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4414 action_cur = &actions_pre; 4415 break; 4416 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4417 raw_encap = actions->conf; 4418 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4419 action_cur = &actions_pre; 4420 break; 4421 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4422 raw_decap = actions->conf; 4423 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4424 action_cur = &actions_pre; 4425 break; 4426 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4427 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4428 copy_vlan = true; 4429 break; 4430 default: 4431 break; 4432 } 4433 if (!action_cur) 4434 action_cur = &actions_sfx; 4435 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 4436 (*action_cur)++; 4437 } 4438 /* Add end action to the actions. */ 4439 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4440 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4441 actions_pre++; 4442 /* Set the tag. */ 4443 set_tag = (void *)actions_pre; 4444 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4445 mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 4446 &tag_id); 4447 if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { 4448 DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", 4449 dev->data->port_id); 4450 mlx5_ipool_free(priv->sh->ipool 4451 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); 4452 return 0; 4453 } else if (!tag_id) { 4454 return 0; 4455 } 4456 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 4457 assert(tag_action); 4458 tag_action->conf = set_tag; 4459 /* Prepare the suffix subflow items. */ 4460 tag_item = sfx_items++; 4461 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4462 int item_type = items->type; 4463 4464 switch (item_type) { 4465 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4466 memcpy(sfx_items, items, sizeof(*sfx_items)); 4467 sfx_items++; 4468 break; 4469 case RTE_FLOW_ITEM_TYPE_VLAN: 4470 if (copy_vlan) { 4471 memcpy(sfx_items, items, sizeof(*sfx_items)); 4472 /* 4473 * Convert to internal match item, it is used 4474 * for vlan push and set vid. 4475 */ 4476 sfx_items->type = (enum rte_flow_item_type) 4477 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4478 sfx_items++; 4479 } 4480 break; 4481 default: 4482 break; 4483 } 4484 } 4485 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4486 sfx_items++; 4487 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4488 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 4489 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4490 tag_mask = tag_spec + 1; 4491 tag_mask->data = 0xffffff00; 4492 tag_item->type = (enum rte_flow_item_type) 4493 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4494 tag_item->spec = tag_spec; 4495 tag_item->last = NULL; 4496 tag_item->mask = tag_mask; 4497 return tag_id; 4498 } 4499 4500 /** 4501 * Split action list having QUEUE/RSS for metadata register copy. 4502 * 4503 * Once Q/RSS action is detected in user's action list, the flow action 4504 * should be split in order to copy metadata registers, which will happen in 4505 * RX_CP_TBL like, 4506 * - CQE->flow_tag := reg_c[1] (MARK) 4507 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4508 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4509 * This is because the last action of each flow must be a terminal action 4510 * (QUEUE, RSS or DROP). 4511 * 4512 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4513 * stored and kept in the mlx5_flow structure per each sub_flow. 4514 * 4515 * The Q/RSS action is replaced with, 4516 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4517 * And the following JUMP action is added at the end, 4518 * - JUMP, to RX_CP_TBL. 4519 * 4520 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4521 * flow_create_split_metadata() routine. The flow will look like, 4522 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4523 * 4524 * @param dev 4525 * Pointer to Ethernet device. 4526 * @param[out] split_actions 4527 * Pointer to store split actions to jump to CP_TBL. 4528 * @param[in] actions 4529 * Pointer to the list of original flow actions. 4530 * @param[in] qrss 4531 * Pointer to the Q/RSS action. 4532 * @param[in] actions_n 4533 * Number of original actions. 4534 * @param[out] error 4535 * Perform verbose error reporting if not NULL. 4536 * 4537 * @return 4538 * non-zero unique flow_id on success, otherwise 0 and 4539 * error/rte_error are set. 4540 */ 4541 static uint32_t 4542 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4543 struct rte_flow_action *split_actions, 4544 const struct rte_flow_action *actions, 4545 const struct rte_flow_action *qrss, 4546 int actions_n, struct rte_flow_error *error) 4547 { 4548 struct mlx5_priv *priv = dev->data->dev_private; 4549 struct mlx5_rte_flow_action_set_tag *set_tag; 4550 struct rte_flow_action_jump *jump; 4551 const int qrss_idx = qrss - actions; 4552 uint32_t flow_id = 0; 4553 int ret = 0; 4554 4555 /* 4556 * Given actions will be split 4557 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4558 * - Add jump to mreg CP_TBL. 4559 * As a result, there will be one more action. 4560 */ 4561 ++actions_n; 4562 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4563 set_tag = (void *)(split_actions + actions_n); 4564 /* 4565 * If tag action is not set to void(it means we are not the meter 4566 * suffix flow), add the tag action. Since meter suffix flow already 4567 * has the tag added. 4568 */ 4569 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4570 /* 4571 * Allocate the new subflow ID. This one is unique within 4572 * device and not shared with representors. Otherwise, 4573 * we would have to resolve multi-thread access synch 4574 * issue. Each flow on the shared device is appended 4575 * with source vport identifier, so the resulting 4576 * flows will be unique in the shared (by master and 4577 * representors) domain even if they have coinciding 4578 * IDs. 4579 */ 4580 mlx5_ipool_malloc(priv->sh->ipool 4581 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 4582 if (!flow_id) 4583 return rte_flow_error_set(error, ENOMEM, 4584 RTE_FLOW_ERROR_TYPE_ACTION, 4585 NULL, "can't allocate id " 4586 "for split Q/RSS subflow"); 4587 /* Internal SET_TAG action to set flow ID. */ 4588 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4589 .data = flow_id, 4590 }; 4591 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4592 if (ret < 0) 4593 return ret; 4594 set_tag->id = ret; 4595 /* Construct new actions array. */ 4596 /* Replace QUEUE/RSS action. */ 4597 split_actions[qrss_idx] = (struct rte_flow_action){ 4598 .type = (enum rte_flow_action_type) 4599 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4600 .conf = set_tag, 4601 }; 4602 } 4603 /* JUMP action to jump to mreg copy table (CP_TBL). */ 4604 jump = (void *)(set_tag + 1); 4605 *jump = (struct rte_flow_action_jump){ 4606 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4607 }; 4608 split_actions[actions_n - 2] = (struct rte_flow_action){ 4609 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4610 .conf = jump, 4611 }; 4612 split_actions[actions_n - 1] = (struct rte_flow_action){ 4613 .type = RTE_FLOW_ACTION_TYPE_END, 4614 }; 4615 return flow_id; 4616 } 4617 4618 /** 4619 * Extend the given action list for Tx metadata copy. 4620 * 4621 * Copy the given action list to the ext_actions and add flow metadata register 4622 * copy action in order to copy reg_a set by WQE to reg_c[0]. 4623 * 4624 * @param[out] ext_actions 4625 * Pointer to the extended action list. 4626 * @param[in] actions 4627 * Pointer to the list of actions. 4628 * @param[in] actions_n 4629 * Number of actions in the list. 4630 * @param[out] error 4631 * Perform verbose error reporting if not NULL. 4632 * @param[in] encap_idx 4633 * The encap action inndex. 4634 * 4635 * @return 4636 * 0 on success, negative value otherwise 4637 */ 4638 static int 4639 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 4640 struct rte_flow_action *ext_actions, 4641 const struct rte_flow_action *actions, 4642 int actions_n, struct rte_flow_error *error, 4643 int encap_idx) 4644 { 4645 struct mlx5_flow_action_copy_mreg *cp_mreg = 4646 (struct mlx5_flow_action_copy_mreg *) 4647 (ext_actions + actions_n + 1); 4648 int ret; 4649 4650 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4651 if (ret < 0) 4652 return ret; 4653 cp_mreg->dst = ret; 4654 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 4655 if (ret < 0) 4656 return ret; 4657 cp_mreg->src = ret; 4658 if (encap_idx != 0) 4659 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 4660 if (encap_idx == actions_n - 1) { 4661 ext_actions[actions_n - 1] = (struct rte_flow_action){ 4662 .type = (enum rte_flow_action_type) 4663 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4664 .conf = cp_mreg, 4665 }; 4666 ext_actions[actions_n] = (struct rte_flow_action){ 4667 .type = RTE_FLOW_ACTION_TYPE_END, 4668 }; 4669 } else { 4670 ext_actions[encap_idx] = (struct rte_flow_action){ 4671 .type = (enum rte_flow_action_type) 4672 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4673 .conf = cp_mreg, 4674 }; 4675 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 4676 sizeof(*ext_actions) * (actions_n - encap_idx)); 4677 } 4678 return 0; 4679 } 4680 4681 /** 4682 * Check the match action from the action list. 4683 * 4684 * @param[in] actions 4685 * Pointer to the list of actions. 4686 * @param[in] attr 4687 * Flow rule attributes. 4688 * @param[in] action 4689 * The action to be check if exist. 4690 * @param[out] match_action_pos 4691 * Pointer to the position of the matched action if exists, otherwise is -1. 4692 * @param[out] qrss_action_pos 4693 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 4694 * 4695 * @return 4696 * > 0 the total number of actions. 4697 * 0 if not found match action in action list. 4698 */ 4699 static int 4700 flow_check_match_action(const struct rte_flow_action actions[], 4701 const struct rte_flow_attr *attr, 4702 enum rte_flow_action_type action, 4703 int *match_action_pos, int *qrss_action_pos) 4704 { 4705 const struct rte_flow_action_sample *sample; 4706 int actions_n = 0; 4707 int jump_flag = 0; 4708 uint32_t ratio = 0; 4709 int sub_type = 0; 4710 int flag = 0; 4711 4712 *match_action_pos = -1; 4713 *qrss_action_pos = -1; 4714 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4715 if (actions->type == action) { 4716 flag = 1; 4717 *match_action_pos = actions_n; 4718 } 4719 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE || 4720 actions->type == RTE_FLOW_ACTION_TYPE_RSS) 4721 *qrss_action_pos = actions_n; 4722 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP) 4723 jump_flag = 1; 4724 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) { 4725 sample = actions->conf; 4726 ratio = sample->ratio; 4727 sub_type = ((const struct rte_flow_action *) 4728 (sample->actions))->type; 4729 } 4730 actions_n++; 4731 } 4732 if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) { 4733 if (ratio == 1) { 4734 /* JUMP Action not support for Mirroring; 4735 * Mirroring support multi-destination; 4736 */ 4737 if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END) 4738 flag = 0; 4739 } 4740 } 4741 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4742 return flag ? actions_n + 1 : 0; 4743 } 4744 4745 #define SAMPLE_SUFFIX_ITEM 2 4746 4747 /** 4748 * Split the sample flow. 4749 * 4750 * As sample flow will split to two sub flow, sample flow with 4751 * sample action, the other actions will move to new suffix flow. 4752 * 4753 * Also add unique tag id with tag action in the sample flow, 4754 * the same tag id will be as match in the suffix flow. 4755 * 4756 * @param dev 4757 * Pointer to Ethernet device. 4758 * @param[in] fdb_tx 4759 * FDB egress flow flag. 4760 * @param[out] sfx_items 4761 * Suffix flow match items (list terminated by the END pattern item). 4762 * @param[in] actions 4763 * Associated actions (list terminated by the END action). 4764 * @param[out] actions_sfx 4765 * Suffix flow actions. 4766 * @param[out] actions_pre 4767 * Prefix flow actions. 4768 * @param[in] actions_n 4769 * The total number of actions. 4770 * @param[in] sample_action_pos 4771 * The sample action position. 4772 * @param[in] qrss_action_pos 4773 * The Queue/RSS action position. 4774 * @param[out] error 4775 * Perform verbose error reporting if not NULL. 4776 * 4777 * @return 4778 * 0 on success, or unique flow_id, a negative errno value 4779 * otherwise and rte_errno is set. 4780 */ 4781 static int 4782 flow_sample_split_prep(struct rte_eth_dev *dev, 4783 uint32_t fdb_tx, 4784 struct rte_flow_item sfx_items[], 4785 const struct rte_flow_action actions[], 4786 struct rte_flow_action actions_sfx[], 4787 struct rte_flow_action actions_pre[], 4788 int actions_n, 4789 int sample_action_pos, 4790 int qrss_action_pos, 4791 struct rte_flow_error *error) 4792 { 4793 struct mlx5_priv *priv = dev->data->dev_private; 4794 struct mlx5_rte_flow_action_set_tag *set_tag; 4795 struct mlx5_rte_flow_item_tag *tag_spec; 4796 struct mlx5_rte_flow_item_tag *tag_mask; 4797 uint32_t tag_id = 0; 4798 int index; 4799 int ret; 4800 4801 if (sample_action_pos < 0) 4802 return rte_flow_error_set(error, EINVAL, 4803 RTE_FLOW_ERROR_TYPE_ACTION, 4804 NULL, "invalid position of sample " 4805 "action in list"); 4806 if (!fdb_tx) { 4807 /* Prepare the prefix tag action. */ 4808 set_tag = (void *)(actions_pre + actions_n + 1); 4809 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 4810 if (ret < 0) 4811 return ret; 4812 set_tag->id = ret; 4813 mlx5_ipool_malloc(priv->sh->ipool 4814 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 4815 set_tag->data = tag_id; 4816 /* Prepare the suffix subflow items. */ 4817 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 4818 tag_spec->data = tag_id; 4819 tag_spec->id = set_tag->id; 4820 tag_mask = tag_spec + 1; 4821 tag_mask->data = UINT32_MAX; 4822 sfx_items[0] = (struct rte_flow_item){ 4823 .type = (enum rte_flow_item_type) 4824 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4825 .spec = tag_spec, 4826 .last = NULL, 4827 .mask = tag_mask, 4828 }; 4829 sfx_items[1] = (struct rte_flow_item){ 4830 .type = (enum rte_flow_item_type) 4831 RTE_FLOW_ITEM_TYPE_END, 4832 }; 4833 } 4834 /* Prepare the actions for prefix and suffix flow. */ 4835 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 4836 index = qrss_action_pos; 4837 /* Put the preceding the Queue/RSS action into prefix flow. */ 4838 if (index != 0) 4839 memcpy(actions_pre, actions, 4840 sizeof(struct rte_flow_action) * index); 4841 /* Put others preceding the sample action into prefix flow. */ 4842 if (sample_action_pos > index + 1) 4843 memcpy(actions_pre + index, actions + index + 1, 4844 sizeof(struct rte_flow_action) * 4845 (sample_action_pos - index - 1)); 4846 index = sample_action_pos - 1; 4847 /* Put Queue/RSS action into Suffix flow. */ 4848 memcpy(actions_sfx, actions + qrss_action_pos, 4849 sizeof(struct rte_flow_action)); 4850 actions_sfx++; 4851 } else { 4852 index = sample_action_pos; 4853 if (index != 0) 4854 memcpy(actions_pre, actions, 4855 sizeof(struct rte_flow_action) * index); 4856 } 4857 /* Add the extra tag action for NIC-RX and E-Switch ingress. */ 4858 if (!fdb_tx) { 4859 actions_pre[index++] = 4860 (struct rte_flow_action){ 4861 .type = (enum rte_flow_action_type) 4862 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4863 .conf = set_tag, 4864 }; 4865 } 4866 memcpy(actions_pre + index, actions + sample_action_pos, 4867 sizeof(struct rte_flow_action)); 4868 index += 1; 4869 actions_pre[index] = (struct rte_flow_action){ 4870 .type = (enum rte_flow_action_type) 4871 RTE_FLOW_ACTION_TYPE_END, 4872 }; 4873 /* Put the actions after sample into Suffix flow. */ 4874 memcpy(actions_sfx, actions + sample_action_pos + 1, 4875 sizeof(struct rte_flow_action) * 4876 (actions_n - sample_action_pos - 1)); 4877 return tag_id; 4878 } 4879 4880 /** 4881 * The splitting for metadata feature. 4882 * 4883 * - Q/RSS action on NIC Rx should be split in order to pass by 4884 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4885 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4886 * 4887 * - All the actions on NIC Tx should have a mreg copy action to 4888 * copy reg_a from WQE to reg_c[0]. 4889 * 4890 * @param dev 4891 * Pointer to Ethernet device. 4892 * @param[in] flow 4893 * Parent flow structure pointer. 4894 * @param[in] prefix_layers 4895 * Prefix flow layer flags. 4896 * @param[in] prefix_mark 4897 * Prefix subflow mark flag, may be 0. 4898 * @param[in] attr 4899 * Flow rule attributes. 4900 * @param[in] items 4901 * Pattern specification (list terminated by the END pattern item). 4902 * @param[in] actions 4903 * Associated actions (list terminated by the END action). 4904 * @param[in] external 4905 * This flow rule is created by request external to PMD. 4906 * @param[in] flow_idx 4907 * This memory pool index to the flow. 4908 * @param[out] error 4909 * Perform verbose error reporting if not NULL. 4910 * @return 4911 * 0 on success, negative value otherwise 4912 */ 4913 static int 4914 flow_create_split_metadata(struct rte_eth_dev *dev, 4915 struct rte_flow *flow, 4916 uint64_t prefix_layers, 4917 uint32_t prefix_mark, 4918 const struct rte_flow_attr *attr, 4919 const struct rte_flow_item items[], 4920 const struct rte_flow_action actions[], 4921 bool external, uint32_t flow_idx, 4922 struct rte_flow_error *error) 4923 { 4924 struct mlx5_priv *priv = dev->data->dev_private; 4925 struct mlx5_dev_config *config = &priv->config; 4926 const struct rte_flow_action *qrss = NULL; 4927 struct rte_flow_action *ext_actions = NULL; 4928 struct mlx5_flow *dev_flow = NULL; 4929 uint32_t qrss_id = 0; 4930 int mtr_sfx = 0; 4931 size_t act_size; 4932 int actions_n; 4933 int encap_idx; 4934 int ret; 4935 4936 /* Check whether extensive metadata feature is engaged. */ 4937 if (!config->dv_flow_en || 4938 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4939 !mlx5_flow_ext_mreg_supported(dev)) 4940 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 4941 prefix_mark, attr, items, 4942 actions, external, flow_idx, 4943 error); 4944 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 4945 &encap_idx); 4946 if (qrss) { 4947 /* Exclude hairpin flows from splitting. */ 4948 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 4949 const struct rte_flow_action_queue *queue; 4950 4951 queue = qrss->conf; 4952 if (mlx5_rxq_get_type(dev, queue->index) == 4953 MLX5_RXQ_TYPE_HAIRPIN) 4954 qrss = NULL; 4955 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 4956 const struct rte_flow_action_rss *rss; 4957 4958 rss = qrss->conf; 4959 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 4960 MLX5_RXQ_TYPE_HAIRPIN) 4961 qrss = NULL; 4962 } 4963 } 4964 if (qrss) { 4965 /* Check if it is in meter suffix table. */ 4966 mtr_sfx = attr->group == (attr->transfer ? 4967 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4968 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 4969 /* 4970 * Q/RSS action on NIC Rx should be split in order to pass by 4971 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4972 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4973 */ 4974 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4975 sizeof(struct rte_flow_action_set_tag) + 4976 sizeof(struct rte_flow_action_jump); 4977 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4978 SOCKET_ID_ANY); 4979 if (!ext_actions) 4980 return rte_flow_error_set(error, ENOMEM, 4981 RTE_FLOW_ERROR_TYPE_ACTION, 4982 NULL, "no memory to split " 4983 "metadata flow"); 4984 /* 4985 * If we are the suffix flow of meter, tag already exist. 4986 * Set the tag action to void. 4987 */ 4988 if (mtr_sfx) 4989 ext_actions[qrss - actions].type = 4990 RTE_FLOW_ACTION_TYPE_VOID; 4991 else 4992 ext_actions[qrss - actions].type = 4993 (enum rte_flow_action_type) 4994 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4995 /* 4996 * Create the new actions list with removed Q/RSS action 4997 * and appended set tag and jump to register copy table 4998 * (RX_CP_TBL). We should preallocate unique tag ID here 4999 * in advance, because it is needed for set tag action. 5000 */ 5001 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5002 qrss, actions_n, error); 5003 if (!mtr_sfx && !qrss_id) { 5004 ret = -rte_errno; 5005 goto exit; 5006 } 5007 } else if (attr->egress && !attr->transfer) { 5008 /* 5009 * All the actions on NIC Tx should have a metadata register 5010 * copy action to copy reg_a from WQE to reg_c[meta] 5011 */ 5012 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5013 sizeof(struct mlx5_flow_action_copy_mreg); 5014 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5015 SOCKET_ID_ANY); 5016 if (!ext_actions) 5017 return rte_flow_error_set(error, ENOMEM, 5018 RTE_FLOW_ERROR_TYPE_ACTION, 5019 NULL, "no memory to split " 5020 "metadata flow"); 5021 /* Create the action list appended with copy register. */ 5022 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5023 actions_n, error, encap_idx); 5024 if (ret < 0) 5025 goto exit; 5026 } 5027 /* Add the unmodified original or prefix subflow. */ 5028 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, 5029 prefix_mark, attr, 5030 items, ext_actions ? ext_actions : 5031 actions, external, flow_idx, error); 5032 if (ret < 0) 5033 goto exit; 5034 MLX5_ASSERT(dev_flow); 5035 if (qrss) { 5036 const struct rte_flow_attr q_attr = { 5037 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5038 .ingress = 1, 5039 }; 5040 /* Internal PMD action to set register. */ 5041 struct mlx5_rte_flow_item_tag q_tag_spec = { 5042 .data = qrss_id, 5043 .id = REG_NON, 5044 }; 5045 struct rte_flow_item q_items[] = { 5046 { 5047 .type = (enum rte_flow_item_type) 5048 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5049 .spec = &q_tag_spec, 5050 .last = NULL, 5051 .mask = NULL, 5052 }, 5053 { 5054 .type = RTE_FLOW_ITEM_TYPE_END, 5055 }, 5056 }; 5057 struct rte_flow_action q_actions[] = { 5058 { 5059 .type = qrss->type, 5060 .conf = qrss->conf, 5061 }, 5062 { 5063 .type = RTE_FLOW_ACTION_TYPE_END, 5064 }, 5065 }; 5066 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5067 5068 /* 5069 * Configure the tag item only if there is no meter subflow. 5070 * Since tag is already marked in the meter suffix subflow 5071 * we can just use the meter suffix items as is. 5072 */ 5073 if (qrss_id) { 5074 /* Not meter subflow. */ 5075 MLX5_ASSERT(!mtr_sfx); 5076 /* 5077 * Put unique id in prefix flow due to it is destroyed 5078 * after suffix flow and id will be freed after there 5079 * is no actual flows with this id and identifier 5080 * reallocation becomes possible (for example, for 5081 * other flows in other threads). 5082 */ 5083 dev_flow->handle->split_flow_id = qrss_id; 5084 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5085 error); 5086 if (ret < 0) 5087 goto exit; 5088 q_tag_spec.id = ret; 5089 } 5090 dev_flow = NULL; 5091 /* Add suffix subflow to execute Q/RSS. */ 5092 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 0, 5093 &q_attr, mtr_sfx ? items : 5094 q_items, q_actions, 5095 external, flow_idx, error); 5096 if (ret < 0) 5097 goto exit; 5098 /* qrss ID should be freed if failed. */ 5099 qrss_id = 0; 5100 MLX5_ASSERT(dev_flow); 5101 } 5102 5103 exit: 5104 /* 5105 * We do not destroy the partially created sub_flows in case of error. 5106 * These ones are included into parent flow list and will be destroyed 5107 * by flow_drv_destroy. 5108 */ 5109 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5110 qrss_id); 5111 mlx5_free(ext_actions); 5112 return ret; 5113 } 5114 5115 /** 5116 * The splitting for meter feature. 5117 * 5118 * - The meter flow will be split to two flows as prefix and 5119 * suffix flow. The packets make sense only it pass the prefix 5120 * meter action. 5121 * 5122 * - Reg_C_5 is used for the packet to match betweend prefix and 5123 * suffix flow. 5124 * 5125 * @param dev 5126 * Pointer to Ethernet device. 5127 * @param[in] flow 5128 * Parent flow structure pointer. 5129 * @param[in] prefix_layers 5130 * Prefix subflow layers, may be 0. 5131 * @param[in] prefix_mark 5132 * Prefix subflow mark flag, may be 0. 5133 * @param[in] attr 5134 * Flow rule attributes. 5135 * @param[in] items 5136 * Pattern specification (list terminated by the END pattern item). 5137 * @param[in] actions 5138 * Associated actions (list terminated by the END action). 5139 * @param[in] external 5140 * This flow rule is created by request external to PMD. 5141 * @param[in] flow_idx 5142 * This memory pool index to the flow. 5143 * @param[out] error 5144 * Perform verbose error reporting if not NULL. 5145 * @return 5146 * 0 on success, negative value otherwise 5147 */ 5148 static int 5149 flow_create_split_meter(struct rte_eth_dev *dev, 5150 struct rte_flow *flow, 5151 uint64_t prefix_layers, 5152 uint32_t prefix_mark, 5153 const struct rte_flow_attr *attr, 5154 const struct rte_flow_item items[], 5155 const struct rte_flow_action actions[], 5156 bool external, uint32_t flow_idx, 5157 struct rte_flow_error *error) 5158 { 5159 struct mlx5_priv *priv = dev->data->dev_private; 5160 struct rte_flow_action *sfx_actions = NULL; 5161 struct rte_flow_action *pre_actions = NULL; 5162 struct rte_flow_item *sfx_items = NULL; 5163 struct mlx5_flow *dev_flow = NULL; 5164 struct rte_flow_attr sfx_attr = *attr; 5165 uint32_t mtr = 0; 5166 uint32_t mtr_tag_id = 0; 5167 size_t act_size; 5168 size_t item_size; 5169 int actions_n = 0; 5170 int ret; 5171 5172 if (priv->mtr_en) 5173 actions_n = flow_check_meter_action(actions, &mtr); 5174 if (mtr) { 5175 /* The five prefix actions: meter, decap, encap, tag, end. */ 5176 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 5177 sizeof(struct mlx5_rte_flow_action_set_tag); 5178 /* tag, vlan, port id, end. */ 5179 #define METER_SUFFIX_ITEM 4 5180 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 5181 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5182 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 5183 0, SOCKET_ID_ANY); 5184 if (!sfx_actions) 5185 return rte_flow_error_set(error, ENOMEM, 5186 RTE_FLOW_ERROR_TYPE_ACTION, 5187 NULL, "no memory to split " 5188 "meter flow"); 5189 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 5190 act_size); 5191 pre_actions = sfx_actions + actions_n; 5192 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 5193 actions, sfx_actions, 5194 pre_actions); 5195 if (!mtr_tag_id) { 5196 ret = -rte_errno; 5197 goto exit; 5198 } 5199 /* Add the prefix subflow. */ 5200 ret = flow_create_split_inner(dev, flow, &dev_flow, 5201 prefix_layers, 0, 5202 attr, items, 5203 pre_actions, external, 5204 flow_idx, error); 5205 if (ret) { 5206 ret = -rte_errno; 5207 goto exit; 5208 } 5209 dev_flow->handle->split_flow_id = mtr_tag_id; 5210 /* Setting the sfx group atrr. */ 5211 sfx_attr.group = sfx_attr.transfer ? 5212 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 5213 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 5214 } 5215 /* Add the prefix subflow. */ 5216 ret = flow_create_split_metadata(dev, flow, dev_flow ? 5217 flow_get_prefix_layer_flags(dev_flow) : 5218 prefix_layers, dev_flow ? 5219 dev_flow->handle->mark : prefix_mark, 5220 &sfx_attr, sfx_items ? 5221 sfx_items : items, 5222 sfx_actions ? sfx_actions : actions, 5223 external, flow_idx, error); 5224 exit: 5225 if (sfx_actions) 5226 mlx5_free(sfx_actions); 5227 return ret; 5228 } 5229 5230 /** 5231 * The splitting for sample feature. 5232 * 5233 * Once Sample action is detected in the action list, the flow actions should 5234 * be split into prefix sub flow and suffix sub flow. 5235 * 5236 * The original items remain in the prefix sub flow, all actions preceding the 5237 * sample action and the sample action itself will be copied to the prefix 5238 * sub flow, the actions following the sample action will be copied to the 5239 * suffix sub flow, Queue action always be located in the suffix sub flow. 5240 * 5241 * In order to make the packet from prefix sub flow matches with suffix sub 5242 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5243 * flow uses tag item with the unique flow id. 5244 * 5245 * @param dev 5246 * Pointer to Ethernet device. 5247 * @param[in] flow 5248 * Parent flow structure pointer. 5249 * @param[in] attr 5250 * Flow rule attributes. 5251 * @param[in] items 5252 * Pattern specification (list terminated by the END pattern item). 5253 * @param[in] actions 5254 * Associated actions (list terminated by the END action). 5255 * @param[in] external 5256 * This flow rule is created by request external to PMD. 5257 * @param[in] flow_idx 5258 * This memory pool index to the flow. 5259 * @param[out] error 5260 * Perform verbose error reporting if not NULL. 5261 * @return 5262 * 0 on success, negative value otherwise 5263 */ 5264 static int 5265 flow_create_split_sample(struct rte_eth_dev *dev, 5266 struct rte_flow *flow, 5267 const struct rte_flow_attr *attr, 5268 const struct rte_flow_item items[], 5269 const struct rte_flow_action actions[], 5270 bool external, uint32_t flow_idx, 5271 struct rte_flow_error *error) 5272 { 5273 struct mlx5_priv *priv = dev->data->dev_private; 5274 struct rte_flow_action *sfx_actions = NULL; 5275 struct rte_flow_action *pre_actions = NULL; 5276 struct rte_flow_item *sfx_items = NULL; 5277 struct mlx5_flow *dev_flow = NULL; 5278 struct rte_flow_attr sfx_attr = *attr; 5279 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5280 struct mlx5_flow_dv_sample_resource *sample_res; 5281 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5282 struct mlx5_flow_tbl_resource *sfx_tbl; 5283 union mlx5_flow_tbl_key sfx_table_key; 5284 #endif 5285 size_t act_size; 5286 size_t item_size; 5287 uint32_t fdb_tx = 0; 5288 int32_t tag_id = 0; 5289 int actions_n = 0; 5290 int sample_action_pos; 5291 int qrss_action_pos; 5292 int ret = 0; 5293 5294 if (priv->sampler_en) 5295 actions_n = flow_check_match_action(actions, attr, 5296 RTE_FLOW_ACTION_TYPE_SAMPLE, 5297 &sample_action_pos, &qrss_action_pos); 5298 if (actions_n) { 5299 /* The prefix actions must includes sample, tag, end. */ 5300 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5301 + sizeof(struct mlx5_rte_flow_action_set_tag); 5302 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5303 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5304 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5305 item_size), 0, SOCKET_ID_ANY); 5306 if (!sfx_actions) 5307 return rte_flow_error_set(error, ENOMEM, 5308 RTE_FLOW_ERROR_TYPE_ACTION, 5309 NULL, "no memory to split " 5310 "sample flow"); 5311 /* The representor_id is -1 for uplink. */ 5312 fdb_tx = (attr->transfer && priv->representor_id != -1); 5313 if (!fdb_tx) 5314 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5315 + act_size); 5316 pre_actions = sfx_actions + actions_n; 5317 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items, 5318 actions, sfx_actions, 5319 pre_actions, actions_n, 5320 sample_action_pos, 5321 qrss_action_pos, error); 5322 if (tag_id < 0 || (!fdb_tx && !tag_id)) { 5323 ret = -rte_errno; 5324 goto exit; 5325 } 5326 /* Add the prefix subflow. */ 5327 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, 0, attr, 5328 items, pre_actions, external, 5329 flow_idx, error); 5330 if (ret) { 5331 ret = -rte_errno; 5332 goto exit; 5333 } 5334 dev_flow->handle->split_flow_id = tag_id; 5335 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5336 /* Set the sfx group attr. */ 5337 sample_res = (struct mlx5_flow_dv_sample_resource *) 5338 dev_flow->dv.sample_res; 5339 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5340 sample_res->normal_path_tbl; 5341 sfx_tbl_data = container_of(sfx_tbl, 5342 struct mlx5_flow_tbl_data_entry, tbl); 5343 sfx_table_key.v64 = sfx_tbl_data->entry.key; 5344 sfx_attr.group = sfx_attr.transfer ? 5345 (sfx_table_key.table_id - 1) : 5346 sfx_table_key.table_id; 5347 #endif 5348 } 5349 /* Add the suffix subflow. */ 5350 ret = flow_create_split_meter(dev, flow, dev_flow ? 5351 flow_get_prefix_layer_flags(dev_flow) : 0, 5352 dev_flow ? dev_flow->handle->mark : 0, 5353 &sfx_attr, sfx_items ? sfx_items : items, 5354 sfx_actions ? sfx_actions : actions, 5355 external, flow_idx, error); 5356 exit: 5357 if (sfx_actions) 5358 mlx5_free(sfx_actions); 5359 return ret; 5360 } 5361 5362 /** 5363 * Split the flow to subflow set. The splitters might be linked 5364 * in the chain, like this: 5365 * flow_create_split_outer() calls: 5366 * flow_create_split_meter() calls: 5367 * flow_create_split_metadata(meter_subflow_0) calls: 5368 * flow_create_split_inner(metadata_subflow_0) 5369 * flow_create_split_inner(metadata_subflow_1) 5370 * flow_create_split_inner(metadata_subflow_2) 5371 * flow_create_split_metadata(meter_subflow_1) calls: 5372 * flow_create_split_inner(metadata_subflow_0) 5373 * flow_create_split_inner(metadata_subflow_1) 5374 * flow_create_split_inner(metadata_subflow_2) 5375 * 5376 * This provide flexible way to add new levels of flow splitting. 5377 * The all of successfully created subflows are included to the 5378 * parent flow dev_flow list. 5379 * 5380 * @param dev 5381 * Pointer to Ethernet device. 5382 * @param[in] flow 5383 * Parent flow structure pointer. 5384 * @param[in] attr 5385 * Flow rule attributes. 5386 * @param[in] items 5387 * Pattern specification (list terminated by the END pattern item). 5388 * @param[in] actions 5389 * Associated actions (list terminated by the END action). 5390 * @param[in] external 5391 * This flow rule is created by request external to PMD. 5392 * @param[in] flow_idx 5393 * This memory pool index to the flow. 5394 * @param[out] error 5395 * Perform verbose error reporting if not NULL. 5396 * @return 5397 * 0 on success, negative value otherwise 5398 */ 5399 static int 5400 flow_create_split_outer(struct rte_eth_dev *dev, 5401 struct rte_flow *flow, 5402 const struct rte_flow_attr *attr, 5403 const struct rte_flow_item items[], 5404 const struct rte_flow_action actions[], 5405 bool external, uint32_t flow_idx, 5406 struct rte_flow_error *error) 5407 { 5408 int ret; 5409 5410 ret = flow_create_split_sample(dev, flow, attr, items, 5411 actions, external, flow_idx, error); 5412 MLX5_ASSERT(ret <= 0); 5413 return ret; 5414 } 5415 5416 static struct mlx5_flow_tunnel * 5417 flow_tunnel_from_rule(struct rte_eth_dev *dev, 5418 const struct rte_flow_attr *attr, 5419 const struct rte_flow_item items[], 5420 const struct rte_flow_action actions[]) 5421 { 5422 struct mlx5_flow_tunnel *tunnel; 5423 5424 #pragma GCC diagnostic push 5425 #pragma GCC diagnostic ignored "-Wcast-qual" 5426 if (is_flow_tunnel_match_rule(dev, attr, items, actions)) 5427 tunnel = (struct mlx5_flow_tunnel *)items[0].spec; 5428 else if (is_flow_tunnel_steer_rule(dev, attr, items, actions)) 5429 tunnel = (struct mlx5_flow_tunnel *)actions[0].conf; 5430 else 5431 tunnel = NULL; 5432 #pragma GCC diagnostic pop 5433 5434 return tunnel; 5435 } 5436 5437 /** 5438 * Adjust flow RSS workspace if needed. 5439 * 5440 * @param wks 5441 * Pointer to thread flow work space. 5442 * @param rss_desc 5443 * Pointer to RSS descriptor. 5444 * @param[in] nrssq_num 5445 * New RSS queue number. 5446 * 5447 * @return 5448 * 0 on success, -1 otherwise and rte_errno is set. 5449 */ 5450 static int 5451 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 5452 struct mlx5_flow_rss_desc *rss_desc, 5453 uint32_t nrssq_num) 5454 { 5455 bool fidx = !!wks->flow_idx; 5456 5457 if (likely(nrssq_num <= wks->rssq_num[fidx])) 5458 return 0; 5459 rss_desc->queue = realloc(rss_desc->queue, 5460 sizeof(rss_desc->queue[0]) * RTE_ALIGN(nrssq_num, 2)); 5461 if (!rss_desc->queue) { 5462 rte_errno = ENOMEM; 5463 return -1; 5464 } 5465 wks->rssq_num[fidx] = RTE_ALIGN(nrssq_num, 2); 5466 return 0; 5467 } 5468 5469 /** 5470 * Create a flow and add it to @p list. 5471 * 5472 * @param dev 5473 * Pointer to Ethernet device. 5474 * @param list 5475 * Pointer to a TAILQ flow list. If this parameter NULL, 5476 * no list insertion occurred, flow is just created, 5477 * this is caller's responsibility to track the 5478 * created flow. 5479 * @param[in] attr 5480 * Flow rule attributes. 5481 * @param[in] items 5482 * Pattern specification (list terminated by the END pattern item). 5483 * @param[in] actions 5484 * Associated actions (list terminated by the END action). 5485 * @param[in] external 5486 * This flow rule is created by request external to PMD. 5487 * @param[out] error 5488 * Perform verbose error reporting if not NULL. 5489 * 5490 * @return 5491 * A flow index on success, 0 otherwise and rte_errno is set. 5492 */ 5493 static uint32_t 5494 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 5495 const struct rte_flow_attr *attr, 5496 const struct rte_flow_item items[], 5497 const struct rte_flow_action original_actions[], 5498 bool external, struct rte_flow_error *error) 5499 { 5500 struct mlx5_priv *priv = dev->data->dev_private; 5501 struct rte_flow *flow = NULL; 5502 struct mlx5_flow *dev_flow; 5503 const struct rte_flow_action_rss *rss; 5504 struct mlx5_translated_shared_action 5505 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5506 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5507 union { 5508 struct mlx5_flow_expand_rss buf; 5509 uint8_t buffer[2048]; 5510 } expand_buffer; 5511 union { 5512 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5513 uint8_t buffer[2048]; 5514 } actions_rx; 5515 union { 5516 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5517 uint8_t buffer[2048]; 5518 } actions_hairpin_tx; 5519 union { 5520 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 5521 uint8_t buffer[2048]; 5522 } items_tx; 5523 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 5524 struct mlx5_flow_rss_desc *rss_desc; 5525 const struct rte_flow_action *p_actions_rx; 5526 uint32_t i; 5527 uint32_t idx = 0; 5528 int hairpin_flow; 5529 struct rte_flow_attr attr_tx = { .priority = 0 }; 5530 struct rte_flow_attr attr_factor = {0}; 5531 const struct rte_flow_action *actions; 5532 struct rte_flow_action *translated_actions = NULL; 5533 struct mlx5_flow_tunnel *tunnel; 5534 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 5535 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5536 bool fidx = !!wks->flow_idx; 5537 int ret; 5538 5539 MLX5_ASSERT(wks); 5540 rss_desc = &wks->rss_desc[fidx]; 5541 ret = flow_shared_actions_translate(original_actions, 5542 shared_actions, 5543 &shared_actions_n, 5544 &translated_actions, error); 5545 if (ret < 0) { 5546 MLX5_ASSERT(translated_actions == NULL); 5547 return 0; 5548 } 5549 actions = translated_actions ? translated_actions : original_actions; 5550 memcpy((void *)&attr_factor, (const void *)attr, sizeof(*attr)); 5551 p_actions_rx = actions; 5552 hairpin_flow = flow_check_hairpin_split(dev, &attr_factor, actions); 5553 ret = flow_drv_validate(dev, &attr_factor, items, p_actions_rx, 5554 external, hairpin_flow, error); 5555 if (ret < 0) 5556 goto error_before_hairpin_split; 5557 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 5558 if (!flow) { 5559 rte_errno = ENOMEM; 5560 goto error_before_hairpin_split; 5561 } 5562 if (hairpin_flow > 0) { 5563 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 5564 rte_errno = EINVAL; 5565 goto error_before_hairpin_split; 5566 } 5567 flow_hairpin_split(dev, actions, actions_rx.actions, 5568 actions_hairpin_tx.actions, items_tx.items, 5569 idx); 5570 p_actions_rx = actions_rx.actions; 5571 } 5572 flow->drv_type = flow_get_drv_type(dev, &attr_factor); 5573 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 5574 flow->drv_type < MLX5_FLOW_TYPE_MAX); 5575 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 5576 rss = flow_get_rss_action(p_actions_rx); 5577 if (rss) { 5578 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 5579 return 0; 5580 /* 5581 * The following information is required by 5582 * mlx5_flow_hashfields_adjust() in advance. 5583 */ 5584 rss_desc->level = rss->level; 5585 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 5586 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 5587 } 5588 flow->dev_handles = 0; 5589 if (rss && rss->types) { 5590 unsigned int graph_root; 5591 5592 graph_root = find_graph_root(items, rss->level); 5593 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 5594 items, rss->types, 5595 mlx5_support_expansion, graph_root); 5596 MLX5_ASSERT(ret > 0 && 5597 (unsigned int)ret < sizeof(expand_buffer.buffer)); 5598 } else { 5599 buf->entries = 1; 5600 buf->entry[0].pattern = (void *)(uintptr_t)items; 5601 } 5602 flow->shared_rss = flow_get_shared_rss_action(shared_actions, 5603 shared_actions_n); 5604 /* 5605 * Record the start index when there is a nested call. All sub-flows 5606 * need to be translated before another calling. 5607 * No need to use ping-pong buffer to save memory here. 5608 */ 5609 if (fidx) { 5610 MLX5_ASSERT(!wks->flow_nested_idx); 5611 wks->flow_nested_idx = fidx; 5612 } 5613 for (i = 0; i < buf->entries; ++i) { 5614 /* 5615 * The splitter may create multiple dev_flows, 5616 * depending on configuration. In the simplest 5617 * case it just creates unmodified original flow. 5618 */ 5619 ret = flow_create_split_outer(dev, flow, &attr_factor, 5620 buf->entry[i].pattern, 5621 p_actions_rx, external, idx, 5622 error); 5623 if (ret < 0) 5624 goto error; 5625 if (is_flow_tunnel_steer_rule(dev, attr, 5626 buf->entry[i].pattern, 5627 p_actions_rx)) { 5628 ret = flow_tunnel_add_default_miss(dev, flow, attr, 5629 p_actions_rx, 5630 idx, 5631 &default_miss_ctx, 5632 error); 5633 if (ret < 0) { 5634 mlx5_free(default_miss_ctx.queue); 5635 goto error; 5636 } 5637 } 5638 } 5639 /* Create the tx flow. */ 5640 if (hairpin_flow) { 5641 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 5642 attr_tx.ingress = 0; 5643 attr_tx.egress = 1; 5644 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 5645 actions_hairpin_tx.actions, 5646 idx, error); 5647 if (!dev_flow) 5648 goto error; 5649 dev_flow->flow = flow; 5650 dev_flow->external = 0; 5651 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5652 dev_flow->handle, next); 5653 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 5654 items_tx.items, 5655 actions_hairpin_tx.actions, error); 5656 if (ret < 0) 5657 goto error; 5658 } 5659 /* 5660 * Update the metadata register copy table. If extensive 5661 * metadata feature is enabled and registers are supported 5662 * we might create the extra rte_flow for each unique 5663 * MARK/FLAG action ID. 5664 * 5665 * The table is updated for ingress Flows only, because 5666 * the egress Flows belong to the different device and 5667 * copy table should be updated in peer NIC Rx domain. 5668 */ 5669 if (attr_factor.ingress && 5670 (external || attr_factor.group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 5671 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 5672 if (ret) 5673 goto error; 5674 } 5675 /* 5676 * If the flow is external (from application) OR device is started, then 5677 * the flow will be applied immediately. 5678 */ 5679 if (external || dev->data->dev_started) { 5680 ret = flow_drv_apply(dev, flow, error); 5681 if (ret < 0) 5682 goto error; 5683 } 5684 if (list) { 5685 rte_spinlock_lock(&priv->flow_list_lock); 5686 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 5687 flow, next); 5688 rte_spinlock_unlock(&priv->flow_list_lock); 5689 } 5690 flow_rxq_flags_set(dev, flow); 5691 rte_free(translated_actions); 5692 /* Nested flow creation index recovery. */ 5693 wks->flow_idx = wks->flow_nested_idx; 5694 if (wks->flow_nested_idx) 5695 wks->flow_nested_idx = 0; 5696 tunnel = flow_tunnel_from_rule(dev, attr, items, actions); 5697 if (tunnel) { 5698 flow->tunnel = 1; 5699 flow->tunnel_id = tunnel->tunnel_id; 5700 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 5701 mlx5_free(default_miss_ctx.queue); 5702 } 5703 return idx; 5704 error: 5705 MLX5_ASSERT(flow); 5706 ret = rte_errno; /* Save rte_errno before cleanup. */ 5707 flow_mreg_del_copy_action(dev, flow); 5708 flow_drv_destroy(dev, flow); 5709 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 5710 rte_errno = ret; /* Restore rte_errno. */ 5711 ret = rte_errno; 5712 rte_errno = ret; 5713 wks->flow_idx = wks->flow_nested_idx; 5714 if (wks->flow_nested_idx) 5715 wks->flow_nested_idx = 0; 5716 error_before_hairpin_split: 5717 rte_free(translated_actions); 5718 return 0; 5719 } 5720 5721 /** 5722 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 5723 * incoming packets to table 1. 5724 * 5725 * Other flow rules, requested for group n, will be created in 5726 * e-switch table n+1. 5727 * Jump action to e-switch group n will be created to group n+1. 5728 * 5729 * Used when working in switchdev mode, to utilise advantages of table 1 5730 * and above. 5731 * 5732 * @param dev 5733 * Pointer to Ethernet device. 5734 * 5735 * @return 5736 * Pointer to flow on success, NULL otherwise and rte_errno is set. 5737 */ 5738 struct rte_flow * 5739 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 5740 { 5741 const struct rte_flow_attr attr = { 5742 .group = 0, 5743 .priority = 0, 5744 .ingress = 1, 5745 .egress = 0, 5746 .transfer = 1, 5747 }; 5748 const struct rte_flow_item pattern = { 5749 .type = RTE_FLOW_ITEM_TYPE_END, 5750 }; 5751 struct rte_flow_action_jump jump = { 5752 .group = 1, 5753 }; 5754 const struct rte_flow_action actions[] = { 5755 { 5756 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5757 .conf = &jump, 5758 }, 5759 { 5760 .type = RTE_FLOW_ACTION_TYPE_END, 5761 }, 5762 }; 5763 struct mlx5_priv *priv = dev->data->dev_private; 5764 struct rte_flow_error error; 5765 5766 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 5767 &attr, &pattern, 5768 actions, false, &error); 5769 } 5770 5771 /** 5772 * Validate a flow supported by the NIC. 5773 * 5774 * @see rte_flow_validate() 5775 * @see rte_flow_ops 5776 */ 5777 int 5778 mlx5_flow_validate(struct rte_eth_dev *dev, 5779 const struct rte_flow_attr *attr, 5780 const struct rte_flow_item items[], 5781 const struct rte_flow_action original_actions[], 5782 struct rte_flow_error *error) 5783 { 5784 int hairpin_flow; 5785 struct mlx5_translated_shared_action 5786 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5787 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5788 const struct rte_flow_action *actions; 5789 struct rte_flow_action *translated_actions = NULL; 5790 int ret = flow_shared_actions_translate(original_actions, 5791 shared_actions, 5792 &shared_actions_n, 5793 &translated_actions, error); 5794 5795 if (ret) 5796 return ret; 5797 actions = translated_actions ? translated_actions : original_actions; 5798 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5799 ret = flow_drv_validate(dev, attr, items, actions, 5800 true, hairpin_flow, error); 5801 rte_free(translated_actions); 5802 return ret; 5803 } 5804 5805 /** 5806 * Create a flow. 5807 * 5808 * @see rte_flow_create() 5809 * @see rte_flow_ops 5810 */ 5811 struct rte_flow * 5812 mlx5_flow_create(struct rte_eth_dev *dev, 5813 const struct rte_flow_attr *attr, 5814 const struct rte_flow_item items[], 5815 const struct rte_flow_action actions[], 5816 struct rte_flow_error *error) 5817 { 5818 struct mlx5_priv *priv = dev->data->dev_private; 5819 5820 /* 5821 * If the device is not started yet, it is not allowed to created a 5822 * flow from application. PMD default flows and traffic control flows 5823 * are not affected. 5824 */ 5825 if (unlikely(!dev->data->dev_started)) { 5826 DRV_LOG(DEBUG, "port %u is not started when " 5827 "inserting a flow", dev->data->port_id); 5828 rte_flow_error_set(error, ENODEV, 5829 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5830 NULL, 5831 "port not started"); 5832 return NULL; 5833 } 5834 5835 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 5836 attr, items, actions, true, error); 5837 } 5838 5839 /** 5840 * Destroy a flow in a list. 5841 * 5842 * @param dev 5843 * Pointer to Ethernet device. 5844 * @param list 5845 * Pointer to the Indexed flow list. If this parameter NULL, 5846 * there is no flow removal from the list. Be noted that as 5847 * flow is add to the indexed list, memory of the indexed 5848 * list points to maybe changed as flow destroyed. 5849 * @param[in] flow_idx 5850 * Index of flow to destroy. 5851 */ 5852 static void 5853 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 5854 uint32_t flow_idx) 5855 { 5856 struct mlx5_priv *priv = dev->data->dev_private; 5857 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5858 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5859 [MLX5_IPOOL_RTE_FLOW], flow_idx); 5860 5861 if (!flow) 5862 return; 5863 /* 5864 * Update RX queue flags only if port is started, otherwise it is 5865 * already clean. 5866 */ 5867 if (dev->data->dev_started) 5868 flow_rxq_flags_trim(dev, flow); 5869 flow_drv_destroy(dev, flow); 5870 if (list) { 5871 rte_spinlock_lock(&priv->flow_list_lock); 5872 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 5873 flow_idx, flow, next); 5874 rte_spinlock_unlock(&priv->flow_list_lock); 5875 } 5876 flow_mreg_del_copy_action(dev, flow); 5877 if (flow->fdir) { 5878 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5879 if (priv_fdir_flow->rix_flow == flow_idx) 5880 break; 5881 } 5882 if (priv_fdir_flow) { 5883 LIST_REMOVE(priv_fdir_flow, next); 5884 mlx5_free(priv_fdir_flow->fdir); 5885 mlx5_free(priv_fdir_flow); 5886 } 5887 } 5888 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5889 if (flow->tunnel) { 5890 struct mlx5_flow_tunnel *tunnel; 5891 5892 rte_spinlock_lock(&mlx5_tunnel_hub(dev)->sl); 5893 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 5894 RTE_VERIFY(tunnel); 5895 LIST_REMOVE(tunnel, chain); 5896 rte_spinlock_unlock(&mlx5_tunnel_hub(dev)->sl); 5897 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 5898 mlx5_flow_tunnel_free(dev, tunnel); 5899 } 5900 } 5901 5902 /** 5903 * Destroy all flows. 5904 * 5905 * @param dev 5906 * Pointer to Ethernet device. 5907 * @param list 5908 * Pointer to the Indexed flow list. 5909 * @param active 5910 * If flushing is called avtively. 5911 */ 5912 void 5913 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 5914 { 5915 uint32_t num_flushed = 0; 5916 5917 while (*list) { 5918 flow_list_destroy(dev, list, *list); 5919 num_flushed++; 5920 } 5921 if (active) { 5922 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 5923 dev->data->port_id, num_flushed); 5924 } 5925 } 5926 5927 /** 5928 * Stop all default actions for flows. 5929 * 5930 * @param dev 5931 * Pointer to Ethernet device. 5932 */ 5933 void 5934 mlx5_flow_stop_default(struct rte_eth_dev *dev) 5935 { 5936 flow_mreg_del_default_copy_action(dev); 5937 flow_rxq_flags_clear(dev); 5938 } 5939 5940 /** 5941 * Start all default actions for flows. 5942 * 5943 * @param dev 5944 * Pointer to Ethernet device. 5945 * @return 5946 * 0 on success, a negative errno value otherwise and rte_errno is set. 5947 */ 5948 int 5949 mlx5_flow_start_default(struct rte_eth_dev *dev) 5950 { 5951 struct rte_flow_error error; 5952 5953 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 5954 return flow_mreg_add_default_copy_action(dev, &error); 5955 } 5956 5957 /** 5958 * Release key of thread specific flow workspace data. 5959 */ 5960 static void 5961 flow_release_workspace(void *data) 5962 { 5963 struct mlx5_flow_workspace *wks = data; 5964 5965 if (!wks) 5966 return; 5967 free(wks->rss_desc[0].queue); 5968 free(wks->rss_desc[1].queue); 5969 free(wks); 5970 } 5971 5972 /** 5973 * Initialize key of thread specific flow workspace data. 5974 */ 5975 static void 5976 flow_alloc_workspace(void) 5977 { 5978 if (pthread_key_create(&key_workspace, flow_release_workspace)) 5979 DRV_LOG(ERR, "Can't create flow workspace data thread key."); 5980 } 5981 5982 /** 5983 * Get thread specific flow workspace. 5984 * 5985 * @return pointer to thread specific flowworkspace data, NULL on error. 5986 */ 5987 struct mlx5_flow_workspace* 5988 mlx5_flow_get_thread_workspace(void) 5989 { 5990 struct mlx5_flow_workspace *data; 5991 5992 if (pthread_once(&key_workspace_init, flow_alloc_workspace)) { 5993 DRV_LOG(ERR, "Failed to init flow workspace data thread key."); 5994 return NULL; 5995 } 5996 data = pthread_getspecific(key_workspace); 5997 if (!data) { 5998 data = calloc(1, sizeof(*data)); 5999 if (!data) { 6000 DRV_LOG(ERR, "Failed to allocate flow workspace " 6001 "memory."); 6002 return NULL; 6003 } 6004 data->rss_desc[0].queue = calloc(1, 6005 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6006 if (!data->rss_desc[0].queue) 6007 goto err; 6008 data->rss_desc[1].queue = calloc(1, 6009 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 6010 if (!data->rss_desc[1].queue) 6011 goto err; 6012 data->rssq_num[0] = MLX5_RSSQ_DEFAULT_NUM; 6013 data->rssq_num[1] = MLX5_RSSQ_DEFAULT_NUM; 6014 if (pthread_setspecific(key_workspace, data)) { 6015 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 6016 goto err; 6017 } 6018 } 6019 return data; 6020 err: 6021 if (data->rss_desc[0].queue) 6022 free(data->rss_desc[0].queue); 6023 if (data->rss_desc[1].queue) 6024 free(data->rss_desc[1].queue); 6025 free(data); 6026 return NULL; 6027 } 6028 6029 /** 6030 * Verify the flow list is empty 6031 * 6032 * @param dev 6033 * Pointer to Ethernet device. 6034 * 6035 * @return the number of flows not released. 6036 */ 6037 int 6038 mlx5_flow_verify(struct rte_eth_dev *dev) 6039 { 6040 struct mlx5_priv *priv = dev->data->dev_private; 6041 struct rte_flow *flow; 6042 uint32_t idx; 6043 int ret = 0; 6044 6045 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 6046 flow, next) { 6047 DRV_LOG(DEBUG, "port %u flow %p still referenced", 6048 dev->data->port_id, (void *)flow); 6049 ++ret; 6050 } 6051 return ret; 6052 } 6053 6054 /** 6055 * Enable default hairpin egress flow. 6056 * 6057 * @param dev 6058 * Pointer to Ethernet device. 6059 * @param queue 6060 * The queue index. 6061 * 6062 * @return 6063 * 0 on success, a negative errno value otherwise and rte_errno is set. 6064 */ 6065 int 6066 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 6067 uint32_t queue) 6068 { 6069 struct mlx5_priv *priv = dev->data->dev_private; 6070 const struct rte_flow_attr attr = { 6071 .egress = 1, 6072 .priority = 0, 6073 }; 6074 struct mlx5_rte_flow_item_tx_queue queue_spec = { 6075 .queue = queue, 6076 }; 6077 struct mlx5_rte_flow_item_tx_queue queue_mask = { 6078 .queue = UINT32_MAX, 6079 }; 6080 struct rte_flow_item items[] = { 6081 { 6082 .type = (enum rte_flow_item_type) 6083 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6084 .spec = &queue_spec, 6085 .last = NULL, 6086 .mask = &queue_mask, 6087 }, 6088 { 6089 .type = RTE_FLOW_ITEM_TYPE_END, 6090 }, 6091 }; 6092 struct rte_flow_action_jump jump = { 6093 .group = MLX5_HAIRPIN_TX_TABLE, 6094 }; 6095 struct rte_flow_action actions[2]; 6096 uint32_t flow_idx; 6097 struct rte_flow_error error; 6098 6099 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 6100 actions[0].conf = &jump; 6101 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 6102 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6103 &attr, items, actions, false, &error); 6104 if (!flow_idx) { 6105 DRV_LOG(DEBUG, 6106 "Failed to create ctrl flow: rte_errno(%d)," 6107 " type(%d), message(%s)", 6108 rte_errno, error.type, 6109 error.message ? error.message : " (no stated reason)"); 6110 return -rte_errno; 6111 } 6112 return 0; 6113 } 6114 6115 /** 6116 * Enable a control flow configured from the control plane. 6117 * 6118 * @param dev 6119 * Pointer to Ethernet device. 6120 * @param eth_spec 6121 * An Ethernet flow spec to apply. 6122 * @param eth_mask 6123 * An Ethernet flow mask to apply. 6124 * @param vlan_spec 6125 * A VLAN flow spec to apply. 6126 * @param vlan_mask 6127 * A VLAN flow mask to apply. 6128 * 6129 * @return 6130 * 0 on success, a negative errno value otherwise and rte_errno is set. 6131 */ 6132 int 6133 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 6134 struct rte_flow_item_eth *eth_spec, 6135 struct rte_flow_item_eth *eth_mask, 6136 struct rte_flow_item_vlan *vlan_spec, 6137 struct rte_flow_item_vlan *vlan_mask) 6138 { 6139 struct mlx5_priv *priv = dev->data->dev_private; 6140 const struct rte_flow_attr attr = { 6141 .ingress = 1, 6142 .priority = MLX5_FLOW_PRIO_RSVD, 6143 }; 6144 struct rte_flow_item items[] = { 6145 { 6146 .type = RTE_FLOW_ITEM_TYPE_ETH, 6147 .spec = eth_spec, 6148 .last = NULL, 6149 .mask = eth_mask, 6150 }, 6151 { 6152 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 6153 RTE_FLOW_ITEM_TYPE_END, 6154 .spec = vlan_spec, 6155 .last = NULL, 6156 .mask = vlan_mask, 6157 }, 6158 { 6159 .type = RTE_FLOW_ITEM_TYPE_END, 6160 }, 6161 }; 6162 uint16_t queue[priv->reta_idx_n]; 6163 struct rte_flow_action_rss action_rss = { 6164 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 6165 .level = 0, 6166 .types = priv->rss_conf.rss_hf, 6167 .key_len = priv->rss_conf.rss_key_len, 6168 .queue_num = priv->reta_idx_n, 6169 .key = priv->rss_conf.rss_key, 6170 .queue = queue, 6171 }; 6172 struct rte_flow_action actions[] = { 6173 { 6174 .type = RTE_FLOW_ACTION_TYPE_RSS, 6175 .conf = &action_rss, 6176 }, 6177 { 6178 .type = RTE_FLOW_ACTION_TYPE_END, 6179 }, 6180 }; 6181 uint32_t flow_idx; 6182 struct rte_flow_error error; 6183 unsigned int i; 6184 6185 if (!priv->reta_idx_n || !priv->rxqs_n) { 6186 return 0; 6187 } 6188 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 6189 action_rss.types = 0; 6190 for (i = 0; i != priv->reta_idx_n; ++i) 6191 queue[i] = (*priv->reta_idx)[i]; 6192 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6193 &attr, items, actions, false, &error); 6194 if (!flow_idx) 6195 return -rte_errno; 6196 return 0; 6197 } 6198 6199 /** 6200 * Enable a flow control configured from the control plane. 6201 * 6202 * @param dev 6203 * Pointer to Ethernet device. 6204 * @param eth_spec 6205 * An Ethernet flow spec to apply. 6206 * @param eth_mask 6207 * An Ethernet flow mask to apply. 6208 * 6209 * @return 6210 * 0 on success, a negative errno value otherwise and rte_errno is set. 6211 */ 6212 int 6213 mlx5_ctrl_flow(struct rte_eth_dev *dev, 6214 struct rte_flow_item_eth *eth_spec, 6215 struct rte_flow_item_eth *eth_mask) 6216 { 6217 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 6218 } 6219 6220 /** 6221 * Create default miss flow rule matching lacp traffic 6222 * 6223 * @param dev 6224 * Pointer to Ethernet device. 6225 * @param eth_spec 6226 * An Ethernet flow spec to apply. 6227 * 6228 * @return 6229 * 0 on success, a negative errno value otherwise and rte_errno is set. 6230 */ 6231 int 6232 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 6233 { 6234 struct mlx5_priv *priv = dev->data->dev_private; 6235 /* 6236 * The LACP matching is done by only using ether type since using 6237 * a multicast dst mac causes kernel to give low priority to this flow. 6238 */ 6239 static const struct rte_flow_item_eth lacp_spec = { 6240 .type = RTE_BE16(0x8809), 6241 }; 6242 static const struct rte_flow_item_eth lacp_mask = { 6243 .type = 0xffff, 6244 }; 6245 const struct rte_flow_attr attr = { 6246 .ingress = 1, 6247 }; 6248 struct rte_flow_item items[] = { 6249 { 6250 .type = RTE_FLOW_ITEM_TYPE_ETH, 6251 .spec = &lacp_spec, 6252 .mask = &lacp_mask, 6253 }, 6254 { 6255 .type = RTE_FLOW_ITEM_TYPE_END, 6256 }, 6257 }; 6258 struct rte_flow_action actions[] = { 6259 { 6260 .type = (enum rte_flow_action_type) 6261 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 6262 }, 6263 { 6264 .type = RTE_FLOW_ACTION_TYPE_END, 6265 }, 6266 }; 6267 struct rte_flow_error error; 6268 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 6269 &attr, items, actions, false, &error); 6270 6271 if (!flow_idx) 6272 return -rte_errno; 6273 return 0; 6274 } 6275 6276 /** 6277 * Destroy a flow. 6278 * 6279 * @see rte_flow_destroy() 6280 * @see rte_flow_ops 6281 */ 6282 int 6283 mlx5_flow_destroy(struct rte_eth_dev *dev, 6284 struct rte_flow *flow, 6285 struct rte_flow_error *error __rte_unused) 6286 { 6287 struct mlx5_priv *priv = dev->data->dev_private; 6288 6289 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6290 return 0; 6291 } 6292 6293 /** 6294 * Destroy all flows. 6295 * 6296 * @see rte_flow_flush() 6297 * @see rte_flow_ops 6298 */ 6299 int 6300 mlx5_flow_flush(struct rte_eth_dev *dev, 6301 struct rte_flow_error *error __rte_unused) 6302 { 6303 struct mlx5_priv *priv = dev->data->dev_private; 6304 6305 mlx5_flow_list_flush(dev, &priv->flows, false); 6306 return 0; 6307 } 6308 6309 /** 6310 * Isolated mode. 6311 * 6312 * @see rte_flow_isolate() 6313 * @see rte_flow_ops 6314 */ 6315 int 6316 mlx5_flow_isolate(struct rte_eth_dev *dev, 6317 int enable, 6318 struct rte_flow_error *error) 6319 { 6320 struct mlx5_priv *priv = dev->data->dev_private; 6321 6322 if (dev->data->dev_started) { 6323 rte_flow_error_set(error, EBUSY, 6324 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6325 NULL, 6326 "port must be stopped first"); 6327 return -rte_errno; 6328 } 6329 priv->isolated = !!enable; 6330 if (enable) 6331 dev->dev_ops = &mlx5_os_dev_ops_isolate; 6332 else 6333 dev->dev_ops = &mlx5_os_dev_ops; 6334 6335 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 6336 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 6337 6338 return 0; 6339 } 6340 6341 /** 6342 * Query a flow. 6343 * 6344 * @see rte_flow_query() 6345 * @see rte_flow_ops 6346 */ 6347 static int 6348 flow_drv_query(struct rte_eth_dev *dev, 6349 uint32_t flow_idx, 6350 const struct rte_flow_action *actions, 6351 void *data, 6352 struct rte_flow_error *error) 6353 { 6354 struct mlx5_priv *priv = dev->data->dev_private; 6355 const struct mlx5_flow_driver_ops *fops; 6356 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6357 [MLX5_IPOOL_RTE_FLOW], 6358 flow_idx); 6359 enum mlx5_flow_drv_type ftype; 6360 6361 if (!flow) { 6362 return rte_flow_error_set(error, ENOENT, 6363 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6364 NULL, 6365 "invalid flow handle"); 6366 } 6367 ftype = flow->drv_type; 6368 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 6369 fops = flow_get_drv_ops(ftype); 6370 6371 return fops->query(dev, flow, actions, data, error); 6372 } 6373 6374 /** 6375 * Query a flow. 6376 * 6377 * @see rte_flow_query() 6378 * @see rte_flow_ops 6379 */ 6380 int 6381 mlx5_flow_query(struct rte_eth_dev *dev, 6382 struct rte_flow *flow, 6383 const struct rte_flow_action *actions, 6384 void *data, 6385 struct rte_flow_error *error) 6386 { 6387 int ret; 6388 6389 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 6390 error); 6391 if (ret < 0) 6392 return ret; 6393 return 0; 6394 } 6395 6396 /** 6397 * Convert a flow director filter to a generic flow. 6398 * 6399 * @param dev 6400 * Pointer to Ethernet device. 6401 * @param fdir_filter 6402 * Flow director filter to add. 6403 * @param attributes 6404 * Generic flow parameters structure. 6405 * 6406 * @return 6407 * 0 on success, a negative errno value otherwise and rte_errno is set. 6408 */ 6409 static int 6410 flow_fdir_filter_convert(struct rte_eth_dev *dev, 6411 const struct rte_eth_fdir_filter *fdir_filter, 6412 struct mlx5_fdir *attributes) 6413 { 6414 struct mlx5_priv *priv = dev->data->dev_private; 6415 const struct rte_eth_fdir_input *input = &fdir_filter->input; 6416 const struct rte_eth_fdir_masks *mask = 6417 &dev->data->dev_conf.fdir_conf.mask; 6418 6419 /* Validate queue number. */ 6420 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 6421 DRV_LOG(ERR, "port %u invalid queue number %d", 6422 dev->data->port_id, fdir_filter->action.rx_queue); 6423 rte_errno = EINVAL; 6424 return -rte_errno; 6425 } 6426 attributes->attr.ingress = 1; 6427 attributes->items[0] = (struct rte_flow_item) { 6428 .type = RTE_FLOW_ITEM_TYPE_ETH, 6429 .spec = &attributes->l2, 6430 .mask = &attributes->l2_mask, 6431 }; 6432 switch (fdir_filter->action.behavior) { 6433 case RTE_ETH_FDIR_ACCEPT: 6434 attributes->actions[0] = (struct rte_flow_action){ 6435 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 6436 .conf = &attributes->queue, 6437 }; 6438 break; 6439 case RTE_ETH_FDIR_REJECT: 6440 attributes->actions[0] = (struct rte_flow_action){ 6441 .type = RTE_FLOW_ACTION_TYPE_DROP, 6442 }; 6443 break; 6444 default: 6445 DRV_LOG(ERR, "port %u invalid behavior %d", 6446 dev->data->port_id, 6447 fdir_filter->action.behavior); 6448 rte_errno = ENOTSUP; 6449 return -rte_errno; 6450 } 6451 attributes->queue.index = fdir_filter->action.rx_queue; 6452 /* Handle L3. */ 6453 switch (fdir_filter->input.flow_type) { 6454 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 6455 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 6456 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 6457 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 6458 .src_addr = input->flow.ip4_flow.src_ip, 6459 .dst_addr = input->flow.ip4_flow.dst_ip, 6460 .time_to_live = input->flow.ip4_flow.ttl, 6461 .type_of_service = input->flow.ip4_flow.tos, 6462 }; 6463 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 6464 .src_addr = mask->ipv4_mask.src_ip, 6465 .dst_addr = mask->ipv4_mask.dst_ip, 6466 .time_to_live = mask->ipv4_mask.ttl, 6467 .type_of_service = mask->ipv4_mask.tos, 6468 .next_proto_id = mask->ipv4_mask.proto, 6469 }; 6470 attributes->items[1] = (struct rte_flow_item){ 6471 .type = RTE_FLOW_ITEM_TYPE_IPV4, 6472 .spec = &attributes->l3, 6473 .mask = &attributes->l3_mask, 6474 }; 6475 break; 6476 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 6477 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 6478 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 6479 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 6480 .hop_limits = input->flow.ipv6_flow.hop_limits, 6481 .proto = input->flow.ipv6_flow.proto, 6482 }; 6483 6484 memcpy(attributes->l3.ipv6.hdr.src_addr, 6485 input->flow.ipv6_flow.src_ip, 6486 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 6487 memcpy(attributes->l3.ipv6.hdr.dst_addr, 6488 input->flow.ipv6_flow.dst_ip, 6489 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 6490 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 6491 mask->ipv6_mask.src_ip, 6492 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 6493 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 6494 mask->ipv6_mask.dst_ip, 6495 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 6496 attributes->items[1] = (struct rte_flow_item){ 6497 .type = RTE_FLOW_ITEM_TYPE_IPV6, 6498 .spec = &attributes->l3, 6499 .mask = &attributes->l3_mask, 6500 }; 6501 break; 6502 default: 6503 DRV_LOG(ERR, "port %u invalid flow type%d", 6504 dev->data->port_id, fdir_filter->input.flow_type); 6505 rte_errno = ENOTSUP; 6506 return -rte_errno; 6507 } 6508 /* Handle L4. */ 6509 switch (fdir_filter->input.flow_type) { 6510 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 6511 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 6512 .src_port = input->flow.udp4_flow.src_port, 6513 .dst_port = input->flow.udp4_flow.dst_port, 6514 }; 6515 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 6516 .src_port = mask->src_port_mask, 6517 .dst_port = mask->dst_port_mask, 6518 }; 6519 attributes->items[2] = (struct rte_flow_item){ 6520 .type = RTE_FLOW_ITEM_TYPE_UDP, 6521 .spec = &attributes->l4, 6522 .mask = &attributes->l4_mask, 6523 }; 6524 break; 6525 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 6526 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 6527 .src_port = input->flow.tcp4_flow.src_port, 6528 .dst_port = input->flow.tcp4_flow.dst_port, 6529 }; 6530 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 6531 .src_port = mask->src_port_mask, 6532 .dst_port = mask->dst_port_mask, 6533 }; 6534 attributes->items[2] = (struct rte_flow_item){ 6535 .type = RTE_FLOW_ITEM_TYPE_TCP, 6536 .spec = &attributes->l4, 6537 .mask = &attributes->l4_mask, 6538 }; 6539 break; 6540 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 6541 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 6542 .src_port = input->flow.udp6_flow.src_port, 6543 .dst_port = input->flow.udp6_flow.dst_port, 6544 }; 6545 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 6546 .src_port = mask->src_port_mask, 6547 .dst_port = mask->dst_port_mask, 6548 }; 6549 attributes->items[2] = (struct rte_flow_item){ 6550 .type = RTE_FLOW_ITEM_TYPE_UDP, 6551 .spec = &attributes->l4, 6552 .mask = &attributes->l4_mask, 6553 }; 6554 break; 6555 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 6556 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 6557 .src_port = input->flow.tcp6_flow.src_port, 6558 .dst_port = input->flow.tcp6_flow.dst_port, 6559 }; 6560 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 6561 .src_port = mask->src_port_mask, 6562 .dst_port = mask->dst_port_mask, 6563 }; 6564 attributes->items[2] = (struct rte_flow_item){ 6565 .type = RTE_FLOW_ITEM_TYPE_TCP, 6566 .spec = &attributes->l4, 6567 .mask = &attributes->l4_mask, 6568 }; 6569 break; 6570 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 6571 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 6572 break; 6573 default: 6574 DRV_LOG(ERR, "port %u invalid flow type%d", 6575 dev->data->port_id, fdir_filter->input.flow_type); 6576 rte_errno = ENOTSUP; 6577 return -rte_errno; 6578 } 6579 return 0; 6580 } 6581 6582 #define FLOW_FDIR_CMP(f1, f2, fld) \ 6583 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 6584 6585 /** 6586 * Compare two FDIR flows. If items and actions are identical, the two flows are 6587 * regarded as same. 6588 * 6589 * @param dev 6590 * Pointer to Ethernet device. 6591 * @param f1 6592 * FDIR flow to compare. 6593 * @param f2 6594 * FDIR flow to compare. 6595 * 6596 * @return 6597 * Zero on match, 1 otherwise. 6598 */ 6599 static int 6600 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 6601 { 6602 if (FLOW_FDIR_CMP(f1, f2, attr) || 6603 FLOW_FDIR_CMP(f1, f2, l2) || 6604 FLOW_FDIR_CMP(f1, f2, l2_mask) || 6605 FLOW_FDIR_CMP(f1, f2, l3) || 6606 FLOW_FDIR_CMP(f1, f2, l3_mask) || 6607 FLOW_FDIR_CMP(f1, f2, l4) || 6608 FLOW_FDIR_CMP(f1, f2, l4_mask) || 6609 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 6610 return 1; 6611 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 6612 FLOW_FDIR_CMP(f1, f2, queue)) 6613 return 1; 6614 return 0; 6615 } 6616 6617 /** 6618 * Search device flow list to find out a matched FDIR flow. 6619 * 6620 * @param dev 6621 * Pointer to Ethernet device. 6622 * @param fdir_flow 6623 * FDIR flow to lookup. 6624 * 6625 * @return 6626 * Index of flow if found, 0 otherwise. 6627 */ 6628 static uint32_t 6629 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 6630 { 6631 struct mlx5_priv *priv = dev->data->dev_private; 6632 uint32_t flow_idx = 0; 6633 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6634 6635 MLX5_ASSERT(fdir_flow); 6636 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 6637 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { 6638 DRV_LOG(DEBUG, "port %u found FDIR flow %u", 6639 dev->data->port_id, flow_idx); 6640 flow_idx = priv_fdir_flow->rix_flow; 6641 break; 6642 } 6643 } 6644 return flow_idx; 6645 } 6646 6647 /** 6648 * Add new flow director filter and store it in list. 6649 * 6650 * @param dev 6651 * Pointer to Ethernet device. 6652 * @param fdir_filter 6653 * Flow director filter to add. 6654 * 6655 * @return 6656 * 0 on success, a negative errno value otherwise and rte_errno is set. 6657 */ 6658 static int 6659 flow_fdir_filter_add(struct rte_eth_dev *dev, 6660 const struct rte_eth_fdir_filter *fdir_filter) 6661 { 6662 struct mlx5_priv *priv = dev->data->dev_private; 6663 struct mlx5_fdir *fdir_flow; 6664 struct rte_flow *flow; 6665 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6666 uint32_t flow_idx; 6667 int ret; 6668 6669 fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*fdir_flow), 0, 6670 SOCKET_ID_ANY); 6671 if (!fdir_flow) { 6672 rte_errno = ENOMEM; 6673 return -rte_errno; 6674 } 6675 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 6676 if (ret) 6677 goto error; 6678 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); 6679 if (flow_idx) { 6680 rte_errno = EEXIST; 6681 goto error; 6682 } 6683 priv_fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, 6684 sizeof(struct mlx5_fdir_flow), 6685 0, SOCKET_ID_ANY); 6686 if (!priv_fdir_flow) { 6687 rte_errno = ENOMEM; 6688 goto error; 6689 } 6690 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 6691 fdir_flow->items, fdir_flow->actions, true, 6692 NULL); 6693 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 6694 if (!flow) 6695 goto error; 6696 flow->fdir = 1; 6697 priv_fdir_flow->fdir = fdir_flow; 6698 priv_fdir_flow->rix_flow = flow_idx; 6699 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); 6700 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 6701 dev->data->port_id, (void *)flow); 6702 return 0; 6703 error: 6704 mlx5_free(priv_fdir_flow); 6705 mlx5_free(fdir_flow); 6706 return -rte_errno; 6707 } 6708 6709 /** 6710 * Delete specific filter. 6711 * 6712 * @param dev 6713 * Pointer to Ethernet device. 6714 * @param fdir_filter 6715 * Filter to be deleted. 6716 * 6717 * @return 6718 * 0 on success, a negative errno value otherwise and rte_errno is set. 6719 */ 6720 static int 6721 flow_fdir_filter_delete(struct rte_eth_dev *dev, 6722 const struct rte_eth_fdir_filter *fdir_filter) 6723 { 6724 struct mlx5_priv *priv = dev->data->dev_private; 6725 uint32_t flow_idx; 6726 struct mlx5_fdir fdir_flow = { 6727 .attr.group = 0, 6728 }; 6729 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6730 int ret; 6731 6732 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 6733 if (ret) 6734 return -rte_errno; 6735 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 6736 /* Find the fdir in priv list */ 6737 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) 6738 break; 6739 } 6740 if (!priv_fdir_flow) 6741 return 0; 6742 LIST_REMOVE(priv_fdir_flow, next); 6743 flow_idx = priv_fdir_flow->rix_flow; 6744 flow_list_destroy(dev, &priv->flows, flow_idx); 6745 mlx5_free(priv_fdir_flow->fdir); 6746 mlx5_free(priv_fdir_flow); 6747 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", 6748 dev->data->port_id, flow_idx); 6749 return 0; 6750 } 6751 6752 /** 6753 * Update queue for specific filter. 6754 * 6755 * @param dev 6756 * Pointer to Ethernet device. 6757 * @param fdir_filter 6758 * Filter to be updated. 6759 * 6760 * @return 6761 * 0 on success, a negative errno value otherwise and rte_errno is set. 6762 */ 6763 static int 6764 flow_fdir_filter_update(struct rte_eth_dev *dev, 6765 const struct rte_eth_fdir_filter *fdir_filter) 6766 { 6767 int ret; 6768 6769 ret = flow_fdir_filter_delete(dev, fdir_filter); 6770 if (ret) 6771 return ret; 6772 return flow_fdir_filter_add(dev, fdir_filter); 6773 } 6774 6775 /** 6776 * Flush all filters. 6777 * 6778 * @param dev 6779 * Pointer to Ethernet device. 6780 */ 6781 static void 6782 flow_fdir_filter_flush(struct rte_eth_dev *dev) 6783 { 6784 struct mlx5_priv *priv = dev->data->dev_private; 6785 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6786 6787 while (!LIST_EMPTY(&priv->fdir_flows)) { 6788 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); 6789 LIST_REMOVE(priv_fdir_flow, next); 6790 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); 6791 mlx5_free(priv_fdir_flow->fdir); 6792 mlx5_free(priv_fdir_flow); 6793 } 6794 } 6795 6796 /** 6797 * Get flow director information. 6798 * 6799 * @param dev 6800 * Pointer to Ethernet device. 6801 * @param[out] fdir_info 6802 * Resulting flow director information. 6803 */ 6804 static void 6805 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 6806 { 6807 struct rte_eth_fdir_masks *mask = 6808 &dev->data->dev_conf.fdir_conf.mask; 6809 6810 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 6811 fdir_info->guarant_spc = 0; 6812 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 6813 fdir_info->max_flexpayload = 0; 6814 fdir_info->flow_types_mask[0] = 0; 6815 fdir_info->flex_payload_unit = 0; 6816 fdir_info->max_flex_payload_segment_num = 0; 6817 fdir_info->flex_payload_limit = 0; 6818 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 6819 } 6820 6821 /** 6822 * Deal with flow director operations. 6823 * 6824 * @param dev 6825 * Pointer to Ethernet device. 6826 * @param filter_op 6827 * Operation to perform. 6828 * @param arg 6829 * Pointer to operation-specific structure. 6830 * 6831 * @return 6832 * 0 on success, a negative errno value otherwise and rte_errno is set. 6833 */ 6834 static int 6835 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 6836 void *arg) 6837 { 6838 enum rte_fdir_mode fdir_mode = 6839 dev->data->dev_conf.fdir_conf.mode; 6840 6841 if (filter_op == RTE_ETH_FILTER_NOP) 6842 return 0; 6843 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 6844 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 6845 DRV_LOG(ERR, "port %u flow director mode %d not supported", 6846 dev->data->port_id, fdir_mode); 6847 rte_errno = EINVAL; 6848 return -rte_errno; 6849 } 6850 switch (filter_op) { 6851 case RTE_ETH_FILTER_ADD: 6852 return flow_fdir_filter_add(dev, arg); 6853 case RTE_ETH_FILTER_UPDATE: 6854 return flow_fdir_filter_update(dev, arg); 6855 case RTE_ETH_FILTER_DELETE: 6856 return flow_fdir_filter_delete(dev, arg); 6857 case RTE_ETH_FILTER_FLUSH: 6858 flow_fdir_filter_flush(dev); 6859 break; 6860 case RTE_ETH_FILTER_INFO: 6861 flow_fdir_info_get(dev, arg); 6862 break; 6863 default: 6864 DRV_LOG(DEBUG, "port %u unknown operation %u", 6865 dev->data->port_id, filter_op); 6866 rte_errno = EINVAL; 6867 return -rte_errno; 6868 } 6869 return 0; 6870 } 6871 6872 /** 6873 * Manage filter operations. 6874 * 6875 * @param dev 6876 * Pointer to Ethernet device structure. 6877 * @param filter_type 6878 * Filter type. 6879 * @param filter_op 6880 * Operation to perform. 6881 * @param arg 6882 * Pointer to operation-specific structure. 6883 * 6884 * @return 6885 * 0 on success, a negative errno value otherwise and rte_errno is set. 6886 */ 6887 int 6888 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 6889 enum rte_filter_type filter_type, 6890 enum rte_filter_op filter_op, 6891 void *arg) 6892 { 6893 switch (filter_type) { 6894 case RTE_ETH_FILTER_GENERIC: 6895 if (filter_op != RTE_ETH_FILTER_GET) { 6896 rte_errno = EINVAL; 6897 return -rte_errno; 6898 } 6899 *(const void **)arg = &mlx5_flow_ops; 6900 return 0; 6901 case RTE_ETH_FILTER_FDIR: 6902 return flow_fdir_ctrl_func(dev, filter_op, arg); 6903 default: 6904 DRV_LOG(ERR, "port %u filter type (%d) not supported", 6905 dev->data->port_id, filter_type); 6906 rte_errno = ENOTSUP; 6907 return -rte_errno; 6908 } 6909 return 0; 6910 } 6911 6912 /** 6913 * Create the needed meter and suffix tables. 6914 * 6915 * @param[in] dev 6916 * Pointer to Ethernet device. 6917 * @param[in] fm 6918 * Pointer to the flow meter. 6919 * 6920 * @return 6921 * Pointer to table set on success, NULL otherwise. 6922 */ 6923 struct mlx5_meter_domains_infos * 6924 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 6925 const struct mlx5_flow_meter *fm) 6926 { 6927 const struct mlx5_flow_driver_ops *fops; 6928 6929 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6930 return fops->create_mtr_tbls(dev, fm); 6931 } 6932 6933 /** 6934 * Destroy the meter table set. 6935 * 6936 * @param[in] dev 6937 * Pointer to Ethernet device. 6938 * @param[in] tbl 6939 * Pointer to the meter table set. 6940 * 6941 * @return 6942 * 0 on success. 6943 */ 6944 int 6945 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 6946 struct mlx5_meter_domains_infos *tbls) 6947 { 6948 const struct mlx5_flow_driver_ops *fops; 6949 6950 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6951 return fops->destroy_mtr_tbls(dev, tbls); 6952 } 6953 6954 /** 6955 * Create policer rules. 6956 * 6957 * @param[in] dev 6958 * Pointer to Ethernet device. 6959 * @param[in] fm 6960 * Pointer to flow meter structure. 6961 * @param[in] attr 6962 * Pointer to flow attributes. 6963 * 6964 * @return 6965 * 0 on success, -1 otherwise. 6966 */ 6967 int 6968 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 6969 struct mlx5_flow_meter *fm, 6970 const struct rte_flow_attr *attr) 6971 { 6972 const struct mlx5_flow_driver_ops *fops; 6973 6974 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6975 return fops->create_policer_rules(dev, fm, attr); 6976 } 6977 6978 /** 6979 * Destroy policer rules. 6980 * 6981 * @param[in] fm 6982 * Pointer to flow meter structure. 6983 * @param[in] attr 6984 * Pointer to flow attributes. 6985 * 6986 * @return 6987 * 0 on success, -1 otherwise. 6988 */ 6989 int 6990 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 6991 struct mlx5_flow_meter *fm, 6992 const struct rte_flow_attr *attr) 6993 { 6994 const struct mlx5_flow_driver_ops *fops; 6995 6996 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6997 return fops->destroy_policer_rules(dev, fm, attr); 6998 } 6999 7000 /** 7001 * Allocate a counter. 7002 * 7003 * @param[in] dev 7004 * Pointer to Ethernet device structure. 7005 * 7006 * @return 7007 * Index to allocated counter on success, 0 otherwise. 7008 */ 7009 uint32_t 7010 mlx5_counter_alloc(struct rte_eth_dev *dev) 7011 { 7012 const struct mlx5_flow_driver_ops *fops; 7013 struct rte_flow_attr attr = { .transfer = 0 }; 7014 7015 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7016 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7017 return fops->counter_alloc(dev); 7018 } 7019 DRV_LOG(ERR, 7020 "port %u counter allocate is not supported.", 7021 dev->data->port_id); 7022 return 0; 7023 } 7024 7025 /** 7026 * Free a counter. 7027 * 7028 * @param[in] dev 7029 * Pointer to Ethernet device structure. 7030 * @param[in] cnt 7031 * Index to counter to be free. 7032 */ 7033 void 7034 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7035 { 7036 const struct mlx5_flow_driver_ops *fops; 7037 struct rte_flow_attr attr = { .transfer = 0 }; 7038 7039 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7040 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7041 fops->counter_free(dev, cnt); 7042 return; 7043 } 7044 DRV_LOG(ERR, 7045 "port %u counter free is not supported.", 7046 dev->data->port_id); 7047 } 7048 7049 /** 7050 * Query counter statistics. 7051 * 7052 * @param[in] dev 7053 * Pointer to Ethernet device structure. 7054 * @param[in] cnt 7055 * Index to counter to query. 7056 * @param[in] clear 7057 * Set to clear counter statistics. 7058 * @param[out] pkts 7059 * The counter hits packets number to save. 7060 * @param[out] bytes 7061 * The counter hits bytes number to save. 7062 * 7063 * @return 7064 * 0 on success, a negative errno value otherwise. 7065 */ 7066 int 7067 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7068 bool clear, uint64_t *pkts, uint64_t *bytes) 7069 { 7070 const struct mlx5_flow_driver_ops *fops; 7071 struct rte_flow_attr attr = { .transfer = 0 }; 7072 7073 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7074 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7075 return fops->counter_query(dev, cnt, clear, pkts, bytes); 7076 } 7077 DRV_LOG(ERR, 7078 "port %u counter query is not supported.", 7079 dev->data->port_id); 7080 return -ENOTSUP; 7081 } 7082 7083 /** 7084 * Allocate a new memory for the counter values wrapped by all the needed 7085 * management. 7086 * 7087 * @param[in] sh 7088 * Pointer to mlx5_dev_ctx_shared object. 7089 * 7090 * @return 7091 * 0 on success, a negative errno value otherwise. 7092 */ 7093 static int 7094 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 7095 { 7096 struct mlx5_devx_mkey_attr mkey_attr; 7097 struct mlx5_counter_stats_mem_mng *mem_mng; 7098 volatile struct flow_counter_stats *raw_data; 7099 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 7100 int size = (sizeof(struct flow_counter_stats) * 7101 MLX5_COUNTERS_PER_POOL + 7102 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 7103 sizeof(struct mlx5_counter_stats_mem_mng); 7104 size_t pgsize = rte_mem_page_size(); 7105 uint8_t *mem; 7106 int i; 7107 7108 if (pgsize == (size_t)-1) { 7109 DRV_LOG(ERR, "Failed to get mem page size"); 7110 rte_errno = ENOMEM; 7111 return -ENOMEM; 7112 } 7113 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 7114 if (!mem) { 7115 rte_errno = ENOMEM; 7116 return -ENOMEM; 7117 } 7118 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 7119 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 7120 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 7121 IBV_ACCESS_LOCAL_WRITE); 7122 if (!mem_mng->umem) { 7123 rte_errno = errno; 7124 mlx5_free(mem); 7125 return -rte_errno; 7126 } 7127 mkey_attr.addr = (uintptr_t)mem; 7128 mkey_attr.size = size; 7129 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 7130 mkey_attr.pd = sh->pdn; 7131 mkey_attr.log_entity_size = 0; 7132 mkey_attr.pg_access = 0; 7133 mkey_attr.klm_array = NULL; 7134 mkey_attr.klm_num = 0; 7135 mkey_attr.relaxed_ordering = sh->cmng.relaxed_ordering; 7136 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 7137 if (!mem_mng->dm) { 7138 mlx5_glue->devx_umem_dereg(mem_mng->umem); 7139 rte_errno = errno; 7140 mlx5_free(mem); 7141 return -rte_errno; 7142 } 7143 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 7144 raw_data = (volatile struct flow_counter_stats *)mem; 7145 for (i = 0; i < raws_n; ++i) { 7146 mem_mng->raws[i].mem_mng = mem_mng; 7147 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 7148 } 7149 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 7150 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 7151 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 7152 next); 7153 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 7154 sh->cmng.mem_mng = mem_mng; 7155 return 0; 7156 } 7157 7158 /** 7159 * Set the statistic memory to the new counter pool. 7160 * 7161 * @param[in] sh 7162 * Pointer to mlx5_dev_ctx_shared object. 7163 * @param[in] pool 7164 * Pointer to the pool to set the statistic memory. 7165 * 7166 * @return 7167 * 0 on success, a negative errno value otherwise. 7168 */ 7169 static int 7170 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 7171 struct mlx5_flow_counter_pool *pool) 7172 { 7173 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7174 /* Resize statistic memory once used out. */ 7175 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 7176 mlx5_flow_create_counter_stat_mem_mng(sh)) { 7177 DRV_LOG(ERR, "Cannot resize counter stat mem."); 7178 return -1; 7179 } 7180 rte_spinlock_lock(&pool->sl); 7181 pool->raw = cmng->mem_mng->raws + pool->index % 7182 MLX5_CNT_CONTAINER_RESIZE; 7183 rte_spinlock_unlock(&pool->sl); 7184 pool->raw_hw = NULL; 7185 return 0; 7186 } 7187 7188 #define MLX5_POOL_QUERY_FREQ_US 1000000 7189 7190 /** 7191 * Set the periodic procedure for triggering asynchronous batch queries for all 7192 * the counter pools. 7193 * 7194 * @param[in] sh 7195 * Pointer to mlx5_dev_ctx_shared object. 7196 */ 7197 void 7198 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 7199 { 7200 uint32_t pools_n, us; 7201 7202 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 7203 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 7204 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 7205 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 7206 sh->cmng.query_thread_on = 0; 7207 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 7208 } else { 7209 sh->cmng.query_thread_on = 1; 7210 } 7211 } 7212 7213 /** 7214 * The periodic procedure for triggering asynchronous batch queries for all the 7215 * counter pools. This function is probably called by the host thread. 7216 * 7217 * @param[in] arg 7218 * The parameter for the alarm process. 7219 */ 7220 void 7221 mlx5_flow_query_alarm(void *arg) 7222 { 7223 struct mlx5_dev_ctx_shared *sh = arg; 7224 int ret; 7225 uint16_t pool_index = sh->cmng.pool_index; 7226 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7227 struct mlx5_flow_counter_pool *pool; 7228 uint16_t n_valid; 7229 7230 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 7231 goto set_alarm; 7232 rte_spinlock_lock(&cmng->pool_update_sl); 7233 pool = cmng->pools[pool_index]; 7234 n_valid = cmng->n_valid; 7235 rte_spinlock_unlock(&cmng->pool_update_sl); 7236 /* Set the statistic memory to the new created pool. */ 7237 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 7238 goto set_alarm; 7239 if (pool->raw_hw) 7240 /* There is a pool query in progress. */ 7241 goto set_alarm; 7242 pool->raw_hw = 7243 LIST_FIRST(&sh->cmng.free_stat_raws); 7244 if (!pool->raw_hw) 7245 /* No free counter statistics raw memory. */ 7246 goto set_alarm; 7247 /* 7248 * Identify the counters released between query trigger and query 7249 * handle more efficiently. The counter released in this gap period 7250 * should wait for a new round of query as the new arrived packets 7251 * will not be taken into account. 7252 */ 7253 pool->query_gen++; 7254 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 7255 MLX5_COUNTERS_PER_POOL, 7256 NULL, NULL, 7257 pool->raw_hw->mem_mng->dm->id, 7258 (void *)(uintptr_t) 7259 pool->raw_hw->data, 7260 sh->devx_comp, 7261 (uint64_t)(uintptr_t)pool); 7262 if (ret) { 7263 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 7264 " %d", pool->min_dcs->id); 7265 pool->raw_hw = NULL; 7266 goto set_alarm; 7267 } 7268 LIST_REMOVE(pool->raw_hw, next); 7269 sh->cmng.pending_queries++; 7270 pool_index++; 7271 if (pool_index >= n_valid) 7272 pool_index = 0; 7273 set_alarm: 7274 sh->cmng.pool_index = pool_index; 7275 mlx5_set_query_alarm(sh); 7276 } 7277 7278 /** 7279 * Check and callback event for new aged flow in the counter pool 7280 * 7281 * @param[in] sh 7282 * Pointer to mlx5_dev_ctx_shared object. 7283 * @param[in] pool 7284 * Pointer to Current counter pool. 7285 */ 7286 static void 7287 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7288 struct mlx5_flow_counter_pool *pool) 7289 { 7290 struct mlx5_priv *priv; 7291 struct mlx5_flow_counter *cnt; 7292 struct mlx5_age_info *age_info; 7293 struct mlx5_age_param *age_param; 7294 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7295 struct mlx5_counter_stats_raw *prev = pool->raw; 7296 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7297 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7298 uint16_t expected = AGE_CANDIDATE; 7299 uint32_t i; 7300 7301 pool->time_of_last_age_check = curr_time; 7302 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7303 cnt = MLX5_POOL_GET_CNT(pool, i); 7304 age_param = MLX5_CNT_TO_AGE(cnt); 7305 if (__atomic_load_n(&age_param->state, 7306 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7307 continue; 7308 if (cur->data[i].hits != prev->data[i].hits) { 7309 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7310 __ATOMIC_RELAXED); 7311 continue; 7312 } 7313 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7314 time_delta, 7315 __ATOMIC_RELAXED) <= age_param->timeout) 7316 continue; 7317 /** 7318 * Hold the lock first, or if between the 7319 * state AGE_TMOUT and tailq operation the 7320 * release happened, the release procedure 7321 * may delete a non-existent tailq node. 7322 */ 7323 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7324 age_info = GET_PORT_AGE_INFO(priv); 7325 rte_spinlock_lock(&age_info->aged_sl); 7326 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7327 AGE_TMOUT, false, 7328 __ATOMIC_RELAXED, 7329 __ATOMIC_RELAXED)) { 7330 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7331 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7332 } 7333 rte_spinlock_unlock(&age_info->aged_sl); 7334 } 7335 for (i = 0; i < sh->max_port; i++) { 7336 age_info = &sh->port[i].age_info; 7337 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 7338 continue; 7339 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) 7340 rte_eth_dev_callback_process 7341 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 7342 RTE_ETH_EVENT_FLOW_AGED, NULL); 7343 age_info->flags = 0; 7344 } 7345 } 7346 7347 /** 7348 * Handler for the HW respond about ready values from an asynchronous batch 7349 * query. This function is probably called by the host thread. 7350 * 7351 * @param[in] sh 7352 * The pointer to the shared device context. 7353 * @param[in] async_id 7354 * The Devx async ID. 7355 * @param[in] status 7356 * The status of the completion. 7357 */ 7358 void 7359 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7360 uint64_t async_id, int status) 7361 { 7362 struct mlx5_flow_counter_pool *pool = 7363 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7364 struct mlx5_counter_stats_raw *raw_to_free; 7365 uint8_t query_gen = pool->query_gen ^ 1; 7366 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7367 enum mlx5_counter_type cnt_type = 7368 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7369 MLX5_COUNTER_TYPE_ORIGIN; 7370 7371 if (unlikely(status)) { 7372 raw_to_free = pool->raw_hw; 7373 } else { 7374 raw_to_free = pool->raw; 7375 if (pool->is_aged) 7376 mlx5_flow_aging_check(sh, pool); 7377 rte_spinlock_lock(&pool->sl); 7378 pool->raw = pool->raw_hw; 7379 rte_spinlock_unlock(&pool->sl); 7380 /* Be sure the new raw counters data is updated in memory. */ 7381 rte_io_wmb(); 7382 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7383 rte_spinlock_lock(&cmng->csl[cnt_type]); 7384 TAILQ_CONCAT(&cmng->counters[cnt_type], 7385 &pool->counters[query_gen], next); 7386 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7387 } 7388 } 7389 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7390 pool->raw_hw = NULL; 7391 sh->cmng.pending_queries--; 7392 } 7393 7394 static const struct mlx5_flow_tbl_data_entry * 7395 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 7396 { 7397 struct mlx5_priv *priv = dev->data->dev_private; 7398 struct mlx5_dev_ctx_shared *sh = priv->sh; 7399 struct mlx5_hlist_entry *he; 7400 union tunnel_offload_mark mbits = { .val = mark }; 7401 union mlx5_flow_tbl_key table_key = { 7402 { 7403 .table_id = tunnel_id_to_flow_tbl(mbits.table_id), 7404 .dummy = 0, 7405 .domain = !!mbits.transfer, 7406 .direction = 0, 7407 } 7408 }; 7409 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); 7410 return he ? 7411 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 7412 } 7413 7414 static void 7415 mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list, 7416 struct mlx5_hlist_entry *entry) 7417 { 7418 struct mlx5_dev_ctx_shared *sh = list->ctx; 7419 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 7420 7421 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7422 tunnel_flow_tbl_to_id(tte->flow_table)); 7423 mlx5_free(tte); 7424 } 7425 7426 static struct mlx5_hlist_entry * 7427 mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, 7428 uint64_t key __rte_unused, 7429 void *ctx __rte_unused) 7430 { 7431 struct mlx5_dev_ctx_shared *sh = list->ctx; 7432 struct tunnel_tbl_entry *tte; 7433 7434 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 7435 sizeof(*tte), 0, 7436 SOCKET_ID_ANY); 7437 if (!tte) 7438 goto err; 7439 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7440 &tte->flow_table); 7441 if (tte->flow_table >= MLX5_MAX_TABLES) { 7442 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 7443 tte->flow_table); 7444 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 7445 tte->flow_table); 7446 goto err; 7447 } else if (!tte->flow_table) { 7448 goto err; 7449 } 7450 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 7451 return &tte->hash; 7452 err: 7453 if (tte) 7454 mlx5_free(tte); 7455 return NULL; 7456 } 7457 7458 static uint32_t 7459 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 7460 const struct mlx5_flow_tunnel *tunnel, 7461 uint32_t group, uint32_t *table, 7462 struct rte_flow_error *error) 7463 { 7464 struct mlx5_hlist_entry *he; 7465 struct tunnel_tbl_entry *tte; 7466 union tunnel_tbl_key key = { 7467 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 7468 .group = group 7469 }; 7470 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7471 struct mlx5_hlist *group_hash; 7472 7473 group_hash = tunnel ? tunnel->groups : thub->groups; 7474 he = mlx5_hlist_register(group_hash, key.val, NULL); 7475 if (!he) 7476 return rte_flow_error_set(error, EINVAL, 7477 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7478 NULL, 7479 "tunnel group index not supported"); 7480 tte = container_of(he, typeof(*tte), hash); 7481 *table = tte->flow_table; 7482 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 7483 dev->data->port_id, key.tunnel_id, group, *table); 7484 return 0; 7485 } 7486 7487 static int 7488 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 7489 struct flow_grp_info grp_info, struct rte_flow_error *error) 7490 { 7491 if (grp_info.transfer && grp_info.external && grp_info.fdb_def_rule) { 7492 if (group == UINT32_MAX) 7493 return rte_flow_error_set 7494 (error, EINVAL, 7495 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7496 NULL, 7497 "group index not supported"); 7498 *table = group + 1; 7499 } else { 7500 *table = group; 7501 } 7502 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 7503 return 0; 7504 } 7505 7506 /** 7507 * Translate the rte_flow group index to HW table value. 7508 * 7509 * If tunnel offload is disabled, all group ids converted to flow table 7510 * id using the standard method. 7511 * If tunnel offload is enabled, group id can be converted using the 7512 * standard or tunnel conversion method. Group conversion method 7513 * selection depends on flags in `grp_info` parameter: 7514 * - Internal (grp_info.external == 0) groups conversion uses the 7515 * standard method. 7516 * - Group ids in JUMP action converted with the tunnel conversion. 7517 * - Group id in rule attribute conversion depends on a rule type and 7518 * group id value: 7519 * ** non zero group attributes converted with the tunnel method 7520 * ** zero group attribute in non-tunnel rule is converted using the 7521 * standard method - there's only one root table 7522 * ** zero group attribute in steer tunnel rule is converted with the 7523 * standard method - single root table 7524 * ** zero group attribute in match tunnel rule is a special OvS 7525 * case: that value is used for portability reasons. That group 7526 * id is converted with the tunnel conversion method. 7527 * 7528 * @param[in] dev 7529 * Port device 7530 * @param[in] tunnel 7531 * PMD tunnel offload object 7532 * @param[in] group 7533 * rte_flow group index value. 7534 * @param[out] table 7535 * HW table value. 7536 * @param[in] grp_info 7537 * flags used for conversion 7538 * @param[out] error 7539 * Pointer to error structure. 7540 * 7541 * @return 7542 * 0 on success, a negative errno value otherwise and rte_errno is set. 7543 */ 7544 int 7545 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 7546 const struct mlx5_flow_tunnel *tunnel, 7547 uint32_t group, uint32_t *table, 7548 struct flow_grp_info grp_info, 7549 struct rte_flow_error *error) 7550 { 7551 int ret; 7552 bool standard_translation; 7553 7554 if (grp_info.external && group < MLX5_MAX_TABLES_EXTERNAL) 7555 group *= MLX5_FLOW_TABLE_FACTOR; 7556 if (is_tunnel_offload_active(dev)) { 7557 standard_translation = !grp_info.external || 7558 grp_info.std_tbl_fix; 7559 } else { 7560 standard_translation = true; 7561 } 7562 DRV_LOG(DEBUG, 7563 "port %u group=%#x transfer=%d external=%d fdb_def_rule=%d translate=%s", 7564 dev->data->port_id, group, grp_info.transfer, 7565 grp_info.external, grp_info.fdb_def_rule, 7566 standard_translation ? "STANDARD" : "TUNNEL"); 7567 if (standard_translation) 7568 ret = flow_group_to_table(dev->data->port_id, group, table, 7569 grp_info, error); 7570 else 7571 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 7572 table, error); 7573 7574 return ret; 7575 } 7576 7577 /** 7578 * Discover availability of metadata reg_c's. 7579 * 7580 * Iteratively use test flows to check availability. 7581 * 7582 * @param[in] dev 7583 * Pointer to the Ethernet device structure. 7584 * 7585 * @return 7586 * 0 on success, a negative errno value otherwise and rte_errno is set. 7587 */ 7588 int 7589 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7590 { 7591 struct mlx5_priv *priv = dev->data->dev_private; 7592 struct mlx5_dev_config *config = &priv->config; 7593 enum modify_reg idx; 7594 int n = 0; 7595 7596 /* reg_c[0] and reg_c[1] are reserved. */ 7597 config->flow_mreg_c[n++] = REG_C_0; 7598 config->flow_mreg_c[n++] = REG_C_1; 7599 /* Discover availability of other reg_c's. */ 7600 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7601 struct rte_flow_attr attr = { 7602 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7603 .priority = MLX5_FLOW_PRIO_RSVD, 7604 .ingress = 1, 7605 }; 7606 struct rte_flow_item items[] = { 7607 [0] = { 7608 .type = RTE_FLOW_ITEM_TYPE_END, 7609 }, 7610 }; 7611 struct rte_flow_action actions[] = { 7612 [0] = { 7613 .type = (enum rte_flow_action_type) 7614 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7615 .conf = &(struct mlx5_flow_action_copy_mreg){ 7616 .src = REG_C_1, 7617 .dst = idx, 7618 }, 7619 }, 7620 [1] = { 7621 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7622 .conf = &(struct rte_flow_action_jump){ 7623 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7624 }, 7625 }, 7626 [2] = { 7627 .type = RTE_FLOW_ACTION_TYPE_END, 7628 }, 7629 }; 7630 uint32_t flow_idx; 7631 struct rte_flow *flow; 7632 struct rte_flow_error error; 7633 7634 if (!config->dv_flow_en) 7635 break; 7636 /* Create internal flow, validation skips copy action. */ 7637 flow_idx = flow_list_create(dev, NULL, &attr, items, 7638 actions, false, &error); 7639 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 7640 flow_idx); 7641 if (!flow) 7642 continue; 7643 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 7644 config->flow_mreg_c[n++] = idx; 7645 flow_list_destroy(dev, NULL, flow_idx); 7646 } 7647 for (; n < MLX5_MREG_C_NUM; ++n) 7648 config->flow_mreg_c[n] = REG_NON; 7649 return 0; 7650 } 7651 7652 /** 7653 * Dump flow raw hw data to file 7654 * 7655 * @param[in] dev 7656 * The pointer to Ethernet device. 7657 * @param[in] file 7658 * A pointer to a file for output. 7659 * @param[out] error 7660 * Perform verbose error reporting if not NULL. PMDs initialize this 7661 * structure in case of error only. 7662 * @return 7663 * 0 on success, a nagative value otherwise. 7664 */ 7665 int 7666 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 7667 FILE *file, 7668 struct rte_flow_error *error __rte_unused) 7669 { 7670 struct mlx5_priv *priv = dev->data->dev_private; 7671 struct mlx5_dev_ctx_shared *sh = priv->sh; 7672 7673 if (!priv->config.dv_flow_en) { 7674 if (fputs("device dv flow disabled\n", file) <= 0) 7675 return -errno; 7676 return -ENOTSUP; 7677 } 7678 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 7679 sh->tx_domain, file); 7680 } 7681 7682 /** 7683 * Get aged-out flows. 7684 * 7685 * @param[in] dev 7686 * Pointer to the Ethernet device structure. 7687 * @param[in] context 7688 * The address of an array of pointers to the aged-out flows contexts. 7689 * @param[in] nb_countexts 7690 * The length of context array pointers. 7691 * @param[out] error 7692 * Perform verbose error reporting if not NULL. Initialized in case of 7693 * error only. 7694 * 7695 * @return 7696 * how many contexts get in success, otherwise negative errno value. 7697 * if nb_contexts is 0, return the amount of all aged contexts. 7698 * if nb_contexts is not 0 , return the amount of aged flows reported 7699 * in the context array. 7700 */ 7701 int 7702 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 7703 uint32_t nb_contexts, struct rte_flow_error *error) 7704 { 7705 const struct mlx5_flow_driver_ops *fops; 7706 struct rte_flow_attr attr = { .transfer = 0 }; 7707 7708 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7709 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7710 return fops->get_aged_flows(dev, contexts, nb_contexts, 7711 error); 7712 } 7713 DRV_LOG(ERR, 7714 "port %u get aged flows is not supported.", 7715 dev->data->port_id); 7716 return -ENOTSUP; 7717 } 7718 7719 /* Wrapper for driver action_validate op callback */ 7720 static int 7721 flow_drv_action_validate(struct rte_eth_dev *dev, 7722 const struct rte_flow_shared_action_conf *conf, 7723 const struct rte_flow_action *action, 7724 const struct mlx5_flow_driver_ops *fops, 7725 struct rte_flow_error *error) 7726 { 7727 static const char err_msg[] = "shared action validation unsupported"; 7728 7729 if (!fops->action_validate) { 7730 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7731 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7732 NULL, err_msg); 7733 return -rte_errno; 7734 } 7735 return fops->action_validate(dev, conf, action, error); 7736 } 7737 7738 /** 7739 * Destroys the shared action by handle. 7740 * 7741 * @param dev 7742 * Pointer to Ethernet device structure. 7743 * @param[in] action 7744 * Handle for the shared action to be destroyed. 7745 * @param[out] error 7746 * Perform verbose error reporting if not NULL. PMDs initialize this 7747 * structure in case of error only. 7748 * 7749 * @return 7750 * 0 on success, a negative errno value otherwise and rte_errno is set. 7751 * 7752 * @note: wrapper for driver action_create op callback. 7753 */ 7754 static int 7755 mlx5_shared_action_destroy(struct rte_eth_dev *dev, 7756 struct rte_flow_shared_action *action, 7757 struct rte_flow_error *error) 7758 { 7759 static const char err_msg[] = "shared action destruction unsupported"; 7760 struct rte_flow_attr attr = { .transfer = 0 }; 7761 const struct mlx5_flow_driver_ops *fops = 7762 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7763 7764 if (!fops->action_destroy) { 7765 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7766 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7767 NULL, err_msg); 7768 return -rte_errno; 7769 } 7770 return fops->action_destroy(dev, action, error); 7771 } 7772 7773 /* Wrapper for driver action_destroy op callback */ 7774 static int 7775 flow_drv_action_update(struct rte_eth_dev *dev, 7776 struct rte_flow_shared_action *action, 7777 const void *action_conf, 7778 const struct mlx5_flow_driver_ops *fops, 7779 struct rte_flow_error *error) 7780 { 7781 static const char err_msg[] = "shared action update unsupported"; 7782 7783 if (!fops->action_update) { 7784 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7785 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7786 NULL, err_msg); 7787 return -rte_errno; 7788 } 7789 return fops->action_update(dev, action, action_conf, error); 7790 } 7791 7792 /** 7793 * Create shared action for reuse in multiple flow rules. 7794 * 7795 * @param dev 7796 * Pointer to Ethernet device structure. 7797 * @param[in] action 7798 * Action configuration for shared action creation. 7799 * @param[out] error 7800 * Perform verbose error reporting if not NULL. PMDs initialize this 7801 * structure in case of error only. 7802 * @return 7803 * A valid handle in case of success, NULL otherwise and rte_errno is set. 7804 */ 7805 static struct rte_flow_shared_action * 7806 mlx5_shared_action_create(struct rte_eth_dev *dev, 7807 const struct rte_flow_shared_action_conf *conf, 7808 const struct rte_flow_action *action, 7809 struct rte_flow_error *error) 7810 { 7811 static const char err_msg[] = "shared action creation unsupported"; 7812 struct rte_flow_attr attr = { .transfer = 0 }; 7813 const struct mlx5_flow_driver_ops *fops = 7814 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7815 7816 if (flow_drv_action_validate(dev, conf, action, fops, error)) 7817 return NULL; 7818 if (!fops->action_create) { 7819 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7820 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7821 NULL, err_msg); 7822 return NULL; 7823 } 7824 return fops->action_create(dev, conf, action, error); 7825 } 7826 7827 /** 7828 * Updates inplace the shared action configuration pointed by *action* handle 7829 * with the configuration provided as *action* argument. 7830 * The update of the shared action configuration effects all flow rules reusing 7831 * the action via handle. 7832 * 7833 * @param dev 7834 * Pointer to Ethernet device structure. 7835 * @param[in] shared_action 7836 * Handle for the shared action to be updated. 7837 * @param[in] action 7838 * Action specification used to modify the action pointed by handle. 7839 * *action* should be of same type with the action pointed by the *action* 7840 * handle argument, otherwise considered as invalid. 7841 * @param[out] error 7842 * Perform verbose error reporting if not NULL. PMDs initialize this 7843 * structure in case of error only. 7844 * 7845 * @return 7846 * 0 on success, a negative errno value otherwise and rte_errno is set. 7847 */ 7848 static int 7849 mlx5_shared_action_update(struct rte_eth_dev *dev, 7850 struct rte_flow_shared_action *shared_action, 7851 const struct rte_flow_action *action, 7852 struct rte_flow_error *error) 7853 { 7854 struct rte_flow_attr attr = { .transfer = 0 }; 7855 const struct mlx5_flow_driver_ops *fops = 7856 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7857 int ret; 7858 7859 switch (shared_action->type) { 7860 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 7861 if (action->type != RTE_FLOW_ACTION_TYPE_RSS) { 7862 return rte_flow_error_set(error, EINVAL, 7863 RTE_FLOW_ERROR_TYPE_ACTION, 7864 NULL, 7865 "update action type invalid"); 7866 } 7867 ret = flow_drv_action_validate(dev, NULL, action, fops, error); 7868 if (ret) 7869 return ret; 7870 return flow_drv_action_update(dev, shared_action, action->conf, 7871 fops, error); 7872 default: 7873 return rte_flow_error_set(error, ENOTSUP, 7874 RTE_FLOW_ERROR_TYPE_ACTION, 7875 NULL, 7876 "action type not supported"); 7877 } 7878 } 7879 7880 /** 7881 * Query the shared action by handle. 7882 * 7883 * This function allows retrieving action-specific data such as counters. 7884 * Data is gathered by special action which may be present/referenced in 7885 * more than one flow rule definition. 7886 * 7887 * \see RTE_FLOW_ACTION_TYPE_COUNT 7888 * 7889 * @param dev 7890 * Pointer to Ethernet device structure. 7891 * @param[in] action 7892 * Handle for the shared action to query. 7893 * @param[in, out] data 7894 * Pointer to storage for the associated query data type. 7895 * @param[out] error 7896 * Perform verbose error reporting if not NULL. PMDs initialize this 7897 * structure in case of error only. 7898 * 7899 * @return 7900 * 0 on success, a negative errno value otherwise and rte_errno is set. 7901 */ 7902 static int 7903 mlx5_shared_action_query(struct rte_eth_dev *dev, 7904 const struct rte_flow_shared_action *action, 7905 void *data, 7906 struct rte_flow_error *error) 7907 { 7908 (void)dev; 7909 switch (action->type) { 7910 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 7911 __atomic_load(&action->refcnt, (uint32_t *)data, 7912 __ATOMIC_RELAXED); 7913 return 0; 7914 default: 7915 return rte_flow_error_set(error, ENOTSUP, 7916 RTE_FLOW_ERROR_TYPE_ACTION, 7917 NULL, 7918 "action type not supported"); 7919 } 7920 } 7921 7922 /** 7923 * Destroy all shared actions. 7924 * 7925 * @param dev 7926 * Pointer to Ethernet device. 7927 * 7928 * @return 7929 * 0 on success, a negative errno value otherwise and rte_errno is set. 7930 */ 7931 int 7932 mlx5_shared_action_flush(struct rte_eth_dev *dev) 7933 { 7934 struct rte_flow_error error; 7935 struct mlx5_priv *priv = dev->data->dev_private; 7936 struct rte_flow_shared_action *action; 7937 int ret = 0; 7938 7939 while (!LIST_EMPTY(&priv->shared_actions)) { 7940 action = LIST_FIRST(&priv->shared_actions); 7941 ret = mlx5_shared_action_destroy(dev, action, &error); 7942 } 7943 return ret; 7944 } 7945 7946 static void 7947 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 7948 struct mlx5_flow_tunnel *tunnel) 7949 { 7950 struct mlx5_priv *priv = dev->data->dev_private; 7951 7952 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 7953 dev->data->port_id, tunnel->tunnel_id); 7954 RTE_VERIFY(!__atomic_load_n(&tunnel->refctn, __ATOMIC_RELAXED)); 7955 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID], 7956 tunnel->tunnel_id); 7957 mlx5_hlist_destroy(tunnel->groups); 7958 mlx5_free(tunnel); 7959 } 7960 7961 static struct mlx5_flow_tunnel * 7962 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 7963 { 7964 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 7965 struct mlx5_flow_tunnel *tun; 7966 7967 LIST_FOREACH(tun, &thub->tunnels, chain) { 7968 if (tun->tunnel_id == id) 7969 break; 7970 } 7971 7972 return tun; 7973 } 7974 7975 static struct mlx5_flow_tunnel * 7976 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 7977 const struct rte_flow_tunnel *app_tunnel) 7978 { 7979 struct mlx5_priv *priv = dev->data->dev_private; 7980 struct mlx5_flow_tunnel *tunnel; 7981 uint32_t id; 7982 7983 mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 7984 &id); 7985 if (id >= MLX5_MAX_TUNNELS) { 7986 mlx5_ipool_free(priv->sh->ipool 7987 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id); 7988 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 7989 return NULL; 7990 } else if (!id) { 7991 return NULL; 7992 } 7993 /** 7994 * mlx5 flow tunnel is an auxlilary data structure 7995 * It's not part of IO. No need to allocate it from 7996 * huge pages pools dedicated for IO 7997 */ 7998 tunnel = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*tunnel), 7999 0, SOCKET_ID_ANY); 8000 if (!tunnel) { 8001 mlx5_ipool_free(priv->sh->ipool 8002 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id); 8003 return NULL; 8004 } 8005 tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, 8006 mlx5_flow_tunnel_grp2tbl_create_cb, 8007 NULL, 8008 mlx5_flow_tunnel_grp2tbl_remove_cb); 8009 if (!tunnel->groups) { 8010 mlx5_ipool_free(priv->sh->ipool 8011 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id); 8012 mlx5_free(tunnel); 8013 return NULL; 8014 } 8015 tunnel->groups->ctx = priv->sh; 8016 /* initiate new PMD tunnel */ 8017 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 8018 tunnel->tunnel_id = id; 8019 tunnel->action.type = (typeof(tunnel->action.type)) 8020 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 8021 tunnel->action.conf = tunnel; 8022 tunnel->item.type = (typeof(tunnel->item.type)) 8023 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 8024 tunnel->item.spec = tunnel; 8025 tunnel->item.last = NULL; 8026 tunnel->item.mask = NULL; 8027 8028 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 8029 dev->data->port_id, tunnel->tunnel_id); 8030 8031 return tunnel; 8032 } 8033 8034 static int 8035 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 8036 const struct rte_flow_tunnel *app_tunnel, 8037 struct mlx5_flow_tunnel **tunnel) 8038 { 8039 int ret; 8040 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 8041 struct mlx5_flow_tunnel *tun; 8042 8043 rte_spinlock_lock(&thub->sl); 8044 LIST_FOREACH(tun, &thub->tunnels, chain) { 8045 if (!memcmp(app_tunnel, &tun->app_tunnel, 8046 sizeof(*app_tunnel))) { 8047 *tunnel = tun; 8048 ret = 0; 8049 break; 8050 } 8051 } 8052 if (!tun) { 8053 tun = mlx5_flow_tunnel_allocate(dev, app_tunnel); 8054 if (tun) { 8055 LIST_INSERT_HEAD(&thub->tunnels, tun, chain); 8056 *tunnel = tun; 8057 } else { 8058 ret = -ENOMEM; 8059 } 8060 } 8061 rte_spinlock_unlock(&thub->sl); 8062 if (tun) 8063 __atomic_add_fetch(&tun->refctn, 1, __ATOMIC_RELAXED); 8064 8065 return ret; 8066 } 8067 8068 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 8069 { 8070 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 8071 8072 if (!thub) 8073 return; 8074 if (!LIST_EMPTY(&thub->tunnels)) 8075 DRV_LOG(WARNING, "port %u tunnels present\n", port_id); 8076 mlx5_hlist_destroy(thub->groups); 8077 mlx5_free(thub); 8078 } 8079 8080 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 8081 { 8082 int err; 8083 struct mlx5_flow_tunnel_hub *thub; 8084 8085 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 8086 0, SOCKET_ID_ANY); 8087 if (!thub) 8088 return -ENOMEM; 8089 LIST_INIT(&thub->tunnels); 8090 rte_spinlock_init(&thub->sl); 8091 thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0, 8092 0, mlx5_flow_tunnel_grp2tbl_create_cb, 8093 NULL, 8094 mlx5_flow_tunnel_grp2tbl_remove_cb); 8095 if (!thub->groups) { 8096 err = -rte_errno; 8097 goto err; 8098 } 8099 thub->groups->ctx = sh; 8100 sh->tunnel_hub = thub; 8101 8102 return 0; 8103 8104 err: 8105 if (thub->groups) 8106 mlx5_hlist_destroy(thub->groups); 8107 if (thub) 8108 mlx5_free(thub); 8109 return err; 8110 } 8111 8112 #ifndef HAVE_MLX5DV_DR 8113 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 8114 #else 8115 #define MLX5_DOMAIN_SYNC_FLOW \ 8116 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 8117 #endif 8118 8119 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 8120 { 8121 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 8122 const struct mlx5_flow_driver_ops *fops; 8123 int ret; 8124 struct rte_flow_attr attr = { .transfer = 0 }; 8125 8126 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8127 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 8128 if (ret > 0) 8129 ret = -ret; 8130 return ret; 8131 } 8132