1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 #include <rte_common.h> 14 #include <rte_ether.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rxtx.h" 32 33 /** Device flow drivers. */ 34 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 35 36 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 37 38 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 39 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 40 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 41 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 42 #endif 43 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 44 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 45 }; 46 47 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 48 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 49 (const int []){ \ 50 __VA_ARGS__, 0, \ 51 } 52 53 /** Node object of input graph for mlx5_flow_expand_rss(). */ 54 struct mlx5_flow_expand_node { 55 const int *const next; 56 /**< 57 * List of next node indexes. Index 0 is interpreted as a terminator. 58 */ 59 const enum rte_flow_item_type type; 60 /**< Pattern item type of current node. */ 61 uint64_t rss_types; 62 /**< 63 * RSS types bit-field associated with this node 64 * (see ETH_RSS_* definitions). 65 */ 66 }; 67 68 /** Object returned by mlx5_flow_expand_rss(). */ 69 struct mlx5_flow_expand_rss { 70 uint32_t entries; 71 /**< Number of entries @p patterns and @p priorities. */ 72 struct { 73 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 74 uint32_t priority; /**< Priority offset for each expansion. */ 75 } entry[]; 76 }; 77 78 static enum rte_flow_item_type 79 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 80 { 81 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 82 uint16_t ether_type = 0; 83 uint16_t ether_type_m; 84 uint8_t ip_next_proto = 0; 85 uint8_t ip_next_proto_m; 86 87 if (item == NULL || item->spec == NULL) 88 return ret; 89 switch (item->type) { 90 case RTE_FLOW_ITEM_TYPE_ETH: 91 if (item->mask) 92 ether_type_m = ((const struct rte_flow_item_eth *) 93 (item->mask))->type; 94 else 95 ether_type_m = rte_flow_item_eth_mask.type; 96 if (ether_type_m != RTE_BE16(0xFFFF)) 97 break; 98 ether_type = ((const struct rte_flow_item_eth *) 99 (item->spec))->type; 100 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 101 ret = RTE_FLOW_ITEM_TYPE_IPV4; 102 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 103 ret = RTE_FLOW_ITEM_TYPE_IPV6; 104 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 105 ret = RTE_FLOW_ITEM_TYPE_VLAN; 106 else 107 ret = RTE_FLOW_ITEM_TYPE_END; 108 break; 109 case RTE_FLOW_ITEM_TYPE_VLAN: 110 if (item->mask) 111 ether_type_m = ((const struct rte_flow_item_vlan *) 112 (item->mask))->inner_type; 113 else 114 ether_type_m = rte_flow_item_vlan_mask.inner_type; 115 if (ether_type_m != RTE_BE16(0xFFFF)) 116 break; 117 ether_type = ((const struct rte_flow_item_vlan *) 118 (item->spec))->inner_type; 119 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 120 ret = RTE_FLOW_ITEM_TYPE_IPV4; 121 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 122 ret = RTE_FLOW_ITEM_TYPE_IPV6; 123 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 124 ret = RTE_FLOW_ITEM_TYPE_VLAN; 125 else 126 ret = RTE_FLOW_ITEM_TYPE_END; 127 break; 128 case RTE_FLOW_ITEM_TYPE_IPV4: 129 if (item->mask) 130 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 131 (item->mask))->hdr.next_proto_id; 132 else 133 ip_next_proto_m = 134 rte_flow_item_ipv4_mask.hdr.next_proto_id; 135 if (ip_next_proto_m != 0xFF) 136 break; 137 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 138 (item->spec))->hdr.next_proto_id; 139 if (ip_next_proto == IPPROTO_UDP) 140 ret = RTE_FLOW_ITEM_TYPE_UDP; 141 else if (ip_next_proto == IPPROTO_TCP) 142 ret = RTE_FLOW_ITEM_TYPE_TCP; 143 else if (ip_next_proto == IPPROTO_IP) 144 ret = RTE_FLOW_ITEM_TYPE_IPV4; 145 else if (ip_next_proto == IPPROTO_IPV6) 146 ret = RTE_FLOW_ITEM_TYPE_IPV6; 147 else 148 ret = RTE_FLOW_ITEM_TYPE_END; 149 break; 150 case RTE_FLOW_ITEM_TYPE_IPV6: 151 if (item->mask) 152 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 153 (item->mask))->hdr.proto; 154 else 155 ip_next_proto_m = 156 rte_flow_item_ipv6_mask.hdr.proto; 157 if (ip_next_proto_m != 0xFF) 158 break; 159 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 160 (item->spec))->hdr.proto; 161 if (ip_next_proto == IPPROTO_UDP) 162 ret = RTE_FLOW_ITEM_TYPE_UDP; 163 else if (ip_next_proto == IPPROTO_TCP) 164 ret = RTE_FLOW_ITEM_TYPE_TCP; 165 else if (ip_next_proto == IPPROTO_IP) 166 ret = RTE_FLOW_ITEM_TYPE_IPV4; 167 else if (ip_next_proto == IPPROTO_IPV6) 168 ret = RTE_FLOW_ITEM_TYPE_IPV6; 169 else 170 ret = RTE_FLOW_ITEM_TYPE_END; 171 break; 172 default: 173 ret = RTE_FLOW_ITEM_TYPE_VOID; 174 break; 175 } 176 return ret; 177 } 178 179 /** 180 * Expand RSS flows into several possible flows according to the RSS hash 181 * fields requested and the driver capabilities. 182 * 183 * @param[out] buf 184 * Buffer to store the result expansion. 185 * @param[in] size 186 * Buffer size in bytes. If 0, @p buf can be NULL. 187 * @param[in] pattern 188 * User flow pattern. 189 * @param[in] types 190 * RSS types to expand (see ETH_RSS_* definitions). 191 * @param[in] graph 192 * Input graph to expand @p pattern according to @p types. 193 * @param[in] graph_root_index 194 * Index of root node in @p graph, typically 0. 195 * 196 * @return 197 * A positive value representing the size of @p buf in bytes regardless of 198 * @p size on success, a negative errno value otherwise and rte_errno is 199 * set, the following errors are defined: 200 * 201 * -E2BIG: graph-depth @p graph is too deep. 202 */ 203 static int 204 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 205 const struct rte_flow_item *pattern, uint64_t types, 206 const struct mlx5_flow_expand_node graph[], 207 int graph_root_index) 208 { 209 const int elt_n = 8; 210 const struct rte_flow_item *item; 211 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 212 const int *next_node; 213 const int *stack[elt_n]; 214 int stack_pos = 0; 215 struct rte_flow_item flow_items[elt_n]; 216 unsigned int i; 217 size_t lsize; 218 size_t user_pattern_size = 0; 219 void *addr = NULL; 220 const struct mlx5_flow_expand_node *next = NULL; 221 struct rte_flow_item missed_item; 222 int missed = 0; 223 int elt = 0; 224 const struct rte_flow_item *last_item = NULL; 225 226 memset(&missed_item, 0, sizeof(missed_item)); 227 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 228 elt_n * sizeof(buf->entry[0]); 229 if (lsize <= size) { 230 buf->entry[0].priority = 0; 231 buf->entry[0].pattern = (void *)&buf->entry[elt_n]; 232 buf->entries = 0; 233 addr = buf->entry[0].pattern; 234 } 235 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 236 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 237 last_item = item; 238 for (i = 0; node->next && node->next[i]; ++i) { 239 next = &graph[node->next[i]]; 240 if (next->type == item->type) 241 break; 242 } 243 if (next) 244 node = next; 245 user_pattern_size += sizeof(*item); 246 } 247 user_pattern_size += sizeof(*item); /* Handle END item. */ 248 lsize += user_pattern_size; 249 /* Copy the user pattern in the first entry of the buffer. */ 250 if (lsize <= size) { 251 rte_memcpy(addr, pattern, user_pattern_size); 252 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 253 buf->entries = 1; 254 } 255 /* Start expanding. */ 256 memset(flow_items, 0, sizeof(flow_items)); 257 user_pattern_size -= sizeof(*item); 258 /* 259 * Check if the last valid item has spec set, need complete pattern, 260 * and the pattern can be used for expansion. 261 */ 262 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 263 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 264 /* Item type END indicates expansion is not required. */ 265 return lsize; 266 } 267 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 268 next = NULL; 269 missed = 1; 270 for (i = 0; node->next && node->next[i]; ++i) { 271 next = &graph[node->next[i]]; 272 if (next->type == missed_item.type) { 273 flow_items[0].type = missed_item.type; 274 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 275 break; 276 } 277 next = NULL; 278 } 279 } 280 if (next && missed) { 281 elt = 2; /* missed item + item end. */ 282 node = next; 283 lsize += elt * sizeof(*item) + user_pattern_size; 284 if ((node->rss_types & types) && lsize <= size) { 285 buf->entry[buf->entries].priority = 1; 286 buf->entry[buf->entries].pattern = addr; 287 buf->entries++; 288 rte_memcpy(addr, buf->entry[0].pattern, 289 user_pattern_size); 290 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 291 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 292 addr = (void *)(((uintptr_t)addr) + 293 elt * sizeof(*item)); 294 } 295 } 296 memset(flow_items, 0, sizeof(flow_items)); 297 next_node = node->next; 298 stack[stack_pos] = next_node; 299 node = next_node ? &graph[*next_node] : NULL; 300 while (node) { 301 flow_items[stack_pos].type = node->type; 302 if (node->rss_types & types) { 303 /* 304 * compute the number of items to copy from the 305 * expansion and copy it. 306 * When the stack_pos is 0, there are 1 element in it, 307 * plus the addition END item. 308 */ 309 elt = stack_pos + 2; 310 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 311 lsize += elt * sizeof(*item) + user_pattern_size; 312 if (lsize <= size) { 313 size_t n = elt * sizeof(*item); 314 315 buf->entry[buf->entries].priority = 316 stack_pos + 1 + missed; 317 buf->entry[buf->entries].pattern = addr; 318 buf->entries++; 319 rte_memcpy(addr, buf->entry[0].pattern, 320 user_pattern_size); 321 addr = (void *)(((uintptr_t)addr) + 322 user_pattern_size); 323 rte_memcpy(addr, &missed_item, 324 missed * sizeof(*item)); 325 addr = (void *)(((uintptr_t)addr) + 326 missed * sizeof(*item)); 327 rte_memcpy(addr, flow_items, n); 328 addr = (void *)(((uintptr_t)addr) + n); 329 } 330 } 331 /* Go deeper. */ 332 if (node->next) { 333 next_node = node->next; 334 if (stack_pos++ == elt_n) { 335 rte_errno = E2BIG; 336 return -rte_errno; 337 } 338 stack[stack_pos] = next_node; 339 } else if (*(next_node + 1)) { 340 /* Follow up with the next possibility. */ 341 ++next_node; 342 } else { 343 /* Move to the next path. */ 344 if (stack_pos) 345 next_node = stack[--stack_pos]; 346 next_node++; 347 stack[stack_pos] = next_node; 348 } 349 node = *next_node ? &graph[*next_node] : NULL; 350 }; 351 /* no expanded flows but we have missed item, create one rule for it */ 352 if (buf->entries == 1 && missed != 0) { 353 elt = 2; 354 lsize += elt * sizeof(*item) + user_pattern_size; 355 if (lsize <= size) { 356 buf->entry[buf->entries].priority = 1; 357 buf->entry[buf->entries].pattern = addr; 358 buf->entries++; 359 flow_items[0].type = missed_item.type; 360 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 361 rte_memcpy(addr, buf->entry[0].pattern, 362 user_pattern_size); 363 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 364 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 365 addr = (void *)(((uintptr_t)addr) + 366 elt * sizeof(*item)); 367 } 368 } 369 return lsize; 370 } 371 372 enum mlx5_expansion { 373 MLX5_EXPANSION_ROOT, 374 MLX5_EXPANSION_ROOT_OUTER, 375 MLX5_EXPANSION_ROOT_ETH_VLAN, 376 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 377 MLX5_EXPANSION_OUTER_ETH, 378 MLX5_EXPANSION_OUTER_ETH_VLAN, 379 MLX5_EXPANSION_OUTER_VLAN, 380 MLX5_EXPANSION_OUTER_IPV4, 381 MLX5_EXPANSION_OUTER_IPV4_UDP, 382 MLX5_EXPANSION_OUTER_IPV4_TCP, 383 MLX5_EXPANSION_OUTER_IPV6, 384 MLX5_EXPANSION_OUTER_IPV6_UDP, 385 MLX5_EXPANSION_OUTER_IPV6_TCP, 386 MLX5_EXPANSION_VXLAN, 387 MLX5_EXPANSION_VXLAN_GPE, 388 MLX5_EXPANSION_GRE, 389 MLX5_EXPANSION_MPLS, 390 MLX5_EXPANSION_ETH, 391 MLX5_EXPANSION_ETH_VLAN, 392 MLX5_EXPANSION_VLAN, 393 MLX5_EXPANSION_IPV4, 394 MLX5_EXPANSION_IPV4_UDP, 395 MLX5_EXPANSION_IPV4_TCP, 396 MLX5_EXPANSION_IPV6, 397 MLX5_EXPANSION_IPV6_UDP, 398 MLX5_EXPANSION_IPV6_TCP, 399 }; 400 401 /** Supported expansion of items. */ 402 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 403 [MLX5_EXPANSION_ROOT] = { 404 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 405 MLX5_EXPANSION_IPV4, 406 MLX5_EXPANSION_IPV6), 407 .type = RTE_FLOW_ITEM_TYPE_END, 408 }, 409 [MLX5_EXPANSION_ROOT_OUTER] = { 410 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 411 MLX5_EXPANSION_OUTER_IPV4, 412 MLX5_EXPANSION_OUTER_IPV6), 413 .type = RTE_FLOW_ITEM_TYPE_END, 414 }, 415 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 416 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 417 .type = RTE_FLOW_ITEM_TYPE_END, 418 }, 419 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 420 .next = MLX5_FLOW_EXPAND_RSS_NEXT 421 (MLX5_EXPANSION_OUTER_ETH_VLAN), 422 .type = RTE_FLOW_ITEM_TYPE_END, 423 }, 424 [MLX5_EXPANSION_OUTER_ETH] = { 425 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 426 MLX5_EXPANSION_OUTER_IPV6, 427 MLX5_EXPANSION_MPLS), 428 .type = RTE_FLOW_ITEM_TYPE_ETH, 429 .rss_types = 0, 430 }, 431 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 432 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 433 .type = RTE_FLOW_ITEM_TYPE_ETH, 434 .rss_types = 0, 435 }, 436 [MLX5_EXPANSION_OUTER_VLAN] = { 437 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 438 MLX5_EXPANSION_OUTER_IPV6), 439 .type = RTE_FLOW_ITEM_TYPE_VLAN, 440 }, 441 [MLX5_EXPANSION_OUTER_IPV4] = { 442 .next = MLX5_FLOW_EXPAND_RSS_NEXT 443 (MLX5_EXPANSION_OUTER_IPV4_UDP, 444 MLX5_EXPANSION_OUTER_IPV4_TCP, 445 MLX5_EXPANSION_GRE, 446 MLX5_EXPANSION_IPV4, 447 MLX5_EXPANSION_IPV6), 448 .type = RTE_FLOW_ITEM_TYPE_IPV4, 449 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 450 ETH_RSS_NONFRAG_IPV4_OTHER, 451 }, 452 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 453 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 454 MLX5_EXPANSION_VXLAN_GPE), 455 .type = RTE_FLOW_ITEM_TYPE_UDP, 456 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 457 }, 458 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 459 .type = RTE_FLOW_ITEM_TYPE_TCP, 460 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 461 }, 462 [MLX5_EXPANSION_OUTER_IPV6] = { 463 .next = MLX5_FLOW_EXPAND_RSS_NEXT 464 (MLX5_EXPANSION_OUTER_IPV6_UDP, 465 MLX5_EXPANSION_OUTER_IPV6_TCP, 466 MLX5_EXPANSION_IPV4, 467 MLX5_EXPANSION_IPV6), 468 .type = RTE_FLOW_ITEM_TYPE_IPV6, 469 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 470 ETH_RSS_NONFRAG_IPV6_OTHER, 471 }, 472 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 473 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 474 MLX5_EXPANSION_VXLAN_GPE), 475 .type = RTE_FLOW_ITEM_TYPE_UDP, 476 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 477 }, 478 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 479 .type = RTE_FLOW_ITEM_TYPE_TCP, 480 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 481 }, 482 [MLX5_EXPANSION_VXLAN] = { 483 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 484 MLX5_EXPANSION_IPV4, 485 MLX5_EXPANSION_IPV6), 486 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 487 }, 488 [MLX5_EXPANSION_VXLAN_GPE] = { 489 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 490 MLX5_EXPANSION_IPV4, 491 MLX5_EXPANSION_IPV6), 492 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 493 }, 494 [MLX5_EXPANSION_GRE] = { 495 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 496 .type = RTE_FLOW_ITEM_TYPE_GRE, 497 }, 498 [MLX5_EXPANSION_MPLS] = { 499 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 500 MLX5_EXPANSION_IPV6), 501 .type = RTE_FLOW_ITEM_TYPE_MPLS, 502 }, 503 [MLX5_EXPANSION_ETH] = { 504 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 505 MLX5_EXPANSION_IPV6), 506 .type = RTE_FLOW_ITEM_TYPE_ETH, 507 }, 508 [MLX5_EXPANSION_ETH_VLAN] = { 509 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 510 .type = RTE_FLOW_ITEM_TYPE_ETH, 511 }, 512 [MLX5_EXPANSION_VLAN] = { 513 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 514 MLX5_EXPANSION_IPV6), 515 .type = RTE_FLOW_ITEM_TYPE_VLAN, 516 }, 517 [MLX5_EXPANSION_IPV4] = { 518 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 519 MLX5_EXPANSION_IPV4_TCP), 520 .type = RTE_FLOW_ITEM_TYPE_IPV4, 521 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 522 ETH_RSS_NONFRAG_IPV4_OTHER, 523 }, 524 [MLX5_EXPANSION_IPV4_UDP] = { 525 .type = RTE_FLOW_ITEM_TYPE_UDP, 526 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 527 }, 528 [MLX5_EXPANSION_IPV4_TCP] = { 529 .type = RTE_FLOW_ITEM_TYPE_TCP, 530 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 531 }, 532 [MLX5_EXPANSION_IPV6] = { 533 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 534 MLX5_EXPANSION_IPV6_TCP), 535 .type = RTE_FLOW_ITEM_TYPE_IPV6, 536 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 537 ETH_RSS_NONFRAG_IPV6_OTHER, 538 }, 539 [MLX5_EXPANSION_IPV6_UDP] = { 540 .type = RTE_FLOW_ITEM_TYPE_UDP, 541 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 542 }, 543 [MLX5_EXPANSION_IPV6_TCP] = { 544 .type = RTE_FLOW_ITEM_TYPE_TCP, 545 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 546 }, 547 }; 548 549 static const struct rte_flow_ops mlx5_flow_ops = { 550 .validate = mlx5_flow_validate, 551 .create = mlx5_flow_create, 552 .destroy = mlx5_flow_destroy, 553 .flush = mlx5_flow_flush, 554 .isolate = mlx5_flow_isolate, 555 .query = mlx5_flow_query, 556 .dev_dump = mlx5_flow_dev_dump, 557 .get_aged_flows = mlx5_flow_get_aged_flows, 558 }; 559 560 /* Convert FDIR request to Generic flow. */ 561 struct mlx5_fdir { 562 struct rte_flow_attr attr; 563 struct rte_flow_item items[4]; 564 struct rte_flow_item_eth l2; 565 struct rte_flow_item_eth l2_mask; 566 union { 567 struct rte_flow_item_ipv4 ipv4; 568 struct rte_flow_item_ipv6 ipv6; 569 } l3; 570 union { 571 struct rte_flow_item_ipv4 ipv4; 572 struct rte_flow_item_ipv6 ipv6; 573 } l3_mask; 574 union { 575 struct rte_flow_item_udp udp; 576 struct rte_flow_item_tcp tcp; 577 } l4; 578 union { 579 struct rte_flow_item_udp udp; 580 struct rte_flow_item_tcp tcp; 581 } l4_mask; 582 struct rte_flow_action actions[2]; 583 struct rte_flow_action_queue queue; 584 }; 585 586 /* Tunnel information. */ 587 struct mlx5_flow_tunnel_info { 588 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 589 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 590 }; 591 592 static struct mlx5_flow_tunnel_info tunnels_info[] = { 593 { 594 .tunnel = MLX5_FLOW_LAYER_VXLAN, 595 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 596 }, 597 { 598 .tunnel = MLX5_FLOW_LAYER_GENEVE, 599 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 600 }, 601 { 602 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 603 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 604 }, 605 { 606 .tunnel = MLX5_FLOW_LAYER_GRE, 607 .ptype = RTE_PTYPE_TUNNEL_GRE, 608 }, 609 { 610 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 611 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 612 }, 613 { 614 .tunnel = MLX5_FLOW_LAYER_MPLS, 615 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 616 }, 617 { 618 .tunnel = MLX5_FLOW_LAYER_NVGRE, 619 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 620 }, 621 { 622 .tunnel = MLX5_FLOW_LAYER_IPIP, 623 .ptype = RTE_PTYPE_TUNNEL_IP, 624 }, 625 { 626 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 627 .ptype = RTE_PTYPE_TUNNEL_IP, 628 }, 629 { 630 .tunnel = MLX5_FLOW_LAYER_GTP, 631 .ptype = RTE_PTYPE_TUNNEL_GTPU, 632 }, 633 }; 634 635 /** 636 * Translate tag ID to register. 637 * 638 * @param[in] dev 639 * Pointer to the Ethernet device structure. 640 * @param[in] feature 641 * The feature that request the register. 642 * @param[in] id 643 * The request register ID. 644 * @param[out] error 645 * Error description in case of any. 646 * 647 * @return 648 * The request register on success, a negative errno 649 * value otherwise and rte_errno is set. 650 */ 651 int 652 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 653 enum mlx5_feature_name feature, 654 uint32_t id, 655 struct rte_flow_error *error) 656 { 657 struct mlx5_priv *priv = dev->data->dev_private; 658 struct mlx5_dev_config *config = &priv->config; 659 enum modify_reg start_reg; 660 bool skip_mtr_reg = false; 661 662 switch (feature) { 663 case MLX5_HAIRPIN_RX: 664 return REG_B; 665 case MLX5_HAIRPIN_TX: 666 return REG_A; 667 case MLX5_METADATA_RX: 668 switch (config->dv_xmeta_en) { 669 case MLX5_XMETA_MODE_LEGACY: 670 return REG_B; 671 case MLX5_XMETA_MODE_META16: 672 return REG_C_0; 673 case MLX5_XMETA_MODE_META32: 674 return REG_C_1; 675 } 676 break; 677 case MLX5_METADATA_TX: 678 return REG_A; 679 case MLX5_METADATA_FDB: 680 switch (config->dv_xmeta_en) { 681 case MLX5_XMETA_MODE_LEGACY: 682 return REG_NON; 683 case MLX5_XMETA_MODE_META16: 684 return REG_C_0; 685 case MLX5_XMETA_MODE_META32: 686 return REG_C_1; 687 } 688 break; 689 case MLX5_FLOW_MARK: 690 switch (config->dv_xmeta_en) { 691 case MLX5_XMETA_MODE_LEGACY: 692 return REG_NON; 693 case MLX5_XMETA_MODE_META16: 694 return REG_C_1; 695 case MLX5_XMETA_MODE_META32: 696 return REG_C_0; 697 } 698 break; 699 case MLX5_MTR_SFX: 700 /* 701 * If meter color and flow match share one register, flow match 702 * should use the meter color register for match. 703 */ 704 if (priv->mtr_reg_share) 705 return priv->mtr_color_reg; 706 else 707 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 708 REG_C_3; 709 case MLX5_MTR_COLOR: 710 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 711 return priv->mtr_color_reg; 712 case MLX5_COPY_MARK: 713 /* 714 * Metadata COPY_MARK register using is in meter suffix sub 715 * flow while with meter. It's safe to share the same register. 716 */ 717 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 718 case MLX5_APP_TAG: 719 /* 720 * If meter is enable, it will engage the register for color 721 * match and flow match. If meter color match is not using the 722 * REG_C_2, need to skip the REG_C_x be used by meter color 723 * match. 724 * If meter is disable, free to use all available registers. 725 */ 726 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 727 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 728 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 729 if (id > (REG_C_7 - start_reg)) 730 return rte_flow_error_set(error, EINVAL, 731 RTE_FLOW_ERROR_TYPE_ITEM, 732 NULL, "invalid tag id"); 733 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 734 return rte_flow_error_set(error, ENOTSUP, 735 RTE_FLOW_ERROR_TYPE_ITEM, 736 NULL, "unsupported tag id"); 737 /* 738 * This case means meter is using the REG_C_x great than 2. 739 * Take care not to conflict with meter color REG_C_x. 740 * If the available index REG_C_y >= REG_C_x, skip the 741 * color register. 742 */ 743 if (skip_mtr_reg && config->flow_mreg_c 744 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 745 if (id >= (REG_C_7 - start_reg)) 746 return rte_flow_error_set(error, EINVAL, 747 RTE_FLOW_ERROR_TYPE_ITEM, 748 NULL, "invalid tag id"); 749 if (config->flow_mreg_c 750 [id + 1 + start_reg - REG_C_0] != REG_NON) 751 return config->flow_mreg_c 752 [id + 1 + start_reg - REG_C_0]; 753 return rte_flow_error_set(error, ENOTSUP, 754 RTE_FLOW_ERROR_TYPE_ITEM, 755 NULL, "unsupported tag id"); 756 } 757 return config->flow_mreg_c[id + start_reg - REG_C_0]; 758 } 759 MLX5_ASSERT(false); 760 return rte_flow_error_set(error, EINVAL, 761 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 762 NULL, "invalid feature name"); 763 } 764 765 /** 766 * Check extensive flow metadata register support. 767 * 768 * @param dev 769 * Pointer to rte_eth_dev structure. 770 * 771 * @return 772 * True if device supports extensive flow metadata register, otherwise false. 773 */ 774 bool 775 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 776 { 777 struct mlx5_priv *priv = dev->data->dev_private; 778 struct mlx5_dev_config *config = &priv->config; 779 780 /* 781 * Having available reg_c can be regarded inclusively as supporting 782 * extensive flow metadata register, which could mean, 783 * - metadata register copy action by modify header. 784 * - 16 modify header actions is supported. 785 * - reg_c's are preserved across different domain (FDB and NIC) on 786 * packet loopback by flow lookup miss. 787 */ 788 return config->flow_mreg_c[2] != REG_NON; 789 } 790 791 /** 792 * Verify the @p item specifications (spec, last, mask) are compatible with the 793 * NIC capabilities. 794 * 795 * @param[in] item 796 * Item specification. 797 * @param[in] mask 798 * @p item->mask or flow default bit-masks. 799 * @param[in] nic_mask 800 * Bit-masks covering supported fields by the NIC to compare with user mask. 801 * @param[in] size 802 * Bit-masks size in bytes. 803 * @param[out] error 804 * Pointer to error structure. 805 * 806 * @return 807 * 0 on success, a negative errno value otherwise and rte_errno is set. 808 */ 809 int 810 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 811 const uint8_t *mask, 812 const uint8_t *nic_mask, 813 unsigned int size, 814 struct rte_flow_error *error) 815 { 816 unsigned int i; 817 818 MLX5_ASSERT(nic_mask); 819 for (i = 0; i < size; ++i) 820 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 821 return rte_flow_error_set(error, ENOTSUP, 822 RTE_FLOW_ERROR_TYPE_ITEM, 823 item, 824 "mask enables non supported" 825 " bits"); 826 if (!item->spec && (item->mask || item->last)) 827 return rte_flow_error_set(error, EINVAL, 828 RTE_FLOW_ERROR_TYPE_ITEM, item, 829 "mask/last without a spec is not" 830 " supported"); 831 if (item->spec && item->last) { 832 uint8_t spec[size]; 833 uint8_t last[size]; 834 unsigned int i; 835 int ret; 836 837 for (i = 0; i < size; ++i) { 838 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 839 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 840 } 841 ret = memcmp(spec, last, size); 842 if (ret != 0) 843 return rte_flow_error_set(error, EINVAL, 844 RTE_FLOW_ERROR_TYPE_ITEM, 845 item, 846 "range is not valid"); 847 } 848 return 0; 849 } 850 851 /** 852 * Adjust the hash fields according to the @p flow information. 853 * 854 * @param[in] dev_flow. 855 * Pointer to the mlx5_flow. 856 * @param[in] tunnel 857 * 1 when the hash field is for a tunnel item. 858 * @param[in] layer_types 859 * ETH_RSS_* types. 860 * @param[in] hash_fields 861 * Item hash fields. 862 * 863 * @return 864 * The hash fields that should be used. 865 */ 866 uint64_t 867 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 868 int tunnel __rte_unused, uint64_t layer_types, 869 uint64_t hash_fields) 870 { 871 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 872 int rss_request_inner = rss_desc->level >= 2; 873 874 /* Check RSS hash level for tunnel. */ 875 if (tunnel && rss_request_inner) 876 hash_fields |= IBV_RX_HASH_INNER; 877 else if (tunnel || rss_request_inner) 878 return 0; 879 #endif 880 /* Check if requested layer matches RSS hash fields. */ 881 if (!(rss_desc->types & layer_types)) 882 return 0; 883 return hash_fields; 884 } 885 886 /** 887 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 888 * if several tunnel rules are used on this queue, the tunnel ptype will be 889 * cleared. 890 * 891 * @param rxq_ctrl 892 * Rx queue to update. 893 */ 894 static void 895 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 896 { 897 unsigned int i; 898 uint32_t tunnel_ptype = 0; 899 900 /* Look up for the ptype to use. */ 901 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 902 if (!rxq_ctrl->flow_tunnels_n[i]) 903 continue; 904 if (!tunnel_ptype) { 905 tunnel_ptype = tunnels_info[i].ptype; 906 } else { 907 tunnel_ptype = 0; 908 break; 909 } 910 } 911 rxq_ctrl->rxq.tunnel = tunnel_ptype; 912 } 913 914 /** 915 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 916 * flow. 917 * 918 * @param[in] dev 919 * Pointer to the Ethernet device structure. 920 * @param[in] dev_handle 921 * Pointer to device flow handle structure. 922 */ 923 static void 924 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 925 struct mlx5_flow_handle *dev_handle) 926 { 927 struct mlx5_priv *priv = dev->data->dev_private; 928 const int mark = dev_handle->mark; 929 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 930 struct mlx5_hrxq *hrxq; 931 unsigned int i; 932 933 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 934 return; 935 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 936 dev_handle->rix_hrxq); 937 if (!hrxq) 938 return; 939 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 940 int idx = hrxq->ind_table->queues[i]; 941 struct mlx5_rxq_ctrl *rxq_ctrl = 942 container_of((*priv->rxqs)[idx], 943 struct mlx5_rxq_ctrl, rxq); 944 945 /* 946 * To support metadata register copy on Tx loopback, 947 * this must be always enabled (metadata may arive 948 * from other port - not from local flows only. 949 */ 950 if (priv->config.dv_flow_en && 951 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 952 mlx5_flow_ext_mreg_supported(dev)) { 953 rxq_ctrl->rxq.mark = 1; 954 rxq_ctrl->flow_mark_n = 1; 955 } else if (mark) { 956 rxq_ctrl->rxq.mark = 1; 957 rxq_ctrl->flow_mark_n++; 958 } 959 if (tunnel) { 960 unsigned int j; 961 962 /* Increase the counter matching the flow. */ 963 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 964 if ((tunnels_info[j].tunnel & 965 dev_handle->layers) == 966 tunnels_info[j].tunnel) { 967 rxq_ctrl->flow_tunnels_n[j]++; 968 break; 969 } 970 } 971 flow_rxq_tunnel_ptype_update(rxq_ctrl); 972 } 973 } 974 } 975 976 /** 977 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 978 * 979 * @param[in] dev 980 * Pointer to the Ethernet device structure. 981 * @param[in] flow 982 * Pointer to flow structure. 983 */ 984 static void 985 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 986 { 987 struct mlx5_priv *priv = dev->data->dev_private; 988 uint32_t handle_idx; 989 struct mlx5_flow_handle *dev_handle; 990 991 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 992 handle_idx, dev_handle, next) 993 flow_drv_rxq_flags_set(dev, dev_handle); 994 } 995 996 /** 997 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 998 * device flow if no other flow uses it with the same kind of request. 999 * 1000 * @param dev 1001 * Pointer to Ethernet device. 1002 * @param[in] dev_handle 1003 * Pointer to the device flow handle structure. 1004 */ 1005 static void 1006 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1007 struct mlx5_flow_handle *dev_handle) 1008 { 1009 struct mlx5_priv *priv = dev->data->dev_private; 1010 const int mark = dev_handle->mark; 1011 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1012 struct mlx5_hrxq *hrxq; 1013 unsigned int i; 1014 1015 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1016 return; 1017 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1018 dev_handle->rix_hrxq); 1019 if (!hrxq) 1020 return; 1021 MLX5_ASSERT(dev->data->dev_started); 1022 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1023 int idx = hrxq->ind_table->queues[i]; 1024 struct mlx5_rxq_ctrl *rxq_ctrl = 1025 container_of((*priv->rxqs)[idx], 1026 struct mlx5_rxq_ctrl, rxq); 1027 1028 if (priv->config.dv_flow_en && 1029 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1030 mlx5_flow_ext_mreg_supported(dev)) { 1031 rxq_ctrl->rxq.mark = 1; 1032 rxq_ctrl->flow_mark_n = 1; 1033 } else if (mark) { 1034 rxq_ctrl->flow_mark_n--; 1035 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1036 } 1037 if (tunnel) { 1038 unsigned int j; 1039 1040 /* Decrease the counter matching the flow. */ 1041 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1042 if ((tunnels_info[j].tunnel & 1043 dev_handle->layers) == 1044 tunnels_info[j].tunnel) { 1045 rxq_ctrl->flow_tunnels_n[j]--; 1046 break; 1047 } 1048 } 1049 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1050 } 1051 } 1052 } 1053 1054 /** 1055 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1056 * @p flow if no other flow uses it with the same kind of request. 1057 * 1058 * @param dev 1059 * Pointer to Ethernet device. 1060 * @param[in] flow 1061 * Pointer to the flow. 1062 */ 1063 static void 1064 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1065 { 1066 struct mlx5_priv *priv = dev->data->dev_private; 1067 uint32_t handle_idx; 1068 struct mlx5_flow_handle *dev_handle; 1069 1070 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1071 handle_idx, dev_handle, next) 1072 flow_drv_rxq_flags_trim(dev, dev_handle); 1073 } 1074 1075 /** 1076 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1077 * 1078 * @param dev 1079 * Pointer to Ethernet device. 1080 */ 1081 static void 1082 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1083 { 1084 struct mlx5_priv *priv = dev->data->dev_private; 1085 unsigned int i; 1086 1087 for (i = 0; i != priv->rxqs_n; ++i) { 1088 struct mlx5_rxq_ctrl *rxq_ctrl; 1089 unsigned int j; 1090 1091 if (!(*priv->rxqs)[i]) 1092 continue; 1093 rxq_ctrl = container_of((*priv->rxqs)[i], 1094 struct mlx5_rxq_ctrl, rxq); 1095 rxq_ctrl->flow_mark_n = 0; 1096 rxq_ctrl->rxq.mark = 0; 1097 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1098 rxq_ctrl->flow_tunnels_n[j] = 0; 1099 rxq_ctrl->rxq.tunnel = 0; 1100 } 1101 } 1102 1103 /** 1104 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1105 * 1106 * @param[in] dev 1107 * Pointer to the Ethernet device structure. 1108 */ 1109 void 1110 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1111 { 1112 struct mlx5_priv *priv = dev->data->dev_private; 1113 struct mlx5_rxq_data *data; 1114 unsigned int i; 1115 1116 for (i = 0; i != priv->rxqs_n; ++i) { 1117 if (!(*priv->rxqs)[i]) 1118 continue; 1119 data = (*priv->rxqs)[i]; 1120 if (!rte_flow_dynf_metadata_avail()) { 1121 data->dynf_meta = 0; 1122 data->flow_meta_mask = 0; 1123 data->flow_meta_offset = -1; 1124 } else { 1125 data->dynf_meta = 1; 1126 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1127 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1128 } 1129 } 1130 } 1131 1132 /* 1133 * return a pointer to the desired action in the list of actions. 1134 * 1135 * @param[in] actions 1136 * The list of actions to search the action in. 1137 * @param[in] action 1138 * The action to find. 1139 * 1140 * @return 1141 * Pointer to the action in the list, if found. NULL otherwise. 1142 */ 1143 const struct rte_flow_action * 1144 mlx5_flow_find_action(const struct rte_flow_action *actions, 1145 enum rte_flow_action_type action) 1146 { 1147 if (actions == NULL) 1148 return NULL; 1149 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1150 if (actions->type == action) 1151 return actions; 1152 return NULL; 1153 } 1154 1155 /* 1156 * Validate the flag action. 1157 * 1158 * @param[in] action_flags 1159 * Bit-fields that holds the actions detected until now. 1160 * @param[in] attr 1161 * Attributes of flow that includes this action. 1162 * @param[out] error 1163 * Pointer to error structure. 1164 * 1165 * @return 1166 * 0 on success, a negative errno value otherwise and rte_errno is set. 1167 */ 1168 int 1169 mlx5_flow_validate_action_flag(uint64_t action_flags, 1170 const struct rte_flow_attr *attr, 1171 struct rte_flow_error *error) 1172 { 1173 if (action_flags & MLX5_FLOW_ACTION_MARK) 1174 return rte_flow_error_set(error, EINVAL, 1175 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1176 "can't mark and flag in same flow"); 1177 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1178 return rte_flow_error_set(error, EINVAL, 1179 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1180 "can't have 2 flag" 1181 " actions in same flow"); 1182 if (attr->egress) 1183 return rte_flow_error_set(error, ENOTSUP, 1184 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1185 "flag action not supported for " 1186 "egress"); 1187 return 0; 1188 } 1189 1190 /* 1191 * Validate the mark action. 1192 * 1193 * @param[in] action 1194 * Pointer to the queue action. 1195 * @param[in] action_flags 1196 * Bit-fields that holds the actions detected until now. 1197 * @param[in] attr 1198 * Attributes of flow that includes this action. 1199 * @param[out] error 1200 * Pointer to error structure. 1201 * 1202 * @return 1203 * 0 on success, a negative errno value otherwise and rte_errno is set. 1204 */ 1205 int 1206 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1207 uint64_t action_flags, 1208 const struct rte_flow_attr *attr, 1209 struct rte_flow_error *error) 1210 { 1211 const struct rte_flow_action_mark *mark = action->conf; 1212 1213 if (!mark) 1214 return rte_flow_error_set(error, EINVAL, 1215 RTE_FLOW_ERROR_TYPE_ACTION, 1216 action, 1217 "configuration cannot be null"); 1218 if (mark->id >= MLX5_FLOW_MARK_MAX) 1219 return rte_flow_error_set(error, EINVAL, 1220 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1221 &mark->id, 1222 "mark id must in 0 <= id < " 1223 RTE_STR(MLX5_FLOW_MARK_MAX)); 1224 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1225 return rte_flow_error_set(error, EINVAL, 1226 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1227 "can't flag and mark in same flow"); 1228 if (action_flags & MLX5_FLOW_ACTION_MARK) 1229 return rte_flow_error_set(error, EINVAL, 1230 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1231 "can't have 2 mark actions in same" 1232 " flow"); 1233 if (attr->egress) 1234 return rte_flow_error_set(error, ENOTSUP, 1235 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1236 "mark action not supported for " 1237 "egress"); 1238 return 0; 1239 } 1240 1241 /* 1242 * Validate the drop action. 1243 * 1244 * @param[in] action_flags 1245 * Bit-fields that holds the actions detected until now. 1246 * @param[in] attr 1247 * Attributes of flow that includes this action. 1248 * @param[out] error 1249 * Pointer to error structure. 1250 * 1251 * @return 1252 * 0 on success, a negative errno value otherwise and rte_errno is set. 1253 */ 1254 int 1255 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1256 const struct rte_flow_attr *attr, 1257 struct rte_flow_error *error) 1258 { 1259 if (attr->egress) 1260 return rte_flow_error_set(error, ENOTSUP, 1261 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1262 "drop action not supported for " 1263 "egress"); 1264 return 0; 1265 } 1266 1267 /* 1268 * Validate the queue action. 1269 * 1270 * @param[in] action 1271 * Pointer to the queue action. 1272 * @param[in] action_flags 1273 * Bit-fields that holds the actions detected until now. 1274 * @param[in] dev 1275 * Pointer to the Ethernet device structure. 1276 * @param[in] attr 1277 * Attributes of flow that includes this action. 1278 * @param[out] error 1279 * Pointer to error structure. 1280 * 1281 * @return 1282 * 0 on success, a negative errno value otherwise and rte_errno is set. 1283 */ 1284 int 1285 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1286 uint64_t action_flags, 1287 struct rte_eth_dev *dev, 1288 const struct rte_flow_attr *attr, 1289 struct rte_flow_error *error) 1290 { 1291 struct mlx5_priv *priv = dev->data->dev_private; 1292 const struct rte_flow_action_queue *queue = action->conf; 1293 1294 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1295 return rte_flow_error_set(error, EINVAL, 1296 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1297 "can't have 2 fate actions in" 1298 " same flow"); 1299 if (!priv->rxqs_n) 1300 return rte_flow_error_set(error, EINVAL, 1301 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1302 NULL, "No Rx queues configured"); 1303 if (queue->index >= priv->rxqs_n) 1304 return rte_flow_error_set(error, EINVAL, 1305 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1306 &queue->index, 1307 "queue index out of range"); 1308 if (!(*priv->rxqs)[queue->index]) 1309 return rte_flow_error_set(error, EINVAL, 1310 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1311 &queue->index, 1312 "queue is not configured"); 1313 if (attr->egress) 1314 return rte_flow_error_set(error, ENOTSUP, 1315 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1316 "queue action not supported for " 1317 "egress"); 1318 return 0; 1319 } 1320 1321 /* 1322 * Validate the rss action. 1323 * 1324 * @param[in] action 1325 * Pointer to the queue action. 1326 * @param[in] action_flags 1327 * Bit-fields that holds the actions detected until now. 1328 * @param[in] dev 1329 * Pointer to the Ethernet device structure. 1330 * @param[in] attr 1331 * Attributes of flow that includes this action. 1332 * @param[in] item_flags 1333 * Items that were detected. 1334 * @param[out] error 1335 * Pointer to error structure. 1336 * 1337 * @return 1338 * 0 on success, a negative errno value otherwise and rte_errno is set. 1339 */ 1340 int 1341 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1342 uint64_t action_flags, 1343 struct rte_eth_dev *dev, 1344 const struct rte_flow_attr *attr, 1345 uint64_t item_flags, 1346 struct rte_flow_error *error) 1347 { 1348 struct mlx5_priv *priv = dev->data->dev_private; 1349 const struct rte_flow_action_rss *rss = action->conf; 1350 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1351 unsigned int i; 1352 1353 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1354 return rte_flow_error_set(error, EINVAL, 1355 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1356 "can't have 2 fate actions" 1357 " in same flow"); 1358 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1359 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1360 return rte_flow_error_set(error, ENOTSUP, 1361 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1362 &rss->func, 1363 "RSS hash function not supported"); 1364 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1365 if (rss->level > 2) 1366 #else 1367 if (rss->level > 1) 1368 #endif 1369 return rte_flow_error_set(error, ENOTSUP, 1370 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1371 &rss->level, 1372 "tunnel RSS is not supported"); 1373 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1374 if (rss->key_len == 0 && rss->key != NULL) 1375 return rte_flow_error_set(error, ENOTSUP, 1376 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1377 &rss->key_len, 1378 "RSS hash key length 0"); 1379 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1380 return rte_flow_error_set(error, ENOTSUP, 1381 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1382 &rss->key_len, 1383 "RSS hash key too small"); 1384 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1385 return rte_flow_error_set(error, ENOTSUP, 1386 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1387 &rss->key_len, 1388 "RSS hash key too large"); 1389 if (rss->queue_num > priv->config.ind_table_max_size) 1390 return rte_flow_error_set(error, ENOTSUP, 1391 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1392 &rss->queue_num, 1393 "number of queues too large"); 1394 if (rss->types & MLX5_RSS_HF_MASK) 1395 return rte_flow_error_set(error, ENOTSUP, 1396 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1397 &rss->types, 1398 "some RSS protocols are not" 1399 " supported"); 1400 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1401 !(rss->types & ETH_RSS_IP)) 1402 return rte_flow_error_set(error, EINVAL, 1403 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1404 "L3 partial RSS requested but L3 RSS" 1405 " type not specified"); 1406 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1407 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1408 return rte_flow_error_set(error, EINVAL, 1409 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1410 "L4 partial RSS requested but L4 RSS" 1411 " type not specified"); 1412 if (!priv->rxqs_n) 1413 return rte_flow_error_set(error, EINVAL, 1414 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1415 NULL, "No Rx queues configured"); 1416 if (!rss->queue_num) 1417 return rte_flow_error_set(error, EINVAL, 1418 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1419 NULL, "No queues configured"); 1420 for (i = 0; i != rss->queue_num; ++i) { 1421 if (rss->queue[i] >= priv->rxqs_n) 1422 return rte_flow_error_set 1423 (error, EINVAL, 1424 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1425 &rss->queue[i], "queue index out of range"); 1426 if (!(*priv->rxqs)[rss->queue[i]]) 1427 return rte_flow_error_set 1428 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1429 &rss->queue[i], "queue is not configured"); 1430 } 1431 if (attr->egress) 1432 return rte_flow_error_set(error, ENOTSUP, 1433 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1434 "rss action not supported for " 1435 "egress"); 1436 if (rss->level > 1 && !tunnel) 1437 return rte_flow_error_set(error, EINVAL, 1438 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1439 "inner RSS is not supported for " 1440 "non-tunnel flows"); 1441 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1442 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1443 return rte_flow_error_set(error, EINVAL, 1444 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1445 "RSS on eCPRI is not supported now"); 1446 } 1447 return 0; 1448 } 1449 1450 /* 1451 * Validate the default miss action. 1452 * 1453 * @param[in] action_flags 1454 * Bit-fields that holds the actions detected until now. 1455 * @param[out] error 1456 * Pointer to error structure. 1457 * 1458 * @return 1459 * 0 on success, a negative errno value otherwise and rte_errno is set. 1460 */ 1461 int 1462 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1463 const struct rte_flow_attr *attr, 1464 struct rte_flow_error *error) 1465 { 1466 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1467 return rte_flow_error_set(error, EINVAL, 1468 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1469 "can't have 2 fate actions in" 1470 " same flow"); 1471 if (attr->egress) 1472 return rte_flow_error_set(error, ENOTSUP, 1473 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1474 "default miss action not supported " 1475 "for egress"); 1476 if (attr->group) 1477 return rte_flow_error_set(error, ENOTSUP, 1478 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1479 "only group 0 is supported"); 1480 if (attr->transfer) 1481 return rte_flow_error_set(error, ENOTSUP, 1482 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1483 NULL, "transfer is not supported"); 1484 return 0; 1485 } 1486 1487 /* 1488 * Validate the count action. 1489 * 1490 * @param[in] dev 1491 * Pointer to the Ethernet device structure. 1492 * @param[in] attr 1493 * Attributes of flow that includes this action. 1494 * @param[out] error 1495 * Pointer to error structure. 1496 * 1497 * @return 1498 * 0 on success, a negative errno value otherwise and rte_errno is set. 1499 */ 1500 int 1501 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1502 const struct rte_flow_attr *attr, 1503 struct rte_flow_error *error) 1504 { 1505 if (attr->egress) 1506 return rte_flow_error_set(error, ENOTSUP, 1507 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1508 "count action not supported for " 1509 "egress"); 1510 return 0; 1511 } 1512 1513 /** 1514 * Verify the @p attributes will be correctly understood by the NIC and store 1515 * them in the @p flow if everything is correct. 1516 * 1517 * @param[in] dev 1518 * Pointer to the Ethernet device structure. 1519 * @param[in] attributes 1520 * Pointer to flow attributes 1521 * @param[out] error 1522 * Pointer to error structure. 1523 * 1524 * @return 1525 * 0 on success, a negative errno value otherwise and rte_errno is set. 1526 */ 1527 int 1528 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1529 const struct rte_flow_attr *attributes, 1530 struct rte_flow_error *error) 1531 { 1532 struct mlx5_priv *priv = dev->data->dev_private; 1533 uint32_t priority_max = priv->config.flow_prio - 1; 1534 1535 if (attributes->group) 1536 return rte_flow_error_set(error, ENOTSUP, 1537 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1538 NULL, "groups is not supported"); 1539 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1540 attributes->priority >= priority_max) 1541 return rte_flow_error_set(error, ENOTSUP, 1542 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1543 NULL, "priority out of range"); 1544 if (attributes->egress) 1545 return rte_flow_error_set(error, ENOTSUP, 1546 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1547 "egress is not supported"); 1548 if (attributes->transfer && !priv->config.dv_esw_en) 1549 return rte_flow_error_set(error, ENOTSUP, 1550 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1551 NULL, "transfer is not supported"); 1552 if (!attributes->ingress) 1553 return rte_flow_error_set(error, EINVAL, 1554 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1555 NULL, 1556 "ingress attribute is mandatory"); 1557 return 0; 1558 } 1559 1560 /** 1561 * Validate ICMP6 item. 1562 * 1563 * @param[in] item 1564 * Item specification. 1565 * @param[in] item_flags 1566 * Bit-fields that holds the items detected until now. 1567 * @param[out] error 1568 * Pointer to error structure. 1569 * 1570 * @return 1571 * 0 on success, a negative errno value otherwise and rte_errno is set. 1572 */ 1573 int 1574 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1575 uint64_t item_flags, 1576 uint8_t target_protocol, 1577 struct rte_flow_error *error) 1578 { 1579 const struct rte_flow_item_icmp6 *mask = item->mask; 1580 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1581 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1582 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1583 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1584 MLX5_FLOW_LAYER_OUTER_L4; 1585 int ret; 1586 1587 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1588 return rte_flow_error_set(error, EINVAL, 1589 RTE_FLOW_ERROR_TYPE_ITEM, item, 1590 "protocol filtering not compatible" 1591 " with ICMP6 layer"); 1592 if (!(item_flags & l3m)) 1593 return rte_flow_error_set(error, EINVAL, 1594 RTE_FLOW_ERROR_TYPE_ITEM, item, 1595 "IPv6 is mandatory to filter on" 1596 " ICMP6"); 1597 if (item_flags & l4m) 1598 return rte_flow_error_set(error, EINVAL, 1599 RTE_FLOW_ERROR_TYPE_ITEM, item, 1600 "multiple L4 layers not supported"); 1601 if (!mask) 1602 mask = &rte_flow_item_icmp6_mask; 1603 ret = mlx5_flow_item_acceptable 1604 (item, (const uint8_t *)mask, 1605 (const uint8_t *)&rte_flow_item_icmp6_mask, 1606 sizeof(struct rte_flow_item_icmp6), error); 1607 if (ret < 0) 1608 return ret; 1609 return 0; 1610 } 1611 1612 /** 1613 * Validate ICMP item. 1614 * 1615 * @param[in] item 1616 * Item specification. 1617 * @param[in] item_flags 1618 * Bit-fields that holds the items detected until now. 1619 * @param[out] error 1620 * Pointer to error structure. 1621 * 1622 * @return 1623 * 0 on success, a negative errno value otherwise and rte_errno is set. 1624 */ 1625 int 1626 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1627 uint64_t item_flags, 1628 uint8_t target_protocol, 1629 struct rte_flow_error *error) 1630 { 1631 const struct rte_flow_item_icmp *mask = item->mask; 1632 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1633 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1634 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1635 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1636 MLX5_FLOW_LAYER_OUTER_L4; 1637 int ret; 1638 1639 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1640 return rte_flow_error_set(error, EINVAL, 1641 RTE_FLOW_ERROR_TYPE_ITEM, item, 1642 "protocol filtering not compatible" 1643 " with ICMP layer"); 1644 if (!(item_flags & l3m)) 1645 return rte_flow_error_set(error, EINVAL, 1646 RTE_FLOW_ERROR_TYPE_ITEM, item, 1647 "IPv4 is mandatory to filter" 1648 " on ICMP"); 1649 if (item_flags & l4m) 1650 return rte_flow_error_set(error, EINVAL, 1651 RTE_FLOW_ERROR_TYPE_ITEM, item, 1652 "multiple L4 layers not supported"); 1653 if (!mask) 1654 mask = &rte_flow_item_icmp_mask; 1655 ret = mlx5_flow_item_acceptable 1656 (item, (const uint8_t *)mask, 1657 (const uint8_t *)&rte_flow_item_icmp_mask, 1658 sizeof(struct rte_flow_item_icmp), error); 1659 if (ret < 0) 1660 return ret; 1661 return 0; 1662 } 1663 1664 /** 1665 * Validate Ethernet item. 1666 * 1667 * @param[in] item 1668 * Item specification. 1669 * @param[in] item_flags 1670 * Bit-fields that holds the items detected until now. 1671 * @param[out] error 1672 * Pointer to error structure. 1673 * 1674 * @return 1675 * 0 on success, a negative errno value otherwise and rte_errno is set. 1676 */ 1677 int 1678 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1679 uint64_t item_flags, 1680 struct rte_flow_error *error) 1681 { 1682 const struct rte_flow_item_eth *mask = item->mask; 1683 const struct rte_flow_item_eth nic_mask = { 1684 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1685 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1686 .type = RTE_BE16(0xffff), 1687 }; 1688 int ret; 1689 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1690 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1691 MLX5_FLOW_LAYER_OUTER_L2; 1692 1693 if (item_flags & ethm) 1694 return rte_flow_error_set(error, ENOTSUP, 1695 RTE_FLOW_ERROR_TYPE_ITEM, item, 1696 "multiple L2 layers not supported"); 1697 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1698 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1699 return rte_flow_error_set(error, EINVAL, 1700 RTE_FLOW_ERROR_TYPE_ITEM, item, 1701 "L2 layer should not follow " 1702 "L3 layers"); 1703 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1704 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1705 return rte_flow_error_set(error, EINVAL, 1706 RTE_FLOW_ERROR_TYPE_ITEM, item, 1707 "L2 layer should not follow VLAN"); 1708 if (!mask) 1709 mask = &rte_flow_item_eth_mask; 1710 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1711 (const uint8_t *)&nic_mask, 1712 sizeof(struct rte_flow_item_eth), 1713 error); 1714 return ret; 1715 } 1716 1717 /** 1718 * Validate VLAN item. 1719 * 1720 * @param[in] item 1721 * Item specification. 1722 * @param[in] item_flags 1723 * Bit-fields that holds the items detected until now. 1724 * @param[in] dev 1725 * Ethernet device flow is being created on. 1726 * @param[out] error 1727 * Pointer to error structure. 1728 * 1729 * @return 1730 * 0 on success, a negative errno value otherwise and rte_errno is set. 1731 */ 1732 int 1733 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1734 uint64_t item_flags, 1735 struct rte_eth_dev *dev, 1736 struct rte_flow_error *error) 1737 { 1738 const struct rte_flow_item_vlan *spec = item->spec; 1739 const struct rte_flow_item_vlan *mask = item->mask; 1740 const struct rte_flow_item_vlan nic_mask = { 1741 .tci = RTE_BE16(UINT16_MAX), 1742 .inner_type = RTE_BE16(UINT16_MAX), 1743 }; 1744 uint16_t vlan_tag = 0; 1745 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1746 int ret; 1747 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1748 MLX5_FLOW_LAYER_INNER_L4) : 1749 (MLX5_FLOW_LAYER_OUTER_L3 | 1750 MLX5_FLOW_LAYER_OUTER_L4); 1751 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1752 MLX5_FLOW_LAYER_OUTER_VLAN; 1753 1754 if (item_flags & vlanm) 1755 return rte_flow_error_set(error, EINVAL, 1756 RTE_FLOW_ERROR_TYPE_ITEM, item, 1757 "multiple VLAN layers not supported"); 1758 else if ((item_flags & l34m) != 0) 1759 return rte_flow_error_set(error, EINVAL, 1760 RTE_FLOW_ERROR_TYPE_ITEM, item, 1761 "VLAN cannot follow L3/L4 layer"); 1762 if (!mask) 1763 mask = &rte_flow_item_vlan_mask; 1764 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1765 (const uint8_t *)&nic_mask, 1766 sizeof(struct rte_flow_item_vlan), 1767 error); 1768 if (ret) 1769 return ret; 1770 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1771 struct mlx5_priv *priv = dev->data->dev_private; 1772 1773 if (priv->vmwa_context) { 1774 /* 1775 * Non-NULL context means we have a virtual machine 1776 * and SR-IOV enabled, we have to create VLAN interface 1777 * to make hypervisor to setup E-Switch vport 1778 * context correctly. We avoid creating the multiple 1779 * VLAN interfaces, so we cannot support VLAN tag mask. 1780 */ 1781 return rte_flow_error_set(error, EINVAL, 1782 RTE_FLOW_ERROR_TYPE_ITEM, 1783 item, 1784 "VLAN tag mask is not" 1785 " supported in virtual" 1786 " environment"); 1787 } 1788 } 1789 if (spec) { 1790 vlan_tag = spec->tci; 1791 vlan_tag &= mask->tci; 1792 } 1793 /* 1794 * From verbs perspective an empty VLAN is equivalent 1795 * to a packet without VLAN layer. 1796 */ 1797 if (!vlan_tag) 1798 return rte_flow_error_set(error, EINVAL, 1799 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1800 item->spec, 1801 "VLAN cannot be empty"); 1802 return 0; 1803 } 1804 1805 /** 1806 * Validate IPV4 item. 1807 * 1808 * @param[in] item 1809 * Item specification. 1810 * @param[in] item_flags 1811 * Bit-fields that holds the items detected until now. 1812 * @param[in] last_item 1813 * Previous validated item in the pattern items. 1814 * @param[in] ether_type 1815 * Type in the ethernet layer header (including dot1q). 1816 * @param[in] acc_mask 1817 * Acceptable mask, if NULL default internal default mask 1818 * will be used to check whether item fields are supported. 1819 * @param[out] error 1820 * Pointer to error structure. 1821 * 1822 * @return 1823 * 0 on success, a negative errno value otherwise and rte_errno is set. 1824 */ 1825 int 1826 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1827 uint64_t item_flags, 1828 uint64_t last_item, 1829 uint16_t ether_type, 1830 const struct rte_flow_item_ipv4 *acc_mask, 1831 struct rte_flow_error *error) 1832 { 1833 const struct rte_flow_item_ipv4 *mask = item->mask; 1834 const struct rte_flow_item_ipv4 *spec = item->spec; 1835 const struct rte_flow_item_ipv4 nic_mask = { 1836 .hdr = { 1837 .src_addr = RTE_BE32(0xffffffff), 1838 .dst_addr = RTE_BE32(0xffffffff), 1839 .type_of_service = 0xff, 1840 .next_proto_id = 0xff, 1841 }, 1842 }; 1843 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1844 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1845 MLX5_FLOW_LAYER_OUTER_L3; 1846 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1847 MLX5_FLOW_LAYER_OUTER_L4; 1848 int ret; 1849 uint8_t next_proto = 0xFF; 1850 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1851 MLX5_FLOW_LAYER_OUTER_VLAN | 1852 MLX5_FLOW_LAYER_INNER_VLAN); 1853 1854 if ((last_item & l2_vlan) && ether_type && 1855 ether_type != RTE_ETHER_TYPE_IPV4) 1856 return rte_flow_error_set(error, EINVAL, 1857 RTE_FLOW_ERROR_TYPE_ITEM, item, 1858 "IPv4 cannot follow L2/VLAN layer " 1859 "which ether type is not IPv4"); 1860 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1861 if (mask && spec) 1862 next_proto = mask->hdr.next_proto_id & 1863 spec->hdr.next_proto_id; 1864 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1865 return rte_flow_error_set(error, EINVAL, 1866 RTE_FLOW_ERROR_TYPE_ITEM, 1867 item, 1868 "multiple tunnel " 1869 "not supported"); 1870 } 1871 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1872 return rte_flow_error_set(error, EINVAL, 1873 RTE_FLOW_ERROR_TYPE_ITEM, item, 1874 "wrong tunnel type - IPv6 specified " 1875 "but IPv4 item provided"); 1876 if (item_flags & l3m) 1877 return rte_flow_error_set(error, ENOTSUP, 1878 RTE_FLOW_ERROR_TYPE_ITEM, item, 1879 "multiple L3 layers not supported"); 1880 else if (item_flags & l4m) 1881 return rte_flow_error_set(error, EINVAL, 1882 RTE_FLOW_ERROR_TYPE_ITEM, item, 1883 "L3 cannot follow an L4 layer."); 1884 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1885 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1886 return rte_flow_error_set(error, EINVAL, 1887 RTE_FLOW_ERROR_TYPE_ITEM, item, 1888 "L3 cannot follow an NVGRE layer."); 1889 if (!mask) 1890 mask = &rte_flow_item_ipv4_mask; 1891 else if (mask->hdr.next_proto_id != 0 && 1892 mask->hdr.next_proto_id != 0xff) 1893 return rte_flow_error_set(error, EINVAL, 1894 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1895 "partial mask is not supported" 1896 " for protocol"); 1897 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1898 acc_mask ? (const uint8_t *)acc_mask 1899 : (const uint8_t *)&nic_mask, 1900 sizeof(struct rte_flow_item_ipv4), 1901 error); 1902 if (ret < 0) 1903 return ret; 1904 return 0; 1905 } 1906 1907 /** 1908 * Validate IPV6 item. 1909 * 1910 * @param[in] item 1911 * Item specification. 1912 * @param[in] item_flags 1913 * Bit-fields that holds the items detected until now. 1914 * @param[in] last_item 1915 * Previous validated item in the pattern items. 1916 * @param[in] ether_type 1917 * Type in the ethernet layer header (including dot1q). 1918 * @param[in] acc_mask 1919 * Acceptable mask, if NULL default internal default mask 1920 * will be used to check whether item fields are supported. 1921 * @param[out] error 1922 * Pointer to error structure. 1923 * 1924 * @return 1925 * 0 on success, a negative errno value otherwise and rte_errno is set. 1926 */ 1927 int 1928 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1929 uint64_t item_flags, 1930 uint64_t last_item, 1931 uint16_t ether_type, 1932 const struct rte_flow_item_ipv6 *acc_mask, 1933 struct rte_flow_error *error) 1934 { 1935 const struct rte_flow_item_ipv6 *mask = item->mask; 1936 const struct rte_flow_item_ipv6 *spec = item->spec; 1937 const struct rte_flow_item_ipv6 nic_mask = { 1938 .hdr = { 1939 .src_addr = 1940 "\xff\xff\xff\xff\xff\xff\xff\xff" 1941 "\xff\xff\xff\xff\xff\xff\xff\xff", 1942 .dst_addr = 1943 "\xff\xff\xff\xff\xff\xff\xff\xff" 1944 "\xff\xff\xff\xff\xff\xff\xff\xff", 1945 .vtc_flow = RTE_BE32(0xffffffff), 1946 .proto = 0xff, 1947 }, 1948 }; 1949 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1950 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1951 MLX5_FLOW_LAYER_OUTER_L3; 1952 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1953 MLX5_FLOW_LAYER_OUTER_L4; 1954 int ret; 1955 uint8_t next_proto = 0xFF; 1956 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1957 MLX5_FLOW_LAYER_OUTER_VLAN | 1958 MLX5_FLOW_LAYER_INNER_VLAN); 1959 1960 if ((last_item & l2_vlan) && ether_type && 1961 ether_type != RTE_ETHER_TYPE_IPV6) 1962 return rte_flow_error_set(error, EINVAL, 1963 RTE_FLOW_ERROR_TYPE_ITEM, item, 1964 "IPv6 cannot follow L2/VLAN layer " 1965 "which ether type is not IPv6"); 1966 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1967 if (mask && spec) 1968 next_proto = mask->hdr.proto & spec->hdr.proto; 1969 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1970 return rte_flow_error_set(error, EINVAL, 1971 RTE_FLOW_ERROR_TYPE_ITEM, 1972 item, 1973 "multiple tunnel " 1974 "not supported"); 1975 } 1976 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1977 return rte_flow_error_set(error, EINVAL, 1978 RTE_FLOW_ERROR_TYPE_ITEM, item, 1979 "wrong tunnel type - IPv4 specified " 1980 "but IPv6 item provided"); 1981 if (item_flags & l3m) 1982 return rte_flow_error_set(error, ENOTSUP, 1983 RTE_FLOW_ERROR_TYPE_ITEM, item, 1984 "multiple L3 layers not supported"); 1985 else if (item_flags & l4m) 1986 return rte_flow_error_set(error, EINVAL, 1987 RTE_FLOW_ERROR_TYPE_ITEM, item, 1988 "L3 cannot follow an L4 layer."); 1989 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1990 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1991 return rte_flow_error_set(error, EINVAL, 1992 RTE_FLOW_ERROR_TYPE_ITEM, item, 1993 "L3 cannot follow an NVGRE layer."); 1994 if (!mask) 1995 mask = &rte_flow_item_ipv6_mask; 1996 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1997 acc_mask ? (const uint8_t *)acc_mask 1998 : (const uint8_t *)&nic_mask, 1999 sizeof(struct rte_flow_item_ipv6), 2000 error); 2001 if (ret < 0) 2002 return ret; 2003 return 0; 2004 } 2005 2006 /** 2007 * Validate UDP item. 2008 * 2009 * @param[in] item 2010 * Item specification. 2011 * @param[in] item_flags 2012 * Bit-fields that holds the items detected until now. 2013 * @param[in] target_protocol 2014 * The next protocol in the previous item. 2015 * @param[in] flow_mask 2016 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2017 * @param[out] error 2018 * Pointer to error structure. 2019 * 2020 * @return 2021 * 0 on success, a negative errno value otherwise and rte_errno is set. 2022 */ 2023 int 2024 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2025 uint64_t item_flags, 2026 uint8_t target_protocol, 2027 struct rte_flow_error *error) 2028 { 2029 const struct rte_flow_item_udp *mask = item->mask; 2030 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2031 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2032 MLX5_FLOW_LAYER_OUTER_L3; 2033 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2034 MLX5_FLOW_LAYER_OUTER_L4; 2035 int ret; 2036 2037 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2038 return rte_flow_error_set(error, EINVAL, 2039 RTE_FLOW_ERROR_TYPE_ITEM, item, 2040 "protocol filtering not compatible" 2041 " with UDP layer"); 2042 if (!(item_flags & l3m)) 2043 return rte_flow_error_set(error, EINVAL, 2044 RTE_FLOW_ERROR_TYPE_ITEM, item, 2045 "L3 is mandatory to filter on L4"); 2046 if (item_flags & l4m) 2047 return rte_flow_error_set(error, EINVAL, 2048 RTE_FLOW_ERROR_TYPE_ITEM, item, 2049 "multiple L4 layers not supported"); 2050 if (!mask) 2051 mask = &rte_flow_item_udp_mask; 2052 ret = mlx5_flow_item_acceptable 2053 (item, (const uint8_t *)mask, 2054 (const uint8_t *)&rte_flow_item_udp_mask, 2055 sizeof(struct rte_flow_item_udp), error); 2056 if (ret < 0) 2057 return ret; 2058 return 0; 2059 } 2060 2061 /** 2062 * Validate TCP item. 2063 * 2064 * @param[in] item 2065 * Item specification. 2066 * @param[in] item_flags 2067 * Bit-fields that holds the items detected until now. 2068 * @param[in] target_protocol 2069 * The next protocol in the previous item. 2070 * @param[out] error 2071 * Pointer to error structure. 2072 * 2073 * @return 2074 * 0 on success, a negative errno value otherwise and rte_errno is set. 2075 */ 2076 int 2077 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2078 uint64_t item_flags, 2079 uint8_t target_protocol, 2080 const struct rte_flow_item_tcp *flow_mask, 2081 struct rte_flow_error *error) 2082 { 2083 const struct rte_flow_item_tcp *mask = item->mask; 2084 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2085 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2086 MLX5_FLOW_LAYER_OUTER_L3; 2087 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2088 MLX5_FLOW_LAYER_OUTER_L4; 2089 int ret; 2090 2091 MLX5_ASSERT(flow_mask); 2092 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2093 return rte_flow_error_set(error, EINVAL, 2094 RTE_FLOW_ERROR_TYPE_ITEM, item, 2095 "protocol filtering not compatible" 2096 " with TCP layer"); 2097 if (!(item_flags & l3m)) 2098 return rte_flow_error_set(error, EINVAL, 2099 RTE_FLOW_ERROR_TYPE_ITEM, item, 2100 "L3 is mandatory to filter on L4"); 2101 if (item_flags & l4m) 2102 return rte_flow_error_set(error, EINVAL, 2103 RTE_FLOW_ERROR_TYPE_ITEM, item, 2104 "multiple L4 layers not supported"); 2105 if (!mask) 2106 mask = &rte_flow_item_tcp_mask; 2107 ret = mlx5_flow_item_acceptable 2108 (item, (const uint8_t *)mask, 2109 (const uint8_t *)flow_mask, 2110 sizeof(struct rte_flow_item_tcp), error); 2111 if (ret < 0) 2112 return ret; 2113 return 0; 2114 } 2115 2116 /** 2117 * Validate VXLAN item. 2118 * 2119 * @param[in] item 2120 * Item specification. 2121 * @param[in] item_flags 2122 * Bit-fields that holds the items detected until now. 2123 * @param[in] target_protocol 2124 * The next protocol in the previous item. 2125 * @param[out] error 2126 * Pointer to error structure. 2127 * 2128 * @return 2129 * 0 on success, a negative errno value otherwise and rte_errno is set. 2130 */ 2131 int 2132 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2133 uint64_t item_flags, 2134 struct rte_flow_error *error) 2135 { 2136 const struct rte_flow_item_vxlan *spec = item->spec; 2137 const struct rte_flow_item_vxlan *mask = item->mask; 2138 int ret; 2139 union vni { 2140 uint32_t vlan_id; 2141 uint8_t vni[4]; 2142 } id = { .vlan_id = 0, }; 2143 2144 2145 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2146 return rte_flow_error_set(error, ENOTSUP, 2147 RTE_FLOW_ERROR_TYPE_ITEM, item, 2148 "multiple tunnel layers not" 2149 " supported"); 2150 /* 2151 * Verify only UDPv4 is present as defined in 2152 * https://tools.ietf.org/html/rfc7348 2153 */ 2154 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2155 return rte_flow_error_set(error, EINVAL, 2156 RTE_FLOW_ERROR_TYPE_ITEM, item, 2157 "no outer UDP layer found"); 2158 if (!mask) 2159 mask = &rte_flow_item_vxlan_mask; 2160 ret = mlx5_flow_item_acceptable 2161 (item, (const uint8_t *)mask, 2162 (const uint8_t *)&rte_flow_item_vxlan_mask, 2163 sizeof(struct rte_flow_item_vxlan), 2164 error); 2165 if (ret < 0) 2166 return ret; 2167 if (spec) { 2168 memcpy(&id.vni[1], spec->vni, 3); 2169 memcpy(&id.vni[1], mask->vni, 3); 2170 } 2171 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2172 return rte_flow_error_set(error, ENOTSUP, 2173 RTE_FLOW_ERROR_TYPE_ITEM, item, 2174 "VXLAN tunnel must be fully defined"); 2175 return 0; 2176 } 2177 2178 /** 2179 * Validate VXLAN_GPE item. 2180 * 2181 * @param[in] item 2182 * Item specification. 2183 * @param[in] item_flags 2184 * Bit-fields that holds the items detected until now. 2185 * @param[in] priv 2186 * Pointer to the private data structure. 2187 * @param[in] target_protocol 2188 * The next protocol in the previous item. 2189 * @param[out] error 2190 * Pointer to error structure. 2191 * 2192 * @return 2193 * 0 on success, a negative errno value otherwise and rte_errno is set. 2194 */ 2195 int 2196 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2197 uint64_t item_flags, 2198 struct rte_eth_dev *dev, 2199 struct rte_flow_error *error) 2200 { 2201 struct mlx5_priv *priv = dev->data->dev_private; 2202 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2203 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2204 int ret; 2205 union vni { 2206 uint32_t vlan_id; 2207 uint8_t vni[4]; 2208 } id = { .vlan_id = 0, }; 2209 2210 if (!priv->config.l3_vxlan_en) 2211 return rte_flow_error_set(error, ENOTSUP, 2212 RTE_FLOW_ERROR_TYPE_ITEM, item, 2213 "L3 VXLAN is not enabled by device" 2214 " parameter and/or not configured in" 2215 " firmware"); 2216 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2217 return rte_flow_error_set(error, ENOTSUP, 2218 RTE_FLOW_ERROR_TYPE_ITEM, item, 2219 "multiple tunnel layers not" 2220 " supported"); 2221 /* 2222 * Verify only UDPv4 is present as defined in 2223 * https://tools.ietf.org/html/rfc7348 2224 */ 2225 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2226 return rte_flow_error_set(error, EINVAL, 2227 RTE_FLOW_ERROR_TYPE_ITEM, item, 2228 "no outer UDP layer found"); 2229 if (!mask) 2230 mask = &rte_flow_item_vxlan_gpe_mask; 2231 ret = mlx5_flow_item_acceptable 2232 (item, (const uint8_t *)mask, 2233 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2234 sizeof(struct rte_flow_item_vxlan_gpe), 2235 error); 2236 if (ret < 0) 2237 return ret; 2238 if (spec) { 2239 if (spec->protocol) 2240 return rte_flow_error_set(error, ENOTSUP, 2241 RTE_FLOW_ERROR_TYPE_ITEM, 2242 item, 2243 "VxLAN-GPE protocol" 2244 " not supported"); 2245 memcpy(&id.vni[1], spec->vni, 3); 2246 memcpy(&id.vni[1], mask->vni, 3); 2247 } 2248 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2249 return rte_flow_error_set(error, ENOTSUP, 2250 RTE_FLOW_ERROR_TYPE_ITEM, item, 2251 "VXLAN-GPE tunnel must be fully" 2252 " defined"); 2253 return 0; 2254 } 2255 /** 2256 * Validate GRE Key item. 2257 * 2258 * @param[in] item 2259 * Item specification. 2260 * @param[in] item_flags 2261 * Bit flags to mark detected items. 2262 * @param[in] gre_item 2263 * Pointer to gre_item 2264 * @param[out] error 2265 * Pointer to error structure. 2266 * 2267 * @return 2268 * 0 on success, a negative errno value otherwise and rte_errno is set. 2269 */ 2270 int 2271 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2272 uint64_t item_flags, 2273 const struct rte_flow_item *gre_item, 2274 struct rte_flow_error *error) 2275 { 2276 const rte_be32_t *mask = item->mask; 2277 int ret = 0; 2278 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2279 const struct rte_flow_item_gre *gre_spec; 2280 const struct rte_flow_item_gre *gre_mask; 2281 2282 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2283 return rte_flow_error_set(error, ENOTSUP, 2284 RTE_FLOW_ERROR_TYPE_ITEM, item, 2285 "Multiple GRE key not support"); 2286 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2287 return rte_flow_error_set(error, ENOTSUP, 2288 RTE_FLOW_ERROR_TYPE_ITEM, item, 2289 "No preceding GRE header"); 2290 if (item_flags & MLX5_FLOW_LAYER_INNER) 2291 return rte_flow_error_set(error, ENOTSUP, 2292 RTE_FLOW_ERROR_TYPE_ITEM, item, 2293 "GRE key following a wrong item"); 2294 gre_mask = gre_item->mask; 2295 if (!gre_mask) 2296 gre_mask = &rte_flow_item_gre_mask; 2297 gre_spec = gre_item->spec; 2298 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2299 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2300 return rte_flow_error_set(error, EINVAL, 2301 RTE_FLOW_ERROR_TYPE_ITEM, item, 2302 "Key bit must be on"); 2303 2304 if (!mask) 2305 mask = &gre_key_default_mask; 2306 ret = mlx5_flow_item_acceptable 2307 (item, (const uint8_t *)mask, 2308 (const uint8_t *)&gre_key_default_mask, 2309 sizeof(rte_be32_t), error); 2310 return ret; 2311 } 2312 2313 /** 2314 * Validate GRE item. 2315 * 2316 * @param[in] item 2317 * Item specification. 2318 * @param[in] item_flags 2319 * Bit flags to mark detected items. 2320 * @param[in] target_protocol 2321 * The next protocol in the previous item. 2322 * @param[out] error 2323 * Pointer to error structure. 2324 * 2325 * @return 2326 * 0 on success, a negative errno value otherwise and rte_errno is set. 2327 */ 2328 int 2329 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2330 uint64_t item_flags, 2331 uint8_t target_protocol, 2332 struct rte_flow_error *error) 2333 { 2334 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2335 const struct rte_flow_item_gre *mask = item->mask; 2336 int ret; 2337 const struct rte_flow_item_gre nic_mask = { 2338 .c_rsvd0_ver = RTE_BE16(0xB000), 2339 .protocol = RTE_BE16(UINT16_MAX), 2340 }; 2341 2342 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2343 return rte_flow_error_set(error, EINVAL, 2344 RTE_FLOW_ERROR_TYPE_ITEM, item, 2345 "protocol filtering not compatible" 2346 " with this GRE layer"); 2347 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2348 return rte_flow_error_set(error, ENOTSUP, 2349 RTE_FLOW_ERROR_TYPE_ITEM, item, 2350 "multiple tunnel layers not" 2351 " supported"); 2352 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2353 return rte_flow_error_set(error, ENOTSUP, 2354 RTE_FLOW_ERROR_TYPE_ITEM, item, 2355 "L3 Layer is missing"); 2356 if (!mask) 2357 mask = &rte_flow_item_gre_mask; 2358 ret = mlx5_flow_item_acceptable 2359 (item, (const uint8_t *)mask, 2360 (const uint8_t *)&nic_mask, 2361 sizeof(struct rte_flow_item_gre), error); 2362 if (ret < 0) 2363 return ret; 2364 #ifndef HAVE_MLX5DV_DR 2365 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2366 if (spec && (spec->protocol & mask->protocol)) 2367 return rte_flow_error_set(error, ENOTSUP, 2368 RTE_FLOW_ERROR_TYPE_ITEM, item, 2369 "without MPLS support the" 2370 " specification cannot be used for" 2371 " filtering"); 2372 #endif 2373 #endif 2374 return 0; 2375 } 2376 2377 /** 2378 * Validate Geneve item. 2379 * 2380 * @param[in] item 2381 * Item specification. 2382 * @param[in] itemFlags 2383 * Bit-fields that holds the items detected until now. 2384 * @param[in] enPriv 2385 * Pointer to the private data structure. 2386 * @param[out] error 2387 * Pointer to error structure. 2388 * 2389 * @return 2390 * 0 on success, a negative errno value otherwise and rte_errno is set. 2391 */ 2392 2393 int 2394 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2395 uint64_t item_flags, 2396 struct rte_eth_dev *dev, 2397 struct rte_flow_error *error) 2398 { 2399 struct mlx5_priv *priv = dev->data->dev_private; 2400 const struct rte_flow_item_geneve *spec = item->spec; 2401 const struct rte_flow_item_geneve *mask = item->mask; 2402 int ret; 2403 uint16_t gbhdr; 2404 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2405 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2406 const struct rte_flow_item_geneve nic_mask = { 2407 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2408 .vni = "\xff\xff\xff", 2409 .protocol = RTE_BE16(UINT16_MAX), 2410 }; 2411 2412 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2413 return rte_flow_error_set(error, ENOTSUP, 2414 RTE_FLOW_ERROR_TYPE_ITEM, item, 2415 "L3 Geneve is not enabled by device" 2416 " parameter and/or not configured in" 2417 " firmware"); 2418 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2419 return rte_flow_error_set(error, ENOTSUP, 2420 RTE_FLOW_ERROR_TYPE_ITEM, item, 2421 "multiple tunnel layers not" 2422 " supported"); 2423 /* 2424 * Verify only UDPv4 is present as defined in 2425 * https://tools.ietf.org/html/rfc7348 2426 */ 2427 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2428 return rte_flow_error_set(error, EINVAL, 2429 RTE_FLOW_ERROR_TYPE_ITEM, item, 2430 "no outer UDP layer found"); 2431 if (!mask) 2432 mask = &rte_flow_item_geneve_mask; 2433 ret = mlx5_flow_item_acceptable 2434 (item, (const uint8_t *)mask, 2435 (const uint8_t *)&nic_mask, 2436 sizeof(struct rte_flow_item_geneve), error); 2437 if (ret) 2438 return ret; 2439 if (spec) { 2440 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2441 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2442 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2443 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2444 return rte_flow_error_set(error, ENOTSUP, 2445 RTE_FLOW_ERROR_TYPE_ITEM, 2446 item, 2447 "Geneve protocol unsupported" 2448 " fields are being used"); 2449 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2450 return rte_flow_error_set 2451 (error, ENOTSUP, 2452 RTE_FLOW_ERROR_TYPE_ITEM, 2453 item, 2454 "Unsupported Geneve options length"); 2455 } 2456 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2457 return rte_flow_error_set 2458 (error, ENOTSUP, 2459 RTE_FLOW_ERROR_TYPE_ITEM, item, 2460 "Geneve tunnel must be fully defined"); 2461 return 0; 2462 } 2463 2464 /** 2465 * Validate MPLS item. 2466 * 2467 * @param[in] dev 2468 * Pointer to the rte_eth_dev structure. 2469 * @param[in] item 2470 * Item specification. 2471 * @param[in] item_flags 2472 * Bit-fields that holds the items detected until now. 2473 * @param[in] prev_layer 2474 * The protocol layer indicated in previous item. 2475 * @param[out] error 2476 * Pointer to error structure. 2477 * 2478 * @return 2479 * 0 on success, a negative errno value otherwise and rte_errno is set. 2480 */ 2481 int 2482 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2483 const struct rte_flow_item *item __rte_unused, 2484 uint64_t item_flags __rte_unused, 2485 uint64_t prev_layer __rte_unused, 2486 struct rte_flow_error *error) 2487 { 2488 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2489 const struct rte_flow_item_mpls *mask = item->mask; 2490 struct mlx5_priv *priv = dev->data->dev_private; 2491 int ret; 2492 2493 if (!priv->config.mpls_en) 2494 return rte_flow_error_set(error, ENOTSUP, 2495 RTE_FLOW_ERROR_TYPE_ITEM, item, 2496 "MPLS not supported or" 2497 " disabled in firmware" 2498 " configuration."); 2499 /* MPLS over IP, UDP, GRE is allowed */ 2500 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2501 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2502 MLX5_FLOW_LAYER_GRE))) 2503 return rte_flow_error_set(error, EINVAL, 2504 RTE_FLOW_ERROR_TYPE_ITEM, item, 2505 "protocol filtering not compatible" 2506 " with MPLS layer"); 2507 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2508 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2509 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2510 return rte_flow_error_set(error, ENOTSUP, 2511 RTE_FLOW_ERROR_TYPE_ITEM, item, 2512 "multiple tunnel layers not" 2513 " supported"); 2514 if (!mask) 2515 mask = &rte_flow_item_mpls_mask; 2516 ret = mlx5_flow_item_acceptable 2517 (item, (const uint8_t *)mask, 2518 (const uint8_t *)&rte_flow_item_mpls_mask, 2519 sizeof(struct rte_flow_item_mpls), error); 2520 if (ret < 0) 2521 return ret; 2522 return 0; 2523 #else 2524 return rte_flow_error_set(error, ENOTSUP, 2525 RTE_FLOW_ERROR_TYPE_ITEM, item, 2526 "MPLS is not supported by Verbs, please" 2527 " update."); 2528 #endif 2529 } 2530 2531 /** 2532 * Validate NVGRE item. 2533 * 2534 * @param[in] item 2535 * Item specification. 2536 * @param[in] item_flags 2537 * Bit flags to mark detected items. 2538 * @param[in] target_protocol 2539 * The next protocol in the previous item. 2540 * @param[out] error 2541 * Pointer to error structure. 2542 * 2543 * @return 2544 * 0 on success, a negative errno value otherwise and rte_errno is set. 2545 */ 2546 int 2547 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2548 uint64_t item_flags, 2549 uint8_t target_protocol, 2550 struct rte_flow_error *error) 2551 { 2552 const struct rte_flow_item_nvgre *mask = item->mask; 2553 int ret; 2554 2555 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2556 return rte_flow_error_set(error, EINVAL, 2557 RTE_FLOW_ERROR_TYPE_ITEM, item, 2558 "protocol filtering not compatible" 2559 " with this GRE layer"); 2560 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2561 return rte_flow_error_set(error, ENOTSUP, 2562 RTE_FLOW_ERROR_TYPE_ITEM, item, 2563 "multiple tunnel layers not" 2564 " supported"); 2565 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2566 return rte_flow_error_set(error, ENOTSUP, 2567 RTE_FLOW_ERROR_TYPE_ITEM, item, 2568 "L3 Layer is missing"); 2569 if (!mask) 2570 mask = &rte_flow_item_nvgre_mask; 2571 ret = mlx5_flow_item_acceptable 2572 (item, (const uint8_t *)mask, 2573 (const uint8_t *)&rte_flow_item_nvgre_mask, 2574 sizeof(struct rte_flow_item_nvgre), error); 2575 if (ret < 0) 2576 return ret; 2577 return 0; 2578 } 2579 2580 /** 2581 * Validate eCPRI item. 2582 * 2583 * @param[in] item 2584 * Item specification. 2585 * @param[in] item_flags 2586 * Bit-fields that holds the items detected until now. 2587 * @param[in] last_item 2588 * Previous validated item in the pattern items. 2589 * @param[in] ether_type 2590 * Type in the ethernet layer header (including dot1q). 2591 * @param[in] acc_mask 2592 * Acceptable mask, if NULL default internal default mask 2593 * will be used to check whether item fields are supported. 2594 * @param[out] error 2595 * Pointer to error structure. 2596 * 2597 * @return 2598 * 0 on success, a negative errno value otherwise and rte_errno is set. 2599 */ 2600 int 2601 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 2602 uint64_t item_flags, 2603 uint64_t last_item, 2604 uint16_t ether_type, 2605 const struct rte_flow_item_ecpri *acc_mask, 2606 struct rte_flow_error *error) 2607 { 2608 const struct rte_flow_item_ecpri *mask = item->mask; 2609 const struct rte_flow_item_ecpri nic_mask = { 2610 .hdr = { 2611 .common = { 2612 .u32 = 2613 RTE_BE32(((const struct rte_ecpri_common_hdr) { 2614 .type = 0xFF, 2615 }).u32), 2616 }, 2617 .dummy[0] = 0xFFFFFFFF, 2618 }, 2619 }; 2620 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 2621 MLX5_FLOW_LAYER_OUTER_VLAN); 2622 struct rte_flow_item_ecpri mask_lo; 2623 2624 if ((last_item & outer_l2_vlan) && ether_type && 2625 ether_type != RTE_ETHER_TYPE_ECPRI) 2626 return rte_flow_error_set(error, EINVAL, 2627 RTE_FLOW_ERROR_TYPE_ITEM, item, 2628 "eCPRI cannot follow L2/VLAN layer " 2629 "which ether type is not 0xAEFE."); 2630 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2631 return rte_flow_error_set(error, EINVAL, 2632 RTE_FLOW_ERROR_TYPE_ITEM, item, 2633 "eCPRI with tunnel is not supported " 2634 "right now."); 2635 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 2636 return rte_flow_error_set(error, ENOTSUP, 2637 RTE_FLOW_ERROR_TYPE_ITEM, item, 2638 "multiple L3 layers not supported"); 2639 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 2640 return rte_flow_error_set(error, EINVAL, 2641 RTE_FLOW_ERROR_TYPE_ITEM, item, 2642 "eCPRI cannot follow a TCP layer."); 2643 /* In specification, eCPRI could be over UDP layer. */ 2644 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 2645 return rte_flow_error_set(error, EINVAL, 2646 RTE_FLOW_ERROR_TYPE_ITEM, item, 2647 "eCPRI over UDP layer is not yet " 2648 "supported right now."); 2649 /* Mask for type field in common header could be zero. */ 2650 if (!mask) 2651 mask = &rte_flow_item_ecpri_mask; 2652 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 2653 /* Input mask is in big-endian format. */ 2654 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 2655 return rte_flow_error_set(error, EINVAL, 2656 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2657 "partial mask is not supported " 2658 "for protocol"); 2659 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 2660 return rte_flow_error_set(error, EINVAL, 2661 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2662 "message header mask must be after " 2663 "a type mask"); 2664 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2665 acc_mask ? (const uint8_t *)acc_mask 2666 : (const uint8_t *)&nic_mask, 2667 sizeof(struct rte_flow_item_ecpri), 2668 error); 2669 } 2670 2671 /* Allocate unique ID for the split Q/RSS subflows. */ 2672 static uint32_t 2673 flow_qrss_get_id(struct rte_eth_dev *dev) 2674 { 2675 struct mlx5_priv *priv = dev->data->dev_private; 2676 uint32_t qrss_id, ret; 2677 2678 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2679 if (ret) 2680 return 0; 2681 MLX5_ASSERT(qrss_id); 2682 return qrss_id; 2683 } 2684 2685 /* Free unique ID for the split Q/RSS subflows. */ 2686 static void 2687 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2688 { 2689 struct mlx5_priv *priv = dev->data->dev_private; 2690 2691 if (qrss_id) 2692 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2693 } 2694 2695 /** 2696 * Release resource related QUEUE/RSS action split. 2697 * 2698 * @param dev 2699 * Pointer to Ethernet device. 2700 * @param flow 2701 * Flow to release id's from. 2702 */ 2703 static void 2704 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2705 struct rte_flow *flow) 2706 { 2707 struct mlx5_priv *priv = dev->data->dev_private; 2708 uint32_t handle_idx; 2709 struct mlx5_flow_handle *dev_handle; 2710 2711 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2712 handle_idx, dev_handle, next) 2713 if (dev_handle->split_flow_id) 2714 flow_qrss_free_id(dev, dev_handle->split_flow_id); 2715 } 2716 2717 static int 2718 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2719 const struct rte_flow_attr *attr __rte_unused, 2720 const struct rte_flow_item items[] __rte_unused, 2721 const struct rte_flow_action actions[] __rte_unused, 2722 bool external __rte_unused, 2723 int hairpin __rte_unused, 2724 struct rte_flow_error *error) 2725 { 2726 return rte_flow_error_set(error, ENOTSUP, 2727 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2728 } 2729 2730 static struct mlx5_flow * 2731 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2732 const struct rte_flow_attr *attr __rte_unused, 2733 const struct rte_flow_item items[] __rte_unused, 2734 const struct rte_flow_action actions[] __rte_unused, 2735 struct rte_flow_error *error) 2736 { 2737 rte_flow_error_set(error, ENOTSUP, 2738 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2739 return NULL; 2740 } 2741 2742 static int 2743 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2744 struct mlx5_flow *dev_flow __rte_unused, 2745 const struct rte_flow_attr *attr __rte_unused, 2746 const struct rte_flow_item items[] __rte_unused, 2747 const struct rte_flow_action actions[] __rte_unused, 2748 struct rte_flow_error *error) 2749 { 2750 return rte_flow_error_set(error, ENOTSUP, 2751 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2752 } 2753 2754 static int 2755 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2756 struct rte_flow *flow __rte_unused, 2757 struct rte_flow_error *error) 2758 { 2759 return rte_flow_error_set(error, ENOTSUP, 2760 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2761 } 2762 2763 static void 2764 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2765 struct rte_flow *flow __rte_unused) 2766 { 2767 } 2768 2769 static void 2770 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2771 struct rte_flow *flow __rte_unused) 2772 { 2773 } 2774 2775 static int 2776 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2777 struct rte_flow *flow __rte_unused, 2778 const struct rte_flow_action *actions __rte_unused, 2779 void *data __rte_unused, 2780 struct rte_flow_error *error) 2781 { 2782 return rte_flow_error_set(error, ENOTSUP, 2783 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2784 } 2785 2786 /* Void driver to protect from null pointer reference. */ 2787 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2788 .validate = flow_null_validate, 2789 .prepare = flow_null_prepare, 2790 .translate = flow_null_translate, 2791 .apply = flow_null_apply, 2792 .remove = flow_null_remove, 2793 .destroy = flow_null_destroy, 2794 .query = flow_null_query, 2795 }; 2796 2797 /** 2798 * Select flow driver type according to flow attributes and device 2799 * configuration. 2800 * 2801 * @param[in] dev 2802 * Pointer to the dev structure. 2803 * @param[in] attr 2804 * Pointer to the flow attributes. 2805 * 2806 * @return 2807 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2808 */ 2809 static enum mlx5_flow_drv_type 2810 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2811 { 2812 struct mlx5_priv *priv = dev->data->dev_private; 2813 /* The OS can determine first a specific flow type (DV, VERBS) */ 2814 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 2815 2816 if (type != MLX5_FLOW_TYPE_MAX) 2817 return type; 2818 /* If no OS specific type - continue with DV/VERBS selection */ 2819 if (attr->transfer && priv->config.dv_esw_en) 2820 type = MLX5_FLOW_TYPE_DV; 2821 if (!attr->transfer) 2822 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2823 MLX5_FLOW_TYPE_VERBS; 2824 return type; 2825 } 2826 2827 #define flow_get_drv_ops(type) flow_drv_ops[type] 2828 2829 /** 2830 * Flow driver validation API. This abstracts calling driver specific functions. 2831 * The type of flow driver is determined according to flow attributes. 2832 * 2833 * @param[in] dev 2834 * Pointer to the dev structure. 2835 * @param[in] attr 2836 * Pointer to the flow attributes. 2837 * @param[in] items 2838 * Pointer to the list of items. 2839 * @param[in] actions 2840 * Pointer to the list of actions. 2841 * @param[in] external 2842 * This flow rule is created by request external to PMD. 2843 * @param[in] hairpin 2844 * Number of hairpin TX actions, 0 means classic flow. 2845 * @param[out] error 2846 * Pointer to the error structure. 2847 * 2848 * @return 2849 * 0 on success, a negative errno value otherwise and rte_errno is set. 2850 */ 2851 static inline int 2852 flow_drv_validate(struct rte_eth_dev *dev, 2853 const struct rte_flow_attr *attr, 2854 const struct rte_flow_item items[], 2855 const struct rte_flow_action actions[], 2856 bool external, int hairpin, struct rte_flow_error *error) 2857 { 2858 const struct mlx5_flow_driver_ops *fops; 2859 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2860 2861 fops = flow_get_drv_ops(type); 2862 return fops->validate(dev, attr, items, actions, external, 2863 hairpin, error); 2864 } 2865 2866 /** 2867 * Flow driver preparation API. This abstracts calling driver specific 2868 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2869 * calculates the size of memory required for device flow, allocates the memory, 2870 * initializes the device flow and returns the pointer. 2871 * 2872 * @note 2873 * This function initializes device flow structure such as dv or verbs in 2874 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2875 * rest. For example, adding returning device flow to flow->dev_flow list and 2876 * setting backward reference to the flow should be done out of this function. 2877 * layers field is not filled either. 2878 * 2879 * @param[in] dev 2880 * Pointer to the dev structure. 2881 * @param[in] attr 2882 * Pointer to the flow attributes. 2883 * @param[in] items 2884 * Pointer to the list of items. 2885 * @param[in] actions 2886 * Pointer to the list of actions. 2887 * @param[in] flow_idx 2888 * This memory pool index to the flow. 2889 * @param[out] error 2890 * Pointer to the error structure. 2891 * 2892 * @return 2893 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2894 */ 2895 static inline struct mlx5_flow * 2896 flow_drv_prepare(struct rte_eth_dev *dev, 2897 const struct rte_flow *flow, 2898 const struct rte_flow_attr *attr, 2899 const struct rte_flow_item items[], 2900 const struct rte_flow_action actions[], 2901 uint32_t flow_idx, 2902 struct rte_flow_error *error) 2903 { 2904 const struct mlx5_flow_driver_ops *fops; 2905 enum mlx5_flow_drv_type type = flow->drv_type; 2906 struct mlx5_flow *mlx5_flow = NULL; 2907 2908 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2909 fops = flow_get_drv_ops(type); 2910 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 2911 if (mlx5_flow) 2912 mlx5_flow->flow_idx = flow_idx; 2913 return mlx5_flow; 2914 } 2915 2916 /** 2917 * Flow driver translation API. This abstracts calling driver specific 2918 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2919 * translates a generic flow into a driver flow. flow_drv_prepare() must 2920 * precede. 2921 * 2922 * @note 2923 * dev_flow->layers could be filled as a result of parsing during translation 2924 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2925 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2926 * flow->actions could be overwritten even though all the expanded dev_flows 2927 * have the same actions. 2928 * 2929 * @param[in] dev 2930 * Pointer to the rte dev structure. 2931 * @param[in, out] dev_flow 2932 * Pointer to the mlx5 flow. 2933 * @param[in] attr 2934 * Pointer to the flow attributes. 2935 * @param[in] items 2936 * Pointer to the list of items. 2937 * @param[in] actions 2938 * Pointer to the list of actions. 2939 * @param[out] error 2940 * Pointer to the error structure. 2941 * 2942 * @return 2943 * 0 on success, a negative errno value otherwise and rte_errno is set. 2944 */ 2945 static inline int 2946 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2947 const struct rte_flow_attr *attr, 2948 const struct rte_flow_item items[], 2949 const struct rte_flow_action actions[], 2950 struct rte_flow_error *error) 2951 { 2952 const struct mlx5_flow_driver_ops *fops; 2953 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2954 2955 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2956 fops = flow_get_drv_ops(type); 2957 return fops->translate(dev, dev_flow, attr, items, actions, error); 2958 } 2959 2960 /** 2961 * Flow driver apply API. This abstracts calling driver specific functions. 2962 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2963 * translated driver flows on to device. flow_drv_translate() must precede. 2964 * 2965 * @param[in] dev 2966 * Pointer to Ethernet device structure. 2967 * @param[in, out] flow 2968 * Pointer to flow structure. 2969 * @param[out] error 2970 * Pointer to error structure. 2971 * 2972 * @return 2973 * 0 on success, a negative errno value otherwise and rte_errno is set. 2974 */ 2975 static inline int 2976 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2977 struct rte_flow_error *error) 2978 { 2979 const struct mlx5_flow_driver_ops *fops; 2980 enum mlx5_flow_drv_type type = flow->drv_type; 2981 2982 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2983 fops = flow_get_drv_ops(type); 2984 return fops->apply(dev, flow, error); 2985 } 2986 2987 /** 2988 * Flow driver remove API. This abstracts calling driver specific functions. 2989 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2990 * on device. All the resources of the flow should be freed by calling 2991 * flow_drv_destroy(). 2992 * 2993 * @param[in] dev 2994 * Pointer to Ethernet device. 2995 * @param[in, out] flow 2996 * Pointer to flow structure. 2997 */ 2998 static inline void 2999 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 3000 { 3001 const struct mlx5_flow_driver_ops *fops; 3002 enum mlx5_flow_drv_type type = flow->drv_type; 3003 3004 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3005 fops = flow_get_drv_ops(type); 3006 fops->remove(dev, flow); 3007 } 3008 3009 /** 3010 * Flow driver destroy API. This abstracts calling driver specific functions. 3011 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3012 * on device and releases resources of the flow. 3013 * 3014 * @param[in] dev 3015 * Pointer to Ethernet device. 3016 * @param[in, out] flow 3017 * Pointer to flow structure. 3018 */ 3019 static inline void 3020 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3021 { 3022 const struct mlx5_flow_driver_ops *fops; 3023 enum mlx5_flow_drv_type type = flow->drv_type; 3024 3025 flow_mreg_split_qrss_release(dev, flow); 3026 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3027 fops = flow_get_drv_ops(type); 3028 fops->destroy(dev, flow); 3029 } 3030 3031 /** 3032 * Get RSS action from the action list. 3033 * 3034 * @param[in] actions 3035 * Pointer to the list of actions. 3036 * 3037 * @return 3038 * Pointer to the RSS action if exist, else return NULL. 3039 */ 3040 static const struct rte_flow_action_rss* 3041 flow_get_rss_action(const struct rte_flow_action actions[]) 3042 { 3043 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3044 switch (actions->type) { 3045 case RTE_FLOW_ACTION_TYPE_RSS: 3046 return (const struct rte_flow_action_rss *) 3047 actions->conf; 3048 default: 3049 break; 3050 } 3051 } 3052 return NULL; 3053 } 3054 3055 static unsigned int 3056 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3057 { 3058 const struct rte_flow_item *item; 3059 unsigned int has_vlan = 0; 3060 3061 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3062 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3063 has_vlan = 1; 3064 break; 3065 } 3066 } 3067 if (has_vlan) 3068 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3069 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3070 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3071 MLX5_EXPANSION_ROOT_OUTER; 3072 } 3073 3074 /** 3075 * Get layer flags from the prefix flow. 3076 * 3077 * Some flows may be split to several subflows, the prefix subflow gets the 3078 * match items and the suffix sub flow gets the actions. 3079 * Some actions need the user defined match item flags to get the detail for 3080 * the action. 3081 * This function helps the suffix flow to get the item layer flags from prefix 3082 * subflow. 3083 * 3084 * @param[in] dev_flow 3085 * Pointer the created preifx subflow. 3086 * 3087 * @return 3088 * The layers get from prefix subflow. 3089 */ 3090 static inline uint64_t 3091 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3092 { 3093 uint64_t layers = 0; 3094 3095 /* 3096 * Layers bits could be localization, but usually the compiler will 3097 * help to do the optimization work for source code. 3098 * If no decap actions, use the layers directly. 3099 */ 3100 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3101 return dev_flow->handle->layers; 3102 /* Convert L3 layers with decap action. */ 3103 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3104 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3105 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3106 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3107 /* Convert L4 layers with decap action. */ 3108 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3109 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3110 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3111 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3112 return layers; 3113 } 3114 3115 /** 3116 * Get metadata split action information. 3117 * 3118 * @param[in] actions 3119 * Pointer to the list of actions. 3120 * @param[out] qrss 3121 * Pointer to the return pointer. 3122 * @param[out] qrss_type 3123 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3124 * if no QUEUE/RSS is found. 3125 * @param[out] encap_idx 3126 * Pointer to the index of the encap action if exists, otherwise the last 3127 * action index. 3128 * 3129 * @return 3130 * Total number of actions. 3131 */ 3132 static int 3133 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3134 const struct rte_flow_action **qrss, 3135 int *encap_idx) 3136 { 3137 const struct rte_flow_action_raw_encap *raw_encap; 3138 int actions_n = 0; 3139 int raw_decap_idx = -1; 3140 3141 *encap_idx = -1; 3142 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3143 switch (actions->type) { 3144 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3145 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3146 *encap_idx = actions_n; 3147 break; 3148 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3149 raw_decap_idx = actions_n; 3150 break; 3151 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3152 raw_encap = actions->conf; 3153 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3154 *encap_idx = raw_decap_idx != -1 ? 3155 raw_decap_idx : actions_n; 3156 break; 3157 case RTE_FLOW_ACTION_TYPE_QUEUE: 3158 case RTE_FLOW_ACTION_TYPE_RSS: 3159 *qrss = actions; 3160 break; 3161 default: 3162 break; 3163 } 3164 actions_n++; 3165 } 3166 if (*encap_idx == -1) 3167 *encap_idx = actions_n; 3168 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3169 return actions_n + 1; 3170 } 3171 3172 /** 3173 * Check meter action from the action list. 3174 * 3175 * @param[in] actions 3176 * Pointer to the list of actions. 3177 * @param[out] mtr 3178 * Pointer to the meter exist flag. 3179 * 3180 * @return 3181 * Total number of actions. 3182 */ 3183 static int 3184 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 3185 { 3186 int actions_n = 0; 3187 3188 MLX5_ASSERT(mtr); 3189 *mtr = 0; 3190 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3191 switch (actions->type) { 3192 case RTE_FLOW_ACTION_TYPE_METER: 3193 *mtr = 1; 3194 break; 3195 default: 3196 break; 3197 } 3198 actions_n++; 3199 } 3200 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3201 return actions_n + 1; 3202 } 3203 3204 /** 3205 * Check if the flow should be split due to hairpin. 3206 * The reason for the split is that in current HW we can't 3207 * support encap and push-vlan on Rx, so if a flow contains 3208 * these actions we move it to Tx. 3209 * 3210 * @param dev 3211 * Pointer to Ethernet device. 3212 * @param[in] attr 3213 * Flow rule attributes. 3214 * @param[in] actions 3215 * Associated actions (list terminated by the END action). 3216 * 3217 * @return 3218 * > 0 the number of actions and the flow should be split, 3219 * 0 when no split required. 3220 */ 3221 static int 3222 flow_check_hairpin_split(struct rte_eth_dev *dev, 3223 const struct rte_flow_attr *attr, 3224 const struct rte_flow_action actions[]) 3225 { 3226 int queue_action = 0; 3227 int action_n = 0; 3228 int split = 0; 3229 const struct rte_flow_action_queue *queue; 3230 const struct rte_flow_action_rss *rss; 3231 const struct rte_flow_action_raw_encap *raw_encap; 3232 3233 if (!attr->ingress) 3234 return 0; 3235 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3236 switch (actions->type) { 3237 case RTE_FLOW_ACTION_TYPE_QUEUE: 3238 queue = actions->conf; 3239 if (queue == NULL) 3240 return 0; 3241 if (mlx5_rxq_get_type(dev, queue->index) != 3242 MLX5_RXQ_TYPE_HAIRPIN) 3243 return 0; 3244 queue_action = 1; 3245 action_n++; 3246 break; 3247 case RTE_FLOW_ACTION_TYPE_RSS: 3248 rss = actions->conf; 3249 if (rss == NULL || rss->queue_num == 0) 3250 return 0; 3251 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 3252 MLX5_RXQ_TYPE_HAIRPIN) 3253 return 0; 3254 queue_action = 1; 3255 action_n++; 3256 break; 3257 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3258 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3259 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3260 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3261 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3262 split++; 3263 action_n++; 3264 break; 3265 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3266 raw_encap = actions->conf; 3267 if (raw_encap->size > 3268 (sizeof(struct rte_flow_item_eth) + 3269 sizeof(struct rte_flow_item_ipv4))) 3270 split++; 3271 action_n++; 3272 break; 3273 default: 3274 action_n++; 3275 break; 3276 } 3277 } 3278 if (split && queue_action) 3279 return action_n; 3280 return 0; 3281 } 3282 3283 /* Declare flow create/destroy prototype in advance. */ 3284 static uint32_t 3285 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3286 const struct rte_flow_attr *attr, 3287 const struct rte_flow_item items[], 3288 const struct rte_flow_action actions[], 3289 bool external, struct rte_flow_error *error); 3290 3291 static void 3292 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3293 uint32_t flow_idx); 3294 3295 /** 3296 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3297 * 3298 * As mark_id is unique, if there's already a registered flow for the mark_id, 3299 * return by increasing the reference counter of the resource. Otherwise, create 3300 * the resource (mcp_res) and flow. 3301 * 3302 * Flow looks like, 3303 * - If ingress port is ANY and reg_c[1] is mark_id, 3304 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3305 * 3306 * For default flow (zero mark_id), flow is like, 3307 * - If ingress port is ANY, 3308 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 3309 * 3310 * @param dev 3311 * Pointer to Ethernet device. 3312 * @param mark_id 3313 * ID of MARK action, zero means default flow for META. 3314 * @param[out] error 3315 * Perform verbose error reporting if not NULL. 3316 * 3317 * @return 3318 * Associated resource on success, NULL otherwise and rte_errno is set. 3319 */ 3320 static struct mlx5_flow_mreg_copy_resource * 3321 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 3322 struct rte_flow_error *error) 3323 { 3324 struct mlx5_priv *priv = dev->data->dev_private; 3325 struct rte_flow_attr attr = { 3326 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3327 .ingress = 1, 3328 }; 3329 struct mlx5_rte_flow_item_tag tag_spec = { 3330 .data = mark_id, 3331 }; 3332 struct rte_flow_item items[] = { 3333 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 3334 }; 3335 struct rte_flow_action_mark ftag = { 3336 .id = mark_id, 3337 }; 3338 struct mlx5_flow_action_copy_mreg cp_mreg = { 3339 .dst = REG_B, 3340 .src = REG_NON, 3341 }; 3342 struct rte_flow_action_jump jump = { 3343 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3344 }; 3345 struct rte_flow_action actions[] = { 3346 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 3347 }; 3348 struct mlx5_flow_mreg_copy_resource *mcp_res; 3349 uint32_t idx = 0; 3350 int ret; 3351 3352 /* Fill the register fileds in the flow. */ 3353 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3354 if (ret < 0) 3355 return NULL; 3356 tag_spec.id = ret; 3357 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3358 if (ret < 0) 3359 return NULL; 3360 cp_mreg.src = ret; 3361 /* Check if already registered. */ 3362 MLX5_ASSERT(priv->mreg_cp_tbl); 3363 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 3364 if (mcp_res) { 3365 /* For non-default rule. */ 3366 if (mark_id != MLX5_DEFAULT_COPY_ID) 3367 mcp_res->refcnt++; 3368 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 3369 mcp_res->refcnt == 1); 3370 return mcp_res; 3371 } 3372 /* Provide the full width of FLAG specific value. */ 3373 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3374 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3375 /* Build a new flow. */ 3376 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3377 items[0] = (struct rte_flow_item){ 3378 .type = (enum rte_flow_item_type) 3379 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3380 .spec = &tag_spec, 3381 }; 3382 items[1] = (struct rte_flow_item){ 3383 .type = RTE_FLOW_ITEM_TYPE_END, 3384 }; 3385 actions[0] = (struct rte_flow_action){ 3386 .type = (enum rte_flow_action_type) 3387 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3388 .conf = &ftag, 3389 }; 3390 actions[1] = (struct rte_flow_action){ 3391 .type = (enum rte_flow_action_type) 3392 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3393 .conf = &cp_mreg, 3394 }; 3395 actions[2] = (struct rte_flow_action){ 3396 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3397 .conf = &jump, 3398 }; 3399 actions[3] = (struct rte_flow_action){ 3400 .type = RTE_FLOW_ACTION_TYPE_END, 3401 }; 3402 } else { 3403 /* Default rule, wildcard match. */ 3404 attr.priority = MLX5_FLOW_PRIO_RSVD; 3405 items[0] = (struct rte_flow_item){ 3406 .type = RTE_FLOW_ITEM_TYPE_END, 3407 }; 3408 actions[0] = (struct rte_flow_action){ 3409 .type = (enum rte_flow_action_type) 3410 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3411 .conf = &cp_mreg, 3412 }; 3413 actions[1] = (struct rte_flow_action){ 3414 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3415 .conf = &jump, 3416 }; 3417 actions[2] = (struct rte_flow_action){ 3418 .type = RTE_FLOW_ACTION_TYPE_END, 3419 }; 3420 } 3421 /* Build a new entry. */ 3422 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3423 if (!mcp_res) { 3424 rte_errno = ENOMEM; 3425 return NULL; 3426 } 3427 mcp_res->idx = idx; 3428 /* 3429 * The copy Flows are not included in any list. There 3430 * ones are referenced from other Flows and can not 3431 * be applied, removed, deleted in ardbitrary order 3432 * by list traversing. 3433 */ 3434 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3435 actions, false, error); 3436 if (!mcp_res->rix_flow) 3437 goto error; 3438 mcp_res->refcnt++; 3439 mcp_res->hlist_ent.key = mark_id; 3440 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3441 &mcp_res->hlist_ent); 3442 MLX5_ASSERT(!ret); 3443 if (ret) 3444 goto error; 3445 return mcp_res; 3446 error: 3447 if (mcp_res->rix_flow) 3448 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3449 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3450 return NULL; 3451 } 3452 3453 /** 3454 * Release flow in RX_CP_TBL. 3455 * 3456 * @param dev 3457 * Pointer to Ethernet device. 3458 * @flow 3459 * Parent flow for wich copying is provided. 3460 */ 3461 static void 3462 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3463 struct rte_flow *flow) 3464 { 3465 struct mlx5_flow_mreg_copy_resource *mcp_res; 3466 struct mlx5_priv *priv = dev->data->dev_private; 3467 3468 if (!flow->rix_mreg_copy) 3469 return; 3470 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3471 flow->rix_mreg_copy); 3472 if (!mcp_res || !priv->mreg_cp_tbl) 3473 return; 3474 if (flow->copy_applied) { 3475 MLX5_ASSERT(mcp_res->appcnt); 3476 flow->copy_applied = 0; 3477 --mcp_res->appcnt; 3478 if (!mcp_res->appcnt) { 3479 struct rte_flow *mcp_flow = mlx5_ipool_get 3480 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3481 mcp_res->rix_flow); 3482 3483 if (mcp_flow) 3484 flow_drv_remove(dev, mcp_flow); 3485 } 3486 } 3487 /* 3488 * We do not check availability of metadata registers here, 3489 * because copy resources are not allocated in this case. 3490 */ 3491 if (--mcp_res->refcnt) 3492 return; 3493 MLX5_ASSERT(mcp_res->rix_flow); 3494 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3495 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3496 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3497 flow->rix_mreg_copy = 0; 3498 } 3499 3500 /** 3501 * Start flow in RX_CP_TBL. 3502 * 3503 * @param dev 3504 * Pointer to Ethernet device. 3505 * @flow 3506 * Parent flow for wich copying is provided. 3507 * 3508 * @return 3509 * 0 on success, a negative errno value otherwise and rte_errno is set. 3510 */ 3511 static int 3512 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3513 struct rte_flow *flow) 3514 { 3515 struct mlx5_flow_mreg_copy_resource *mcp_res; 3516 struct mlx5_priv *priv = dev->data->dev_private; 3517 int ret; 3518 3519 if (!flow->rix_mreg_copy || flow->copy_applied) 3520 return 0; 3521 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3522 flow->rix_mreg_copy); 3523 if (!mcp_res) 3524 return 0; 3525 if (!mcp_res->appcnt) { 3526 struct rte_flow *mcp_flow = mlx5_ipool_get 3527 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3528 mcp_res->rix_flow); 3529 3530 if (mcp_flow) { 3531 ret = flow_drv_apply(dev, mcp_flow, NULL); 3532 if (ret) 3533 return ret; 3534 } 3535 } 3536 ++mcp_res->appcnt; 3537 flow->copy_applied = 1; 3538 return 0; 3539 } 3540 3541 /** 3542 * Stop flow in RX_CP_TBL. 3543 * 3544 * @param dev 3545 * Pointer to Ethernet device. 3546 * @flow 3547 * Parent flow for wich copying is provided. 3548 */ 3549 static void 3550 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3551 struct rte_flow *flow) 3552 { 3553 struct mlx5_flow_mreg_copy_resource *mcp_res; 3554 struct mlx5_priv *priv = dev->data->dev_private; 3555 3556 if (!flow->rix_mreg_copy || !flow->copy_applied) 3557 return; 3558 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3559 flow->rix_mreg_copy); 3560 if (!mcp_res) 3561 return; 3562 MLX5_ASSERT(mcp_res->appcnt); 3563 --mcp_res->appcnt; 3564 flow->copy_applied = 0; 3565 if (!mcp_res->appcnt) { 3566 struct rte_flow *mcp_flow = mlx5_ipool_get 3567 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3568 mcp_res->rix_flow); 3569 3570 if (mcp_flow) 3571 flow_drv_remove(dev, mcp_flow); 3572 } 3573 } 3574 3575 /** 3576 * Remove the default copy action from RX_CP_TBL. 3577 * 3578 * @param dev 3579 * Pointer to Ethernet device. 3580 */ 3581 static void 3582 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3583 { 3584 struct mlx5_flow_mreg_copy_resource *mcp_res; 3585 struct mlx5_priv *priv = dev->data->dev_private; 3586 3587 /* Check if default flow is registered. */ 3588 if (!priv->mreg_cp_tbl) 3589 return; 3590 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3591 MLX5_DEFAULT_COPY_ID); 3592 if (!mcp_res) 3593 return; 3594 MLX5_ASSERT(mcp_res->rix_flow); 3595 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3596 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3597 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3598 } 3599 3600 /** 3601 * Add the default copy action in in RX_CP_TBL. 3602 * 3603 * @param dev 3604 * Pointer to Ethernet device. 3605 * @param[out] error 3606 * Perform verbose error reporting if not NULL. 3607 * 3608 * @return 3609 * 0 for success, negative value otherwise and rte_errno is set. 3610 */ 3611 static int 3612 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3613 struct rte_flow_error *error) 3614 { 3615 struct mlx5_priv *priv = dev->data->dev_private; 3616 struct mlx5_flow_mreg_copy_resource *mcp_res; 3617 3618 /* Check whether extensive metadata feature is engaged. */ 3619 if (!priv->config.dv_flow_en || 3620 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3621 !mlx5_flow_ext_mreg_supported(dev) || 3622 !priv->sh->dv_regc0_mask) 3623 return 0; 3624 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3625 if (!mcp_res) 3626 return -rte_errno; 3627 return 0; 3628 } 3629 3630 /** 3631 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3632 * 3633 * All the flow having Q/RSS action should be split by 3634 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3635 * performs the following, 3636 * - CQE->flow_tag := reg_c[1] (MARK) 3637 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3638 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3639 * but there should be a flow per each MARK ID set by MARK action. 3640 * 3641 * For the aforementioned reason, if there's a MARK action in flow's action 3642 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3643 * the MARK ID to CQE's flow_tag like, 3644 * - If reg_c[1] is mark_id, 3645 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3646 * 3647 * For SET_META action which stores value in reg_c[0], as the destination is 3648 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3649 * MARK ID means the default flow. The default flow looks like, 3650 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3651 * 3652 * @param dev 3653 * Pointer to Ethernet device. 3654 * @param flow 3655 * Pointer to flow structure. 3656 * @param[in] actions 3657 * Pointer to the list of actions. 3658 * @param[out] error 3659 * Perform verbose error reporting if not NULL. 3660 * 3661 * @return 3662 * 0 on success, negative value otherwise and rte_errno is set. 3663 */ 3664 static int 3665 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3666 struct rte_flow *flow, 3667 const struct rte_flow_action *actions, 3668 struct rte_flow_error *error) 3669 { 3670 struct mlx5_priv *priv = dev->data->dev_private; 3671 struct mlx5_dev_config *config = &priv->config; 3672 struct mlx5_flow_mreg_copy_resource *mcp_res; 3673 const struct rte_flow_action_mark *mark; 3674 3675 /* Check whether extensive metadata feature is engaged. */ 3676 if (!config->dv_flow_en || 3677 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3678 !mlx5_flow_ext_mreg_supported(dev) || 3679 !priv->sh->dv_regc0_mask) 3680 return 0; 3681 /* Find MARK action. */ 3682 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3683 switch (actions->type) { 3684 case RTE_FLOW_ACTION_TYPE_FLAG: 3685 mcp_res = flow_mreg_add_copy_action 3686 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3687 if (!mcp_res) 3688 return -rte_errno; 3689 flow->rix_mreg_copy = mcp_res->idx; 3690 if (dev->data->dev_started) { 3691 mcp_res->appcnt++; 3692 flow->copy_applied = 1; 3693 } 3694 return 0; 3695 case RTE_FLOW_ACTION_TYPE_MARK: 3696 mark = (const struct rte_flow_action_mark *) 3697 actions->conf; 3698 mcp_res = 3699 flow_mreg_add_copy_action(dev, mark->id, error); 3700 if (!mcp_res) 3701 return -rte_errno; 3702 flow->rix_mreg_copy = mcp_res->idx; 3703 if (dev->data->dev_started) { 3704 mcp_res->appcnt++; 3705 flow->copy_applied = 1; 3706 } 3707 return 0; 3708 default: 3709 break; 3710 } 3711 } 3712 return 0; 3713 } 3714 3715 #define MLX5_MAX_SPLIT_ACTIONS 24 3716 #define MLX5_MAX_SPLIT_ITEMS 24 3717 3718 /** 3719 * Split the hairpin flow. 3720 * Since HW can't support encap and push-vlan on Rx, we move these 3721 * actions to Tx. 3722 * If the count action is after the encap then we also 3723 * move the count action. in this case the count will also measure 3724 * the outer bytes. 3725 * 3726 * @param dev 3727 * Pointer to Ethernet device. 3728 * @param[in] actions 3729 * Associated actions (list terminated by the END action). 3730 * @param[out] actions_rx 3731 * Rx flow actions. 3732 * @param[out] actions_tx 3733 * Tx flow actions.. 3734 * @param[out] pattern_tx 3735 * The pattern items for the Tx flow. 3736 * @param[out] flow_id 3737 * The flow ID connected to this flow. 3738 * 3739 * @return 3740 * 0 on success. 3741 */ 3742 static int 3743 flow_hairpin_split(struct rte_eth_dev *dev, 3744 const struct rte_flow_action actions[], 3745 struct rte_flow_action actions_rx[], 3746 struct rte_flow_action actions_tx[], 3747 struct rte_flow_item pattern_tx[], 3748 uint32_t *flow_id) 3749 { 3750 struct mlx5_priv *priv = dev->data->dev_private; 3751 const struct rte_flow_action_raw_encap *raw_encap; 3752 const struct rte_flow_action_raw_decap *raw_decap; 3753 struct mlx5_rte_flow_action_set_tag *set_tag; 3754 struct rte_flow_action *tag_action; 3755 struct mlx5_rte_flow_item_tag *tag_item; 3756 struct rte_flow_item *item; 3757 char *addr; 3758 int encap = 0; 3759 3760 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3761 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3762 switch (actions->type) { 3763 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3764 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3765 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3766 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3767 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3768 rte_memcpy(actions_tx, actions, 3769 sizeof(struct rte_flow_action)); 3770 actions_tx++; 3771 break; 3772 case RTE_FLOW_ACTION_TYPE_COUNT: 3773 if (encap) { 3774 rte_memcpy(actions_tx, actions, 3775 sizeof(struct rte_flow_action)); 3776 actions_tx++; 3777 } else { 3778 rte_memcpy(actions_rx, actions, 3779 sizeof(struct rte_flow_action)); 3780 actions_rx++; 3781 } 3782 break; 3783 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3784 raw_encap = actions->conf; 3785 if (raw_encap->size > 3786 (sizeof(struct rte_flow_item_eth) + 3787 sizeof(struct rte_flow_item_ipv4))) { 3788 memcpy(actions_tx, actions, 3789 sizeof(struct rte_flow_action)); 3790 actions_tx++; 3791 encap = 1; 3792 } else { 3793 rte_memcpy(actions_rx, actions, 3794 sizeof(struct rte_flow_action)); 3795 actions_rx++; 3796 } 3797 break; 3798 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3799 raw_decap = actions->conf; 3800 if (raw_decap->size < 3801 (sizeof(struct rte_flow_item_eth) + 3802 sizeof(struct rte_flow_item_ipv4))) { 3803 memcpy(actions_tx, actions, 3804 sizeof(struct rte_flow_action)); 3805 actions_tx++; 3806 } else { 3807 rte_memcpy(actions_rx, actions, 3808 sizeof(struct rte_flow_action)); 3809 actions_rx++; 3810 } 3811 break; 3812 default: 3813 rte_memcpy(actions_rx, actions, 3814 sizeof(struct rte_flow_action)); 3815 actions_rx++; 3816 break; 3817 } 3818 } 3819 /* Add set meta action and end action for the Rx flow. */ 3820 tag_action = actions_rx; 3821 tag_action->type = (enum rte_flow_action_type) 3822 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3823 actions_rx++; 3824 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3825 actions_rx++; 3826 set_tag = (void *)actions_rx; 3827 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3828 MLX5_ASSERT(set_tag->id > REG_NON); 3829 set_tag->data = *flow_id; 3830 tag_action->conf = set_tag; 3831 /* Create Tx item list. */ 3832 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3833 addr = (void *)&pattern_tx[2]; 3834 item = pattern_tx; 3835 item->type = (enum rte_flow_item_type) 3836 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3837 tag_item = (void *)addr; 3838 tag_item->data = *flow_id; 3839 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3840 MLX5_ASSERT(set_tag->id > REG_NON); 3841 item->spec = tag_item; 3842 addr += sizeof(struct mlx5_rte_flow_item_tag); 3843 tag_item = (void *)addr; 3844 tag_item->data = UINT32_MAX; 3845 tag_item->id = UINT16_MAX; 3846 item->mask = tag_item; 3847 item->last = NULL; 3848 item++; 3849 item->type = RTE_FLOW_ITEM_TYPE_END; 3850 return 0; 3851 } 3852 3853 /** 3854 * The last stage of splitting chain, just creates the subflow 3855 * without any modification. 3856 * 3857 * @param[in] dev 3858 * Pointer to Ethernet device. 3859 * @param[in] flow 3860 * Parent flow structure pointer. 3861 * @param[in, out] sub_flow 3862 * Pointer to return the created subflow, may be NULL. 3863 * @param[in] prefix_layers 3864 * Prefix subflow layers, may be 0. 3865 * @param[in] attr 3866 * Flow rule attributes. 3867 * @param[in] items 3868 * Pattern specification (list terminated by the END pattern item). 3869 * @param[in] actions 3870 * Associated actions (list terminated by the END action). 3871 * @param[in] external 3872 * This flow rule is created by request external to PMD. 3873 * @param[in] flow_idx 3874 * This memory pool index to the flow. 3875 * @param[out] error 3876 * Perform verbose error reporting if not NULL. 3877 * @return 3878 * 0 on success, negative value otherwise 3879 */ 3880 static int 3881 flow_create_split_inner(struct rte_eth_dev *dev, 3882 struct rte_flow *flow, 3883 struct mlx5_flow **sub_flow, 3884 uint64_t prefix_layers, 3885 const struct rte_flow_attr *attr, 3886 const struct rte_flow_item items[], 3887 const struct rte_flow_action actions[], 3888 bool external, uint32_t flow_idx, 3889 struct rte_flow_error *error) 3890 { 3891 struct mlx5_flow *dev_flow; 3892 3893 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 3894 flow_idx, error); 3895 if (!dev_flow) 3896 return -rte_errno; 3897 dev_flow->flow = flow; 3898 dev_flow->external = external; 3899 /* Subflow object was created, we must include one in the list. */ 3900 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 3901 dev_flow->handle, next); 3902 /* 3903 * If dev_flow is as one of the suffix flow, some actions in suffix 3904 * flow may need some user defined item layer flags. 3905 */ 3906 if (prefix_layers) 3907 dev_flow->handle->layers = prefix_layers; 3908 if (sub_flow) 3909 *sub_flow = dev_flow; 3910 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3911 } 3912 3913 /** 3914 * Split the meter flow. 3915 * 3916 * As meter flow will split to three sub flow, other than meter 3917 * action, the other actions make sense to only meter accepts 3918 * the packet. If it need to be dropped, no other additional 3919 * actions should be take. 3920 * 3921 * One kind of special action which decapsulates the L3 tunnel 3922 * header will be in the prefix sub flow, as not to take the 3923 * L3 tunnel header into account. 3924 * 3925 * @param dev 3926 * Pointer to Ethernet device. 3927 * @param[in] items 3928 * Pattern specification (list terminated by the END pattern item). 3929 * @param[out] sfx_items 3930 * Suffix flow match items (list terminated by the END pattern item). 3931 * @param[in] actions 3932 * Associated actions (list terminated by the END action). 3933 * @param[out] actions_sfx 3934 * Suffix flow actions. 3935 * @param[out] actions_pre 3936 * Prefix flow actions. 3937 * @param[out] pattern_sfx 3938 * The pattern items for the suffix flow. 3939 * @param[out] tag_sfx 3940 * Pointer to suffix flow tag. 3941 * 3942 * @return 3943 * 0 on success. 3944 */ 3945 static int 3946 flow_meter_split_prep(struct rte_eth_dev *dev, 3947 const struct rte_flow_item items[], 3948 struct rte_flow_item sfx_items[], 3949 const struct rte_flow_action actions[], 3950 struct rte_flow_action actions_sfx[], 3951 struct rte_flow_action actions_pre[]) 3952 { 3953 struct rte_flow_action *tag_action = NULL; 3954 struct rte_flow_item *tag_item; 3955 struct mlx5_rte_flow_action_set_tag *set_tag; 3956 struct rte_flow_error error; 3957 const struct rte_flow_action_raw_encap *raw_encap; 3958 const struct rte_flow_action_raw_decap *raw_decap; 3959 struct mlx5_rte_flow_item_tag *tag_spec; 3960 struct mlx5_rte_flow_item_tag *tag_mask; 3961 uint32_t tag_id; 3962 bool copy_vlan = false; 3963 3964 /* Prepare the actions for prefix and suffix flow. */ 3965 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3966 struct rte_flow_action **action_cur = NULL; 3967 3968 switch (actions->type) { 3969 case RTE_FLOW_ACTION_TYPE_METER: 3970 /* Add the extra tag action first. */ 3971 tag_action = actions_pre; 3972 tag_action->type = (enum rte_flow_action_type) 3973 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3974 actions_pre++; 3975 action_cur = &actions_pre; 3976 break; 3977 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3978 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3979 action_cur = &actions_pre; 3980 break; 3981 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3982 raw_encap = actions->conf; 3983 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 3984 action_cur = &actions_pre; 3985 break; 3986 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3987 raw_decap = actions->conf; 3988 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3989 action_cur = &actions_pre; 3990 break; 3991 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3992 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3993 copy_vlan = true; 3994 break; 3995 default: 3996 break; 3997 } 3998 if (!action_cur) 3999 action_cur = &actions_sfx; 4000 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 4001 (*action_cur)++; 4002 } 4003 /* Add end action to the actions. */ 4004 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4005 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4006 actions_pre++; 4007 /* Set the tag. */ 4008 set_tag = (void *)actions_pre; 4009 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4010 /* 4011 * Get the id from the qrss_pool to make qrss share the id with meter. 4012 */ 4013 tag_id = flow_qrss_get_id(dev); 4014 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 4015 assert(tag_action); 4016 tag_action->conf = set_tag; 4017 /* Prepare the suffix subflow items. */ 4018 tag_item = sfx_items++; 4019 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4020 int item_type = items->type; 4021 4022 switch (item_type) { 4023 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4024 memcpy(sfx_items, items, sizeof(*sfx_items)); 4025 sfx_items++; 4026 break; 4027 case RTE_FLOW_ITEM_TYPE_VLAN: 4028 if (copy_vlan) { 4029 memcpy(sfx_items, items, sizeof(*sfx_items)); 4030 /* 4031 * Convert to internal match item, it is used 4032 * for vlan push and set vid. 4033 */ 4034 sfx_items->type = (enum rte_flow_item_type) 4035 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4036 sfx_items++; 4037 } 4038 break; 4039 default: 4040 break; 4041 } 4042 } 4043 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4044 sfx_items++; 4045 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4046 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 4047 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4048 tag_mask = tag_spec + 1; 4049 tag_mask->data = 0xffffff00; 4050 tag_item->type = (enum rte_flow_item_type) 4051 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4052 tag_item->spec = tag_spec; 4053 tag_item->last = NULL; 4054 tag_item->mask = tag_mask; 4055 return tag_id; 4056 } 4057 4058 /** 4059 * Split action list having QUEUE/RSS for metadata register copy. 4060 * 4061 * Once Q/RSS action is detected in user's action list, the flow action 4062 * should be split in order to copy metadata registers, which will happen in 4063 * RX_CP_TBL like, 4064 * - CQE->flow_tag := reg_c[1] (MARK) 4065 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4066 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4067 * This is because the last action of each flow must be a terminal action 4068 * (QUEUE, RSS or DROP). 4069 * 4070 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4071 * stored and kept in the mlx5_flow structure per each sub_flow. 4072 * 4073 * The Q/RSS action is replaced with, 4074 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4075 * And the following JUMP action is added at the end, 4076 * - JUMP, to RX_CP_TBL. 4077 * 4078 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4079 * flow_create_split_metadata() routine. The flow will look like, 4080 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4081 * 4082 * @param dev 4083 * Pointer to Ethernet device. 4084 * @param[out] split_actions 4085 * Pointer to store split actions to jump to CP_TBL. 4086 * @param[in] actions 4087 * Pointer to the list of original flow actions. 4088 * @param[in] qrss 4089 * Pointer to the Q/RSS action. 4090 * @param[in] actions_n 4091 * Number of original actions. 4092 * @param[out] error 4093 * Perform verbose error reporting if not NULL. 4094 * 4095 * @return 4096 * non-zero unique flow_id on success, otherwise 0 and 4097 * error/rte_error are set. 4098 */ 4099 static uint32_t 4100 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4101 struct rte_flow_action *split_actions, 4102 const struct rte_flow_action *actions, 4103 const struct rte_flow_action *qrss, 4104 int actions_n, struct rte_flow_error *error) 4105 { 4106 struct mlx5_rte_flow_action_set_tag *set_tag; 4107 struct rte_flow_action_jump *jump; 4108 const int qrss_idx = qrss - actions; 4109 uint32_t flow_id = 0; 4110 int ret = 0; 4111 4112 /* 4113 * Given actions will be split 4114 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4115 * - Add jump to mreg CP_TBL. 4116 * As a result, there will be one more action. 4117 */ 4118 ++actions_n; 4119 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4120 set_tag = (void *)(split_actions + actions_n); 4121 /* 4122 * If tag action is not set to void(it means we are not the meter 4123 * suffix flow), add the tag action. Since meter suffix flow already 4124 * has the tag added. 4125 */ 4126 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4127 /* 4128 * Allocate the new subflow ID. This one is unique within 4129 * device and not shared with representors. Otherwise, 4130 * we would have to resolve multi-thread access synch 4131 * issue. Each flow on the shared device is appended 4132 * with source vport identifier, so the resulting 4133 * flows will be unique in the shared (by master and 4134 * representors) domain even if they have coinciding 4135 * IDs. 4136 */ 4137 flow_id = flow_qrss_get_id(dev); 4138 if (!flow_id) 4139 return rte_flow_error_set(error, ENOMEM, 4140 RTE_FLOW_ERROR_TYPE_ACTION, 4141 NULL, "can't allocate id " 4142 "for split Q/RSS subflow"); 4143 /* Internal SET_TAG action to set flow ID. */ 4144 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4145 .data = flow_id, 4146 }; 4147 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4148 if (ret < 0) 4149 return ret; 4150 set_tag->id = ret; 4151 /* Construct new actions array. */ 4152 /* Replace QUEUE/RSS action. */ 4153 split_actions[qrss_idx] = (struct rte_flow_action){ 4154 .type = (enum rte_flow_action_type) 4155 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4156 .conf = set_tag, 4157 }; 4158 } 4159 /* JUMP action to jump to mreg copy table (CP_TBL). */ 4160 jump = (void *)(set_tag + 1); 4161 *jump = (struct rte_flow_action_jump){ 4162 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4163 }; 4164 split_actions[actions_n - 2] = (struct rte_flow_action){ 4165 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4166 .conf = jump, 4167 }; 4168 split_actions[actions_n - 1] = (struct rte_flow_action){ 4169 .type = RTE_FLOW_ACTION_TYPE_END, 4170 }; 4171 return flow_id; 4172 } 4173 4174 /** 4175 * Extend the given action list for Tx metadata copy. 4176 * 4177 * Copy the given action list to the ext_actions and add flow metadata register 4178 * copy action in order to copy reg_a set by WQE to reg_c[0]. 4179 * 4180 * @param[out] ext_actions 4181 * Pointer to the extended action list. 4182 * @param[in] actions 4183 * Pointer to the list of actions. 4184 * @param[in] actions_n 4185 * Number of actions in the list. 4186 * @param[out] error 4187 * Perform verbose error reporting if not NULL. 4188 * @param[in] encap_idx 4189 * The encap action inndex. 4190 * 4191 * @return 4192 * 0 on success, negative value otherwise 4193 */ 4194 static int 4195 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 4196 struct rte_flow_action *ext_actions, 4197 const struct rte_flow_action *actions, 4198 int actions_n, struct rte_flow_error *error, 4199 int encap_idx) 4200 { 4201 struct mlx5_flow_action_copy_mreg *cp_mreg = 4202 (struct mlx5_flow_action_copy_mreg *) 4203 (ext_actions + actions_n + 1); 4204 int ret; 4205 4206 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4207 if (ret < 0) 4208 return ret; 4209 cp_mreg->dst = ret; 4210 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 4211 if (ret < 0) 4212 return ret; 4213 cp_mreg->src = ret; 4214 if (encap_idx != 0) 4215 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 4216 if (encap_idx == actions_n - 1) { 4217 ext_actions[actions_n - 1] = (struct rte_flow_action){ 4218 .type = (enum rte_flow_action_type) 4219 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4220 .conf = cp_mreg, 4221 }; 4222 ext_actions[actions_n] = (struct rte_flow_action){ 4223 .type = RTE_FLOW_ACTION_TYPE_END, 4224 }; 4225 } else { 4226 ext_actions[encap_idx] = (struct rte_flow_action){ 4227 .type = (enum rte_flow_action_type) 4228 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4229 .conf = cp_mreg, 4230 }; 4231 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 4232 sizeof(*ext_actions) * (actions_n - encap_idx)); 4233 } 4234 return 0; 4235 } 4236 4237 /** 4238 * The splitting for metadata feature. 4239 * 4240 * - Q/RSS action on NIC Rx should be split in order to pass by 4241 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4242 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4243 * 4244 * - All the actions on NIC Tx should have a mreg copy action to 4245 * copy reg_a from WQE to reg_c[0]. 4246 * 4247 * @param dev 4248 * Pointer to Ethernet device. 4249 * @param[in] flow 4250 * Parent flow structure pointer. 4251 * @param[in] prefix_layers 4252 * Prefix flow layer flags. 4253 * @param[in] attr 4254 * Flow rule attributes. 4255 * @param[in] items 4256 * Pattern specification (list terminated by the END pattern item). 4257 * @param[in] actions 4258 * Associated actions (list terminated by the END action). 4259 * @param[in] external 4260 * This flow rule is created by request external to PMD. 4261 * @param[in] flow_idx 4262 * This memory pool index to the flow. 4263 * @param[out] error 4264 * Perform verbose error reporting if not NULL. 4265 * @return 4266 * 0 on success, negative value otherwise 4267 */ 4268 static int 4269 flow_create_split_metadata(struct rte_eth_dev *dev, 4270 struct rte_flow *flow, 4271 uint64_t prefix_layers, 4272 const struct rte_flow_attr *attr, 4273 const struct rte_flow_item items[], 4274 const struct rte_flow_action actions[], 4275 bool external, uint32_t flow_idx, 4276 struct rte_flow_error *error) 4277 { 4278 struct mlx5_priv *priv = dev->data->dev_private; 4279 struct mlx5_dev_config *config = &priv->config; 4280 const struct rte_flow_action *qrss = NULL; 4281 struct rte_flow_action *ext_actions = NULL; 4282 struct mlx5_flow *dev_flow = NULL; 4283 uint32_t qrss_id = 0; 4284 int mtr_sfx = 0; 4285 size_t act_size; 4286 int actions_n; 4287 int encap_idx; 4288 int ret; 4289 4290 /* Check whether extensive metadata feature is engaged. */ 4291 if (!config->dv_flow_en || 4292 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4293 !mlx5_flow_ext_mreg_supported(dev)) 4294 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 4295 attr, items, actions, external, 4296 flow_idx, error); 4297 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 4298 &encap_idx); 4299 if (qrss) { 4300 /* Exclude hairpin flows from splitting. */ 4301 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 4302 const struct rte_flow_action_queue *queue; 4303 4304 queue = qrss->conf; 4305 if (mlx5_rxq_get_type(dev, queue->index) == 4306 MLX5_RXQ_TYPE_HAIRPIN) 4307 qrss = NULL; 4308 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 4309 const struct rte_flow_action_rss *rss; 4310 4311 rss = qrss->conf; 4312 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 4313 MLX5_RXQ_TYPE_HAIRPIN) 4314 qrss = NULL; 4315 } 4316 } 4317 if (qrss) { 4318 /* Check if it is in meter suffix table. */ 4319 mtr_sfx = attr->group == (attr->transfer ? 4320 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4321 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 4322 /* 4323 * Q/RSS action on NIC Rx should be split in order to pass by 4324 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4325 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4326 */ 4327 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4328 sizeof(struct rte_flow_action_set_tag) + 4329 sizeof(struct rte_flow_action_jump); 4330 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4331 SOCKET_ID_ANY); 4332 if (!ext_actions) 4333 return rte_flow_error_set(error, ENOMEM, 4334 RTE_FLOW_ERROR_TYPE_ACTION, 4335 NULL, "no memory to split " 4336 "metadata flow"); 4337 /* 4338 * If we are the suffix flow of meter, tag already exist. 4339 * Set the tag action to void. 4340 */ 4341 if (mtr_sfx) 4342 ext_actions[qrss - actions].type = 4343 RTE_FLOW_ACTION_TYPE_VOID; 4344 else 4345 ext_actions[qrss - actions].type = 4346 (enum rte_flow_action_type) 4347 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4348 /* 4349 * Create the new actions list with removed Q/RSS action 4350 * and appended set tag and jump to register copy table 4351 * (RX_CP_TBL). We should preallocate unique tag ID here 4352 * in advance, because it is needed for set tag action. 4353 */ 4354 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 4355 qrss, actions_n, error); 4356 if (!mtr_sfx && !qrss_id) { 4357 ret = -rte_errno; 4358 goto exit; 4359 } 4360 } else if (attr->egress && !attr->transfer) { 4361 /* 4362 * All the actions on NIC Tx should have a metadata register 4363 * copy action to copy reg_a from WQE to reg_c[meta] 4364 */ 4365 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4366 sizeof(struct mlx5_flow_action_copy_mreg); 4367 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4368 SOCKET_ID_ANY); 4369 if (!ext_actions) 4370 return rte_flow_error_set(error, ENOMEM, 4371 RTE_FLOW_ERROR_TYPE_ACTION, 4372 NULL, "no memory to split " 4373 "metadata flow"); 4374 /* Create the action list appended with copy register. */ 4375 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 4376 actions_n, error, encap_idx); 4377 if (ret < 0) 4378 goto exit; 4379 } 4380 /* Add the unmodified original or prefix subflow. */ 4381 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr, 4382 items, ext_actions ? ext_actions : 4383 actions, external, flow_idx, error); 4384 if (ret < 0) 4385 goto exit; 4386 MLX5_ASSERT(dev_flow); 4387 if (qrss) { 4388 const struct rte_flow_attr q_attr = { 4389 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4390 .ingress = 1, 4391 }; 4392 /* Internal PMD action to set register. */ 4393 struct mlx5_rte_flow_item_tag q_tag_spec = { 4394 .data = qrss_id, 4395 .id = REG_NON, 4396 }; 4397 struct rte_flow_item q_items[] = { 4398 { 4399 .type = (enum rte_flow_item_type) 4400 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4401 .spec = &q_tag_spec, 4402 .last = NULL, 4403 .mask = NULL, 4404 }, 4405 { 4406 .type = RTE_FLOW_ITEM_TYPE_END, 4407 }, 4408 }; 4409 struct rte_flow_action q_actions[] = { 4410 { 4411 .type = qrss->type, 4412 .conf = qrss->conf, 4413 }, 4414 { 4415 .type = RTE_FLOW_ACTION_TYPE_END, 4416 }, 4417 }; 4418 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4419 4420 /* 4421 * Configure the tag item only if there is no meter subflow. 4422 * Since tag is already marked in the meter suffix subflow 4423 * we can just use the meter suffix items as is. 4424 */ 4425 if (qrss_id) { 4426 /* Not meter subflow. */ 4427 MLX5_ASSERT(!mtr_sfx); 4428 /* 4429 * Put unique id in prefix flow due to it is destroyed 4430 * after suffix flow and id will be freed after there 4431 * is no actual flows with this id and identifier 4432 * reallocation becomes possible (for example, for 4433 * other flows in other threads). 4434 */ 4435 dev_flow->handle->split_flow_id = qrss_id; 4436 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4437 error); 4438 if (ret < 0) 4439 goto exit; 4440 q_tag_spec.id = ret; 4441 } 4442 dev_flow = NULL; 4443 /* Add suffix subflow to execute Q/RSS. */ 4444 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 4445 &q_attr, mtr_sfx ? items : 4446 q_items, q_actions, 4447 external, flow_idx, error); 4448 if (ret < 0) 4449 goto exit; 4450 /* qrss ID should be freed if failed. */ 4451 qrss_id = 0; 4452 MLX5_ASSERT(dev_flow); 4453 } 4454 4455 exit: 4456 /* 4457 * We do not destroy the partially created sub_flows in case of error. 4458 * These ones are included into parent flow list and will be destroyed 4459 * by flow_drv_destroy. 4460 */ 4461 flow_qrss_free_id(dev, qrss_id); 4462 mlx5_free(ext_actions); 4463 return ret; 4464 } 4465 4466 /** 4467 * The splitting for meter feature. 4468 * 4469 * - The meter flow will be split to two flows as prefix and 4470 * suffix flow. The packets make sense only it pass the prefix 4471 * meter action. 4472 * 4473 * - Reg_C_5 is used for the packet to match betweend prefix and 4474 * suffix flow. 4475 * 4476 * @param dev 4477 * Pointer to Ethernet device. 4478 * @param[in] flow 4479 * Parent flow structure pointer. 4480 * @param[in] attr 4481 * Flow rule attributes. 4482 * @param[in] items 4483 * Pattern specification (list terminated by the END pattern item). 4484 * @param[in] actions 4485 * Associated actions (list terminated by the END action). 4486 * @param[in] external 4487 * This flow rule is created by request external to PMD. 4488 * @param[in] flow_idx 4489 * This memory pool index to the flow. 4490 * @param[out] error 4491 * Perform verbose error reporting if not NULL. 4492 * @return 4493 * 0 on success, negative value otherwise 4494 */ 4495 static int 4496 flow_create_split_meter(struct rte_eth_dev *dev, 4497 struct rte_flow *flow, 4498 const struct rte_flow_attr *attr, 4499 const struct rte_flow_item items[], 4500 const struct rte_flow_action actions[], 4501 bool external, uint32_t flow_idx, 4502 struct rte_flow_error *error) 4503 { 4504 struct mlx5_priv *priv = dev->data->dev_private; 4505 struct rte_flow_action *sfx_actions = NULL; 4506 struct rte_flow_action *pre_actions = NULL; 4507 struct rte_flow_item *sfx_items = NULL; 4508 struct mlx5_flow *dev_flow = NULL; 4509 struct rte_flow_attr sfx_attr = *attr; 4510 uint32_t mtr = 0; 4511 uint32_t mtr_tag_id = 0; 4512 size_t act_size; 4513 size_t item_size; 4514 int actions_n = 0; 4515 int ret; 4516 4517 if (priv->mtr_en) 4518 actions_n = flow_check_meter_action(actions, &mtr); 4519 if (mtr) { 4520 /* The five prefix actions: meter, decap, encap, tag, end. */ 4521 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4522 sizeof(struct mlx5_rte_flow_action_set_tag); 4523 /* tag, vlan, port id, end. */ 4524 #define METER_SUFFIX_ITEM 4 4525 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4526 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4527 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 4528 0, SOCKET_ID_ANY); 4529 if (!sfx_actions) 4530 return rte_flow_error_set(error, ENOMEM, 4531 RTE_FLOW_ERROR_TYPE_ACTION, 4532 NULL, "no memory to split " 4533 "meter flow"); 4534 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4535 act_size); 4536 pre_actions = sfx_actions + actions_n; 4537 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4538 actions, sfx_actions, 4539 pre_actions); 4540 if (!mtr_tag_id) { 4541 ret = -rte_errno; 4542 goto exit; 4543 } 4544 /* Add the prefix subflow. */ 4545 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr, 4546 items, pre_actions, external, 4547 flow_idx, error); 4548 if (ret) { 4549 ret = -rte_errno; 4550 goto exit; 4551 } 4552 dev_flow->handle->split_flow_id = mtr_tag_id; 4553 /* Setting the sfx group atrr. */ 4554 sfx_attr.group = sfx_attr.transfer ? 4555 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4556 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4557 } 4558 /* Add the prefix subflow. */ 4559 ret = flow_create_split_metadata(dev, flow, dev_flow ? 4560 flow_get_prefix_layer_flags(dev_flow) : 4561 0, &sfx_attr, 4562 sfx_items ? sfx_items : items, 4563 sfx_actions ? sfx_actions : actions, 4564 external, flow_idx, error); 4565 exit: 4566 if (sfx_actions) 4567 mlx5_free(sfx_actions); 4568 return ret; 4569 } 4570 4571 /** 4572 * Split the flow to subflow set. The splitters might be linked 4573 * in the chain, like this: 4574 * flow_create_split_outer() calls: 4575 * flow_create_split_meter() calls: 4576 * flow_create_split_metadata(meter_subflow_0) calls: 4577 * flow_create_split_inner(metadata_subflow_0) 4578 * flow_create_split_inner(metadata_subflow_1) 4579 * flow_create_split_inner(metadata_subflow_2) 4580 * flow_create_split_metadata(meter_subflow_1) calls: 4581 * flow_create_split_inner(metadata_subflow_0) 4582 * flow_create_split_inner(metadata_subflow_1) 4583 * flow_create_split_inner(metadata_subflow_2) 4584 * 4585 * This provide flexible way to add new levels of flow splitting. 4586 * The all of successfully created subflows are included to the 4587 * parent flow dev_flow list. 4588 * 4589 * @param dev 4590 * Pointer to Ethernet device. 4591 * @param[in] flow 4592 * Parent flow structure pointer. 4593 * @param[in] attr 4594 * Flow rule attributes. 4595 * @param[in] items 4596 * Pattern specification (list terminated by the END pattern item). 4597 * @param[in] actions 4598 * Associated actions (list terminated by the END action). 4599 * @param[in] external 4600 * This flow rule is created by request external to PMD. 4601 * @param[in] flow_idx 4602 * This memory pool index to the flow. 4603 * @param[out] error 4604 * Perform verbose error reporting if not NULL. 4605 * @return 4606 * 0 on success, negative value otherwise 4607 */ 4608 static int 4609 flow_create_split_outer(struct rte_eth_dev *dev, 4610 struct rte_flow *flow, 4611 const struct rte_flow_attr *attr, 4612 const struct rte_flow_item items[], 4613 const struct rte_flow_action actions[], 4614 bool external, uint32_t flow_idx, 4615 struct rte_flow_error *error) 4616 { 4617 int ret; 4618 4619 ret = flow_create_split_meter(dev, flow, attr, items, 4620 actions, external, flow_idx, error); 4621 MLX5_ASSERT(ret <= 0); 4622 return ret; 4623 } 4624 4625 /** 4626 * Create a flow and add it to @p list. 4627 * 4628 * @param dev 4629 * Pointer to Ethernet device. 4630 * @param list 4631 * Pointer to a TAILQ flow list. If this parameter NULL, 4632 * no list insertion occurred, flow is just created, 4633 * this is caller's responsibility to track the 4634 * created flow. 4635 * @param[in] attr 4636 * Flow rule attributes. 4637 * @param[in] items 4638 * Pattern specification (list terminated by the END pattern item). 4639 * @param[in] actions 4640 * Associated actions (list terminated by the END action). 4641 * @param[in] external 4642 * This flow rule is created by request external to PMD. 4643 * @param[out] error 4644 * Perform verbose error reporting if not NULL. 4645 * 4646 * @return 4647 * A flow index on success, 0 otherwise and rte_errno is set. 4648 */ 4649 static uint32_t 4650 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 4651 const struct rte_flow_attr *attr, 4652 const struct rte_flow_item items[], 4653 const struct rte_flow_action actions[], 4654 bool external, struct rte_flow_error *error) 4655 { 4656 struct mlx5_priv *priv = dev->data->dev_private; 4657 struct rte_flow *flow = NULL; 4658 struct mlx5_flow *dev_flow; 4659 const struct rte_flow_action_rss *rss; 4660 union { 4661 struct mlx5_flow_expand_rss buf; 4662 uint8_t buffer[2048]; 4663 } expand_buffer; 4664 union { 4665 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4666 uint8_t buffer[2048]; 4667 } actions_rx; 4668 union { 4669 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4670 uint8_t buffer[2048]; 4671 } actions_hairpin_tx; 4672 union { 4673 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4674 uint8_t buffer[2048]; 4675 } items_tx; 4676 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 4677 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 4678 priv->rss_desc)[!!priv->flow_idx]; 4679 const struct rte_flow_action *p_actions_rx = actions; 4680 uint32_t i; 4681 uint32_t idx = 0; 4682 int hairpin_flow; 4683 uint32_t hairpin_id = 0; 4684 struct rte_flow_attr attr_tx = { .priority = 0 }; 4685 int ret; 4686 4687 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4688 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 4689 external, hairpin_flow, error); 4690 if (ret < 0) 4691 return 0; 4692 if (hairpin_flow > 0) { 4693 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4694 rte_errno = EINVAL; 4695 return 0; 4696 } 4697 flow_hairpin_split(dev, actions, actions_rx.actions, 4698 actions_hairpin_tx.actions, items_tx.items, 4699 &hairpin_id); 4700 p_actions_rx = actions_rx.actions; 4701 } 4702 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 4703 if (!flow) { 4704 rte_errno = ENOMEM; 4705 goto error_before_flow; 4706 } 4707 flow->drv_type = flow_get_drv_type(dev, attr); 4708 if (hairpin_id != 0) 4709 flow->hairpin_flow_id = hairpin_id; 4710 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4711 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4712 memset(rss_desc, 0, sizeof(*rss_desc)); 4713 rss = flow_get_rss_action(p_actions_rx); 4714 if (rss) { 4715 /* 4716 * The following information is required by 4717 * mlx5_flow_hashfields_adjust() in advance. 4718 */ 4719 rss_desc->level = rss->level; 4720 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4721 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 4722 } 4723 flow->dev_handles = 0; 4724 if (rss && rss->types) { 4725 unsigned int graph_root; 4726 4727 graph_root = find_graph_root(items, rss->level); 4728 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4729 items, rss->types, 4730 mlx5_support_expansion, graph_root); 4731 MLX5_ASSERT(ret > 0 && 4732 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4733 } else { 4734 buf->entries = 1; 4735 buf->entry[0].pattern = (void *)(uintptr_t)items; 4736 } 4737 /* 4738 * Record the start index when there is a nested call. All sub-flows 4739 * need to be translated before another calling. 4740 * No need to use ping-pong buffer to save memory here. 4741 */ 4742 if (priv->flow_idx) { 4743 MLX5_ASSERT(!priv->flow_nested_idx); 4744 priv->flow_nested_idx = priv->flow_idx; 4745 } 4746 for (i = 0; i < buf->entries; ++i) { 4747 /* 4748 * The splitter may create multiple dev_flows, 4749 * depending on configuration. In the simplest 4750 * case it just creates unmodified original flow. 4751 */ 4752 ret = flow_create_split_outer(dev, flow, attr, 4753 buf->entry[i].pattern, 4754 p_actions_rx, external, idx, 4755 error); 4756 if (ret < 0) 4757 goto error; 4758 } 4759 /* Create the tx flow. */ 4760 if (hairpin_flow) { 4761 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4762 attr_tx.ingress = 0; 4763 attr_tx.egress = 1; 4764 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 4765 actions_hairpin_tx.actions, 4766 idx, error); 4767 if (!dev_flow) 4768 goto error; 4769 dev_flow->flow = flow; 4770 dev_flow->external = 0; 4771 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4772 dev_flow->handle, next); 4773 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4774 items_tx.items, 4775 actions_hairpin_tx.actions, error); 4776 if (ret < 0) 4777 goto error; 4778 } 4779 /* 4780 * Update the metadata register copy table. If extensive 4781 * metadata feature is enabled and registers are supported 4782 * we might create the extra rte_flow for each unique 4783 * MARK/FLAG action ID. 4784 * 4785 * The table is updated for ingress Flows only, because 4786 * the egress Flows belong to the different device and 4787 * copy table should be updated in peer NIC Rx domain. 4788 */ 4789 if (attr->ingress && 4790 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4791 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4792 if (ret) 4793 goto error; 4794 } 4795 /* 4796 * If the flow is external (from application) OR device is started, then 4797 * the flow will be applied immediately. 4798 */ 4799 if (external || dev->data->dev_started) { 4800 ret = flow_drv_apply(dev, flow, error); 4801 if (ret < 0) 4802 goto error; 4803 } 4804 if (list) 4805 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 4806 flow, next); 4807 flow_rxq_flags_set(dev, flow); 4808 /* Nested flow creation index recovery. */ 4809 priv->flow_idx = priv->flow_nested_idx; 4810 if (priv->flow_nested_idx) 4811 priv->flow_nested_idx = 0; 4812 return idx; 4813 error: 4814 MLX5_ASSERT(flow); 4815 ret = rte_errno; /* Save rte_errno before cleanup. */ 4816 flow_mreg_del_copy_action(dev, flow); 4817 flow_drv_destroy(dev, flow); 4818 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 4819 rte_errno = ret; /* Restore rte_errno. */ 4820 error_before_flow: 4821 ret = rte_errno; 4822 if (hairpin_id) 4823 mlx5_flow_id_release(priv->sh->flow_id_pool, 4824 hairpin_id); 4825 rte_errno = ret; 4826 priv->flow_idx = priv->flow_nested_idx; 4827 if (priv->flow_nested_idx) 4828 priv->flow_nested_idx = 0; 4829 return 0; 4830 } 4831 4832 /** 4833 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4834 * incoming packets to table 1. 4835 * 4836 * Other flow rules, requested for group n, will be created in 4837 * e-switch table n+1. 4838 * Jump action to e-switch group n will be created to group n+1. 4839 * 4840 * Used when working in switchdev mode, to utilise advantages of table 1 4841 * and above. 4842 * 4843 * @param dev 4844 * Pointer to Ethernet device. 4845 * 4846 * @return 4847 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4848 */ 4849 struct rte_flow * 4850 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4851 { 4852 const struct rte_flow_attr attr = { 4853 .group = 0, 4854 .priority = 0, 4855 .ingress = 1, 4856 .egress = 0, 4857 .transfer = 1, 4858 }; 4859 const struct rte_flow_item pattern = { 4860 .type = RTE_FLOW_ITEM_TYPE_END, 4861 }; 4862 struct rte_flow_action_jump jump = { 4863 .group = 1, 4864 }; 4865 const struct rte_flow_action actions[] = { 4866 { 4867 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4868 .conf = &jump, 4869 }, 4870 { 4871 .type = RTE_FLOW_ACTION_TYPE_END, 4872 }, 4873 }; 4874 struct mlx5_priv *priv = dev->data->dev_private; 4875 struct rte_flow_error error; 4876 4877 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 4878 &attr, &pattern, 4879 actions, false, &error); 4880 } 4881 4882 /** 4883 * Validate a flow supported by the NIC. 4884 * 4885 * @see rte_flow_validate() 4886 * @see rte_flow_ops 4887 */ 4888 int 4889 mlx5_flow_validate(struct rte_eth_dev *dev, 4890 const struct rte_flow_attr *attr, 4891 const struct rte_flow_item items[], 4892 const struct rte_flow_action actions[], 4893 struct rte_flow_error *error) 4894 { 4895 int hairpin_flow; 4896 4897 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4898 return flow_drv_validate(dev, attr, items, actions, 4899 true, hairpin_flow, error); 4900 } 4901 4902 /** 4903 * Create a flow. 4904 * 4905 * @see rte_flow_create() 4906 * @see rte_flow_ops 4907 */ 4908 struct rte_flow * 4909 mlx5_flow_create(struct rte_eth_dev *dev, 4910 const struct rte_flow_attr *attr, 4911 const struct rte_flow_item items[], 4912 const struct rte_flow_action actions[], 4913 struct rte_flow_error *error) 4914 { 4915 struct mlx5_priv *priv = dev->data->dev_private; 4916 4917 /* 4918 * If the device is not started yet, it is not allowed to created a 4919 * flow from application. PMD default flows and traffic control flows 4920 * are not affected. 4921 */ 4922 if (unlikely(!dev->data->dev_started)) { 4923 DRV_LOG(DEBUG, "port %u is not started when " 4924 "inserting a flow", dev->data->port_id); 4925 rte_flow_error_set(error, ENODEV, 4926 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4927 NULL, 4928 "port not started"); 4929 return NULL; 4930 } 4931 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 4932 attr, items, actions, true, error); 4933 } 4934 4935 /** 4936 * Destroy a flow in a list. 4937 * 4938 * @param dev 4939 * Pointer to Ethernet device. 4940 * @param list 4941 * Pointer to the Indexed flow list. If this parameter NULL, 4942 * there is no flow removal from the list. Be noted that as 4943 * flow is add to the indexed list, memory of the indexed 4944 * list points to maybe changed as flow destroyed. 4945 * @param[in] flow_idx 4946 * Index of flow to destroy. 4947 */ 4948 static void 4949 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 4950 uint32_t flow_idx) 4951 { 4952 struct mlx5_priv *priv = dev->data->dev_private; 4953 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 4954 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 4955 [MLX5_IPOOL_RTE_FLOW], flow_idx); 4956 4957 if (!flow) 4958 return; 4959 /* 4960 * Update RX queue flags only if port is started, otherwise it is 4961 * already clean. 4962 */ 4963 if (dev->data->dev_started) 4964 flow_rxq_flags_trim(dev, flow); 4965 if (flow->hairpin_flow_id) 4966 mlx5_flow_id_release(priv->sh->flow_id_pool, 4967 flow->hairpin_flow_id); 4968 flow_drv_destroy(dev, flow); 4969 if (list) 4970 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 4971 flow_idx, flow, next); 4972 flow_mreg_del_copy_action(dev, flow); 4973 if (flow->fdir) { 4974 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 4975 if (priv_fdir_flow->rix_flow == flow_idx) 4976 break; 4977 } 4978 if (priv_fdir_flow) { 4979 LIST_REMOVE(priv_fdir_flow, next); 4980 mlx5_free(priv_fdir_flow->fdir); 4981 mlx5_free(priv_fdir_flow); 4982 } 4983 } 4984 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 4985 } 4986 4987 /** 4988 * Destroy all flows. 4989 * 4990 * @param dev 4991 * Pointer to Ethernet device. 4992 * @param list 4993 * Pointer to the Indexed flow list. 4994 * @param active 4995 * If flushing is called avtively. 4996 */ 4997 void 4998 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 4999 { 5000 uint32_t num_flushed = 0; 5001 5002 while (*list) { 5003 flow_list_destroy(dev, list, *list); 5004 num_flushed++; 5005 } 5006 if (active) { 5007 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 5008 dev->data->port_id, num_flushed); 5009 } 5010 } 5011 5012 /** 5013 * Remove all flows. 5014 * 5015 * @param dev 5016 * Pointer to Ethernet device. 5017 * @param list 5018 * Pointer to the Indexed flow list. 5019 */ 5020 void 5021 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list) 5022 { 5023 struct mlx5_priv *priv = dev->data->dev_private; 5024 struct rte_flow *flow = NULL; 5025 uint32_t idx; 5026 5027 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 5028 flow, next) { 5029 flow_drv_remove(dev, flow); 5030 flow_mreg_stop_copy_action(dev, flow); 5031 } 5032 flow_mreg_del_default_copy_action(dev); 5033 flow_rxq_flags_clear(dev); 5034 } 5035 5036 /** 5037 * Add all flows. 5038 * 5039 * @param dev 5040 * Pointer to Ethernet device. 5041 * @param list 5042 * Pointer to the Indexed flow list. 5043 * 5044 * @return 5045 * 0 on success, a negative errno value otherwise and rte_errno is set. 5046 */ 5047 int 5048 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list) 5049 { 5050 struct mlx5_priv *priv = dev->data->dev_private; 5051 struct rte_flow *flow = NULL; 5052 struct rte_flow_error error; 5053 uint32_t idx; 5054 int ret = 0; 5055 5056 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 5057 ret = flow_mreg_add_default_copy_action(dev, &error); 5058 if (ret < 0) 5059 return -rte_errno; 5060 /* Apply Flows created by application. */ 5061 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 5062 flow, next) { 5063 ret = flow_mreg_start_copy_action(dev, flow); 5064 if (ret < 0) 5065 goto error; 5066 ret = flow_drv_apply(dev, flow, &error); 5067 if (ret < 0) 5068 goto error; 5069 flow_rxq_flags_set(dev, flow); 5070 } 5071 return 0; 5072 error: 5073 ret = rte_errno; /* Save rte_errno before cleanup. */ 5074 mlx5_flow_stop(dev, list); 5075 rte_errno = ret; /* Restore rte_errno. */ 5076 return -rte_errno; 5077 } 5078 5079 /** 5080 * Stop all default actions for flows. 5081 * 5082 * @param dev 5083 * Pointer to Ethernet device. 5084 */ 5085 void 5086 mlx5_flow_stop_default(struct rte_eth_dev *dev) 5087 { 5088 flow_mreg_del_default_copy_action(dev); 5089 flow_rxq_flags_clear(dev); 5090 } 5091 5092 /** 5093 * Start all default actions for flows. 5094 * 5095 * @param dev 5096 * Pointer to Ethernet device. 5097 * @return 5098 * 0 on success, a negative errno value otherwise and rte_errno is set. 5099 */ 5100 int 5101 mlx5_flow_start_default(struct rte_eth_dev *dev) 5102 { 5103 struct rte_flow_error error; 5104 5105 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 5106 return flow_mreg_add_default_copy_action(dev, &error); 5107 } 5108 5109 /** 5110 * Allocate intermediate resources for flow creation. 5111 * 5112 * @param dev 5113 * Pointer to Ethernet device. 5114 */ 5115 void 5116 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) 5117 { 5118 struct mlx5_priv *priv = dev->data->dev_private; 5119 5120 if (!priv->inter_flows) { 5121 priv->inter_flows = mlx5_malloc(MLX5_MEM_ZERO, 5122 MLX5_NUM_MAX_DEV_FLOWS * 5123 sizeof(struct mlx5_flow) + 5124 (sizeof(struct mlx5_flow_rss_desc) + 5125 sizeof(uint16_t) * UINT16_MAX) * 2, 0, 5126 SOCKET_ID_ANY); 5127 if (!priv->inter_flows) { 5128 DRV_LOG(ERR, "can't allocate intermediate memory."); 5129 return; 5130 } 5131 } 5132 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows) 5133 [MLX5_NUM_MAX_DEV_FLOWS]; 5134 /* Reset the index. */ 5135 priv->flow_idx = 0; 5136 priv->flow_nested_idx = 0; 5137 } 5138 5139 /** 5140 * Free intermediate resources for flows. 5141 * 5142 * @param dev 5143 * Pointer to Ethernet device. 5144 */ 5145 void 5146 mlx5_flow_free_intermediate(struct rte_eth_dev *dev) 5147 { 5148 struct mlx5_priv *priv = dev->data->dev_private; 5149 5150 mlx5_free(priv->inter_flows); 5151 priv->inter_flows = NULL; 5152 } 5153 5154 /** 5155 * Verify the flow list is empty 5156 * 5157 * @param dev 5158 * Pointer to Ethernet device. 5159 * 5160 * @return the number of flows not released. 5161 */ 5162 int 5163 mlx5_flow_verify(struct rte_eth_dev *dev) 5164 { 5165 struct mlx5_priv *priv = dev->data->dev_private; 5166 struct rte_flow *flow; 5167 uint32_t idx; 5168 int ret = 0; 5169 5170 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 5171 flow, next) { 5172 DRV_LOG(DEBUG, "port %u flow %p still referenced", 5173 dev->data->port_id, (void *)flow); 5174 ++ret; 5175 } 5176 return ret; 5177 } 5178 5179 /** 5180 * Enable default hairpin egress flow. 5181 * 5182 * @param dev 5183 * Pointer to Ethernet device. 5184 * @param queue 5185 * The queue index. 5186 * 5187 * @return 5188 * 0 on success, a negative errno value otherwise and rte_errno is set. 5189 */ 5190 int 5191 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 5192 uint32_t queue) 5193 { 5194 struct mlx5_priv *priv = dev->data->dev_private; 5195 const struct rte_flow_attr attr = { 5196 .egress = 1, 5197 .priority = 0, 5198 }; 5199 struct mlx5_rte_flow_item_tx_queue queue_spec = { 5200 .queue = queue, 5201 }; 5202 struct mlx5_rte_flow_item_tx_queue queue_mask = { 5203 .queue = UINT32_MAX, 5204 }; 5205 struct rte_flow_item items[] = { 5206 { 5207 .type = (enum rte_flow_item_type) 5208 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 5209 .spec = &queue_spec, 5210 .last = NULL, 5211 .mask = &queue_mask, 5212 }, 5213 { 5214 .type = RTE_FLOW_ITEM_TYPE_END, 5215 }, 5216 }; 5217 struct rte_flow_action_jump jump = { 5218 .group = MLX5_HAIRPIN_TX_TABLE, 5219 }; 5220 struct rte_flow_action actions[2]; 5221 uint32_t flow_idx; 5222 struct rte_flow_error error; 5223 5224 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 5225 actions[0].conf = &jump; 5226 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 5227 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5228 &attr, items, actions, false, &error); 5229 if (!flow_idx) { 5230 DRV_LOG(DEBUG, 5231 "Failed to create ctrl flow: rte_errno(%d)," 5232 " type(%d), message(%s)", 5233 rte_errno, error.type, 5234 error.message ? error.message : " (no stated reason)"); 5235 return -rte_errno; 5236 } 5237 return 0; 5238 } 5239 5240 /** 5241 * Enable a control flow configured from the control plane. 5242 * 5243 * @param dev 5244 * Pointer to Ethernet device. 5245 * @param eth_spec 5246 * An Ethernet flow spec to apply. 5247 * @param eth_mask 5248 * An Ethernet flow mask to apply. 5249 * @param vlan_spec 5250 * A VLAN flow spec to apply. 5251 * @param vlan_mask 5252 * A VLAN flow mask to apply. 5253 * 5254 * @return 5255 * 0 on success, a negative errno value otherwise and rte_errno is set. 5256 */ 5257 int 5258 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 5259 struct rte_flow_item_eth *eth_spec, 5260 struct rte_flow_item_eth *eth_mask, 5261 struct rte_flow_item_vlan *vlan_spec, 5262 struct rte_flow_item_vlan *vlan_mask) 5263 { 5264 struct mlx5_priv *priv = dev->data->dev_private; 5265 const struct rte_flow_attr attr = { 5266 .ingress = 1, 5267 .priority = MLX5_FLOW_PRIO_RSVD, 5268 }; 5269 struct rte_flow_item items[] = { 5270 { 5271 .type = RTE_FLOW_ITEM_TYPE_ETH, 5272 .spec = eth_spec, 5273 .last = NULL, 5274 .mask = eth_mask, 5275 }, 5276 { 5277 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 5278 RTE_FLOW_ITEM_TYPE_END, 5279 .spec = vlan_spec, 5280 .last = NULL, 5281 .mask = vlan_mask, 5282 }, 5283 { 5284 .type = RTE_FLOW_ITEM_TYPE_END, 5285 }, 5286 }; 5287 uint16_t queue[priv->reta_idx_n]; 5288 struct rte_flow_action_rss action_rss = { 5289 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 5290 .level = 0, 5291 .types = priv->rss_conf.rss_hf, 5292 .key_len = priv->rss_conf.rss_key_len, 5293 .queue_num = priv->reta_idx_n, 5294 .key = priv->rss_conf.rss_key, 5295 .queue = queue, 5296 }; 5297 struct rte_flow_action actions[] = { 5298 { 5299 .type = RTE_FLOW_ACTION_TYPE_RSS, 5300 .conf = &action_rss, 5301 }, 5302 { 5303 .type = RTE_FLOW_ACTION_TYPE_END, 5304 }, 5305 }; 5306 uint32_t flow_idx; 5307 struct rte_flow_error error; 5308 unsigned int i; 5309 5310 if (!priv->reta_idx_n || !priv->rxqs_n) { 5311 return 0; 5312 } 5313 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 5314 action_rss.types = 0; 5315 for (i = 0; i != priv->reta_idx_n; ++i) 5316 queue[i] = (*priv->reta_idx)[i]; 5317 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5318 &attr, items, actions, false, &error); 5319 if (!flow_idx) 5320 return -rte_errno; 5321 return 0; 5322 } 5323 5324 /** 5325 * Enable a flow control configured from the control plane. 5326 * 5327 * @param dev 5328 * Pointer to Ethernet device. 5329 * @param eth_spec 5330 * An Ethernet flow spec to apply. 5331 * @param eth_mask 5332 * An Ethernet flow mask to apply. 5333 * 5334 * @return 5335 * 0 on success, a negative errno value otherwise and rte_errno is set. 5336 */ 5337 int 5338 mlx5_ctrl_flow(struct rte_eth_dev *dev, 5339 struct rte_flow_item_eth *eth_spec, 5340 struct rte_flow_item_eth *eth_mask) 5341 { 5342 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 5343 } 5344 5345 /** 5346 * Create default miss flow rule matching lacp traffic 5347 * 5348 * @param dev 5349 * Pointer to Ethernet device. 5350 * @param eth_spec 5351 * An Ethernet flow spec to apply. 5352 * 5353 * @return 5354 * 0 on success, a negative errno value otherwise and rte_errno is set. 5355 */ 5356 int 5357 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 5358 { 5359 struct mlx5_priv *priv = dev->data->dev_private; 5360 /* 5361 * The LACP matching is done by only using ether type since using 5362 * a multicast dst mac causes kernel to give low priority to this flow. 5363 */ 5364 static const struct rte_flow_item_eth lacp_spec = { 5365 .type = RTE_BE16(0x8809), 5366 }; 5367 static const struct rte_flow_item_eth lacp_mask = { 5368 .type = 0xffff, 5369 }; 5370 const struct rte_flow_attr attr = { 5371 .ingress = 1, 5372 }; 5373 struct rte_flow_item items[] = { 5374 { 5375 .type = RTE_FLOW_ITEM_TYPE_ETH, 5376 .spec = &lacp_spec, 5377 .mask = &lacp_mask, 5378 }, 5379 { 5380 .type = RTE_FLOW_ITEM_TYPE_END, 5381 }, 5382 }; 5383 struct rte_flow_action actions[] = { 5384 { 5385 .type = (enum rte_flow_action_type) 5386 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 5387 }, 5388 { 5389 .type = RTE_FLOW_ACTION_TYPE_END, 5390 }, 5391 }; 5392 struct rte_flow_error error; 5393 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5394 &attr, items, actions, false, &error); 5395 5396 if (!flow_idx) 5397 return -rte_errno; 5398 return 0; 5399 } 5400 5401 /** 5402 * Destroy a flow. 5403 * 5404 * @see rte_flow_destroy() 5405 * @see rte_flow_ops 5406 */ 5407 int 5408 mlx5_flow_destroy(struct rte_eth_dev *dev, 5409 struct rte_flow *flow, 5410 struct rte_flow_error *error __rte_unused) 5411 { 5412 struct mlx5_priv *priv = dev->data->dev_private; 5413 5414 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 5415 return 0; 5416 } 5417 5418 /** 5419 * Destroy all flows. 5420 * 5421 * @see rte_flow_flush() 5422 * @see rte_flow_ops 5423 */ 5424 int 5425 mlx5_flow_flush(struct rte_eth_dev *dev, 5426 struct rte_flow_error *error __rte_unused) 5427 { 5428 struct mlx5_priv *priv = dev->data->dev_private; 5429 5430 mlx5_flow_list_flush(dev, &priv->flows, false); 5431 return 0; 5432 } 5433 5434 /** 5435 * Isolated mode. 5436 * 5437 * @see rte_flow_isolate() 5438 * @see rte_flow_ops 5439 */ 5440 int 5441 mlx5_flow_isolate(struct rte_eth_dev *dev, 5442 int enable, 5443 struct rte_flow_error *error) 5444 { 5445 struct mlx5_priv *priv = dev->data->dev_private; 5446 5447 if (dev->data->dev_started) { 5448 rte_flow_error_set(error, EBUSY, 5449 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5450 NULL, 5451 "port must be stopped first"); 5452 return -rte_errno; 5453 } 5454 priv->isolated = !!enable; 5455 if (enable) 5456 dev->dev_ops = &mlx5_os_dev_ops_isolate; 5457 else 5458 dev->dev_ops = &mlx5_os_dev_ops; 5459 5460 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 5461 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 5462 5463 return 0; 5464 } 5465 5466 /** 5467 * Query a flow. 5468 * 5469 * @see rte_flow_query() 5470 * @see rte_flow_ops 5471 */ 5472 static int 5473 flow_drv_query(struct rte_eth_dev *dev, 5474 uint32_t flow_idx, 5475 const struct rte_flow_action *actions, 5476 void *data, 5477 struct rte_flow_error *error) 5478 { 5479 struct mlx5_priv *priv = dev->data->dev_private; 5480 const struct mlx5_flow_driver_ops *fops; 5481 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5482 [MLX5_IPOOL_RTE_FLOW], 5483 flow_idx); 5484 enum mlx5_flow_drv_type ftype; 5485 5486 if (!flow) { 5487 return rte_flow_error_set(error, ENOENT, 5488 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5489 NULL, 5490 "invalid flow handle"); 5491 } 5492 ftype = flow->drv_type; 5493 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 5494 fops = flow_get_drv_ops(ftype); 5495 5496 return fops->query(dev, flow, actions, data, error); 5497 } 5498 5499 /** 5500 * Query a flow. 5501 * 5502 * @see rte_flow_query() 5503 * @see rte_flow_ops 5504 */ 5505 int 5506 mlx5_flow_query(struct rte_eth_dev *dev, 5507 struct rte_flow *flow, 5508 const struct rte_flow_action *actions, 5509 void *data, 5510 struct rte_flow_error *error) 5511 { 5512 int ret; 5513 5514 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 5515 error); 5516 if (ret < 0) 5517 return ret; 5518 return 0; 5519 } 5520 5521 /** 5522 * Convert a flow director filter to a generic flow. 5523 * 5524 * @param dev 5525 * Pointer to Ethernet device. 5526 * @param fdir_filter 5527 * Flow director filter to add. 5528 * @param attributes 5529 * Generic flow parameters structure. 5530 * 5531 * @return 5532 * 0 on success, a negative errno value otherwise and rte_errno is set. 5533 */ 5534 static int 5535 flow_fdir_filter_convert(struct rte_eth_dev *dev, 5536 const struct rte_eth_fdir_filter *fdir_filter, 5537 struct mlx5_fdir *attributes) 5538 { 5539 struct mlx5_priv *priv = dev->data->dev_private; 5540 const struct rte_eth_fdir_input *input = &fdir_filter->input; 5541 const struct rte_eth_fdir_masks *mask = 5542 &dev->data->dev_conf.fdir_conf.mask; 5543 5544 /* Validate queue number. */ 5545 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 5546 DRV_LOG(ERR, "port %u invalid queue number %d", 5547 dev->data->port_id, fdir_filter->action.rx_queue); 5548 rte_errno = EINVAL; 5549 return -rte_errno; 5550 } 5551 attributes->attr.ingress = 1; 5552 attributes->items[0] = (struct rte_flow_item) { 5553 .type = RTE_FLOW_ITEM_TYPE_ETH, 5554 .spec = &attributes->l2, 5555 .mask = &attributes->l2_mask, 5556 }; 5557 switch (fdir_filter->action.behavior) { 5558 case RTE_ETH_FDIR_ACCEPT: 5559 attributes->actions[0] = (struct rte_flow_action){ 5560 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 5561 .conf = &attributes->queue, 5562 }; 5563 break; 5564 case RTE_ETH_FDIR_REJECT: 5565 attributes->actions[0] = (struct rte_flow_action){ 5566 .type = RTE_FLOW_ACTION_TYPE_DROP, 5567 }; 5568 break; 5569 default: 5570 DRV_LOG(ERR, "port %u invalid behavior %d", 5571 dev->data->port_id, 5572 fdir_filter->action.behavior); 5573 rte_errno = ENOTSUP; 5574 return -rte_errno; 5575 } 5576 attributes->queue.index = fdir_filter->action.rx_queue; 5577 /* Handle L3. */ 5578 switch (fdir_filter->input.flow_type) { 5579 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5580 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5581 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5582 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 5583 .src_addr = input->flow.ip4_flow.src_ip, 5584 .dst_addr = input->flow.ip4_flow.dst_ip, 5585 .time_to_live = input->flow.ip4_flow.ttl, 5586 .type_of_service = input->flow.ip4_flow.tos, 5587 }; 5588 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 5589 .src_addr = mask->ipv4_mask.src_ip, 5590 .dst_addr = mask->ipv4_mask.dst_ip, 5591 .time_to_live = mask->ipv4_mask.ttl, 5592 .type_of_service = mask->ipv4_mask.tos, 5593 .next_proto_id = mask->ipv4_mask.proto, 5594 }; 5595 attributes->items[1] = (struct rte_flow_item){ 5596 .type = RTE_FLOW_ITEM_TYPE_IPV4, 5597 .spec = &attributes->l3, 5598 .mask = &attributes->l3_mask, 5599 }; 5600 break; 5601 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5602 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5603 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5604 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 5605 .hop_limits = input->flow.ipv6_flow.hop_limits, 5606 .proto = input->flow.ipv6_flow.proto, 5607 }; 5608 5609 memcpy(attributes->l3.ipv6.hdr.src_addr, 5610 input->flow.ipv6_flow.src_ip, 5611 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5612 memcpy(attributes->l3.ipv6.hdr.dst_addr, 5613 input->flow.ipv6_flow.dst_ip, 5614 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5615 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 5616 mask->ipv6_mask.src_ip, 5617 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5618 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 5619 mask->ipv6_mask.dst_ip, 5620 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5621 attributes->items[1] = (struct rte_flow_item){ 5622 .type = RTE_FLOW_ITEM_TYPE_IPV6, 5623 .spec = &attributes->l3, 5624 .mask = &attributes->l3_mask, 5625 }; 5626 break; 5627 default: 5628 DRV_LOG(ERR, "port %u invalid flow type%d", 5629 dev->data->port_id, fdir_filter->input.flow_type); 5630 rte_errno = ENOTSUP; 5631 return -rte_errno; 5632 } 5633 /* Handle L4. */ 5634 switch (fdir_filter->input.flow_type) { 5635 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5636 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5637 .src_port = input->flow.udp4_flow.src_port, 5638 .dst_port = input->flow.udp4_flow.dst_port, 5639 }; 5640 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5641 .src_port = mask->src_port_mask, 5642 .dst_port = mask->dst_port_mask, 5643 }; 5644 attributes->items[2] = (struct rte_flow_item){ 5645 .type = RTE_FLOW_ITEM_TYPE_UDP, 5646 .spec = &attributes->l4, 5647 .mask = &attributes->l4_mask, 5648 }; 5649 break; 5650 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5651 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5652 .src_port = input->flow.tcp4_flow.src_port, 5653 .dst_port = input->flow.tcp4_flow.dst_port, 5654 }; 5655 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5656 .src_port = mask->src_port_mask, 5657 .dst_port = mask->dst_port_mask, 5658 }; 5659 attributes->items[2] = (struct rte_flow_item){ 5660 .type = RTE_FLOW_ITEM_TYPE_TCP, 5661 .spec = &attributes->l4, 5662 .mask = &attributes->l4_mask, 5663 }; 5664 break; 5665 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5666 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5667 .src_port = input->flow.udp6_flow.src_port, 5668 .dst_port = input->flow.udp6_flow.dst_port, 5669 }; 5670 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5671 .src_port = mask->src_port_mask, 5672 .dst_port = mask->dst_port_mask, 5673 }; 5674 attributes->items[2] = (struct rte_flow_item){ 5675 .type = RTE_FLOW_ITEM_TYPE_UDP, 5676 .spec = &attributes->l4, 5677 .mask = &attributes->l4_mask, 5678 }; 5679 break; 5680 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5681 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5682 .src_port = input->flow.tcp6_flow.src_port, 5683 .dst_port = input->flow.tcp6_flow.dst_port, 5684 }; 5685 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5686 .src_port = mask->src_port_mask, 5687 .dst_port = mask->dst_port_mask, 5688 }; 5689 attributes->items[2] = (struct rte_flow_item){ 5690 .type = RTE_FLOW_ITEM_TYPE_TCP, 5691 .spec = &attributes->l4, 5692 .mask = &attributes->l4_mask, 5693 }; 5694 break; 5695 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5696 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5697 break; 5698 default: 5699 DRV_LOG(ERR, "port %u invalid flow type%d", 5700 dev->data->port_id, fdir_filter->input.flow_type); 5701 rte_errno = ENOTSUP; 5702 return -rte_errno; 5703 } 5704 return 0; 5705 } 5706 5707 #define FLOW_FDIR_CMP(f1, f2, fld) \ 5708 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 5709 5710 /** 5711 * Compare two FDIR flows. If items and actions are identical, the two flows are 5712 * regarded as same. 5713 * 5714 * @param dev 5715 * Pointer to Ethernet device. 5716 * @param f1 5717 * FDIR flow to compare. 5718 * @param f2 5719 * FDIR flow to compare. 5720 * 5721 * @return 5722 * Zero on match, 1 otherwise. 5723 */ 5724 static int 5725 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 5726 { 5727 if (FLOW_FDIR_CMP(f1, f2, attr) || 5728 FLOW_FDIR_CMP(f1, f2, l2) || 5729 FLOW_FDIR_CMP(f1, f2, l2_mask) || 5730 FLOW_FDIR_CMP(f1, f2, l3) || 5731 FLOW_FDIR_CMP(f1, f2, l3_mask) || 5732 FLOW_FDIR_CMP(f1, f2, l4) || 5733 FLOW_FDIR_CMP(f1, f2, l4_mask) || 5734 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 5735 return 1; 5736 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 5737 FLOW_FDIR_CMP(f1, f2, queue)) 5738 return 1; 5739 return 0; 5740 } 5741 5742 /** 5743 * Search device flow list to find out a matched FDIR flow. 5744 * 5745 * @param dev 5746 * Pointer to Ethernet device. 5747 * @param fdir_flow 5748 * FDIR flow to lookup. 5749 * 5750 * @return 5751 * Index of flow if found, 0 otherwise. 5752 */ 5753 static uint32_t 5754 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5755 { 5756 struct mlx5_priv *priv = dev->data->dev_private; 5757 uint32_t flow_idx = 0; 5758 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5759 5760 MLX5_ASSERT(fdir_flow); 5761 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5762 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { 5763 DRV_LOG(DEBUG, "port %u found FDIR flow %u", 5764 dev->data->port_id, flow_idx); 5765 flow_idx = priv_fdir_flow->rix_flow; 5766 break; 5767 } 5768 } 5769 return flow_idx; 5770 } 5771 5772 /** 5773 * Add new flow director filter and store it in list. 5774 * 5775 * @param dev 5776 * Pointer to Ethernet device. 5777 * @param fdir_filter 5778 * Flow director filter to add. 5779 * 5780 * @return 5781 * 0 on success, a negative errno value otherwise and rte_errno is set. 5782 */ 5783 static int 5784 flow_fdir_filter_add(struct rte_eth_dev *dev, 5785 const struct rte_eth_fdir_filter *fdir_filter) 5786 { 5787 struct mlx5_priv *priv = dev->data->dev_private; 5788 struct mlx5_fdir *fdir_flow; 5789 struct rte_flow *flow; 5790 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5791 uint32_t flow_idx; 5792 int ret; 5793 5794 fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*fdir_flow), 0, 5795 SOCKET_ID_ANY); 5796 if (!fdir_flow) { 5797 rte_errno = ENOMEM; 5798 return -rte_errno; 5799 } 5800 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5801 if (ret) 5802 goto error; 5803 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); 5804 if (flow_idx) { 5805 rte_errno = EEXIST; 5806 goto error; 5807 } 5808 priv_fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, 5809 sizeof(struct mlx5_fdir_flow), 5810 0, SOCKET_ID_ANY); 5811 if (!priv_fdir_flow) { 5812 rte_errno = ENOMEM; 5813 goto error; 5814 } 5815 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5816 fdir_flow->items, fdir_flow->actions, true, 5817 NULL); 5818 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5819 if (!flow) 5820 goto error; 5821 flow->fdir = 1; 5822 priv_fdir_flow->fdir = fdir_flow; 5823 priv_fdir_flow->rix_flow = flow_idx; 5824 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); 5825 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5826 dev->data->port_id, (void *)flow); 5827 return 0; 5828 error: 5829 mlx5_free(priv_fdir_flow); 5830 mlx5_free(fdir_flow); 5831 return -rte_errno; 5832 } 5833 5834 /** 5835 * Delete specific filter. 5836 * 5837 * @param dev 5838 * Pointer to Ethernet device. 5839 * @param fdir_filter 5840 * Filter to be deleted. 5841 * 5842 * @return 5843 * 0 on success, a negative errno value otherwise and rte_errno is set. 5844 */ 5845 static int 5846 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5847 const struct rte_eth_fdir_filter *fdir_filter) 5848 { 5849 struct mlx5_priv *priv = dev->data->dev_private; 5850 uint32_t flow_idx; 5851 struct mlx5_fdir fdir_flow = { 5852 .attr.group = 0, 5853 }; 5854 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5855 int ret; 5856 5857 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5858 if (ret) 5859 return -rte_errno; 5860 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5861 /* Find the fdir in priv list */ 5862 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) 5863 break; 5864 } 5865 if (!priv_fdir_flow) 5866 return 0; 5867 LIST_REMOVE(priv_fdir_flow, next); 5868 flow_idx = priv_fdir_flow->rix_flow; 5869 flow_list_destroy(dev, &priv->flows, flow_idx); 5870 mlx5_free(priv_fdir_flow->fdir); 5871 mlx5_free(priv_fdir_flow); 5872 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", 5873 dev->data->port_id, flow_idx); 5874 return 0; 5875 } 5876 5877 /** 5878 * Update queue for specific filter. 5879 * 5880 * @param dev 5881 * Pointer to Ethernet device. 5882 * @param fdir_filter 5883 * Filter to be updated. 5884 * 5885 * @return 5886 * 0 on success, a negative errno value otherwise and rte_errno is set. 5887 */ 5888 static int 5889 flow_fdir_filter_update(struct rte_eth_dev *dev, 5890 const struct rte_eth_fdir_filter *fdir_filter) 5891 { 5892 int ret; 5893 5894 ret = flow_fdir_filter_delete(dev, fdir_filter); 5895 if (ret) 5896 return ret; 5897 return flow_fdir_filter_add(dev, fdir_filter); 5898 } 5899 5900 /** 5901 * Flush all filters. 5902 * 5903 * @param dev 5904 * Pointer to Ethernet device. 5905 */ 5906 static void 5907 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5908 { 5909 struct mlx5_priv *priv = dev->data->dev_private; 5910 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5911 5912 while (!LIST_EMPTY(&priv->fdir_flows)) { 5913 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); 5914 LIST_REMOVE(priv_fdir_flow, next); 5915 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); 5916 mlx5_free(priv_fdir_flow->fdir); 5917 mlx5_free(priv_fdir_flow); 5918 } 5919 } 5920 5921 /** 5922 * Get flow director information. 5923 * 5924 * @param dev 5925 * Pointer to Ethernet device. 5926 * @param[out] fdir_info 5927 * Resulting flow director information. 5928 */ 5929 static void 5930 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5931 { 5932 struct rte_eth_fdir_masks *mask = 5933 &dev->data->dev_conf.fdir_conf.mask; 5934 5935 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5936 fdir_info->guarant_spc = 0; 5937 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5938 fdir_info->max_flexpayload = 0; 5939 fdir_info->flow_types_mask[0] = 0; 5940 fdir_info->flex_payload_unit = 0; 5941 fdir_info->max_flex_payload_segment_num = 0; 5942 fdir_info->flex_payload_limit = 0; 5943 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5944 } 5945 5946 /** 5947 * Deal with flow director operations. 5948 * 5949 * @param dev 5950 * Pointer to Ethernet device. 5951 * @param filter_op 5952 * Operation to perform. 5953 * @param arg 5954 * Pointer to operation-specific structure. 5955 * 5956 * @return 5957 * 0 on success, a negative errno value otherwise and rte_errno is set. 5958 */ 5959 static int 5960 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5961 void *arg) 5962 { 5963 enum rte_fdir_mode fdir_mode = 5964 dev->data->dev_conf.fdir_conf.mode; 5965 5966 if (filter_op == RTE_ETH_FILTER_NOP) 5967 return 0; 5968 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5969 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5970 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5971 dev->data->port_id, fdir_mode); 5972 rte_errno = EINVAL; 5973 return -rte_errno; 5974 } 5975 switch (filter_op) { 5976 case RTE_ETH_FILTER_ADD: 5977 return flow_fdir_filter_add(dev, arg); 5978 case RTE_ETH_FILTER_UPDATE: 5979 return flow_fdir_filter_update(dev, arg); 5980 case RTE_ETH_FILTER_DELETE: 5981 return flow_fdir_filter_delete(dev, arg); 5982 case RTE_ETH_FILTER_FLUSH: 5983 flow_fdir_filter_flush(dev); 5984 break; 5985 case RTE_ETH_FILTER_INFO: 5986 flow_fdir_info_get(dev, arg); 5987 break; 5988 default: 5989 DRV_LOG(DEBUG, "port %u unknown operation %u", 5990 dev->data->port_id, filter_op); 5991 rte_errno = EINVAL; 5992 return -rte_errno; 5993 } 5994 return 0; 5995 } 5996 5997 /** 5998 * Manage filter operations. 5999 * 6000 * @param dev 6001 * Pointer to Ethernet device structure. 6002 * @param filter_type 6003 * Filter type. 6004 * @param filter_op 6005 * Operation to perform. 6006 * @param arg 6007 * Pointer to operation-specific structure. 6008 * 6009 * @return 6010 * 0 on success, a negative errno value otherwise and rte_errno is set. 6011 */ 6012 int 6013 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 6014 enum rte_filter_type filter_type, 6015 enum rte_filter_op filter_op, 6016 void *arg) 6017 { 6018 switch (filter_type) { 6019 case RTE_ETH_FILTER_GENERIC: 6020 if (filter_op != RTE_ETH_FILTER_GET) { 6021 rte_errno = EINVAL; 6022 return -rte_errno; 6023 } 6024 *(const void **)arg = &mlx5_flow_ops; 6025 return 0; 6026 case RTE_ETH_FILTER_FDIR: 6027 return flow_fdir_ctrl_func(dev, filter_op, arg); 6028 default: 6029 DRV_LOG(ERR, "port %u filter type (%d) not supported", 6030 dev->data->port_id, filter_type); 6031 rte_errno = ENOTSUP; 6032 return -rte_errno; 6033 } 6034 return 0; 6035 } 6036 6037 /** 6038 * Create the needed meter and suffix tables. 6039 * 6040 * @param[in] dev 6041 * Pointer to Ethernet device. 6042 * @param[in] fm 6043 * Pointer to the flow meter. 6044 * 6045 * @return 6046 * Pointer to table set on success, NULL otherwise. 6047 */ 6048 struct mlx5_meter_domains_infos * 6049 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 6050 const struct mlx5_flow_meter *fm) 6051 { 6052 const struct mlx5_flow_driver_ops *fops; 6053 6054 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6055 return fops->create_mtr_tbls(dev, fm); 6056 } 6057 6058 /** 6059 * Destroy the meter table set. 6060 * 6061 * @param[in] dev 6062 * Pointer to Ethernet device. 6063 * @param[in] tbl 6064 * Pointer to the meter table set. 6065 * 6066 * @return 6067 * 0 on success. 6068 */ 6069 int 6070 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 6071 struct mlx5_meter_domains_infos *tbls) 6072 { 6073 const struct mlx5_flow_driver_ops *fops; 6074 6075 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6076 return fops->destroy_mtr_tbls(dev, tbls); 6077 } 6078 6079 /** 6080 * Create policer rules. 6081 * 6082 * @param[in] dev 6083 * Pointer to Ethernet device. 6084 * @param[in] fm 6085 * Pointer to flow meter structure. 6086 * @param[in] attr 6087 * Pointer to flow attributes. 6088 * 6089 * @return 6090 * 0 on success, -1 otherwise. 6091 */ 6092 int 6093 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 6094 struct mlx5_flow_meter *fm, 6095 const struct rte_flow_attr *attr) 6096 { 6097 const struct mlx5_flow_driver_ops *fops; 6098 6099 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6100 return fops->create_policer_rules(dev, fm, attr); 6101 } 6102 6103 /** 6104 * Destroy policer rules. 6105 * 6106 * @param[in] fm 6107 * Pointer to flow meter structure. 6108 * @param[in] attr 6109 * Pointer to flow attributes. 6110 * 6111 * @return 6112 * 0 on success, -1 otherwise. 6113 */ 6114 int 6115 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 6116 struct mlx5_flow_meter *fm, 6117 const struct rte_flow_attr *attr) 6118 { 6119 const struct mlx5_flow_driver_ops *fops; 6120 6121 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6122 return fops->destroy_policer_rules(dev, fm, attr); 6123 } 6124 6125 /** 6126 * Allocate a counter. 6127 * 6128 * @param[in] dev 6129 * Pointer to Ethernet device structure. 6130 * 6131 * @return 6132 * Index to allocated counter on success, 0 otherwise. 6133 */ 6134 uint32_t 6135 mlx5_counter_alloc(struct rte_eth_dev *dev) 6136 { 6137 const struct mlx5_flow_driver_ops *fops; 6138 struct rte_flow_attr attr = { .transfer = 0 }; 6139 6140 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6141 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6142 return fops->counter_alloc(dev); 6143 } 6144 DRV_LOG(ERR, 6145 "port %u counter allocate is not supported.", 6146 dev->data->port_id); 6147 return 0; 6148 } 6149 6150 /** 6151 * Free a counter. 6152 * 6153 * @param[in] dev 6154 * Pointer to Ethernet device structure. 6155 * @param[in] cnt 6156 * Index to counter to be free. 6157 */ 6158 void 6159 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 6160 { 6161 const struct mlx5_flow_driver_ops *fops; 6162 struct rte_flow_attr attr = { .transfer = 0 }; 6163 6164 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6165 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6166 fops->counter_free(dev, cnt); 6167 return; 6168 } 6169 DRV_LOG(ERR, 6170 "port %u counter free is not supported.", 6171 dev->data->port_id); 6172 } 6173 6174 /** 6175 * Query counter statistics. 6176 * 6177 * @param[in] dev 6178 * Pointer to Ethernet device structure. 6179 * @param[in] cnt 6180 * Index to counter to query. 6181 * @param[in] clear 6182 * Set to clear counter statistics. 6183 * @param[out] pkts 6184 * The counter hits packets number to save. 6185 * @param[out] bytes 6186 * The counter hits bytes number to save. 6187 * 6188 * @return 6189 * 0 on success, a negative errno value otherwise. 6190 */ 6191 int 6192 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 6193 bool clear, uint64_t *pkts, uint64_t *bytes) 6194 { 6195 const struct mlx5_flow_driver_ops *fops; 6196 struct rte_flow_attr attr = { .transfer = 0 }; 6197 6198 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6199 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6200 return fops->counter_query(dev, cnt, clear, pkts, bytes); 6201 } 6202 DRV_LOG(ERR, 6203 "port %u counter query is not supported.", 6204 dev->data->port_id); 6205 return -ENOTSUP; 6206 } 6207 6208 #define MLX5_POOL_QUERY_FREQ_US 1000000 6209 6210 /** 6211 * Get number of all validate pools. 6212 * 6213 * @param[in] sh 6214 * Pointer to mlx5_dev_ctx_shared object. 6215 * 6216 * @return 6217 * The number of all validate pools. 6218 */ 6219 static uint32_t 6220 mlx5_get_all_valid_pool_count(struct mlx5_dev_ctx_shared *sh) 6221 { 6222 int i; 6223 uint32_t pools_n = 0; 6224 6225 for (i = 0; i < MLX5_CCONT_TYPE_MAX; ++i) 6226 pools_n += rte_atomic16_read(&sh->cmng.ccont[i].n_valid); 6227 return pools_n; 6228 } 6229 6230 /** 6231 * Set the periodic procedure for triggering asynchronous batch queries for all 6232 * the counter pools. 6233 * 6234 * @param[in] sh 6235 * Pointer to mlx5_dev_ctx_shared object. 6236 */ 6237 void 6238 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 6239 { 6240 uint32_t pools_n, us; 6241 6242 pools_n = mlx5_get_all_valid_pool_count(sh); 6243 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 6244 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 6245 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 6246 sh->cmng.query_thread_on = 0; 6247 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 6248 } else { 6249 sh->cmng.query_thread_on = 1; 6250 } 6251 } 6252 6253 /** 6254 * The periodic procedure for triggering asynchronous batch queries for all the 6255 * counter pools. This function is probably called by the host thread. 6256 * 6257 * @param[in] arg 6258 * The parameter for the alarm process. 6259 */ 6260 void 6261 mlx5_flow_query_alarm(void *arg) 6262 { 6263 struct mlx5_dev_ctx_shared *sh = arg; 6264 struct mlx5_devx_obj *dcs; 6265 uint16_t offset; 6266 int ret; 6267 uint8_t batch = sh->cmng.batch; 6268 uint8_t age = sh->cmng.age; 6269 uint16_t pool_index = sh->cmng.pool_index; 6270 struct mlx5_pools_container *cont; 6271 struct mlx5_flow_counter_pool *pool; 6272 int cont_loop = MLX5_CCONT_TYPE_MAX; 6273 6274 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 6275 goto set_alarm; 6276 next_container: 6277 cont = MLX5_CNT_CONTAINER(sh, batch, age); 6278 rte_spinlock_lock(&cont->resize_sl); 6279 if (!cont->pools) { 6280 rte_spinlock_unlock(&cont->resize_sl); 6281 /* Check if all the containers are empty. */ 6282 if (unlikely(--cont_loop == 0)) 6283 goto set_alarm; 6284 batch ^= 0x1; 6285 pool_index = 0; 6286 if (batch == 0 && pool_index == 0) { 6287 age ^= 0x1; 6288 sh->cmng.batch = batch; 6289 sh->cmng.age = age; 6290 } 6291 goto next_container; 6292 } 6293 pool = cont->pools[pool_index]; 6294 rte_spinlock_unlock(&cont->resize_sl); 6295 if (pool->raw_hw) 6296 /* There is a pool query in progress. */ 6297 goto set_alarm; 6298 pool->raw_hw = 6299 LIST_FIRST(&sh->cmng.free_stat_raws); 6300 if (!pool->raw_hw) 6301 /* No free counter statistics raw memory. */ 6302 goto set_alarm; 6303 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 6304 (&pool->a64_dcs); 6305 if (dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) { 6306 /* Pool without valid counter. */ 6307 pool->raw_hw = NULL; 6308 goto next_pool; 6309 } 6310 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 6311 /* 6312 * Identify the counters released between query trigger and query 6313 * handle more effiecntly. The counter released in this gap period 6314 * should wait for a new round of query as the new arrived packets 6315 * will not be taken into account. 6316 */ 6317 pool->query_gen++; 6318 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 6319 offset, NULL, NULL, 6320 pool->raw_hw->mem_mng->dm->id, 6321 (void *)(uintptr_t) 6322 (pool->raw_hw->data + offset), 6323 sh->devx_comp, 6324 (uint64_t)(uintptr_t)pool); 6325 if (ret) { 6326 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 6327 " %d", pool->min_dcs->id); 6328 pool->raw_hw = NULL; 6329 goto set_alarm; 6330 } 6331 pool->raw_hw->min_dcs_id = dcs->id; 6332 LIST_REMOVE(pool->raw_hw, next); 6333 sh->cmng.pending_queries++; 6334 next_pool: 6335 pool_index++; 6336 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 6337 batch ^= 0x1; 6338 pool_index = 0; 6339 if (batch == 0 && pool_index == 0) 6340 age ^= 0x1; 6341 } 6342 set_alarm: 6343 sh->cmng.batch = batch; 6344 sh->cmng.pool_index = pool_index; 6345 sh->cmng.age = age; 6346 mlx5_set_query_alarm(sh); 6347 } 6348 6349 /** 6350 * Check and callback event for new aged flow in the counter pool 6351 * 6352 * @param[in] sh 6353 * Pointer to mlx5_dev_ctx_shared object. 6354 * @param[in] pool 6355 * Pointer to Current counter pool. 6356 */ 6357 static void 6358 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 6359 struct mlx5_flow_counter_pool *pool) 6360 { 6361 struct mlx5_priv *priv; 6362 struct mlx5_flow_counter *cnt; 6363 struct mlx5_age_info *age_info; 6364 struct mlx5_age_param *age_param; 6365 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 6366 struct mlx5_counter_stats_raw *prev = pool->raw; 6367 uint16_t curr = rte_rdtsc() / (rte_get_tsc_hz() / 10); 6368 uint32_t i; 6369 6370 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 6371 cnt = MLX5_POOL_GET_CNT(pool, i); 6372 age_param = MLX5_CNT_TO_AGE(cnt); 6373 if (rte_atomic16_read(&age_param->state) != AGE_CANDIDATE) 6374 continue; 6375 if (cur->data[i].hits != prev->data[i].hits) { 6376 age_param->expire = curr + age_param->timeout; 6377 continue; 6378 } 6379 if ((uint16_t)(curr - age_param->expire) >= (UINT16_MAX / 2)) 6380 continue; 6381 /** 6382 * Hold the lock first, or if between the 6383 * state AGE_TMOUT and tailq operation the 6384 * release happened, the release procedure 6385 * may delete a non-existent tailq node. 6386 */ 6387 priv = rte_eth_devices[age_param->port_id].data->dev_private; 6388 age_info = GET_PORT_AGE_INFO(priv); 6389 rte_spinlock_lock(&age_info->aged_sl); 6390 /* If the cpmset fails, release happens. */ 6391 if (rte_atomic16_cmpset((volatile uint16_t *) 6392 &age_param->state, 6393 AGE_CANDIDATE, 6394 AGE_TMOUT) == 6395 AGE_CANDIDATE) { 6396 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 6397 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 6398 } 6399 rte_spinlock_unlock(&age_info->aged_sl); 6400 } 6401 for (i = 0; i < sh->max_port; i++) { 6402 age_info = &sh->port[i].age_info; 6403 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 6404 continue; 6405 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) 6406 rte_eth_dev_callback_process 6407 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 6408 RTE_ETH_EVENT_FLOW_AGED, NULL); 6409 age_info->flags = 0; 6410 } 6411 } 6412 6413 /** 6414 * Handler for the HW respond about ready values from an asynchronous batch 6415 * query. This function is probably called by the host thread. 6416 * 6417 * @param[in] sh 6418 * The pointer to the shared device context. 6419 * @param[in] async_id 6420 * The Devx async ID. 6421 * @param[in] status 6422 * The status of the completion. 6423 */ 6424 void 6425 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 6426 uint64_t async_id, int status) 6427 { 6428 struct mlx5_flow_counter_pool *pool = 6429 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 6430 struct mlx5_counter_stats_raw *raw_to_free; 6431 uint8_t age = !!IS_AGE_POOL(pool); 6432 uint8_t query_gen = pool->query_gen ^ 1; 6433 struct mlx5_pools_container *cont = 6434 MLX5_CNT_CONTAINER(sh, !IS_EXT_POOL(pool), age); 6435 6436 if (unlikely(status)) { 6437 raw_to_free = pool->raw_hw; 6438 } else { 6439 raw_to_free = pool->raw; 6440 if (IS_AGE_POOL(pool)) 6441 mlx5_flow_aging_check(sh, pool); 6442 rte_spinlock_lock(&pool->sl); 6443 pool->raw = pool->raw_hw; 6444 rte_spinlock_unlock(&pool->sl); 6445 /* Be sure the new raw counters data is updated in memory. */ 6446 rte_io_wmb(); 6447 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 6448 rte_spinlock_lock(&cont->csl); 6449 TAILQ_CONCAT(&cont->counters, 6450 &pool->counters[query_gen], next); 6451 rte_spinlock_unlock(&cont->csl); 6452 } 6453 } 6454 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 6455 pool->raw_hw = NULL; 6456 sh->cmng.pending_queries--; 6457 } 6458 6459 /** 6460 * Translate the rte_flow group index to HW table value. 6461 * 6462 * @param[in] attributes 6463 * Pointer to flow attributes 6464 * @param[in] external 6465 * Value is part of flow rule created by request external to PMD. 6466 * @param[in] group 6467 * rte_flow group index value. 6468 * @param[out] fdb_def_rule 6469 * Whether fdb jump to table 1 is configured. 6470 * @param[out] table 6471 * HW table value. 6472 * @param[out] error 6473 * Pointer to error structure. 6474 * 6475 * @return 6476 * 0 on success, a negative errno value otherwise and rte_errno is set. 6477 */ 6478 int 6479 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 6480 uint32_t group, bool fdb_def_rule, uint32_t *table, 6481 struct rte_flow_error *error) 6482 { 6483 if (attributes->transfer && external && fdb_def_rule) { 6484 if (group == UINT32_MAX) 6485 return rte_flow_error_set 6486 (error, EINVAL, 6487 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6488 NULL, 6489 "group index not supported"); 6490 *table = group + 1; 6491 } else { 6492 *table = group; 6493 } 6494 return 0; 6495 } 6496 6497 /** 6498 * Discover availability of metadata reg_c's. 6499 * 6500 * Iteratively use test flows to check availability. 6501 * 6502 * @param[in] dev 6503 * Pointer to the Ethernet device structure. 6504 * 6505 * @return 6506 * 0 on success, a negative errno value otherwise and rte_errno is set. 6507 */ 6508 int 6509 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 6510 { 6511 struct mlx5_priv *priv = dev->data->dev_private; 6512 struct mlx5_dev_config *config = &priv->config; 6513 enum modify_reg idx; 6514 int n = 0; 6515 6516 /* reg_c[0] and reg_c[1] are reserved. */ 6517 config->flow_mreg_c[n++] = REG_C_0; 6518 config->flow_mreg_c[n++] = REG_C_1; 6519 /* Discover availability of other reg_c's. */ 6520 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 6521 struct rte_flow_attr attr = { 6522 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6523 .priority = MLX5_FLOW_PRIO_RSVD, 6524 .ingress = 1, 6525 }; 6526 struct rte_flow_item items[] = { 6527 [0] = { 6528 .type = RTE_FLOW_ITEM_TYPE_END, 6529 }, 6530 }; 6531 struct rte_flow_action actions[] = { 6532 [0] = { 6533 .type = (enum rte_flow_action_type) 6534 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6535 .conf = &(struct mlx5_flow_action_copy_mreg){ 6536 .src = REG_C_1, 6537 .dst = idx, 6538 }, 6539 }, 6540 [1] = { 6541 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6542 .conf = &(struct rte_flow_action_jump){ 6543 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6544 }, 6545 }, 6546 [2] = { 6547 .type = RTE_FLOW_ACTION_TYPE_END, 6548 }, 6549 }; 6550 uint32_t flow_idx; 6551 struct rte_flow *flow; 6552 struct rte_flow_error error; 6553 6554 if (!config->dv_flow_en) 6555 break; 6556 /* Create internal flow, validation skips copy action. */ 6557 flow_idx = flow_list_create(dev, NULL, &attr, items, 6558 actions, false, &error); 6559 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 6560 flow_idx); 6561 if (!flow) 6562 continue; 6563 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 6564 config->flow_mreg_c[n++] = idx; 6565 flow_list_destroy(dev, NULL, flow_idx); 6566 } 6567 for (; n < MLX5_MREG_C_NUM; ++n) 6568 config->flow_mreg_c[n] = REG_NON; 6569 return 0; 6570 } 6571 6572 /** 6573 * Dump flow raw hw data to file 6574 * 6575 * @param[in] dev 6576 * The pointer to Ethernet device. 6577 * @param[in] file 6578 * A pointer to a file for output. 6579 * @param[out] error 6580 * Perform verbose error reporting if not NULL. PMDs initialize this 6581 * structure in case of error only. 6582 * @return 6583 * 0 on success, a nagative value otherwise. 6584 */ 6585 int 6586 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 6587 FILE *file, 6588 struct rte_flow_error *error __rte_unused) 6589 { 6590 struct mlx5_priv *priv = dev->data->dev_private; 6591 struct mlx5_dev_ctx_shared *sh = priv->sh; 6592 6593 if (!priv->config.dv_flow_en) { 6594 if (fputs("device dv flow disabled\n", file) <= 0) 6595 return -errno; 6596 return -ENOTSUP; 6597 } 6598 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 6599 sh->tx_domain, file); 6600 } 6601 6602 /** 6603 * Get aged-out flows. 6604 * 6605 * @param[in] dev 6606 * Pointer to the Ethernet device structure. 6607 * @param[in] context 6608 * The address of an array of pointers to the aged-out flows contexts. 6609 * @param[in] nb_countexts 6610 * The length of context array pointers. 6611 * @param[out] error 6612 * Perform verbose error reporting if not NULL. Initialized in case of 6613 * error only. 6614 * 6615 * @return 6616 * how many contexts get in success, otherwise negative errno value. 6617 * if nb_contexts is 0, return the amount of all aged contexts. 6618 * if nb_contexts is not 0 , return the amount of aged flows reported 6619 * in the context array. 6620 */ 6621 int 6622 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 6623 uint32_t nb_contexts, struct rte_flow_error *error) 6624 { 6625 const struct mlx5_flow_driver_ops *fops; 6626 struct rte_flow_attr attr = { .transfer = 0 }; 6627 6628 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6629 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6630 return fops->get_aged_flows(dev, contexts, nb_contexts, 6631 error); 6632 } 6633 DRV_LOG(ERR, 6634 "port %u get aged flows is not supported.", 6635 dev->data->port_id); 6636 return -ENOTSUP; 6637 } 6638