1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 #include <rte_common.h> 14 #include <rte_ether.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_eal_paging.h> 17 #include <rte_flow.h> 18 #include <rte_cycles.h> 19 #include <rte_flow_driver.h> 20 #include <rte_malloc.h> 21 #include <rte_ip.h> 22 23 #include <mlx5_glue.h> 24 #include <mlx5_devx_cmds.h> 25 #include <mlx5_prm.h> 26 #include <mlx5_malloc.h> 27 28 #include "mlx5_defs.h" 29 #include "mlx5.h" 30 #include "mlx5_flow.h" 31 #include "mlx5_flow_os.h" 32 #include "mlx5_rxtx.h" 33 #include "mlx5_common_os.h" 34 35 /** Device flow drivers. */ 36 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 37 38 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 39 40 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 41 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 42 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 43 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 44 #endif 45 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 46 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 47 }; 48 49 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 50 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 51 (const int []){ \ 52 __VA_ARGS__, 0, \ 53 } 54 55 /** Node object of input graph for mlx5_flow_expand_rss(). */ 56 struct mlx5_flow_expand_node { 57 const int *const next; 58 /**< 59 * List of next node indexes. Index 0 is interpreted as a terminator. 60 */ 61 const enum rte_flow_item_type type; 62 /**< Pattern item type of current node. */ 63 uint64_t rss_types; 64 /**< 65 * RSS types bit-field associated with this node 66 * (see ETH_RSS_* definitions). 67 */ 68 }; 69 70 /** Object returned by mlx5_flow_expand_rss(). */ 71 struct mlx5_flow_expand_rss { 72 uint32_t entries; 73 /**< Number of entries @p patterns and @p priorities. */ 74 struct { 75 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 76 uint32_t priority; /**< Priority offset for each expansion. */ 77 } entry[]; 78 }; 79 80 static enum rte_flow_item_type 81 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 82 { 83 enum rte_flow_item_type ret = RTE_FLOW_ITEM_TYPE_VOID; 84 uint16_t ether_type = 0; 85 uint16_t ether_type_m; 86 uint8_t ip_next_proto = 0; 87 uint8_t ip_next_proto_m; 88 89 if (item == NULL || item->spec == NULL) 90 return ret; 91 switch (item->type) { 92 case RTE_FLOW_ITEM_TYPE_ETH: 93 if (item->mask) 94 ether_type_m = ((const struct rte_flow_item_eth *) 95 (item->mask))->type; 96 else 97 ether_type_m = rte_flow_item_eth_mask.type; 98 if (ether_type_m != RTE_BE16(0xFFFF)) 99 break; 100 ether_type = ((const struct rte_flow_item_eth *) 101 (item->spec))->type; 102 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 103 ret = RTE_FLOW_ITEM_TYPE_IPV4; 104 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 105 ret = RTE_FLOW_ITEM_TYPE_IPV6; 106 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 107 ret = RTE_FLOW_ITEM_TYPE_VLAN; 108 else 109 ret = RTE_FLOW_ITEM_TYPE_END; 110 break; 111 case RTE_FLOW_ITEM_TYPE_VLAN: 112 if (item->mask) 113 ether_type_m = ((const struct rte_flow_item_vlan *) 114 (item->mask))->inner_type; 115 else 116 ether_type_m = rte_flow_item_vlan_mask.inner_type; 117 if (ether_type_m != RTE_BE16(0xFFFF)) 118 break; 119 ether_type = ((const struct rte_flow_item_vlan *) 120 (item->spec))->inner_type; 121 if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV4) 122 ret = RTE_FLOW_ITEM_TYPE_IPV4; 123 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_IPV6) 124 ret = RTE_FLOW_ITEM_TYPE_IPV6; 125 else if (rte_be_to_cpu_16(ether_type) == RTE_ETHER_TYPE_VLAN) 126 ret = RTE_FLOW_ITEM_TYPE_VLAN; 127 else 128 ret = RTE_FLOW_ITEM_TYPE_END; 129 break; 130 case RTE_FLOW_ITEM_TYPE_IPV4: 131 if (item->mask) 132 ip_next_proto_m = ((const struct rte_flow_item_ipv4 *) 133 (item->mask))->hdr.next_proto_id; 134 else 135 ip_next_proto_m = 136 rte_flow_item_ipv4_mask.hdr.next_proto_id; 137 if (ip_next_proto_m != 0xFF) 138 break; 139 ip_next_proto = ((const struct rte_flow_item_ipv4 *) 140 (item->spec))->hdr.next_proto_id; 141 if (ip_next_proto == IPPROTO_UDP) 142 ret = RTE_FLOW_ITEM_TYPE_UDP; 143 else if (ip_next_proto == IPPROTO_TCP) 144 ret = RTE_FLOW_ITEM_TYPE_TCP; 145 else if (ip_next_proto == IPPROTO_IP) 146 ret = RTE_FLOW_ITEM_TYPE_IPV4; 147 else if (ip_next_proto == IPPROTO_IPV6) 148 ret = RTE_FLOW_ITEM_TYPE_IPV6; 149 else 150 ret = RTE_FLOW_ITEM_TYPE_END; 151 break; 152 case RTE_FLOW_ITEM_TYPE_IPV6: 153 if (item->mask) 154 ip_next_proto_m = ((const struct rte_flow_item_ipv6 *) 155 (item->mask))->hdr.proto; 156 else 157 ip_next_proto_m = 158 rte_flow_item_ipv6_mask.hdr.proto; 159 if (ip_next_proto_m != 0xFF) 160 break; 161 ip_next_proto = ((const struct rte_flow_item_ipv6 *) 162 (item->spec))->hdr.proto; 163 if (ip_next_proto == IPPROTO_UDP) 164 ret = RTE_FLOW_ITEM_TYPE_UDP; 165 else if (ip_next_proto == IPPROTO_TCP) 166 ret = RTE_FLOW_ITEM_TYPE_TCP; 167 else if (ip_next_proto == IPPROTO_IP) 168 ret = RTE_FLOW_ITEM_TYPE_IPV4; 169 else if (ip_next_proto == IPPROTO_IPV6) 170 ret = RTE_FLOW_ITEM_TYPE_IPV6; 171 else 172 ret = RTE_FLOW_ITEM_TYPE_END; 173 break; 174 default: 175 ret = RTE_FLOW_ITEM_TYPE_VOID; 176 break; 177 } 178 return ret; 179 } 180 181 /** 182 * Expand RSS flows into several possible flows according to the RSS hash 183 * fields requested and the driver capabilities. 184 * 185 * @param[out] buf 186 * Buffer to store the result expansion. 187 * @param[in] size 188 * Buffer size in bytes. If 0, @p buf can be NULL. 189 * @param[in] pattern 190 * User flow pattern. 191 * @param[in] types 192 * RSS types to expand (see ETH_RSS_* definitions). 193 * @param[in] graph 194 * Input graph to expand @p pattern according to @p types. 195 * @param[in] graph_root_index 196 * Index of root node in @p graph, typically 0. 197 * 198 * @return 199 * A positive value representing the size of @p buf in bytes regardless of 200 * @p size on success, a negative errno value otherwise and rte_errno is 201 * set, the following errors are defined: 202 * 203 * -E2BIG: graph-depth @p graph is too deep. 204 */ 205 static int 206 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 207 const struct rte_flow_item *pattern, uint64_t types, 208 const struct mlx5_flow_expand_node graph[], 209 int graph_root_index) 210 { 211 const int elt_n = 8; 212 const struct rte_flow_item *item; 213 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 214 const int *next_node; 215 const int *stack[elt_n]; 216 int stack_pos = 0; 217 struct rte_flow_item flow_items[elt_n]; 218 unsigned int i; 219 size_t lsize; 220 size_t user_pattern_size = 0; 221 void *addr = NULL; 222 const struct mlx5_flow_expand_node *next = NULL; 223 struct rte_flow_item missed_item; 224 int missed = 0; 225 int elt = 0; 226 const struct rte_flow_item *last_item = NULL; 227 228 memset(&missed_item, 0, sizeof(missed_item)); 229 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 230 elt_n * sizeof(buf->entry[0]); 231 if (lsize <= size) { 232 buf->entry[0].priority = 0; 233 buf->entry[0].pattern = (void *)&buf->entry[elt_n]; 234 buf->entries = 0; 235 addr = buf->entry[0].pattern; 236 } 237 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 238 if (item->type != RTE_FLOW_ITEM_TYPE_VOID) 239 last_item = item; 240 for (i = 0; node->next && node->next[i]; ++i) { 241 next = &graph[node->next[i]]; 242 if (next->type == item->type) 243 break; 244 } 245 if (next) 246 node = next; 247 user_pattern_size += sizeof(*item); 248 } 249 user_pattern_size += sizeof(*item); /* Handle END item. */ 250 lsize += user_pattern_size; 251 /* Copy the user pattern in the first entry of the buffer. */ 252 if (lsize <= size) { 253 rte_memcpy(addr, pattern, user_pattern_size); 254 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 255 buf->entries = 1; 256 } 257 /* Start expanding. */ 258 memset(flow_items, 0, sizeof(flow_items)); 259 user_pattern_size -= sizeof(*item); 260 /* 261 * Check if the last valid item has spec set, need complete pattern, 262 * and the pattern can be used for expansion. 263 */ 264 missed_item.type = mlx5_flow_expand_rss_item_complete(last_item); 265 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 266 /* Item type END indicates expansion is not required. */ 267 return lsize; 268 } 269 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 270 next = NULL; 271 missed = 1; 272 for (i = 0; node->next && node->next[i]; ++i) { 273 next = &graph[node->next[i]]; 274 if (next->type == missed_item.type) { 275 flow_items[0].type = missed_item.type; 276 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 277 break; 278 } 279 next = NULL; 280 } 281 } 282 if (next && missed) { 283 elt = 2; /* missed item + item end. */ 284 node = next; 285 lsize += elt * sizeof(*item) + user_pattern_size; 286 if ((node->rss_types & types) && lsize <= size) { 287 buf->entry[buf->entries].priority = 1; 288 buf->entry[buf->entries].pattern = addr; 289 buf->entries++; 290 rte_memcpy(addr, buf->entry[0].pattern, 291 user_pattern_size); 292 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 293 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 294 addr = (void *)(((uintptr_t)addr) + 295 elt * sizeof(*item)); 296 } 297 } 298 memset(flow_items, 0, sizeof(flow_items)); 299 next_node = node->next; 300 stack[stack_pos] = next_node; 301 node = next_node ? &graph[*next_node] : NULL; 302 while (node) { 303 flow_items[stack_pos].type = node->type; 304 if (node->rss_types & types) { 305 /* 306 * compute the number of items to copy from the 307 * expansion and copy it. 308 * When the stack_pos is 0, there are 1 element in it, 309 * plus the addition END item. 310 */ 311 elt = stack_pos + 2; 312 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 313 lsize += elt * sizeof(*item) + user_pattern_size; 314 if (lsize <= size) { 315 size_t n = elt * sizeof(*item); 316 317 buf->entry[buf->entries].priority = 318 stack_pos + 1 + missed; 319 buf->entry[buf->entries].pattern = addr; 320 buf->entries++; 321 rte_memcpy(addr, buf->entry[0].pattern, 322 user_pattern_size); 323 addr = (void *)(((uintptr_t)addr) + 324 user_pattern_size); 325 rte_memcpy(addr, &missed_item, 326 missed * sizeof(*item)); 327 addr = (void *)(((uintptr_t)addr) + 328 missed * sizeof(*item)); 329 rte_memcpy(addr, flow_items, n); 330 addr = (void *)(((uintptr_t)addr) + n); 331 } 332 } 333 /* Go deeper. */ 334 if (node->next) { 335 next_node = node->next; 336 if (stack_pos++ == elt_n) { 337 rte_errno = E2BIG; 338 return -rte_errno; 339 } 340 stack[stack_pos] = next_node; 341 } else if (*(next_node + 1)) { 342 /* Follow up with the next possibility. */ 343 ++next_node; 344 } else { 345 /* Move to the next path. */ 346 if (stack_pos) 347 next_node = stack[--stack_pos]; 348 next_node++; 349 stack[stack_pos] = next_node; 350 } 351 node = *next_node ? &graph[*next_node] : NULL; 352 }; 353 /* no expanded flows but we have missed item, create one rule for it */ 354 if (buf->entries == 1 && missed != 0) { 355 elt = 2; 356 lsize += elt * sizeof(*item) + user_pattern_size; 357 if (lsize <= size) { 358 buf->entry[buf->entries].priority = 1; 359 buf->entry[buf->entries].pattern = addr; 360 buf->entries++; 361 flow_items[0].type = missed_item.type; 362 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 363 rte_memcpy(addr, buf->entry[0].pattern, 364 user_pattern_size); 365 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 366 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 367 addr = (void *)(((uintptr_t)addr) + 368 elt * sizeof(*item)); 369 } 370 } 371 return lsize; 372 } 373 374 enum mlx5_expansion { 375 MLX5_EXPANSION_ROOT, 376 MLX5_EXPANSION_ROOT_OUTER, 377 MLX5_EXPANSION_ROOT_ETH_VLAN, 378 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 379 MLX5_EXPANSION_OUTER_ETH, 380 MLX5_EXPANSION_OUTER_ETH_VLAN, 381 MLX5_EXPANSION_OUTER_VLAN, 382 MLX5_EXPANSION_OUTER_IPV4, 383 MLX5_EXPANSION_OUTER_IPV4_UDP, 384 MLX5_EXPANSION_OUTER_IPV4_TCP, 385 MLX5_EXPANSION_OUTER_IPV6, 386 MLX5_EXPANSION_OUTER_IPV6_UDP, 387 MLX5_EXPANSION_OUTER_IPV6_TCP, 388 MLX5_EXPANSION_VXLAN, 389 MLX5_EXPANSION_VXLAN_GPE, 390 MLX5_EXPANSION_GRE, 391 MLX5_EXPANSION_MPLS, 392 MLX5_EXPANSION_ETH, 393 MLX5_EXPANSION_ETH_VLAN, 394 MLX5_EXPANSION_VLAN, 395 MLX5_EXPANSION_IPV4, 396 MLX5_EXPANSION_IPV4_UDP, 397 MLX5_EXPANSION_IPV4_TCP, 398 MLX5_EXPANSION_IPV6, 399 MLX5_EXPANSION_IPV6_UDP, 400 MLX5_EXPANSION_IPV6_TCP, 401 }; 402 403 /** Supported expansion of items. */ 404 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 405 [MLX5_EXPANSION_ROOT] = { 406 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 407 MLX5_EXPANSION_IPV4, 408 MLX5_EXPANSION_IPV6), 409 .type = RTE_FLOW_ITEM_TYPE_END, 410 }, 411 [MLX5_EXPANSION_ROOT_OUTER] = { 412 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 413 MLX5_EXPANSION_OUTER_IPV4, 414 MLX5_EXPANSION_OUTER_IPV6), 415 .type = RTE_FLOW_ITEM_TYPE_END, 416 }, 417 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 418 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 419 .type = RTE_FLOW_ITEM_TYPE_END, 420 }, 421 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 422 .next = MLX5_FLOW_EXPAND_RSS_NEXT 423 (MLX5_EXPANSION_OUTER_ETH_VLAN), 424 .type = RTE_FLOW_ITEM_TYPE_END, 425 }, 426 [MLX5_EXPANSION_OUTER_ETH] = { 427 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 428 MLX5_EXPANSION_OUTER_IPV6, 429 MLX5_EXPANSION_MPLS), 430 .type = RTE_FLOW_ITEM_TYPE_ETH, 431 .rss_types = 0, 432 }, 433 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 434 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 435 .type = RTE_FLOW_ITEM_TYPE_ETH, 436 .rss_types = 0, 437 }, 438 [MLX5_EXPANSION_OUTER_VLAN] = { 439 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 440 MLX5_EXPANSION_OUTER_IPV6), 441 .type = RTE_FLOW_ITEM_TYPE_VLAN, 442 }, 443 [MLX5_EXPANSION_OUTER_IPV4] = { 444 .next = MLX5_FLOW_EXPAND_RSS_NEXT 445 (MLX5_EXPANSION_OUTER_IPV4_UDP, 446 MLX5_EXPANSION_OUTER_IPV4_TCP, 447 MLX5_EXPANSION_GRE, 448 MLX5_EXPANSION_IPV4, 449 MLX5_EXPANSION_IPV6), 450 .type = RTE_FLOW_ITEM_TYPE_IPV4, 451 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 452 ETH_RSS_NONFRAG_IPV4_OTHER, 453 }, 454 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 455 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 456 MLX5_EXPANSION_VXLAN_GPE), 457 .type = RTE_FLOW_ITEM_TYPE_UDP, 458 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 459 }, 460 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 461 .type = RTE_FLOW_ITEM_TYPE_TCP, 462 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 463 }, 464 [MLX5_EXPANSION_OUTER_IPV6] = { 465 .next = MLX5_FLOW_EXPAND_RSS_NEXT 466 (MLX5_EXPANSION_OUTER_IPV6_UDP, 467 MLX5_EXPANSION_OUTER_IPV6_TCP, 468 MLX5_EXPANSION_IPV4, 469 MLX5_EXPANSION_IPV6), 470 .type = RTE_FLOW_ITEM_TYPE_IPV6, 471 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 472 ETH_RSS_NONFRAG_IPV6_OTHER, 473 }, 474 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 475 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 476 MLX5_EXPANSION_VXLAN_GPE), 477 .type = RTE_FLOW_ITEM_TYPE_UDP, 478 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 479 }, 480 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 481 .type = RTE_FLOW_ITEM_TYPE_TCP, 482 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 483 }, 484 [MLX5_EXPANSION_VXLAN] = { 485 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 486 MLX5_EXPANSION_IPV4, 487 MLX5_EXPANSION_IPV6), 488 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 489 }, 490 [MLX5_EXPANSION_VXLAN_GPE] = { 491 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 492 MLX5_EXPANSION_IPV4, 493 MLX5_EXPANSION_IPV6), 494 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 495 }, 496 [MLX5_EXPANSION_GRE] = { 497 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 498 .type = RTE_FLOW_ITEM_TYPE_GRE, 499 }, 500 [MLX5_EXPANSION_MPLS] = { 501 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 502 MLX5_EXPANSION_IPV6), 503 .type = RTE_FLOW_ITEM_TYPE_MPLS, 504 }, 505 [MLX5_EXPANSION_ETH] = { 506 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 507 MLX5_EXPANSION_IPV6), 508 .type = RTE_FLOW_ITEM_TYPE_ETH, 509 }, 510 [MLX5_EXPANSION_ETH_VLAN] = { 511 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 512 .type = RTE_FLOW_ITEM_TYPE_ETH, 513 }, 514 [MLX5_EXPANSION_VLAN] = { 515 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 516 MLX5_EXPANSION_IPV6), 517 .type = RTE_FLOW_ITEM_TYPE_VLAN, 518 }, 519 [MLX5_EXPANSION_IPV4] = { 520 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 521 MLX5_EXPANSION_IPV4_TCP), 522 .type = RTE_FLOW_ITEM_TYPE_IPV4, 523 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 524 ETH_RSS_NONFRAG_IPV4_OTHER, 525 }, 526 [MLX5_EXPANSION_IPV4_UDP] = { 527 .type = RTE_FLOW_ITEM_TYPE_UDP, 528 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 529 }, 530 [MLX5_EXPANSION_IPV4_TCP] = { 531 .type = RTE_FLOW_ITEM_TYPE_TCP, 532 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 533 }, 534 [MLX5_EXPANSION_IPV6] = { 535 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 536 MLX5_EXPANSION_IPV6_TCP), 537 .type = RTE_FLOW_ITEM_TYPE_IPV6, 538 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 539 ETH_RSS_NONFRAG_IPV6_OTHER, 540 }, 541 [MLX5_EXPANSION_IPV6_UDP] = { 542 .type = RTE_FLOW_ITEM_TYPE_UDP, 543 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 544 }, 545 [MLX5_EXPANSION_IPV6_TCP] = { 546 .type = RTE_FLOW_ITEM_TYPE_TCP, 547 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 548 }, 549 }; 550 551 static struct rte_flow_shared_action * 552 mlx5_shared_action_create(struct rte_eth_dev *dev, 553 const struct rte_flow_shared_action_conf *conf, 554 const struct rte_flow_action *action, 555 struct rte_flow_error *error); 556 static int mlx5_shared_action_destroy 557 (struct rte_eth_dev *dev, 558 struct rte_flow_shared_action *shared_action, 559 struct rte_flow_error *error); 560 static int mlx5_shared_action_update 561 (struct rte_eth_dev *dev, 562 struct rte_flow_shared_action *shared_action, 563 const struct rte_flow_action *action, 564 struct rte_flow_error *error); 565 static int mlx5_shared_action_query 566 (struct rte_eth_dev *dev, 567 const struct rte_flow_shared_action *action, 568 void *data, 569 struct rte_flow_error *error); 570 571 static const struct rte_flow_ops mlx5_flow_ops = { 572 .validate = mlx5_flow_validate, 573 .create = mlx5_flow_create, 574 .destroy = mlx5_flow_destroy, 575 .flush = mlx5_flow_flush, 576 .isolate = mlx5_flow_isolate, 577 .query = mlx5_flow_query, 578 .dev_dump = mlx5_flow_dev_dump, 579 .get_aged_flows = mlx5_flow_get_aged_flows, 580 .shared_action_create = mlx5_shared_action_create, 581 .shared_action_destroy = mlx5_shared_action_destroy, 582 .shared_action_update = mlx5_shared_action_update, 583 .shared_action_query = mlx5_shared_action_query, 584 }; 585 586 /* Convert FDIR request to Generic flow. */ 587 struct mlx5_fdir { 588 struct rte_flow_attr attr; 589 struct rte_flow_item items[4]; 590 struct rte_flow_item_eth l2; 591 struct rte_flow_item_eth l2_mask; 592 union { 593 struct rte_flow_item_ipv4 ipv4; 594 struct rte_flow_item_ipv6 ipv6; 595 } l3; 596 union { 597 struct rte_flow_item_ipv4 ipv4; 598 struct rte_flow_item_ipv6 ipv6; 599 } l3_mask; 600 union { 601 struct rte_flow_item_udp udp; 602 struct rte_flow_item_tcp tcp; 603 } l4; 604 union { 605 struct rte_flow_item_udp udp; 606 struct rte_flow_item_tcp tcp; 607 } l4_mask; 608 struct rte_flow_action actions[2]; 609 struct rte_flow_action_queue queue; 610 }; 611 612 /* Tunnel information. */ 613 struct mlx5_flow_tunnel_info { 614 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 615 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 616 }; 617 618 static struct mlx5_flow_tunnel_info tunnels_info[] = { 619 { 620 .tunnel = MLX5_FLOW_LAYER_VXLAN, 621 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 622 }, 623 { 624 .tunnel = MLX5_FLOW_LAYER_GENEVE, 625 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 626 }, 627 { 628 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 629 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 630 }, 631 { 632 .tunnel = MLX5_FLOW_LAYER_GRE, 633 .ptype = RTE_PTYPE_TUNNEL_GRE, 634 }, 635 { 636 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 637 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 638 }, 639 { 640 .tunnel = MLX5_FLOW_LAYER_MPLS, 641 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 642 }, 643 { 644 .tunnel = MLX5_FLOW_LAYER_NVGRE, 645 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 646 }, 647 { 648 .tunnel = MLX5_FLOW_LAYER_IPIP, 649 .ptype = RTE_PTYPE_TUNNEL_IP, 650 }, 651 { 652 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 653 .ptype = RTE_PTYPE_TUNNEL_IP, 654 }, 655 { 656 .tunnel = MLX5_FLOW_LAYER_GTP, 657 .ptype = RTE_PTYPE_TUNNEL_GTPU, 658 }, 659 }; 660 661 /** 662 * Translate tag ID to register. 663 * 664 * @param[in] dev 665 * Pointer to the Ethernet device structure. 666 * @param[in] feature 667 * The feature that request the register. 668 * @param[in] id 669 * The request register ID. 670 * @param[out] error 671 * Error description in case of any. 672 * 673 * @return 674 * The request register on success, a negative errno 675 * value otherwise and rte_errno is set. 676 */ 677 int 678 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 679 enum mlx5_feature_name feature, 680 uint32_t id, 681 struct rte_flow_error *error) 682 { 683 struct mlx5_priv *priv = dev->data->dev_private; 684 struct mlx5_dev_config *config = &priv->config; 685 enum modify_reg start_reg; 686 bool skip_mtr_reg = false; 687 688 switch (feature) { 689 case MLX5_HAIRPIN_RX: 690 return REG_B; 691 case MLX5_HAIRPIN_TX: 692 return REG_A; 693 case MLX5_METADATA_RX: 694 switch (config->dv_xmeta_en) { 695 case MLX5_XMETA_MODE_LEGACY: 696 return REG_B; 697 case MLX5_XMETA_MODE_META16: 698 return REG_C_0; 699 case MLX5_XMETA_MODE_META32: 700 return REG_C_1; 701 } 702 break; 703 case MLX5_METADATA_TX: 704 return REG_A; 705 case MLX5_METADATA_FDB: 706 switch (config->dv_xmeta_en) { 707 case MLX5_XMETA_MODE_LEGACY: 708 return REG_NON; 709 case MLX5_XMETA_MODE_META16: 710 return REG_C_0; 711 case MLX5_XMETA_MODE_META32: 712 return REG_C_1; 713 } 714 break; 715 case MLX5_FLOW_MARK: 716 switch (config->dv_xmeta_en) { 717 case MLX5_XMETA_MODE_LEGACY: 718 return REG_NON; 719 case MLX5_XMETA_MODE_META16: 720 return REG_C_1; 721 case MLX5_XMETA_MODE_META32: 722 return REG_C_0; 723 } 724 break; 725 case MLX5_MTR_SFX: 726 /* 727 * If meter color and flow match share one register, flow match 728 * should use the meter color register for match. 729 */ 730 if (priv->mtr_reg_share) 731 return priv->mtr_color_reg; 732 else 733 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 734 REG_C_3; 735 case MLX5_MTR_COLOR: 736 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 737 return priv->mtr_color_reg; 738 case MLX5_COPY_MARK: 739 /* 740 * Metadata COPY_MARK register using is in meter suffix sub 741 * flow while with meter. It's safe to share the same register. 742 */ 743 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 744 case MLX5_APP_TAG: 745 /* 746 * If meter is enable, it will engage the register for color 747 * match and flow match. If meter color match is not using the 748 * REG_C_2, need to skip the REG_C_x be used by meter color 749 * match. 750 * If meter is disable, free to use all available registers. 751 */ 752 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 753 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 754 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 755 if (id > (REG_C_7 - start_reg)) 756 return rte_flow_error_set(error, EINVAL, 757 RTE_FLOW_ERROR_TYPE_ITEM, 758 NULL, "invalid tag id"); 759 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 760 return rte_flow_error_set(error, ENOTSUP, 761 RTE_FLOW_ERROR_TYPE_ITEM, 762 NULL, "unsupported tag id"); 763 /* 764 * This case means meter is using the REG_C_x great than 2. 765 * Take care not to conflict with meter color REG_C_x. 766 * If the available index REG_C_y >= REG_C_x, skip the 767 * color register. 768 */ 769 if (skip_mtr_reg && config->flow_mreg_c 770 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 771 if (id >= (REG_C_7 - start_reg)) 772 return rte_flow_error_set(error, EINVAL, 773 RTE_FLOW_ERROR_TYPE_ITEM, 774 NULL, "invalid tag id"); 775 if (config->flow_mreg_c 776 [id + 1 + start_reg - REG_C_0] != REG_NON) 777 return config->flow_mreg_c 778 [id + 1 + start_reg - REG_C_0]; 779 return rte_flow_error_set(error, ENOTSUP, 780 RTE_FLOW_ERROR_TYPE_ITEM, 781 NULL, "unsupported tag id"); 782 } 783 return config->flow_mreg_c[id + start_reg - REG_C_0]; 784 } 785 MLX5_ASSERT(false); 786 return rte_flow_error_set(error, EINVAL, 787 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 788 NULL, "invalid feature name"); 789 } 790 791 /** 792 * Check extensive flow metadata register support. 793 * 794 * @param dev 795 * Pointer to rte_eth_dev structure. 796 * 797 * @return 798 * True if device supports extensive flow metadata register, otherwise false. 799 */ 800 bool 801 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 802 { 803 struct mlx5_priv *priv = dev->data->dev_private; 804 struct mlx5_dev_config *config = &priv->config; 805 806 /* 807 * Having available reg_c can be regarded inclusively as supporting 808 * extensive flow metadata register, which could mean, 809 * - metadata register copy action by modify header. 810 * - 16 modify header actions is supported. 811 * - reg_c's are preserved across different domain (FDB and NIC) on 812 * packet loopback by flow lookup miss. 813 */ 814 return config->flow_mreg_c[2] != REG_NON; 815 } 816 817 /** 818 * Verify the @p item specifications (spec, last, mask) are compatible with the 819 * NIC capabilities. 820 * 821 * @param[in] item 822 * Item specification. 823 * @param[in] mask 824 * @p item->mask or flow default bit-masks. 825 * @param[in] nic_mask 826 * Bit-masks covering supported fields by the NIC to compare with user mask. 827 * @param[in] size 828 * Bit-masks size in bytes. 829 * @param[in] range_accepted 830 * True if range of values is accepted for specific fields, false otherwise. 831 * @param[out] error 832 * Pointer to error structure. 833 * 834 * @return 835 * 0 on success, a negative errno value otherwise and rte_errno is set. 836 */ 837 int 838 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 839 const uint8_t *mask, 840 const uint8_t *nic_mask, 841 unsigned int size, 842 bool range_accepted, 843 struct rte_flow_error *error) 844 { 845 unsigned int i; 846 847 MLX5_ASSERT(nic_mask); 848 for (i = 0; i < size; ++i) 849 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 850 return rte_flow_error_set(error, ENOTSUP, 851 RTE_FLOW_ERROR_TYPE_ITEM, 852 item, 853 "mask enables non supported" 854 " bits"); 855 if (!item->spec && (item->mask || item->last)) 856 return rte_flow_error_set(error, EINVAL, 857 RTE_FLOW_ERROR_TYPE_ITEM, item, 858 "mask/last without a spec is not" 859 " supported"); 860 if (item->spec && item->last && !range_accepted) { 861 uint8_t spec[size]; 862 uint8_t last[size]; 863 unsigned int i; 864 int ret; 865 866 for (i = 0; i < size; ++i) { 867 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 868 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 869 } 870 ret = memcmp(spec, last, size); 871 if (ret != 0) 872 return rte_flow_error_set(error, EINVAL, 873 RTE_FLOW_ERROR_TYPE_ITEM, 874 item, 875 "range is not valid"); 876 } 877 return 0; 878 } 879 880 /** 881 * Adjust the hash fields according to the @p flow information. 882 * 883 * @param[in] dev_flow. 884 * Pointer to the mlx5_flow. 885 * @param[in] tunnel 886 * 1 when the hash field is for a tunnel item. 887 * @param[in] layer_types 888 * ETH_RSS_* types. 889 * @param[in] hash_fields 890 * Item hash fields. 891 * 892 * @return 893 * The hash fields that should be used. 894 */ 895 uint64_t 896 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 897 int tunnel __rte_unused, uint64_t layer_types, 898 uint64_t hash_fields) 899 { 900 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 901 int rss_request_inner = rss_desc->level >= 2; 902 903 /* Check RSS hash level for tunnel. */ 904 if (tunnel && rss_request_inner) 905 hash_fields |= IBV_RX_HASH_INNER; 906 else if (tunnel || rss_request_inner) 907 return 0; 908 #endif 909 /* Check if requested layer matches RSS hash fields. */ 910 if (!(rss_desc->types & layer_types)) 911 return 0; 912 return hash_fields; 913 } 914 915 /** 916 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 917 * if several tunnel rules are used on this queue, the tunnel ptype will be 918 * cleared. 919 * 920 * @param rxq_ctrl 921 * Rx queue to update. 922 */ 923 static void 924 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 925 { 926 unsigned int i; 927 uint32_t tunnel_ptype = 0; 928 929 /* Look up for the ptype to use. */ 930 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 931 if (!rxq_ctrl->flow_tunnels_n[i]) 932 continue; 933 if (!tunnel_ptype) { 934 tunnel_ptype = tunnels_info[i].ptype; 935 } else { 936 tunnel_ptype = 0; 937 break; 938 } 939 } 940 rxq_ctrl->rxq.tunnel = tunnel_ptype; 941 } 942 943 /** 944 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 945 * flow. 946 * 947 * @param[in] dev 948 * Pointer to the Ethernet device structure. 949 * @param[in] dev_handle 950 * Pointer to device flow handle structure. 951 */ 952 static void 953 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 954 struct mlx5_flow_handle *dev_handle) 955 { 956 struct mlx5_priv *priv = dev->data->dev_private; 957 const int mark = dev_handle->mark; 958 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 959 struct mlx5_hrxq *hrxq; 960 unsigned int i; 961 962 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 963 return; 964 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 965 dev_handle->rix_hrxq); 966 if (!hrxq) 967 return; 968 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 969 int idx = hrxq->ind_table->queues[i]; 970 struct mlx5_rxq_ctrl *rxq_ctrl = 971 container_of((*priv->rxqs)[idx], 972 struct mlx5_rxq_ctrl, rxq); 973 974 /* 975 * To support metadata register copy on Tx loopback, 976 * this must be always enabled (metadata may arive 977 * from other port - not from local flows only. 978 */ 979 if (priv->config.dv_flow_en && 980 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 981 mlx5_flow_ext_mreg_supported(dev)) { 982 rxq_ctrl->rxq.mark = 1; 983 rxq_ctrl->flow_mark_n = 1; 984 } else if (mark) { 985 rxq_ctrl->rxq.mark = 1; 986 rxq_ctrl->flow_mark_n++; 987 } 988 if (tunnel) { 989 unsigned int j; 990 991 /* Increase the counter matching the flow. */ 992 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 993 if ((tunnels_info[j].tunnel & 994 dev_handle->layers) == 995 tunnels_info[j].tunnel) { 996 rxq_ctrl->flow_tunnels_n[j]++; 997 break; 998 } 999 } 1000 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1001 } 1002 } 1003 } 1004 1005 /** 1006 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1007 * 1008 * @param[in] dev 1009 * Pointer to the Ethernet device structure. 1010 * @param[in] flow 1011 * Pointer to flow structure. 1012 */ 1013 static void 1014 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1015 { 1016 struct mlx5_priv *priv = dev->data->dev_private; 1017 uint32_t handle_idx; 1018 struct mlx5_flow_handle *dev_handle; 1019 1020 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1021 handle_idx, dev_handle, next) 1022 flow_drv_rxq_flags_set(dev, dev_handle); 1023 } 1024 1025 /** 1026 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1027 * device flow if no other flow uses it with the same kind of request. 1028 * 1029 * @param dev 1030 * Pointer to Ethernet device. 1031 * @param[in] dev_handle 1032 * Pointer to the device flow handle structure. 1033 */ 1034 static void 1035 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1036 struct mlx5_flow_handle *dev_handle) 1037 { 1038 struct mlx5_priv *priv = dev->data->dev_private; 1039 const int mark = dev_handle->mark; 1040 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1041 struct mlx5_hrxq *hrxq; 1042 unsigned int i; 1043 1044 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 1045 return; 1046 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1047 dev_handle->rix_hrxq); 1048 if (!hrxq) 1049 return; 1050 MLX5_ASSERT(dev->data->dev_started); 1051 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 1052 int idx = hrxq->ind_table->queues[i]; 1053 struct mlx5_rxq_ctrl *rxq_ctrl = 1054 container_of((*priv->rxqs)[idx], 1055 struct mlx5_rxq_ctrl, rxq); 1056 1057 if (priv->config.dv_flow_en && 1058 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 1059 mlx5_flow_ext_mreg_supported(dev)) { 1060 rxq_ctrl->rxq.mark = 1; 1061 rxq_ctrl->flow_mark_n = 1; 1062 } else if (mark) { 1063 rxq_ctrl->flow_mark_n--; 1064 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 1065 } 1066 if (tunnel) { 1067 unsigned int j; 1068 1069 /* Decrease the counter matching the flow. */ 1070 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1071 if ((tunnels_info[j].tunnel & 1072 dev_handle->layers) == 1073 tunnels_info[j].tunnel) { 1074 rxq_ctrl->flow_tunnels_n[j]--; 1075 break; 1076 } 1077 } 1078 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1079 } 1080 } 1081 } 1082 1083 /** 1084 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1085 * @p flow if no other flow uses it with the same kind of request. 1086 * 1087 * @param dev 1088 * Pointer to Ethernet device. 1089 * @param[in] flow 1090 * Pointer to the flow. 1091 */ 1092 static void 1093 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1094 { 1095 struct mlx5_priv *priv = dev->data->dev_private; 1096 uint32_t handle_idx; 1097 struct mlx5_flow_handle *dev_handle; 1098 1099 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1100 handle_idx, dev_handle, next) 1101 flow_drv_rxq_flags_trim(dev, dev_handle); 1102 } 1103 1104 /** 1105 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1106 * 1107 * @param dev 1108 * Pointer to Ethernet device. 1109 */ 1110 static void 1111 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1112 { 1113 struct mlx5_priv *priv = dev->data->dev_private; 1114 unsigned int i; 1115 1116 for (i = 0; i != priv->rxqs_n; ++i) { 1117 struct mlx5_rxq_ctrl *rxq_ctrl; 1118 unsigned int j; 1119 1120 if (!(*priv->rxqs)[i]) 1121 continue; 1122 rxq_ctrl = container_of((*priv->rxqs)[i], 1123 struct mlx5_rxq_ctrl, rxq); 1124 rxq_ctrl->flow_mark_n = 0; 1125 rxq_ctrl->rxq.mark = 0; 1126 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1127 rxq_ctrl->flow_tunnels_n[j] = 0; 1128 rxq_ctrl->rxq.tunnel = 0; 1129 } 1130 } 1131 1132 /** 1133 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1134 * 1135 * @param[in] dev 1136 * Pointer to the Ethernet device structure. 1137 */ 1138 void 1139 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1140 { 1141 struct mlx5_priv *priv = dev->data->dev_private; 1142 struct mlx5_rxq_data *data; 1143 unsigned int i; 1144 1145 for (i = 0; i != priv->rxqs_n; ++i) { 1146 if (!(*priv->rxqs)[i]) 1147 continue; 1148 data = (*priv->rxqs)[i]; 1149 if (!rte_flow_dynf_metadata_avail()) { 1150 data->dynf_meta = 0; 1151 data->flow_meta_mask = 0; 1152 data->flow_meta_offset = -1; 1153 } else { 1154 data->dynf_meta = 1; 1155 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1156 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1157 } 1158 } 1159 } 1160 1161 /* 1162 * return a pointer to the desired action in the list of actions. 1163 * 1164 * @param[in] actions 1165 * The list of actions to search the action in. 1166 * @param[in] action 1167 * The action to find. 1168 * 1169 * @return 1170 * Pointer to the action in the list, if found. NULL otherwise. 1171 */ 1172 const struct rte_flow_action * 1173 mlx5_flow_find_action(const struct rte_flow_action *actions, 1174 enum rte_flow_action_type action) 1175 { 1176 if (actions == NULL) 1177 return NULL; 1178 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1179 if (actions->type == action) 1180 return actions; 1181 return NULL; 1182 } 1183 1184 /* 1185 * Validate the flag action. 1186 * 1187 * @param[in] action_flags 1188 * Bit-fields that holds the actions detected until now. 1189 * @param[in] attr 1190 * Attributes of flow that includes this action. 1191 * @param[out] error 1192 * Pointer to error structure. 1193 * 1194 * @return 1195 * 0 on success, a negative errno value otherwise and rte_errno is set. 1196 */ 1197 int 1198 mlx5_flow_validate_action_flag(uint64_t action_flags, 1199 const struct rte_flow_attr *attr, 1200 struct rte_flow_error *error) 1201 { 1202 if (action_flags & MLX5_FLOW_ACTION_MARK) 1203 return rte_flow_error_set(error, EINVAL, 1204 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1205 "can't mark and flag in same flow"); 1206 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1207 return rte_flow_error_set(error, EINVAL, 1208 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1209 "can't have 2 flag" 1210 " actions in same flow"); 1211 if (attr->egress) 1212 return rte_flow_error_set(error, ENOTSUP, 1213 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1214 "flag action not supported for " 1215 "egress"); 1216 return 0; 1217 } 1218 1219 /* 1220 * Validate the mark action. 1221 * 1222 * @param[in] action 1223 * Pointer to the queue action. 1224 * @param[in] action_flags 1225 * Bit-fields that holds the actions detected until now. 1226 * @param[in] attr 1227 * Attributes of flow that includes this action. 1228 * @param[out] error 1229 * Pointer to error structure. 1230 * 1231 * @return 1232 * 0 on success, a negative errno value otherwise and rte_errno is set. 1233 */ 1234 int 1235 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1236 uint64_t action_flags, 1237 const struct rte_flow_attr *attr, 1238 struct rte_flow_error *error) 1239 { 1240 const struct rte_flow_action_mark *mark = action->conf; 1241 1242 if (!mark) 1243 return rte_flow_error_set(error, EINVAL, 1244 RTE_FLOW_ERROR_TYPE_ACTION, 1245 action, 1246 "configuration cannot be null"); 1247 if (mark->id >= MLX5_FLOW_MARK_MAX) 1248 return rte_flow_error_set(error, EINVAL, 1249 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1250 &mark->id, 1251 "mark id must in 0 <= id < " 1252 RTE_STR(MLX5_FLOW_MARK_MAX)); 1253 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1254 return rte_flow_error_set(error, EINVAL, 1255 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1256 "can't flag and mark in same flow"); 1257 if (action_flags & MLX5_FLOW_ACTION_MARK) 1258 return rte_flow_error_set(error, EINVAL, 1259 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1260 "can't have 2 mark actions in same" 1261 " flow"); 1262 if (attr->egress) 1263 return rte_flow_error_set(error, ENOTSUP, 1264 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1265 "mark action not supported for " 1266 "egress"); 1267 return 0; 1268 } 1269 1270 /* 1271 * Validate the drop action. 1272 * 1273 * @param[in] action_flags 1274 * Bit-fields that holds the actions detected until now. 1275 * @param[in] attr 1276 * Attributes of flow that includes this action. 1277 * @param[out] error 1278 * Pointer to error structure. 1279 * 1280 * @return 1281 * 0 on success, a negative errno value otherwise and rte_errno is set. 1282 */ 1283 int 1284 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1285 const struct rte_flow_attr *attr, 1286 struct rte_flow_error *error) 1287 { 1288 if (attr->egress) 1289 return rte_flow_error_set(error, ENOTSUP, 1290 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1291 "drop action not supported for " 1292 "egress"); 1293 return 0; 1294 } 1295 1296 /* 1297 * Validate the queue action. 1298 * 1299 * @param[in] action 1300 * Pointer to the queue action. 1301 * @param[in] action_flags 1302 * Bit-fields that holds the actions detected until now. 1303 * @param[in] dev 1304 * Pointer to the Ethernet device structure. 1305 * @param[in] attr 1306 * Attributes of flow that includes this action. 1307 * @param[out] error 1308 * Pointer to error structure. 1309 * 1310 * @return 1311 * 0 on success, a negative errno value otherwise and rte_errno is set. 1312 */ 1313 int 1314 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1315 uint64_t action_flags, 1316 struct rte_eth_dev *dev, 1317 const struct rte_flow_attr *attr, 1318 struct rte_flow_error *error) 1319 { 1320 struct mlx5_priv *priv = dev->data->dev_private; 1321 const struct rte_flow_action_queue *queue = action->conf; 1322 1323 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1324 return rte_flow_error_set(error, EINVAL, 1325 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1326 "can't have 2 fate actions in" 1327 " same flow"); 1328 if (!priv->rxqs_n) 1329 return rte_flow_error_set(error, EINVAL, 1330 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1331 NULL, "No Rx queues configured"); 1332 if (queue->index >= priv->rxqs_n) 1333 return rte_flow_error_set(error, EINVAL, 1334 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1335 &queue->index, 1336 "queue index out of range"); 1337 if (!(*priv->rxqs)[queue->index]) 1338 return rte_flow_error_set(error, EINVAL, 1339 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1340 &queue->index, 1341 "queue is not configured"); 1342 if (attr->egress) 1343 return rte_flow_error_set(error, ENOTSUP, 1344 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1345 "queue action not supported for " 1346 "egress"); 1347 return 0; 1348 } 1349 1350 /* 1351 * Validate the rss action. 1352 * 1353 * @param[in] dev 1354 * Pointer to the Ethernet device structure. 1355 * @param[in] action 1356 * Pointer to the queue action. 1357 * @param[out] error 1358 * Pointer to error structure. 1359 * 1360 * @return 1361 * 0 on success, a negative errno value otherwise and rte_errno is set. 1362 */ 1363 int 1364 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1365 const struct rte_flow_action *action, 1366 struct rte_flow_error *error) 1367 { 1368 struct mlx5_priv *priv = dev->data->dev_private; 1369 const struct rte_flow_action_rss *rss = action->conf; 1370 unsigned int i; 1371 1372 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1373 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1374 return rte_flow_error_set(error, ENOTSUP, 1375 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1376 &rss->func, 1377 "RSS hash function not supported"); 1378 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1379 if (rss->level > 2) 1380 #else 1381 if (rss->level > 1) 1382 #endif 1383 return rte_flow_error_set(error, ENOTSUP, 1384 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1385 &rss->level, 1386 "tunnel RSS is not supported"); 1387 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1388 if (rss->key_len == 0 && rss->key != NULL) 1389 return rte_flow_error_set(error, ENOTSUP, 1390 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1391 &rss->key_len, 1392 "RSS hash key length 0"); 1393 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1394 return rte_flow_error_set(error, ENOTSUP, 1395 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1396 &rss->key_len, 1397 "RSS hash key too small"); 1398 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1399 return rte_flow_error_set(error, ENOTSUP, 1400 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1401 &rss->key_len, 1402 "RSS hash key too large"); 1403 if (rss->queue_num > priv->config.ind_table_max_size) 1404 return rte_flow_error_set(error, ENOTSUP, 1405 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1406 &rss->queue_num, 1407 "number of queues too large"); 1408 if (rss->types & MLX5_RSS_HF_MASK) 1409 return rte_flow_error_set(error, ENOTSUP, 1410 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1411 &rss->types, 1412 "some RSS protocols are not" 1413 " supported"); 1414 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1415 !(rss->types & ETH_RSS_IP)) 1416 return rte_flow_error_set(error, EINVAL, 1417 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1418 "L3 partial RSS requested but L3 RSS" 1419 " type not specified"); 1420 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1421 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1422 return rte_flow_error_set(error, EINVAL, 1423 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1424 "L4 partial RSS requested but L4 RSS" 1425 " type not specified"); 1426 if (!priv->rxqs_n) 1427 return rte_flow_error_set(error, EINVAL, 1428 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1429 NULL, "No Rx queues configured"); 1430 if (!rss->queue_num) 1431 return rte_flow_error_set(error, EINVAL, 1432 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1433 NULL, "No queues configured"); 1434 for (i = 0; i != rss->queue_num; ++i) { 1435 if (rss->queue[i] >= priv->rxqs_n) 1436 return rte_flow_error_set 1437 (error, EINVAL, 1438 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1439 &rss->queue[i], "queue index out of range"); 1440 if (!(*priv->rxqs)[rss->queue[i]]) 1441 return rte_flow_error_set 1442 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1443 &rss->queue[i], "queue is not configured"); 1444 } 1445 return 0; 1446 } 1447 1448 /* 1449 * Validate the rss action. 1450 * 1451 * @param[in] action 1452 * Pointer to the queue action. 1453 * @param[in] action_flags 1454 * Bit-fields that holds the actions detected until now. 1455 * @param[in] dev 1456 * Pointer to the Ethernet device structure. 1457 * @param[in] attr 1458 * Attributes of flow that includes this action. 1459 * @param[in] item_flags 1460 * Items that were detected. 1461 * @param[out] error 1462 * Pointer to error structure. 1463 * 1464 * @return 1465 * 0 on success, a negative errno value otherwise and rte_errno is set. 1466 */ 1467 int 1468 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1469 uint64_t action_flags, 1470 struct rte_eth_dev *dev, 1471 const struct rte_flow_attr *attr, 1472 uint64_t item_flags, 1473 struct rte_flow_error *error) 1474 { 1475 const struct rte_flow_action_rss *rss = action->conf; 1476 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1477 int ret; 1478 1479 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1480 return rte_flow_error_set(error, EINVAL, 1481 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1482 "can't have 2 fate actions" 1483 " in same flow"); 1484 ret = mlx5_validate_action_rss(dev, action, error); 1485 if (ret) 1486 return ret; 1487 if (attr->egress) 1488 return rte_flow_error_set(error, ENOTSUP, 1489 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1490 "rss action not supported for " 1491 "egress"); 1492 if (rss->level > 1 && !tunnel) 1493 return rte_flow_error_set(error, EINVAL, 1494 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1495 "inner RSS is not supported for " 1496 "non-tunnel flows"); 1497 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1498 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1499 return rte_flow_error_set(error, EINVAL, 1500 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1501 "RSS on eCPRI is not supported now"); 1502 } 1503 return 0; 1504 } 1505 1506 /* 1507 * Validate the default miss action. 1508 * 1509 * @param[in] action_flags 1510 * Bit-fields that holds the actions detected until now. 1511 * @param[out] error 1512 * Pointer to error structure. 1513 * 1514 * @return 1515 * 0 on success, a negative errno value otherwise and rte_errno is set. 1516 */ 1517 int 1518 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1519 const struct rte_flow_attr *attr, 1520 struct rte_flow_error *error) 1521 { 1522 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1523 return rte_flow_error_set(error, EINVAL, 1524 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1525 "can't have 2 fate actions in" 1526 " same flow"); 1527 if (attr->egress) 1528 return rte_flow_error_set(error, ENOTSUP, 1529 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1530 "default miss action not supported " 1531 "for egress"); 1532 if (attr->group) 1533 return rte_flow_error_set(error, ENOTSUP, 1534 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1535 "only group 0 is supported"); 1536 if (attr->transfer) 1537 return rte_flow_error_set(error, ENOTSUP, 1538 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1539 NULL, "transfer is not supported"); 1540 return 0; 1541 } 1542 1543 /* 1544 * Validate the count action. 1545 * 1546 * @param[in] dev 1547 * Pointer to the Ethernet device structure. 1548 * @param[in] attr 1549 * Attributes of flow that includes this action. 1550 * @param[out] error 1551 * Pointer to error structure. 1552 * 1553 * @return 1554 * 0 on success, a negative errno value otherwise and rte_errno is set. 1555 */ 1556 int 1557 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1558 const struct rte_flow_attr *attr, 1559 struct rte_flow_error *error) 1560 { 1561 if (attr->egress) 1562 return rte_flow_error_set(error, ENOTSUP, 1563 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1564 "count action not supported for " 1565 "egress"); 1566 return 0; 1567 } 1568 1569 /** 1570 * Verify the @p attributes will be correctly understood by the NIC and store 1571 * them in the @p flow if everything is correct. 1572 * 1573 * @param[in] dev 1574 * Pointer to the Ethernet device structure. 1575 * @param[in] attributes 1576 * Pointer to flow attributes 1577 * @param[out] error 1578 * Pointer to error structure. 1579 * 1580 * @return 1581 * 0 on success, a negative errno value otherwise and rte_errno is set. 1582 */ 1583 int 1584 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1585 const struct rte_flow_attr *attributes, 1586 struct rte_flow_error *error) 1587 { 1588 struct mlx5_priv *priv = dev->data->dev_private; 1589 uint32_t priority_max = priv->config.flow_prio - 1; 1590 1591 if (attributes->group) 1592 return rte_flow_error_set(error, ENOTSUP, 1593 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1594 NULL, "groups is not supported"); 1595 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1596 attributes->priority >= priority_max) 1597 return rte_flow_error_set(error, ENOTSUP, 1598 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1599 NULL, "priority out of range"); 1600 if (attributes->egress) 1601 return rte_flow_error_set(error, ENOTSUP, 1602 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1603 "egress is not supported"); 1604 if (attributes->transfer && !priv->config.dv_esw_en) 1605 return rte_flow_error_set(error, ENOTSUP, 1606 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1607 NULL, "transfer is not supported"); 1608 if (!attributes->ingress) 1609 return rte_flow_error_set(error, EINVAL, 1610 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1611 NULL, 1612 "ingress attribute is mandatory"); 1613 return 0; 1614 } 1615 1616 /** 1617 * Validate ICMP6 item. 1618 * 1619 * @param[in] item 1620 * Item specification. 1621 * @param[in] item_flags 1622 * Bit-fields that holds the items detected until now. 1623 * @param[out] error 1624 * Pointer to error structure. 1625 * 1626 * @return 1627 * 0 on success, a negative errno value otherwise and rte_errno is set. 1628 */ 1629 int 1630 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1631 uint64_t item_flags, 1632 uint8_t target_protocol, 1633 struct rte_flow_error *error) 1634 { 1635 const struct rte_flow_item_icmp6 *mask = item->mask; 1636 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1637 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1638 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1639 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1640 MLX5_FLOW_LAYER_OUTER_L4; 1641 int ret; 1642 1643 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1644 return rte_flow_error_set(error, EINVAL, 1645 RTE_FLOW_ERROR_TYPE_ITEM, item, 1646 "protocol filtering not compatible" 1647 " with ICMP6 layer"); 1648 if (!(item_flags & l3m)) 1649 return rte_flow_error_set(error, EINVAL, 1650 RTE_FLOW_ERROR_TYPE_ITEM, item, 1651 "IPv6 is mandatory to filter on" 1652 " ICMP6"); 1653 if (item_flags & l4m) 1654 return rte_flow_error_set(error, EINVAL, 1655 RTE_FLOW_ERROR_TYPE_ITEM, item, 1656 "multiple L4 layers not supported"); 1657 if (!mask) 1658 mask = &rte_flow_item_icmp6_mask; 1659 ret = mlx5_flow_item_acceptable 1660 (item, (const uint8_t *)mask, 1661 (const uint8_t *)&rte_flow_item_icmp6_mask, 1662 sizeof(struct rte_flow_item_icmp6), 1663 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1664 if (ret < 0) 1665 return ret; 1666 return 0; 1667 } 1668 1669 /** 1670 * Validate ICMP item. 1671 * 1672 * @param[in] item 1673 * Item specification. 1674 * @param[in] item_flags 1675 * Bit-fields that holds the items detected until now. 1676 * @param[out] error 1677 * Pointer to error structure. 1678 * 1679 * @return 1680 * 0 on success, a negative errno value otherwise and rte_errno is set. 1681 */ 1682 int 1683 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1684 uint64_t item_flags, 1685 uint8_t target_protocol, 1686 struct rte_flow_error *error) 1687 { 1688 const struct rte_flow_item_icmp *mask = item->mask; 1689 const struct rte_flow_item_icmp nic_mask = { 1690 .hdr.icmp_type = 0xff, 1691 .hdr.icmp_code = 0xff, 1692 .hdr.icmp_ident = RTE_BE16(0xffff), 1693 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 1694 }; 1695 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1696 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1697 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1698 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1699 MLX5_FLOW_LAYER_OUTER_L4; 1700 int ret; 1701 1702 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1703 return rte_flow_error_set(error, EINVAL, 1704 RTE_FLOW_ERROR_TYPE_ITEM, item, 1705 "protocol filtering not compatible" 1706 " with ICMP layer"); 1707 if (!(item_flags & l3m)) 1708 return rte_flow_error_set(error, EINVAL, 1709 RTE_FLOW_ERROR_TYPE_ITEM, item, 1710 "IPv4 is mandatory to filter" 1711 " on ICMP"); 1712 if (item_flags & l4m) 1713 return rte_flow_error_set(error, EINVAL, 1714 RTE_FLOW_ERROR_TYPE_ITEM, item, 1715 "multiple L4 layers not supported"); 1716 if (!mask) 1717 mask = &nic_mask; 1718 ret = mlx5_flow_item_acceptable 1719 (item, (const uint8_t *)mask, 1720 (const uint8_t *)&nic_mask, 1721 sizeof(struct rte_flow_item_icmp), 1722 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1723 if (ret < 0) 1724 return ret; 1725 return 0; 1726 } 1727 1728 /** 1729 * Validate Ethernet item. 1730 * 1731 * @param[in] item 1732 * Item specification. 1733 * @param[in] item_flags 1734 * Bit-fields that holds the items detected until now. 1735 * @param[out] error 1736 * Pointer to error structure. 1737 * 1738 * @return 1739 * 0 on success, a negative errno value otherwise and rte_errno is set. 1740 */ 1741 int 1742 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1743 uint64_t item_flags, 1744 struct rte_flow_error *error) 1745 { 1746 const struct rte_flow_item_eth *mask = item->mask; 1747 const struct rte_flow_item_eth nic_mask = { 1748 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1749 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1750 .type = RTE_BE16(0xffff), 1751 }; 1752 int ret; 1753 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1754 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1755 MLX5_FLOW_LAYER_OUTER_L2; 1756 1757 if (item_flags & ethm) 1758 return rte_flow_error_set(error, ENOTSUP, 1759 RTE_FLOW_ERROR_TYPE_ITEM, item, 1760 "multiple L2 layers not supported"); 1761 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1762 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1763 return rte_flow_error_set(error, EINVAL, 1764 RTE_FLOW_ERROR_TYPE_ITEM, item, 1765 "L2 layer should not follow " 1766 "L3 layers"); 1767 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1768 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1769 return rte_flow_error_set(error, EINVAL, 1770 RTE_FLOW_ERROR_TYPE_ITEM, item, 1771 "L2 layer should not follow VLAN"); 1772 if (!mask) 1773 mask = &rte_flow_item_eth_mask; 1774 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1775 (const uint8_t *)&nic_mask, 1776 sizeof(struct rte_flow_item_eth), 1777 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1778 return ret; 1779 } 1780 1781 /** 1782 * Validate VLAN item. 1783 * 1784 * @param[in] item 1785 * Item specification. 1786 * @param[in] item_flags 1787 * Bit-fields that holds the items detected until now. 1788 * @param[in] dev 1789 * Ethernet device flow is being created on. 1790 * @param[out] error 1791 * Pointer to error structure. 1792 * 1793 * @return 1794 * 0 on success, a negative errno value otherwise and rte_errno is set. 1795 */ 1796 int 1797 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1798 uint64_t item_flags, 1799 struct rte_eth_dev *dev, 1800 struct rte_flow_error *error) 1801 { 1802 const struct rte_flow_item_vlan *spec = item->spec; 1803 const struct rte_flow_item_vlan *mask = item->mask; 1804 const struct rte_flow_item_vlan nic_mask = { 1805 .tci = RTE_BE16(UINT16_MAX), 1806 .inner_type = RTE_BE16(UINT16_MAX), 1807 }; 1808 uint16_t vlan_tag = 0; 1809 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1810 int ret; 1811 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1812 MLX5_FLOW_LAYER_INNER_L4) : 1813 (MLX5_FLOW_LAYER_OUTER_L3 | 1814 MLX5_FLOW_LAYER_OUTER_L4); 1815 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1816 MLX5_FLOW_LAYER_OUTER_VLAN; 1817 1818 if (item_flags & vlanm) 1819 return rte_flow_error_set(error, EINVAL, 1820 RTE_FLOW_ERROR_TYPE_ITEM, item, 1821 "multiple VLAN layers not supported"); 1822 else if ((item_flags & l34m) != 0) 1823 return rte_flow_error_set(error, EINVAL, 1824 RTE_FLOW_ERROR_TYPE_ITEM, item, 1825 "VLAN cannot follow L3/L4 layer"); 1826 if (!mask) 1827 mask = &rte_flow_item_vlan_mask; 1828 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1829 (const uint8_t *)&nic_mask, 1830 sizeof(struct rte_flow_item_vlan), 1831 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 1832 if (ret) 1833 return ret; 1834 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1835 struct mlx5_priv *priv = dev->data->dev_private; 1836 1837 if (priv->vmwa_context) { 1838 /* 1839 * Non-NULL context means we have a virtual machine 1840 * and SR-IOV enabled, we have to create VLAN interface 1841 * to make hypervisor to setup E-Switch vport 1842 * context correctly. We avoid creating the multiple 1843 * VLAN interfaces, so we cannot support VLAN tag mask. 1844 */ 1845 return rte_flow_error_set(error, EINVAL, 1846 RTE_FLOW_ERROR_TYPE_ITEM, 1847 item, 1848 "VLAN tag mask is not" 1849 " supported in virtual" 1850 " environment"); 1851 } 1852 } 1853 if (spec) { 1854 vlan_tag = spec->tci; 1855 vlan_tag &= mask->tci; 1856 } 1857 /* 1858 * From verbs perspective an empty VLAN is equivalent 1859 * to a packet without VLAN layer. 1860 */ 1861 if (!vlan_tag) 1862 return rte_flow_error_set(error, EINVAL, 1863 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1864 item->spec, 1865 "VLAN cannot be empty"); 1866 return 0; 1867 } 1868 1869 /** 1870 * Validate IPV4 item. 1871 * 1872 * @param[in] item 1873 * Item specification. 1874 * @param[in] item_flags 1875 * Bit-fields that holds the items detected until now. 1876 * @param[in] last_item 1877 * Previous validated item in the pattern items. 1878 * @param[in] ether_type 1879 * Type in the ethernet layer header (including dot1q). 1880 * @param[in] acc_mask 1881 * Acceptable mask, if NULL default internal default mask 1882 * will be used to check whether item fields are supported. 1883 * @param[in] range_accepted 1884 * True if range of values is accepted for specific fields, false otherwise. 1885 * @param[out] error 1886 * Pointer to error structure. 1887 * 1888 * @return 1889 * 0 on success, a negative errno value otherwise and rte_errno is set. 1890 */ 1891 int 1892 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1893 uint64_t item_flags, 1894 uint64_t last_item, 1895 uint16_t ether_type, 1896 const struct rte_flow_item_ipv4 *acc_mask, 1897 bool range_accepted, 1898 struct rte_flow_error *error) 1899 { 1900 const struct rte_flow_item_ipv4 *mask = item->mask; 1901 const struct rte_flow_item_ipv4 *spec = item->spec; 1902 const struct rte_flow_item_ipv4 nic_mask = { 1903 .hdr = { 1904 .src_addr = RTE_BE32(0xffffffff), 1905 .dst_addr = RTE_BE32(0xffffffff), 1906 .type_of_service = 0xff, 1907 .next_proto_id = 0xff, 1908 }, 1909 }; 1910 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1911 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1912 MLX5_FLOW_LAYER_OUTER_L3; 1913 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1914 MLX5_FLOW_LAYER_OUTER_L4; 1915 int ret; 1916 uint8_t next_proto = 0xFF; 1917 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1918 MLX5_FLOW_LAYER_OUTER_VLAN | 1919 MLX5_FLOW_LAYER_INNER_VLAN); 1920 1921 if ((last_item & l2_vlan) && ether_type && 1922 ether_type != RTE_ETHER_TYPE_IPV4) 1923 return rte_flow_error_set(error, EINVAL, 1924 RTE_FLOW_ERROR_TYPE_ITEM, item, 1925 "IPv4 cannot follow L2/VLAN layer " 1926 "which ether type is not IPv4"); 1927 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1928 if (mask && spec) 1929 next_proto = mask->hdr.next_proto_id & 1930 spec->hdr.next_proto_id; 1931 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1932 return rte_flow_error_set(error, EINVAL, 1933 RTE_FLOW_ERROR_TYPE_ITEM, 1934 item, 1935 "multiple tunnel " 1936 "not supported"); 1937 } 1938 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1939 return rte_flow_error_set(error, EINVAL, 1940 RTE_FLOW_ERROR_TYPE_ITEM, item, 1941 "wrong tunnel type - IPv6 specified " 1942 "but IPv4 item provided"); 1943 if (item_flags & l3m) 1944 return rte_flow_error_set(error, ENOTSUP, 1945 RTE_FLOW_ERROR_TYPE_ITEM, item, 1946 "multiple L3 layers not supported"); 1947 else if (item_flags & l4m) 1948 return rte_flow_error_set(error, EINVAL, 1949 RTE_FLOW_ERROR_TYPE_ITEM, item, 1950 "L3 cannot follow an L4 layer."); 1951 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1952 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1953 return rte_flow_error_set(error, EINVAL, 1954 RTE_FLOW_ERROR_TYPE_ITEM, item, 1955 "L3 cannot follow an NVGRE layer."); 1956 if (!mask) 1957 mask = &rte_flow_item_ipv4_mask; 1958 else if (mask->hdr.next_proto_id != 0 && 1959 mask->hdr.next_proto_id != 0xff) 1960 return rte_flow_error_set(error, EINVAL, 1961 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1962 "partial mask is not supported" 1963 " for protocol"); 1964 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1965 acc_mask ? (const uint8_t *)acc_mask 1966 : (const uint8_t *)&nic_mask, 1967 sizeof(struct rte_flow_item_ipv4), 1968 range_accepted, error); 1969 if (ret < 0) 1970 return ret; 1971 return 0; 1972 } 1973 1974 /** 1975 * Validate IPV6 item. 1976 * 1977 * @param[in] item 1978 * Item specification. 1979 * @param[in] item_flags 1980 * Bit-fields that holds the items detected until now. 1981 * @param[in] last_item 1982 * Previous validated item in the pattern items. 1983 * @param[in] ether_type 1984 * Type in the ethernet layer header (including dot1q). 1985 * @param[in] acc_mask 1986 * Acceptable mask, if NULL default internal default mask 1987 * will be used to check whether item fields are supported. 1988 * @param[out] error 1989 * Pointer to error structure. 1990 * 1991 * @return 1992 * 0 on success, a negative errno value otherwise and rte_errno is set. 1993 */ 1994 int 1995 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1996 uint64_t item_flags, 1997 uint64_t last_item, 1998 uint16_t ether_type, 1999 const struct rte_flow_item_ipv6 *acc_mask, 2000 struct rte_flow_error *error) 2001 { 2002 const struct rte_flow_item_ipv6 *mask = item->mask; 2003 const struct rte_flow_item_ipv6 *spec = item->spec; 2004 const struct rte_flow_item_ipv6 nic_mask = { 2005 .hdr = { 2006 .src_addr = 2007 "\xff\xff\xff\xff\xff\xff\xff\xff" 2008 "\xff\xff\xff\xff\xff\xff\xff\xff", 2009 .dst_addr = 2010 "\xff\xff\xff\xff\xff\xff\xff\xff" 2011 "\xff\xff\xff\xff\xff\xff\xff\xff", 2012 .vtc_flow = RTE_BE32(0xffffffff), 2013 .proto = 0xff, 2014 }, 2015 }; 2016 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2017 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2018 MLX5_FLOW_LAYER_OUTER_L3; 2019 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2020 MLX5_FLOW_LAYER_OUTER_L4; 2021 int ret; 2022 uint8_t next_proto = 0xFF; 2023 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2024 MLX5_FLOW_LAYER_OUTER_VLAN | 2025 MLX5_FLOW_LAYER_INNER_VLAN); 2026 2027 if ((last_item & l2_vlan) && ether_type && 2028 ether_type != RTE_ETHER_TYPE_IPV6) 2029 return rte_flow_error_set(error, EINVAL, 2030 RTE_FLOW_ERROR_TYPE_ITEM, item, 2031 "IPv6 cannot follow L2/VLAN layer " 2032 "which ether type is not IPv6"); 2033 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2034 next_proto = spec->hdr.proto; 2035 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 2036 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2037 return rte_flow_error_set(error, EINVAL, 2038 RTE_FLOW_ERROR_TYPE_ITEM, 2039 item, 2040 "multiple tunnel " 2041 "not supported"); 2042 } 2043 if (next_proto == IPPROTO_HOPOPTS || 2044 next_proto == IPPROTO_ROUTING || 2045 next_proto == IPPROTO_FRAGMENT || 2046 next_proto == IPPROTO_ESP || 2047 next_proto == IPPROTO_AH || 2048 next_proto == IPPROTO_DSTOPTS) 2049 return rte_flow_error_set(error, EINVAL, 2050 RTE_FLOW_ERROR_TYPE_ITEM, item, 2051 "IPv6 proto (next header) should " 2052 "not be set as extension header"); 2053 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2054 return rte_flow_error_set(error, EINVAL, 2055 RTE_FLOW_ERROR_TYPE_ITEM, item, 2056 "wrong tunnel type - IPv4 specified " 2057 "but IPv6 item provided"); 2058 if (item_flags & l3m) 2059 return rte_flow_error_set(error, ENOTSUP, 2060 RTE_FLOW_ERROR_TYPE_ITEM, item, 2061 "multiple L3 layers not supported"); 2062 else if (item_flags & l4m) 2063 return rte_flow_error_set(error, EINVAL, 2064 RTE_FLOW_ERROR_TYPE_ITEM, item, 2065 "L3 cannot follow an L4 layer."); 2066 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2067 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2068 return rte_flow_error_set(error, EINVAL, 2069 RTE_FLOW_ERROR_TYPE_ITEM, item, 2070 "L3 cannot follow an NVGRE layer."); 2071 if (!mask) 2072 mask = &rte_flow_item_ipv6_mask; 2073 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2074 acc_mask ? (const uint8_t *)acc_mask 2075 : (const uint8_t *)&nic_mask, 2076 sizeof(struct rte_flow_item_ipv6), 2077 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2078 if (ret < 0) 2079 return ret; 2080 return 0; 2081 } 2082 2083 /** 2084 * Validate UDP item. 2085 * 2086 * @param[in] item 2087 * Item specification. 2088 * @param[in] item_flags 2089 * Bit-fields that holds the items detected until now. 2090 * @param[in] target_protocol 2091 * The next protocol in the previous item. 2092 * @param[in] flow_mask 2093 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2094 * @param[out] error 2095 * Pointer to error structure. 2096 * 2097 * @return 2098 * 0 on success, a negative errno value otherwise and rte_errno is set. 2099 */ 2100 int 2101 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2102 uint64_t item_flags, 2103 uint8_t target_protocol, 2104 struct rte_flow_error *error) 2105 { 2106 const struct rte_flow_item_udp *mask = item->mask; 2107 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2108 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2109 MLX5_FLOW_LAYER_OUTER_L3; 2110 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2111 MLX5_FLOW_LAYER_OUTER_L4; 2112 int ret; 2113 2114 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2115 return rte_flow_error_set(error, EINVAL, 2116 RTE_FLOW_ERROR_TYPE_ITEM, item, 2117 "protocol filtering not compatible" 2118 " with UDP layer"); 2119 if (!(item_flags & l3m)) 2120 return rte_flow_error_set(error, EINVAL, 2121 RTE_FLOW_ERROR_TYPE_ITEM, item, 2122 "L3 is mandatory to filter on L4"); 2123 if (item_flags & l4m) 2124 return rte_flow_error_set(error, EINVAL, 2125 RTE_FLOW_ERROR_TYPE_ITEM, item, 2126 "multiple L4 layers not supported"); 2127 if (!mask) 2128 mask = &rte_flow_item_udp_mask; 2129 ret = mlx5_flow_item_acceptable 2130 (item, (const uint8_t *)mask, 2131 (const uint8_t *)&rte_flow_item_udp_mask, 2132 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2133 error); 2134 if (ret < 0) 2135 return ret; 2136 return 0; 2137 } 2138 2139 /** 2140 * Validate TCP item. 2141 * 2142 * @param[in] item 2143 * Item specification. 2144 * @param[in] item_flags 2145 * Bit-fields that holds the items detected until now. 2146 * @param[in] target_protocol 2147 * The next protocol in the previous item. 2148 * @param[out] error 2149 * Pointer to error structure. 2150 * 2151 * @return 2152 * 0 on success, a negative errno value otherwise and rte_errno is set. 2153 */ 2154 int 2155 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2156 uint64_t item_flags, 2157 uint8_t target_protocol, 2158 const struct rte_flow_item_tcp *flow_mask, 2159 struct rte_flow_error *error) 2160 { 2161 const struct rte_flow_item_tcp *mask = item->mask; 2162 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2163 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2164 MLX5_FLOW_LAYER_OUTER_L3; 2165 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2166 MLX5_FLOW_LAYER_OUTER_L4; 2167 int ret; 2168 2169 MLX5_ASSERT(flow_mask); 2170 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2171 return rte_flow_error_set(error, EINVAL, 2172 RTE_FLOW_ERROR_TYPE_ITEM, item, 2173 "protocol filtering not compatible" 2174 " with TCP layer"); 2175 if (!(item_flags & l3m)) 2176 return rte_flow_error_set(error, EINVAL, 2177 RTE_FLOW_ERROR_TYPE_ITEM, item, 2178 "L3 is mandatory to filter on L4"); 2179 if (item_flags & l4m) 2180 return rte_flow_error_set(error, EINVAL, 2181 RTE_FLOW_ERROR_TYPE_ITEM, item, 2182 "multiple L4 layers not supported"); 2183 if (!mask) 2184 mask = &rte_flow_item_tcp_mask; 2185 ret = mlx5_flow_item_acceptable 2186 (item, (const uint8_t *)mask, 2187 (const uint8_t *)flow_mask, 2188 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2189 error); 2190 if (ret < 0) 2191 return ret; 2192 return 0; 2193 } 2194 2195 /** 2196 * Validate VXLAN item. 2197 * 2198 * @param[in] item 2199 * Item specification. 2200 * @param[in] item_flags 2201 * Bit-fields that holds the items detected until now. 2202 * @param[in] target_protocol 2203 * The next protocol in the previous item. 2204 * @param[out] error 2205 * Pointer to error structure. 2206 * 2207 * @return 2208 * 0 on success, a negative errno value otherwise and rte_errno is set. 2209 */ 2210 int 2211 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 2212 uint64_t item_flags, 2213 struct rte_flow_error *error) 2214 { 2215 const struct rte_flow_item_vxlan *spec = item->spec; 2216 const struct rte_flow_item_vxlan *mask = item->mask; 2217 int ret; 2218 union vni { 2219 uint32_t vlan_id; 2220 uint8_t vni[4]; 2221 } id = { .vlan_id = 0, }; 2222 2223 2224 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2225 return rte_flow_error_set(error, ENOTSUP, 2226 RTE_FLOW_ERROR_TYPE_ITEM, item, 2227 "multiple tunnel layers not" 2228 " supported"); 2229 /* 2230 * Verify only UDPv4 is present as defined in 2231 * https://tools.ietf.org/html/rfc7348 2232 */ 2233 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2234 return rte_flow_error_set(error, EINVAL, 2235 RTE_FLOW_ERROR_TYPE_ITEM, item, 2236 "no outer UDP layer found"); 2237 if (!mask) 2238 mask = &rte_flow_item_vxlan_mask; 2239 ret = mlx5_flow_item_acceptable 2240 (item, (const uint8_t *)mask, 2241 (const uint8_t *)&rte_flow_item_vxlan_mask, 2242 sizeof(struct rte_flow_item_vxlan), 2243 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2244 if (ret < 0) 2245 return ret; 2246 if (spec) { 2247 memcpy(&id.vni[1], spec->vni, 3); 2248 memcpy(&id.vni[1], mask->vni, 3); 2249 } 2250 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2251 return rte_flow_error_set(error, ENOTSUP, 2252 RTE_FLOW_ERROR_TYPE_ITEM, item, 2253 "VXLAN tunnel must be fully defined"); 2254 return 0; 2255 } 2256 2257 /** 2258 * Validate VXLAN_GPE item. 2259 * 2260 * @param[in] item 2261 * Item specification. 2262 * @param[in] item_flags 2263 * Bit-fields that holds the items detected until now. 2264 * @param[in] priv 2265 * Pointer to the private data structure. 2266 * @param[in] target_protocol 2267 * The next protocol in the previous item. 2268 * @param[out] error 2269 * Pointer to error structure. 2270 * 2271 * @return 2272 * 0 on success, a negative errno value otherwise and rte_errno is set. 2273 */ 2274 int 2275 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2276 uint64_t item_flags, 2277 struct rte_eth_dev *dev, 2278 struct rte_flow_error *error) 2279 { 2280 struct mlx5_priv *priv = dev->data->dev_private; 2281 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2282 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2283 int ret; 2284 union vni { 2285 uint32_t vlan_id; 2286 uint8_t vni[4]; 2287 } id = { .vlan_id = 0, }; 2288 2289 if (!priv->config.l3_vxlan_en) 2290 return rte_flow_error_set(error, ENOTSUP, 2291 RTE_FLOW_ERROR_TYPE_ITEM, item, 2292 "L3 VXLAN is not enabled by device" 2293 " parameter and/or not configured in" 2294 " firmware"); 2295 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2296 return rte_flow_error_set(error, ENOTSUP, 2297 RTE_FLOW_ERROR_TYPE_ITEM, item, 2298 "multiple tunnel layers not" 2299 " supported"); 2300 /* 2301 * Verify only UDPv4 is present as defined in 2302 * https://tools.ietf.org/html/rfc7348 2303 */ 2304 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2305 return rte_flow_error_set(error, EINVAL, 2306 RTE_FLOW_ERROR_TYPE_ITEM, item, 2307 "no outer UDP layer found"); 2308 if (!mask) 2309 mask = &rte_flow_item_vxlan_gpe_mask; 2310 ret = mlx5_flow_item_acceptable 2311 (item, (const uint8_t *)mask, 2312 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2313 sizeof(struct rte_flow_item_vxlan_gpe), 2314 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2315 if (ret < 0) 2316 return ret; 2317 if (spec) { 2318 if (spec->protocol) 2319 return rte_flow_error_set(error, ENOTSUP, 2320 RTE_FLOW_ERROR_TYPE_ITEM, 2321 item, 2322 "VxLAN-GPE protocol" 2323 " not supported"); 2324 memcpy(&id.vni[1], spec->vni, 3); 2325 memcpy(&id.vni[1], mask->vni, 3); 2326 } 2327 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2328 return rte_flow_error_set(error, ENOTSUP, 2329 RTE_FLOW_ERROR_TYPE_ITEM, item, 2330 "VXLAN-GPE tunnel must be fully" 2331 " defined"); 2332 return 0; 2333 } 2334 /** 2335 * Validate GRE Key item. 2336 * 2337 * @param[in] item 2338 * Item specification. 2339 * @param[in] item_flags 2340 * Bit flags to mark detected items. 2341 * @param[in] gre_item 2342 * Pointer to gre_item 2343 * @param[out] error 2344 * Pointer to error structure. 2345 * 2346 * @return 2347 * 0 on success, a negative errno value otherwise and rte_errno is set. 2348 */ 2349 int 2350 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2351 uint64_t item_flags, 2352 const struct rte_flow_item *gre_item, 2353 struct rte_flow_error *error) 2354 { 2355 const rte_be32_t *mask = item->mask; 2356 int ret = 0; 2357 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2358 const struct rte_flow_item_gre *gre_spec; 2359 const struct rte_flow_item_gre *gre_mask; 2360 2361 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2362 return rte_flow_error_set(error, ENOTSUP, 2363 RTE_FLOW_ERROR_TYPE_ITEM, item, 2364 "Multiple GRE key not support"); 2365 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2366 return rte_flow_error_set(error, ENOTSUP, 2367 RTE_FLOW_ERROR_TYPE_ITEM, item, 2368 "No preceding GRE header"); 2369 if (item_flags & MLX5_FLOW_LAYER_INNER) 2370 return rte_flow_error_set(error, ENOTSUP, 2371 RTE_FLOW_ERROR_TYPE_ITEM, item, 2372 "GRE key following a wrong item"); 2373 gre_mask = gre_item->mask; 2374 if (!gre_mask) 2375 gre_mask = &rte_flow_item_gre_mask; 2376 gre_spec = gre_item->spec; 2377 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2378 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2379 return rte_flow_error_set(error, EINVAL, 2380 RTE_FLOW_ERROR_TYPE_ITEM, item, 2381 "Key bit must be on"); 2382 2383 if (!mask) 2384 mask = &gre_key_default_mask; 2385 ret = mlx5_flow_item_acceptable 2386 (item, (const uint8_t *)mask, 2387 (const uint8_t *)&gre_key_default_mask, 2388 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2389 return ret; 2390 } 2391 2392 /** 2393 * Validate GRE item. 2394 * 2395 * @param[in] item 2396 * Item specification. 2397 * @param[in] item_flags 2398 * Bit flags to mark detected items. 2399 * @param[in] target_protocol 2400 * The next protocol in the previous item. 2401 * @param[out] error 2402 * Pointer to error structure. 2403 * 2404 * @return 2405 * 0 on success, a negative errno value otherwise and rte_errno is set. 2406 */ 2407 int 2408 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2409 uint64_t item_flags, 2410 uint8_t target_protocol, 2411 struct rte_flow_error *error) 2412 { 2413 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2414 const struct rte_flow_item_gre *mask = item->mask; 2415 int ret; 2416 const struct rte_flow_item_gre nic_mask = { 2417 .c_rsvd0_ver = RTE_BE16(0xB000), 2418 .protocol = RTE_BE16(UINT16_MAX), 2419 }; 2420 2421 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2422 return rte_flow_error_set(error, EINVAL, 2423 RTE_FLOW_ERROR_TYPE_ITEM, item, 2424 "protocol filtering not compatible" 2425 " with this GRE layer"); 2426 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2427 return rte_flow_error_set(error, ENOTSUP, 2428 RTE_FLOW_ERROR_TYPE_ITEM, item, 2429 "multiple tunnel layers not" 2430 " supported"); 2431 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2432 return rte_flow_error_set(error, ENOTSUP, 2433 RTE_FLOW_ERROR_TYPE_ITEM, item, 2434 "L3 Layer is missing"); 2435 if (!mask) 2436 mask = &rte_flow_item_gre_mask; 2437 ret = mlx5_flow_item_acceptable 2438 (item, (const uint8_t *)mask, 2439 (const uint8_t *)&nic_mask, 2440 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2441 error); 2442 if (ret < 0) 2443 return ret; 2444 #ifndef HAVE_MLX5DV_DR 2445 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2446 if (spec && (spec->protocol & mask->protocol)) 2447 return rte_flow_error_set(error, ENOTSUP, 2448 RTE_FLOW_ERROR_TYPE_ITEM, item, 2449 "without MPLS support the" 2450 " specification cannot be used for" 2451 " filtering"); 2452 #endif 2453 #endif 2454 return 0; 2455 } 2456 2457 /** 2458 * Validate Geneve item. 2459 * 2460 * @param[in] item 2461 * Item specification. 2462 * @param[in] itemFlags 2463 * Bit-fields that holds the items detected until now. 2464 * @param[in] enPriv 2465 * Pointer to the private data structure. 2466 * @param[out] error 2467 * Pointer to error structure. 2468 * 2469 * @return 2470 * 0 on success, a negative errno value otherwise and rte_errno is set. 2471 */ 2472 2473 int 2474 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2475 uint64_t item_flags, 2476 struct rte_eth_dev *dev, 2477 struct rte_flow_error *error) 2478 { 2479 struct mlx5_priv *priv = dev->data->dev_private; 2480 const struct rte_flow_item_geneve *spec = item->spec; 2481 const struct rte_flow_item_geneve *mask = item->mask; 2482 int ret; 2483 uint16_t gbhdr; 2484 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2485 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2486 const struct rte_flow_item_geneve nic_mask = { 2487 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2488 .vni = "\xff\xff\xff", 2489 .protocol = RTE_BE16(UINT16_MAX), 2490 }; 2491 2492 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2493 return rte_flow_error_set(error, ENOTSUP, 2494 RTE_FLOW_ERROR_TYPE_ITEM, item, 2495 "L3 Geneve is not enabled by device" 2496 " parameter and/or not configured in" 2497 " firmware"); 2498 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2499 return rte_flow_error_set(error, ENOTSUP, 2500 RTE_FLOW_ERROR_TYPE_ITEM, item, 2501 "multiple tunnel layers not" 2502 " supported"); 2503 /* 2504 * Verify only UDPv4 is present as defined in 2505 * https://tools.ietf.org/html/rfc7348 2506 */ 2507 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2508 return rte_flow_error_set(error, EINVAL, 2509 RTE_FLOW_ERROR_TYPE_ITEM, item, 2510 "no outer UDP layer found"); 2511 if (!mask) 2512 mask = &rte_flow_item_geneve_mask; 2513 ret = mlx5_flow_item_acceptable 2514 (item, (const uint8_t *)mask, 2515 (const uint8_t *)&nic_mask, 2516 sizeof(struct rte_flow_item_geneve), 2517 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2518 if (ret) 2519 return ret; 2520 if (spec) { 2521 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2522 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2523 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2524 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2525 return rte_flow_error_set(error, ENOTSUP, 2526 RTE_FLOW_ERROR_TYPE_ITEM, 2527 item, 2528 "Geneve protocol unsupported" 2529 " fields are being used"); 2530 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2531 return rte_flow_error_set 2532 (error, ENOTSUP, 2533 RTE_FLOW_ERROR_TYPE_ITEM, 2534 item, 2535 "Unsupported Geneve options length"); 2536 } 2537 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2538 return rte_flow_error_set 2539 (error, ENOTSUP, 2540 RTE_FLOW_ERROR_TYPE_ITEM, item, 2541 "Geneve tunnel must be fully defined"); 2542 return 0; 2543 } 2544 2545 /** 2546 * Validate MPLS item. 2547 * 2548 * @param[in] dev 2549 * Pointer to the rte_eth_dev structure. 2550 * @param[in] item 2551 * Item specification. 2552 * @param[in] item_flags 2553 * Bit-fields that holds the items detected until now. 2554 * @param[in] prev_layer 2555 * The protocol layer indicated in previous item. 2556 * @param[out] error 2557 * Pointer to error structure. 2558 * 2559 * @return 2560 * 0 on success, a negative errno value otherwise and rte_errno is set. 2561 */ 2562 int 2563 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2564 const struct rte_flow_item *item __rte_unused, 2565 uint64_t item_flags __rte_unused, 2566 uint64_t prev_layer __rte_unused, 2567 struct rte_flow_error *error) 2568 { 2569 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2570 const struct rte_flow_item_mpls *mask = item->mask; 2571 struct mlx5_priv *priv = dev->data->dev_private; 2572 int ret; 2573 2574 if (!priv->config.mpls_en) 2575 return rte_flow_error_set(error, ENOTSUP, 2576 RTE_FLOW_ERROR_TYPE_ITEM, item, 2577 "MPLS not supported or" 2578 " disabled in firmware" 2579 " configuration."); 2580 /* MPLS over IP, UDP, GRE is allowed */ 2581 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2582 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2583 MLX5_FLOW_LAYER_GRE))) 2584 return rte_flow_error_set(error, EINVAL, 2585 RTE_FLOW_ERROR_TYPE_ITEM, item, 2586 "protocol filtering not compatible" 2587 " with MPLS layer"); 2588 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2589 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2590 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2591 return rte_flow_error_set(error, ENOTSUP, 2592 RTE_FLOW_ERROR_TYPE_ITEM, item, 2593 "multiple tunnel layers not" 2594 " supported"); 2595 if (!mask) 2596 mask = &rte_flow_item_mpls_mask; 2597 ret = mlx5_flow_item_acceptable 2598 (item, (const uint8_t *)mask, 2599 (const uint8_t *)&rte_flow_item_mpls_mask, 2600 sizeof(struct rte_flow_item_mpls), 2601 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2602 if (ret < 0) 2603 return ret; 2604 return 0; 2605 #else 2606 return rte_flow_error_set(error, ENOTSUP, 2607 RTE_FLOW_ERROR_TYPE_ITEM, item, 2608 "MPLS is not supported by Verbs, please" 2609 " update."); 2610 #endif 2611 } 2612 2613 /** 2614 * Validate NVGRE item. 2615 * 2616 * @param[in] item 2617 * Item specification. 2618 * @param[in] item_flags 2619 * Bit flags to mark detected items. 2620 * @param[in] target_protocol 2621 * The next protocol in the previous item. 2622 * @param[out] error 2623 * Pointer to error structure. 2624 * 2625 * @return 2626 * 0 on success, a negative errno value otherwise and rte_errno is set. 2627 */ 2628 int 2629 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2630 uint64_t item_flags, 2631 uint8_t target_protocol, 2632 struct rte_flow_error *error) 2633 { 2634 const struct rte_flow_item_nvgre *mask = item->mask; 2635 int ret; 2636 2637 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2638 return rte_flow_error_set(error, EINVAL, 2639 RTE_FLOW_ERROR_TYPE_ITEM, item, 2640 "protocol filtering not compatible" 2641 " with this GRE layer"); 2642 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2643 return rte_flow_error_set(error, ENOTSUP, 2644 RTE_FLOW_ERROR_TYPE_ITEM, item, 2645 "multiple tunnel layers not" 2646 " supported"); 2647 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2648 return rte_flow_error_set(error, ENOTSUP, 2649 RTE_FLOW_ERROR_TYPE_ITEM, item, 2650 "L3 Layer is missing"); 2651 if (!mask) 2652 mask = &rte_flow_item_nvgre_mask; 2653 ret = mlx5_flow_item_acceptable 2654 (item, (const uint8_t *)mask, 2655 (const uint8_t *)&rte_flow_item_nvgre_mask, 2656 sizeof(struct rte_flow_item_nvgre), 2657 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2658 if (ret < 0) 2659 return ret; 2660 return 0; 2661 } 2662 2663 /** 2664 * Validate eCPRI item. 2665 * 2666 * @param[in] item 2667 * Item specification. 2668 * @param[in] item_flags 2669 * Bit-fields that holds the items detected until now. 2670 * @param[in] last_item 2671 * Previous validated item in the pattern items. 2672 * @param[in] ether_type 2673 * Type in the ethernet layer header (including dot1q). 2674 * @param[in] acc_mask 2675 * Acceptable mask, if NULL default internal default mask 2676 * will be used to check whether item fields are supported. 2677 * @param[out] error 2678 * Pointer to error structure. 2679 * 2680 * @return 2681 * 0 on success, a negative errno value otherwise and rte_errno is set. 2682 */ 2683 int 2684 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 2685 uint64_t item_flags, 2686 uint64_t last_item, 2687 uint16_t ether_type, 2688 const struct rte_flow_item_ecpri *acc_mask, 2689 struct rte_flow_error *error) 2690 { 2691 const struct rte_flow_item_ecpri *mask = item->mask; 2692 const struct rte_flow_item_ecpri nic_mask = { 2693 .hdr = { 2694 .common = { 2695 .u32 = 2696 RTE_BE32(((const struct rte_ecpri_common_hdr) { 2697 .type = 0xFF, 2698 }).u32), 2699 }, 2700 .dummy[0] = 0xFFFFFFFF, 2701 }, 2702 }; 2703 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 2704 MLX5_FLOW_LAYER_OUTER_VLAN); 2705 struct rte_flow_item_ecpri mask_lo; 2706 2707 if ((last_item & outer_l2_vlan) && ether_type && 2708 ether_type != RTE_ETHER_TYPE_ECPRI) 2709 return rte_flow_error_set(error, EINVAL, 2710 RTE_FLOW_ERROR_TYPE_ITEM, item, 2711 "eCPRI cannot follow L2/VLAN layer " 2712 "which ether type is not 0xAEFE."); 2713 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2714 return rte_flow_error_set(error, EINVAL, 2715 RTE_FLOW_ERROR_TYPE_ITEM, item, 2716 "eCPRI with tunnel is not supported " 2717 "right now."); 2718 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 2719 return rte_flow_error_set(error, ENOTSUP, 2720 RTE_FLOW_ERROR_TYPE_ITEM, item, 2721 "multiple L3 layers not supported"); 2722 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 2723 return rte_flow_error_set(error, EINVAL, 2724 RTE_FLOW_ERROR_TYPE_ITEM, item, 2725 "eCPRI cannot follow a TCP layer."); 2726 /* In specification, eCPRI could be over UDP layer. */ 2727 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 2728 return rte_flow_error_set(error, EINVAL, 2729 RTE_FLOW_ERROR_TYPE_ITEM, item, 2730 "eCPRI over UDP layer is not yet " 2731 "supported right now."); 2732 /* Mask for type field in common header could be zero. */ 2733 if (!mask) 2734 mask = &rte_flow_item_ecpri_mask; 2735 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 2736 /* Input mask is in big-endian format. */ 2737 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 2738 return rte_flow_error_set(error, EINVAL, 2739 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2740 "partial mask is not supported " 2741 "for protocol"); 2742 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 2743 return rte_flow_error_set(error, EINVAL, 2744 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2745 "message header mask must be after " 2746 "a type mask"); 2747 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2748 acc_mask ? (const uint8_t *)acc_mask 2749 : (const uint8_t *)&nic_mask, 2750 sizeof(struct rte_flow_item_ecpri), 2751 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2752 } 2753 2754 /* Allocate unique ID for the split Q/RSS subflows. */ 2755 static uint32_t 2756 flow_qrss_get_id(struct rte_eth_dev *dev) 2757 { 2758 struct mlx5_priv *priv = dev->data->dev_private; 2759 uint32_t qrss_id, ret; 2760 2761 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2762 if (ret) 2763 return 0; 2764 MLX5_ASSERT(qrss_id); 2765 return qrss_id; 2766 } 2767 2768 /* Free unique ID for the split Q/RSS subflows. */ 2769 static void 2770 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2771 { 2772 struct mlx5_priv *priv = dev->data->dev_private; 2773 2774 if (qrss_id) 2775 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2776 } 2777 2778 /** 2779 * Release resource related QUEUE/RSS action split. 2780 * 2781 * @param dev 2782 * Pointer to Ethernet device. 2783 * @param flow 2784 * Flow to release id's from. 2785 */ 2786 static void 2787 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2788 struct rte_flow *flow) 2789 { 2790 struct mlx5_priv *priv = dev->data->dev_private; 2791 uint32_t handle_idx; 2792 struct mlx5_flow_handle *dev_handle; 2793 2794 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2795 handle_idx, dev_handle, next) 2796 if (dev_handle->split_flow_id) 2797 flow_qrss_free_id(dev, dev_handle->split_flow_id); 2798 } 2799 2800 static int 2801 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2802 const struct rte_flow_attr *attr __rte_unused, 2803 const struct rte_flow_item items[] __rte_unused, 2804 const struct rte_flow_action actions[] __rte_unused, 2805 bool external __rte_unused, 2806 int hairpin __rte_unused, 2807 struct rte_flow_error *error) 2808 { 2809 return rte_flow_error_set(error, ENOTSUP, 2810 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2811 } 2812 2813 static struct mlx5_flow * 2814 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2815 const struct rte_flow_attr *attr __rte_unused, 2816 const struct rte_flow_item items[] __rte_unused, 2817 const struct rte_flow_action actions[] __rte_unused, 2818 struct rte_flow_error *error) 2819 { 2820 rte_flow_error_set(error, ENOTSUP, 2821 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2822 return NULL; 2823 } 2824 2825 static int 2826 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2827 struct mlx5_flow *dev_flow __rte_unused, 2828 const struct rte_flow_attr *attr __rte_unused, 2829 const struct rte_flow_item items[] __rte_unused, 2830 const struct rte_flow_action actions[] __rte_unused, 2831 struct rte_flow_error *error) 2832 { 2833 return rte_flow_error_set(error, ENOTSUP, 2834 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2835 } 2836 2837 static int 2838 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2839 struct rte_flow *flow __rte_unused, 2840 struct rte_flow_error *error) 2841 { 2842 return rte_flow_error_set(error, ENOTSUP, 2843 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2844 } 2845 2846 static void 2847 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2848 struct rte_flow *flow __rte_unused) 2849 { 2850 } 2851 2852 static void 2853 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2854 struct rte_flow *flow __rte_unused) 2855 { 2856 } 2857 2858 static int 2859 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2860 struct rte_flow *flow __rte_unused, 2861 const struct rte_flow_action *actions __rte_unused, 2862 void *data __rte_unused, 2863 struct rte_flow_error *error) 2864 { 2865 return rte_flow_error_set(error, ENOTSUP, 2866 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2867 } 2868 2869 /* Void driver to protect from null pointer reference. */ 2870 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2871 .validate = flow_null_validate, 2872 .prepare = flow_null_prepare, 2873 .translate = flow_null_translate, 2874 .apply = flow_null_apply, 2875 .remove = flow_null_remove, 2876 .destroy = flow_null_destroy, 2877 .query = flow_null_query, 2878 }; 2879 2880 /** 2881 * Select flow driver type according to flow attributes and device 2882 * configuration. 2883 * 2884 * @param[in] dev 2885 * Pointer to the dev structure. 2886 * @param[in] attr 2887 * Pointer to the flow attributes. 2888 * 2889 * @return 2890 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2891 */ 2892 static enum mlx5_flow_drv_type 2893 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2894 { 2895 struct mlx5_priv *priv = dev->data->dev_private; 2896 /* The OS can determine first a specific flow type (DV, VERBS) */ 2897 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 2898 2899 if (type != MLX5_FLOW_TYPE_MAX) 2900 return type; 2901 /* If no OS specific type - continue with DV/VERBS selection */ 2902 if (attr->transfer && priv->config.dv_esw_en) 2903 type = MLX5_FLOW_TYPE_DV; 2904 if (!attr->transfer) 2905 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2906 MLX5_FLOW_TYPE_VERBS; 2907 return type; 2908 } 2909 2910 #define flow_get_drv_ops(type) flow_drv_ops[type] 2911 2912 /** 2913 * Flow driver validation API. This abstracts calling driver specific functions. 2914 * The type of flow driver is determined according to flow attributes. 2915 * 2916 * @param[in] dev 2917 * Pointer to the dev structure. 2918 * @param[in] attr 2919 * Pointer to the flow attributes. 2920 * @param[in] items 2921 * Pointer to the list of items. 2922 * @param[in] actions 2923 * Pointer to the list of actions. 2924 * @param[in] external 2925 * This flow rule is created by request external to PMD. 2926 * @param[in] hairpin 2927 * Number of hairpin TX actions, 0 means classic flow. 2928 * @param[out] error 2929 * Pointer to the error structure. 2930 * 2931 * @return 2932 * 0 on success, a negative errno value otherwise and rte_errno is set. 2933 */ 2934 static inline int 2935 flow_drv_validate(struct rte_eth_dev *dev, 2936 const struct rte_flow_attr *attr, 2937 const struct rte_flow_item items[], 2938 const struct rte_flow_action actions[], 2939 bool external, int hairpin, struct rte_flow_error *error) 2940 { 2941 const struct mlx5_flow_driver_ops *fops; 2942 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2943 2944 fops = flow_get_drv_ops(type); 2945 return fops->validate(dev, attr, items, actions, external, 2946 hairpin, error); 2947 } 2948 2949 /** 2950 * Flow driver preparation API. This abstracts calling driver specific 2951 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2952 * calculates the size of memory required for device flow, allocates the memory, 2953 * initializes the device flow and returns the pointer. 2954 * 2955 * @note 2956 * This function initializes device flow structure such as dv or verbs in 2957 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2958 * rest. For example, adding returning device flow to flow->dev_flow list and 2959 * setting backward reference to the flow should be done out of this function. 2960 * layers field is not filled either. 2961 * 2962 * @param[in] dev 2963 * Pointer to the dev structure. 2964 * @param[in] attr 2965 * Pointer to the flow attributes. 2966 * @param[in] items 2967 * Pointer to the list of items. 2968 * @param[in] actions 2969 * Pointer to the list of actions. 2970 * @param[in] flow_idx 2971 * This memory pool index to the flow. 2972 * @param[out] error 2973 * Pointer to the error structure. 2974 * 2975 * @return 2976 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2977 */ 2978 static inline struct mlx5_flow * 2979 flow_drv_prepare(struct rte_eth_dev *dev, 2980 const struct rte_flow *flow, 2981 const struct rte_flow_attr *attr, 2982 const struct rte_flow_item items[], 2983 const struct rte_flow_action actions[], 2984 uint32_t flow_idx, 2985 struct rte_flow_error *error) 2986 { 2987 const struct mlx5_flow_driver_ops *fops; 2988 enum mlx5_flow_drv_type type = flow->drv_type; 2989 struct mlx5_flow *mlx5_flow = NULL; 2990 2991 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2992 fops = flow_get_drv_ops(type); 2993 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 2994 if (mlx5_flow) 2995 mlx5_flow->flow_idx = flow_idx; 2996 return mlx5_flow; 2997 } 2998 2999 /** 3000 * Flow driver translation API. This abstracts calling driver specific 3001 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3002 * translates a generic flow into a driver flow. flow_drv_prepare() must 3003 * precede. 3004 * 3005 * @note 3006 * dev_flow->layers could be filled as a result of parsing during translation 3007 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3008 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3009 * flow->actions could be overwritten even though all the expanded dev_flows 3010 * have the same actions. 3011 * 3012 * @param[in] dev 3013 * Pointer to the rte dev structure. 3014 * @param[in, out] dev_flow 3015 * Pointer to the mlx5 flow. 3016 * @param[in] attr 3017 * Pointer to the flow attributes. 3018 * @param[in] items 3019 * Pointer to the list of items. 3020 * @param[in] actions 3021 * Pointer to the list of actions. 3022 * @param[out] error 3023 * Pointer to the error structure. 3024 * 3025 * @return 3026 * 0 on success, a negative errno value otherwise and rte_errno is set. 3027 */ 3028 static inline int 3029 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3030 const struct rte_flow_attr *attr, 3031 const struct rte_flow_item items[], 3032 const struct rte_flow_action actions[], 3033 struct rte_flow_error *error) 3034 { 3035 const struct mlx5_flow_driver_ops *fops; 3036 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3037 3038 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3039 fops = flow_get_drv_ops(type); 3040 return fops->translate(dev, dev_flow, attr, items, actions, error); 3041 } 3042 3043 /** 3044 * Flow driver apply API. This abstracts calling driver specific functions. 3045 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3046 * translated driver flows on to device. flow_drv_translate() must precede. 3047 * 3048 * @param[in] dev 3049 * Pointer to Ethernet device structure. 3050 * @param[in, out] flow 3051 * Pointer to flow structure. 3052 * @param[out] error 3053 * Pointer to error structure. 3054 * 3055 * @return 3056 * 0 on success, a negative errno value otherwise and rte_errno is set. 3057 */ 3058 static inline int 3059 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3060 struct rte_flow_error *error) 3061 { 3062 const struct mlx5_flow_driver_ops *fops; 3063 enum mlx5_flow_drv_type type = flow->drv_type; 3064 3065 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3066 fops = flow_get_drv_ops(type); 3067 return fops->apply(dev, flow, error); 3068 } 3069 3070 /** 3071 * Flow driver remove API. This abstracts calling driver specific functions. 3072 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3073 * on device. All the resources of the flow should be freed by calling 3074 * flow_drv_destroy(). 3075 * 3076 * @param[in] dev 3077 * Pointer to Ethernet device. 3078 * @param[in, out] flow 3079 * Pointer to flow structure. 3080 */ 3081 static inline void 3082 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 3083 { 3084 const struct mlx5_flow_driver_ops *fops; 3085 enum mlx5_flow_drv_type type = flow->drv_type; 3086 3087 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3088 fops = flow_get_drv_ops(type); 3089 fops->remove(dev, flow); 3090 } 3091 3092 /** 3093 * Flow driver destroy API. This abstracts calling driver specific functions. 3094 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3095 * on device and releases resources of the flow. 3096 * 3097 * @param[in] dev 3098 * Pointer to Ethernet device. 3099 * @param[in, out] flow 3100 * Pointer to flow structure. 3101 */ 3102 static inline void 3103 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3104 { 3105 const struct mlx5_flow_driver_ops *fops; 3106 enum mlx5_flow_drv_type type = flow->drv_type; 3107 3108 flow_mreg_split_qrss_release(dev, flow); 3109 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3110 fops = flow_get_drv_ops(type); 3111 fops->destroy(dev, flow); 3112 } 3113 3114 /** 3115 * Get RSS action from the action list. 3116 * 3117 * @param[in] actions 3118 * Pointer to the list of actions. 3119 * 3120 * @return 3121 * Pointer to the RSS action if exist, else return NULL. 3122 */ 3123 static const struct rte_flow_action_rss* 3124 flow_get_rss_action(const struct rte_flow_action actions[]) 3125 { 3126 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3127 switch (actions->type) { 3128 case RTE_FLOW_ACTION_TYPE_RSS: 3129 return (const struct rte_flow_action_rss *) 3130 actions->conf; 3131 default: 3132 break; 3133 } 3134 } 3135 return NULL; 3136 } 3137 3138 /* maps shared action to translated non shared in some actions array */ 3139 struct mlx5_translated_shared_action { 3140 struct rte_flow_shared_action *action; /**< Shared action */ 3141 int index; /**< Index in related array of rte_flow_action */ 3142 }; 3143 3144 /** 3145 * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related 3146 * non shared action if translation possible. 3147 * This functionality used to run same execution path for both shared & non 3148 * shared actions on flow create. All necessary preparations for shared 3149 * action handling should be preformed on *shared* actions list returned 3150 * from this call. 3151 * 3152 * @param[in] actions 3153 * List of actions to translate. 3154 * @param[out] shared 3155 * List to store translated shared actions. 3156 * @param[in, out] shared_n 3157 * Size of *shared* array. On return should be updated with number of shared 3158 * actions retrieved from the *actions* list. 3159 * @param[out] translated_actions 3160 * List of actions where all shared actions were translated to non shared 3161 * if possible. NULL if no translation took place. 3162 * @param[out] error 3163 * Pointer to the error structure. 3164 * 3165 * @return 3166 * 0 on success, a negative errno value otherwise and rte_errno is set. 3167 */ 3168 static int 3169 flow_shared_actions_translate(const struct rte_flow_action actions[], 3170 struct mlx5_translated_shared_action *shared, 3171 int *shared_n, 3172 struct rte_flow_action **translated_actions, 3173 struct rte_flow_error *error) 3174 { 3175 struct rte_flow_action *translated = NULL; 3176 size_t actions_size; 3177 int n; 3178 int copied_n = 0; 3179 struct mlx5_translated_shared_action *shared_end = NULL; 3180 3181 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3182 if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED) 3183 continue; 3184 if (copied_n == *shared_n) { 3185 return rte_flow_error_set 3186 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3187 NULL, "too many shared actions"); 3188 } 3189 rte_memcpy(&shared[copied_n].action, &actions[n].conf, 3190 sizeof(actions[n].conf)); 3191 shared[copied_n].index = n; 3192 copied_n++; 3193 } 3194 n++; 3195 *shared_n = copied_n; 3196 if (!copied_n) 3197 return 0; 3198 actions_size = sizeof(struct rte_flow_action) * n; 3199 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3200 if (!translated) { 3201 rte_errno = ENOMEM; 3202 return -ENOMEM; 3203 } 3204 memcpy(translated, actions, actions_size); 3205 for (shared_end = shared + copied_n; shared < shared_end; shared++) { 3206 const struct rte_flow_shared_action *shared_action; 3207 3208 shared_action = shared->action; 3209 switch (shared_action->type) { 3210 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 3211 translated[shared->index].type = 3212 RTE_FLOW_ACTION_TYPE_RSS; 3213 translated[shared->index].conf = 3214 &shared_action->rss.origin; 3215 break; 3216 default: 3217 mlx5_free(translated); 3218 return rte_flow_error_set 3219 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3220 NULL, "invalid shared action type"); 3221 } 3222 } 3223 *translated_actions = translated; 3224 return 0; 3225 } 3226 3227 /** 3228 * Get Shared RSS action from the action list. 3229 * 3230 * @param[in] shared 3231 * Pointer to the list of actions. 3232 * @param[in] shared_n 3233 * Actions list length. 3234 * 3235 * @return 3236 * Pointer to the MLX5 RSS action if exists, otherwise return NULL. 3237 */ 3238 static struct mlx5_shared_action_rss * 3239 flow_get_shared_rss_action(struct mlx5_translated_shared_action *shared, 3240 int shared_n) 3241 { 3242 struct mlx5_translated_shared_action *shared_end; 3243 3244 for (shared_end = shared + shared_n; shared < shared_end; shared++) { 3245 struct rte_flow_shared_action *shared_action; 3246 3247 shared_action = shared->action; 3248 switch (shared_action->type) { 3249 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 3250 __atomic_add_fetch(&shared_action->refcnt, 1, 3251 __ATOMIC_RELAXED); 3252 return &shared_action->rss; 3253 default: 3254 break; 3255 } 3256 } 3257 return NULL; 3258 } 3259 3260 struct rte_flow_shared_action * 3261 mlx5_flow_get_shared_rss(struct rte_flow *flow) 3262 { 3263 if (flow->shared_rss) 3264 return container_of(flow->shared_rss, 3265 struct rte_flow_shared_action, rss); 3266 else 3267 return NULL; 3268 } 3269 3270 static unsigned int 3271 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 3272 { 3273 const struct rte_flow_item *item; 3274 unsigned int has_vlan = 0; 3275 3276 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 3277 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 3278 has_vlan = 1; 3279 break; 3280 } 3281 } 3282 if (has_vlan) 3283 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 3284 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 3285 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3286 MLX5_EXPANSION_ROOT_OUTER; 3287 } 3288 3289 /** 3290 * Get layer flags from the prefix flow. 3291 * 3292 * Some flows may be split to several subflows, the prefix subflow gets the 3293 * match items and the suffix sub flow gets the actions. 3294 * Some actions need the user defined match item flags to get the detail for 3295 * the action. 3296 * This function helps the suffix flow to get the item layer flags from prefix 3297 * subflow. 3298 * 3299 * @param[in] dev_flow 3300 * Pointer the created preifx subflow. 3301 * 3302 * @return 3303 * The layers get from prefix subflow. 3304 */ 3305 static inline uint64_t 3306 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 3307 { 3308 uint64_t layers = 0; 3309 3310 /* 3311 * Layers bits could be localization, but usually the compiler will 3312 * help to do the optimization work for source code. 3313 * If no decap actions, use the layers directly. 3314 */ 3315 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 3316 return dev_flow->handle->layers; 3317 /* Convert L3 layers with decap action. */ 3318 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 3319 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3320 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 3321 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3322 /* Convert L4 layers with decap action. */ 3323 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 3324 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 3325 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 3326 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 3327 return layers; 3328 } 3329 3330 /** 3331 * Get metadata split action information. 3332 * 3333 * @param[in] actions 3334 * Pointer to the list of actions. 3335 * @param[out] qrss 3336 * Pointer to the return pointer. 3337 * @param[out] qrss_type 3338 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 3339 * if no QUEUE/RSS is found. 3340 * @param[out] encap_idx 3341 * Pointer to the index of the encap action if exists, otherwise the last 3342 * action index. 3343 * 3344 * @return 3345 * Total number of actions. 3346 */ 3347 static int 3348 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 3349 const struct rte_flow_action **qrss, 3350 int *encap_idx) 3351 { 3352 const struct rte_flow_action_raw_encap *raw_encap; 3353 int actions_n = 0; 3354 int raw_decap_idx = -1; 3355 3356 *encap_idx = -1; 3357 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3358 switch (actions->type) { 3359 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3360 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3361 *encap_idx = actions_n; 3362 break; 3363 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3364 raw_decap_idx = actions_n; 3365 break; 3366 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3367 raw_encap = actions->conf; 3368 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3369 *encap_idx = raw_decap_idx != -1 ? 3370 raw_decap_idx : actions_n; 3371 break; 3372 case RTE_FLOW_ACTION_TYPE_QUEUE: 3373 case RTE_FLOW_ACTION_TYPE_RSS: 3374 *qrss = actions; 3375 break; 3376 default: 3377 break; 3378 } 3379 actions_n++; 3380 } 3381 if (*encap_idx == -1) 3382 *encap_idx = actions_n; 3383 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3384 return actions_n + 1; 3385 } 3386 3387 /** 3388 * Check meter action from the action list. 3389 * 3390 * @param[in] actions 3391 * Pointer to the list of actions. 3392 * @param[out] mtr 3393 * Pointer to the meter exist flag. 3394 * 3395 * @return 3396 * Total number of actions. 3397 */ 3398 static int 3399 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 3400 { 3401 int actions_n = 0; 3402 3403 MLX5_ASSERT(mtr); 3404 *mtr = 0; 3405 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3406 switch (actions->type) { 3407 case RTE_FLOW_ACTION_TYPE_METER: 3408 *mtr = 1; 3409 break; 3410 default: 3411 break; 3412 } 3413 actions_n++; 3414 } 3415 /* Count RTE_FLOW_ACTION_TYPE_END. */ 3416 return actions_n + 1; 3417 } 3418 3419 /** 3420 * Check if the flow should be split due to hairpin. 3421 * The reason for the split is that in current HW we can't 3422 * support encap and push-vlan on Rx, so if a flow contains 3423 * these actions we move it to Tx. 3424 * 3425 * @param dev 3426 * Pointer to Ethernet device. 3427 * @param[in] attr 3428 * Flow rule attributes. 3429 * @param[in] actions 3430 * Associated actions (list terminated by the END action). 3431 * 3432 * @return 3433 * > 0 the number of actions and the flow should be split, 3434 * 0 when no split required. 3435 */ 3436 static int 3437 flow_check_hairpin_split(struct rte_eth_dev *dev, 3438 const struct rte_flow_attr *attr, 3439 const struct rte_flow_action actions[]) 3440 { 3441 int queue_action = 0; 3442 int action_n = 0; 3443 int split = 0; 3444 const struct rte_flow_action_queue *queue; 3445 const struct rte_flow_action_rss *rss; 3446 const struct rte_flow_action_raw_encap *raw_encap; 3447 3448 if (!attr->ingress) 3449 return 0; 3450 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3451 switch (actions->type) { 3452 case RTE_FLOW_ACTION_TYPE_QUEUE: 3453 queue = actions->conf; 3454 if (queue == NULL) 3455 return 0; 3456 if (mlx5_rxq_get_type(dev, queue->index) != 3457 MLX5_RXQ_TYPE_HAIRPIN) 3458 return 0; 3459 queue_action = 1; 3460 action_n++; 3461 break; 3462 case RTE_FLOW_ACTION_TYPE_RSS: 3463 rss = actions->conf; 3464 if (rss == NULL || rss->queue_num == 0) 3465 return 0; 3466 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 3467 MLX5_RXQ_TYPE_HAIRPIN) 3468 return 0; 3469 queue_action = 1; 3470 action_n++; 3471 break; 3472 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3473 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3474 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3475 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3476 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3477 split++; 3478 action_n++; 3479 break; 3480 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3481 raw_encap = actions->conf; 3482 if (raw_encap->size > 3483 (sizeof(struct rte_flow_item_eth) + 3484 sizeof(struct rte_flow_item_ipv4))) 3485 split++; 3486 action_n++; 3487 break; 3488 default: 3489 action_n++; 3490 break; 3491 } 3492 } 3493 if (split && queue_action) 3494 return action_n; 3495 return 0; 3496 } 3497 3498 /* Declare flow create/destroy prototype in advance. */ 3499 static uint32_t 3500 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 3501 const struct rte_flow_attr *attr, 3502 const struct rte_flow_item items[], 3503 const struct rte_flow_action actions[], 3504 bool external, struct rte_flow_error *error); 3505 3506 static void 3507 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 3508 uint32_t flow_idx); 3509 3510 /** 3511 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3512 * 3513 * As mark_id is unique, if there's already a registered flow for the mark_id, 3514 * return by increasing the reference counter of the resource. Otherwise, create 3515 * the resource (mcp_res) and flow. 3516 * 3517 * Flow looks like, 3518 * - If ingress port is ANY and reg_c[1] is mark_id, 3519 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3520 * 3521 * For default flow (zero mark_id), flow is like, 3522 * - If ingress port is ANY, 3523 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 3524 * 3525 * @param dev 3526 * Pointer to Ethernet device. 3527 * @param mark_id 3528 * ID of MARK action, zero means default flow for META. 3529 * @param[out] error 3530 * Perform verbose error reporting if not NULL. 3531 * 3532 * @return 3533 * Associated resource on success, NULL otherwise and rte_errno is set. 3534 */ 3535 static struct mlx5_flow_mreg_copy_resource * 3536 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 3537 struct rte_flow_error *error) 3538 { 3539 struct mlx5_priv *priv = dev->data->dev_private; 3540 struct rte_flow_attr attr = { 3541 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3542 .ingress = 1, 3543 }; 3544 struct mlx5_rte_flow_item_tag tag_spec = { 3545 .data = mark_id, 3546 }; 3547 struct rte_flow_item items[] = { 3548 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 3549 }; 3550 struct rte_flow_action_mark ftag = { 3551 .id = mark_id, 3552 }; 3553 struct mlx5_flow_action_copy_mreg cp_mreg = { 3554 .dst = REG_B, 3555 .src = REG_NON, 3556 }; 3557 struct rte_flow_action_jump jump = { 3558 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3559 }; 3560 struct rte_flow_action actions[] = { 3561 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 3562 }; 3563 struct mlx5_flow_mreg_copy_resource *mcp_res; 3564 uint32_t idx = 0; 3565 int ret; 3566 3567 /* Fill the register fileds in the flow. */ 3568 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3569 if (ret < 0) 3570 return NULL; 3571 tag_spec.id = ret; 3572 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3573 if (ret < 0) 3574 return NULL; 3575 cp_mreg.src = ret; 3576 /* Check if already registered. */ 3577 MLX5_ASSERT(priv->mreg_cp_tbl); 3578 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 3579 if (mcp_res) { 3580 /* For non-default rule. */ 3581 if (mark_id != MLX5_DEFAULT_COPY_ID) 3582 mcp_res->refcnt++; 3583 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 3584 mcp_res->refcnt == 1); 3585 return mcp_res; 3586 } 3587 /* Provide the full width of FLAG specific value. */ 3588 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3589 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3590 /* Build a new flow. */ 3591 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3592 items[0] = (struct rte_flow_item){ 3593 .type = (enum rte_flow_item_type) 3594 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3595 .spec = &tag_spec, 3596 }; 3597 items[1] = (struct rte_flow_item){ 3598 .type = RTE_FLOW_ITEM_TYPE_END, 3599 }; 3600 actions[0] = (struct rte_flow_action){ 3601 .type = (enum rte_flow_action_type) 3602 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3603 .conf = &ftag, 3604 }; 3605 actions[1] = (struct rte_flow_action){ 3606 .type = (enum rte_flow_action_type) 3607 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3608 .conf = &cp_mreg, 3609 }; 3610 actions[2] = (struct rte_flow_action){ 3611 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3612 .conf = &jump, 3613 }; 3614 actions[3] = (struct rte_flow_action){ 3615 .type = RTE_FLOW_ACTION_TYPE_END, 3616 }; 3617 } else { 3618 /* Default rule, wildcard match. */ 3619 attr.priority = MLX5_FLOW_PRIO_RSVD; 3620 items[0] = (struct rte_flow_item){ 3621 .type = RTE_FLOW_ITEM_TYPE_END, 3622 }; 3623 actions[0] = (struct rte_flow_action){ 3624 .type = (enum rte_flow_action_type) 3625 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3626 .conf = &cp_mreg, 3627 }; 3628 actions[1] = (struct rte_flow_action){ 3629 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3630 .conf = &jump, 3631 }; 3632 actions[2] = (struct rte_flow_action){ 3633 .type = RTE_FLOW_ACTION_TYPE_END, 3634 }; 3635 } 3636 /* Build a new entry. */ 3637 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3638 if (!mcp_res) { 3639 rte_errno = ENOMEM; 3640 return NULL; 3641 } 3642 mcp_res->idx = idx; 3643 /* 3644 * The copy Flows are not included in any list. There 3645 * ones are referenced from other Flows and can not 3646 * be applied, removed, deleted in ardbitrary order 3647 * by list traversing. 3648 */ 3649 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3650 actions, false, error); 3651 if (!mcp_res->rix_flow) 3652 goto error; 3653 mcp_res->refcnt++; 3654 mcp_res->hlist_ent.key = mark_id; 3655 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3656 &mcp_res->hlist_ent); 3657 MLX5_ASSERT(!ret); 3658 if (ret) 3659 goto error; 3660 return mcp_res; 3661 error: 3662 if (mcp_res->rix_flow) 3663 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3664 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3665 return NULL; 3666 } 3667 3668 /** 3669 * Release flow in RX_CP_TBL. 3670 * 3671 * @param dev 3672 * Pointer to Ethernet device. 3673 * @flow 3674 * Parent flow for wich copying is provided. 3675 */ 3676 static void 3677 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3678 struct rte_flow *flow) 3679 { 3680 struct mlx5_flow_mreg_copy_resource *mcp_res; 3681 struct mlx5_priv *priv = dev->data->dev_private; 3682 3683 if (!flow->rix_mreg_copy) 3684 return; 3685 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3686 flow->rix_mreg_copy); 3687 if (!mcp_res || !priv->mreg_cp_tbl) 3688 return; 3689 if (flow->copy_applied) { 3690 MLX5_ASSERT(mcp_res->appcnt); 3691 flow->copy_applied = 0; 3692 --mcp_res->appcnt; 3693 if (!mcp_res->appcnt) { 3694 struct rte_flow *mcp_flow = mlx5_ipool_get 3695 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3696 mcp_res->rix_flow); 3697 3698 if (mcp_flow) 3699 flow_drv_remove(dev, mcp_flow); 3700 } 3701 } 3702 /* 3703 * We do not check availability of metadata registers here, 3704 * because copy resources are not allocated in this case. 3705 */ 3706 if (--mcp_res->refcnt) 3707 return; 3708 MLX5_ASSERT(mcp_res->rix_flow); 3709 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3710 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3711 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3712 flow->rix_mreg_copy = 0; 3713 } 3714 3715 /** 3716 * Start flow in RX_CP_TBL. 3717 * 3718 * @param dev 3719 * Pointer to Ethernet device. 3720 * @flow 3721 * Parent flow for wich copying is provided. 3722 * 3723 * @return 3724 * 0 on success, a negative errno value otherwise and rte_errno is set. 3725 */ 3726 static int 3727 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3728 struct rte_flow *flow) 3729 { 3730 struct mlx5_flow_mreg_copy_resource *mcp_res; 3731 struct mlx5_priv *priv = dev->data->dev_private; 3732 int ret; 3733 3734 if (!flow->rix_mreg_copy || flow->copy_applied) 3735 return 0; 3736 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3737 flow->rix_mreg_copy); 3738 if (!mcp_res) 3739 return 0; 3740 if (!mcp_res->appcnt) { 3741 struct rte_flow *mcp_flow = mlx5_ipool_get 3742 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3743 mcp_res->rix_flow); 3744 3745 if (mcp_flow) { 3746 ret = flow_drv_apply(dev, mcp_flow, NULL); 3747 if (ret) 3748 return ret; 3749 } 3750 } 3751 ++mcp_res->appcnt; 3752 flow->copy_applied = 1; 3753 return 0; 3754 } 3755 3756 /** 3757 * Stop flow in RX_CP_TBL. 3758 * 3759 * @param dev 3760 * Pointer to Ethernet device. 3761 * @flow 3762 * Parent flow for wich copying is provided. 3763 */ 3764 static void 3765 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3766 struct rte_flow *flow) 3767 { 3768 struct mlx5_flow_mreg_copy_resource *mcp_res; 3769 struct mlx5_priv *priv = dev->data->dev_private; 3770 3771 if (!flow->rix_mreg_copy || !flow->copy_applied) 3772 return; 3773 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3774 flow->rix_mreg_copy); 3775 if (!mcp_res) 3776 return; 3777 MLX5_ASSERT(mcp_res->appcnt); 3778 --mcp_res->appcnt; 3779 flow->copy_applied = 0; 3780 if (!mcp_res->appcnt) { 3781 struct rte_flow *mcp_flow = mlx5_ipool_get 3782 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3783 mcp_res->rix_flow); 3784 3785 if (mcp_flow) 3786 flow_drv_remove(dev, mcp_flow); 3787 } 3788 } 3789 3790 /** 3791 * Remove the default copy action from RX_CP_TBL. 3792 * 3793 * @param dev 3794 * Pointer to Ethernet device. 3795 */ 3796 static void 3797 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3798 { 3799 struct mlx5_flow_mreg_copy_resource *mcp_res; 3800 struct mlx5_priv *priv = dev->data->dev_private; 3801 3802 /* Check if default flow is registered. */ 3803 if (!priv->mreg_cp_tbl) 3804 return; 3805 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3806 MLX5_DEFAULT_COPY_ID); 3807 if (!mcp_res) 3808 return; 3809 MLX5_ASSERT(mcp_res->rix_flow); 3810 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3811 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3812 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3813 } 3814 3815 /** 3816 * Add the default copy action in in RX_CP_TBL. 3817 * 3818 * @param dev 3819 * Pointer to Ethernet device. 3820 * @param[out] error 3821 * Perform verbose error reporting if not NULL. 3822 * 3823 * @return 3824 * 0 for success, negative value otherwise and rte_errno is set. 3825 */ 3826 static int 3827 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3828 struct rte_flow_error *error) 3829 { 3830 struct mlx5_priv *priv = dev->data->dev_private; 3831 struct mlx5_flow_mreg_copy_resource *mcp_res; 3832 3833 /* Check whether extensive metadata feature is engaged. */ 3834 if (!priv->config.dv_flow_en || 3835 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3836 !mlx5_flow_ext_mreg_supported(dev) || 3837 !priv->sh->dv_regc0_mask) 3838 return 0; 3839 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3840 if (!mcp_res) 3841 return -rte_errno; 3842 return 0; 3843 } 3844 3845 /** 3846 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3847 * 3848 * All the flow having Q/RSS action should be split by 3849 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3850 * performs the following, 3851 * - CQE->flow_tag := reg_c[1] (MARK) 3852 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3853 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3854 * but there should be a flow per each MARK ID set by MARK action. 3855 * 3856 * For the aforementioned reason, if there's a MARK action in flow's action 3857 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3858 * the MARK ID to CQE's flow_tag like, 3859 * - If reg_c[1] is mark_id, 3860 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3861 * 3862 * For SET_META action which stores value in reg_c[0], as the destination is 3863 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3864 * MARK ID means the default flow. The default flow looks like, 3865 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3866 * 3867 * @param dev 3868 * Pointer to Ethernet device. 3869 * @param flow 3870 * Pointer to flow structure. 3871 * @param[in] actions 3872 * Pointer to the list of actions. 3873 * @param[out] error 3874 * Perform verbose error reporting if not NULL. 3875 * 3876 * @return 3877 * 0 on success, negative value otherwise and rte_errno is set. 3878 */ 3879 static int 3880 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3881 struct rte_flow *flow, 3882 const struct rte_flow_action *actions, 3883 struct rte_flow_error *error) 3884 { 3885 struct mlx5_priv *priv = dev->data->dev_private; 3886 struct mlx5_dev_config *config = &priv->config; 3887 struct mlx5_flow_mreg_copy_resource *mcp_res; 3888 const struct rte_flow_action_mark *mark; 3889 3890 /* Check whether extensive metadata feature is engaged. */ 3891 if (!config->dv_flow_en || 3892 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3893 !mlx5_flow_ext_mreg_supported(dev) || 3894 !priv->sh->dv_regc0_mask) 3895 return 0; 3896 /* Find MARK action. */ 3897 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3898 switch (actions->type) { 3899 case RTE_FLOW_ACTION_TYPE_FLAG: 3900 mcp_res = flow_mreg_add_copy_action 3901 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3902 if (!mcp_res) 3903 return -rte_errno; 3904 flow->rix_mreg_copy = mcp_res->idx; 3905 if (dev->data->dev_started) { 3906 mcp_res->appcnt++; 3907 flow->copy_applied = 1; 3908 } 3909 return 0; 3910 case RTE_FLOW_ACTION_TYPE_MARK: 3911 mark = (const struct rte_flow_action_mark *) 3912 actions->conf; 3913 mcp_res = 3914 flow_mreg_add_copy_action(dev, mark->id, error); 3915 if (!mcp_res) 3916 return -rte_errno; 3917 flow->rix_mreg_copy = mcp_res->idx; 3918 if (dev->data->dev_started) { 3919 mcp_res->appcnt++; 3920 flow->copy_applied = 1; 3921 } 3922 return 0; 3923 default: 3924 break; 3925 } 3926 } 3927 return 0; 3928 } 3929 3930 #define MLX5_MAX_SPLIT_ACTIONS 24 3931 #define MLX5_MAX_SPLIT_ITEMS 24 3932 3933 /** 3934 * Split the hairpin flow. 3935 * Since HW can't support encap and push-vlan on Rx, we move these 3936 * actions to Tx. 3937 * If the count action is after the encap then we also 3938 * move the count action. in this case the count will also measure 3939 * the outer bytes. 3940 * 3941 * @param dev 3942 * Pointer to Ethernet device. 3943 * @param[in] actions 3944 * Associated actions (list terminated by the END action). 3945 * @param[out] actions_rx 3946 * Rx flow actions. 3947 * @param[out] actions_tx 3948 * Tx flow actions.. 3949 * @param[out] pattern_tx 3950 * The pattern items for the Tx flow. 3951 * @param[out] flow_id 3952 * The flow ID connected to this flow. 3953 * 3954 * @return 3955 * 0 on success. 3956 */ 3957 static int 3958 flow_hairpin_split(struct rte_eth_dev *dev, 3959 const struct rte_flow_action actions[], 3960 struct rte_flow_action actions_rx[], 3961 struct rte_flow_action actions_tx[], 3962 struct rte_flow_item pattern_tx[], 3963 uint32_t *flow_id) 3964 { 3965 struct mlx5_priv *priv = dev->data->dev_private; 3966 const struct rte_flow_action_raw_encap *raw_encap; 3967 const struct rte_flow_action_raw_decap *raw_decap; 3968 struct mlx5_rte_flow_action_set_tag *set_tag; 3969 struct rte_flow_action *tag_action; 3970 struct mlx5_rte_flow_item_tag *tag_item; 3971 struct rte_flow_item *item; 3972 char *addr; 3973 int encap = 0; 3974 3975 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3976 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3977 switch (actions->type) { 3978 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3979 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3980 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3981 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3982 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 3983 rte_memcpy(actions_tx, actions, 3984 sizeof(struct rte_flow_action)); 3985 actions_tx++; 3986 break; 3987 case RTE_FLOW_ACTION_TYPE_COUNT: 3988 if (encap) { 3989 rte_memcpy(actions_tx, actions, 3990 sizeof(struct rte_flow_action)); 3991 actions_tx++; 3992 } else { 3993 rte_memcpy(actions_rx, actions, 3994 sizeof(struct rte_flow_action)); 3995 actions_rx++; 3996 } 3997 break; 3998 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3999 raw_encap = actions->conf; 4000 if (raw_encap->size > 4001 (sizeof(struct rte_flow_item_eth) + 4002 sizeof(struct rte_flow_item_ipv4))) { 4003 memcpy(actions_tx, actions, 4004 sizeof(struct rte_flow_action)); 4005 actions_tx++; 4006 encap = 1; 4007 } else { 4008 rte_memcpy(actions_rx, actions, 4009 sizeof(struct rte_flow_action)); 4010 actions_rx++; 4011 } 4012 break; 4013 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4014 raw_decap = actions->conf; 4015 if (raw_decap->size < 4016 (sizeof(struct rte_flow_item_eth) + 4017 sizeof(struct rte_flow_item_ipv4))) { 4018 memcpy(actions_tx, actions, 4019 sizeof(struct rte_flow_action)); 4020 actions_tx++; 4021 } else { 4022 rte_memcpy(actions_rx, actions, 4023 sizeof(struct rte_flow_action)); 4024 actions_rx++; 4025 } 4026 break; 4027 default: 4028 rte_memcpy(actions_rx, actions, 4029 sizeof(struct rte_flow_action)); 4030 actions_rx++; 4031 break; 4032 } 4033 } 4034 /* Add set meta action and end action for the Rx flow. */ 4035 tag_action = actions_rx; 4036 tag_action->type = (enum rte_flow_action_type) 4037 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4038 actions_rx++; 4039 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4040 actions_rx++; 4041 set_tag = (void *)actions_rx; 4042 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 4043 MLX5_ASSERT(set_tag->id > REG_NON); 4044 set_tag->data = *flow_id; 4045 tag_action->conf = set_tag; 4046 /* Create Tx item list. */ 4047 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4048 addr = (void *)&pattern_tx[2]; 4049 item = pattern_tx; 4050 item->type = (enum rte_flow_item_type) 4051 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4052 tag_item = (void *)addr; 4053 tag_item->data = *flow_id; 4054 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4055 MLX5_ASSERT(set_tag->id > REG_NON); 4056 item->spec = tag_item; 4057 addr += sizeof(struct mlx5_rte_flow_item_tag); 4058 tag_item = (void *)addr; 4059 tag_item->data = UINT32_MAX; 4060 tag_item->id = UINT16_MAX; 4061 item->mask = tag_item; 4062 item->last = NULL; 4063 item++; 4064 item->type = RTE_FLOW_ITEM_TYPE_END; 4065 return 0; 4066 } 4067 4068 /** 4069 * The last stage of splitting chain, just creates the subflow 4070 * without any modification. 4071 * 4072 * @param[in] dev 4073 * Pointer to Ethernet device. 4074 * @param[in] flow 4075 * Parent flow structure pointer. 4076 * @param[in, out] sub_flow 4077 * Pointer to return the created subflow, may be NULL. 4078 * @param[in] prefix_layers 4079 * Prefix subflow layers, may be 0. 4080 * @param[in] prefix_mark 4081 * Prefix subflow mark flag, may be 0. 4082 * @param[in] attr 4083 * Flow rule attributes. 4084 * @param[in] items 4085 * Pattern specification (list terminated by the END pattern item). 4086 * @param[in] actions 4087 * Associated actions (list terminated by the END action). 4088 * @param[in] external 4089 * This flow rule is created by request external to PMD. 4090 * @param[in] flow_idx 4091 * This memory pool index to the flow. 4092 * @param[out] error 4093 * Perform verbose error reporting if not NULL. 4094 * @return 4095 * 0 on success, negative value otherwise 4096 */ 4097 static int 4098 flow_create_split_inner(struct rte_eth_dev *dev, 4099 struct rte_flow *flow, 4100 struct mlx5_flow **sub_flow, 4101 uint64_t prefix_layers, 4102 uint32_t prefix_mark, 4103 const struct rte_flow_attr *attr, 4104 const struct rte_flow_item items[], 4105 const struct rte_flow_action actions[], 4106 bool external, uint32_t flow_idx, 4107 struct rte_flow_error *error) 4108 { 4109 struct mlx5_flow *dev_flow; 4110 4111 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4112 flow_idx, error); 4113 if (!dev_flow) 4114 return -rte_errno; 4115 dev_flow->flow = flow; 4116 dev_flow->external = external; 4117 /* Subflow object was created, we must include one in the list. */ 4118 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4119 dev_flow->handle, next); 4120 /* 4121 * If dev_flow is as one of the suffix flow, some actions in suffix 4122 * flow may need some user defined item layer flags, and pass the 4123 * Metadate rxq mark flag to suffix flow as well. 4124 */ 4125 if (prefix_layers) 4126 dev_flow->handle->layers = prefix_layers; 4127 if (prefix_mark) 4128 dev_flow->handle->mark = 1; 4129 if (sub_flow) 4130 *sub_flow = dev_flow; 4131 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4132 } 4133 4134 /** 4135 * Split the meter flow. 4136 * 4137 * As meter flow will split to three sub flow, other than meter 4138 * action, the other actions make sense to only meter accepts 4139 * the packet. If it need to be dropped, no other additional 4140 * actions should be take. 4141 * 4142 * One kind of special action which decapsulates the L3 tunnel 4143 * header will be in the prefix sub flow, as not to take the 4144 * L3 tunnel header into account. 4145 * 4146 * @param dev 4147 * Pointer to Ethernet device. 4148 * @param[in] items 4149 * Pattern specification (list terminated by the END pattern item). 4150 * @param[out] sfx_items 4151 * Suffix flow match items (list terminated by the END pattern item). 4152 * @param[in] actions 4153 * Associated actions (list terminated by the END action). 4154 * @param[out] actions_sfx 4155 * Suffix flow actions. 4156 * @param[out] actions_pre 4157 * Prefix flow actions. 4158 * @param[out] pattern_sfx 4159 * The pattern items for the suffix flow. 4160 * @param[out] tag_sfx 4161 * Pointer to suffix flow tag. 4162 * 4163 * @return 4164 * 0 on success. 4165 */ 4166 static int 4167 flow_meter_split_prep(struct rte_eth_dev *dev, 4168 const struct rte_flow_item items[], 4169 struct rte_flow_item sfx_items[], 4170 const struct rte_flow_action actions[], 4171 struct rte_flow_action actions_sfx[], 4172 struct rte_flow_action actions_pre[]) 4173 { 4174 struct rte_flow_action *tag_action = NULL; 4175 struct rte_flow_item *tag_item; 4176 struct mlx5_rte_flow_action_set_tag *set_tag; 4177 struct rte_flow_error error; 4178 const struct rte_flow_action_raw_encap *raw_encap; 4179 const struct rte_flow_action_raw_decap *raw_decap; 4180 struct mlx5_rte_flow_item_tag *tag_spec; 4181 struct mlx5_rte_flow_item_tag *tag_mask; 4182 uint32_t tag_id; 4183 bool copy_vlan = false; 4184 4185 /* Prepare the actions for prefix and suffix flow. */ 4186 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4187 struct rte_flow_action **action_cur = NULL; 4188 4189 switch (actions->type) { 4190 case RTE_FLOW_ACTION_TYPE_METER: 4191 /* Add the extra tag action first. */ 4192 tag_action = actions_pre; 4193 tag_action->type = (enum rte_flow_action_type) 4194 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4195 actions_pre++; 4196 action_cur = &actions_pre; 4197 break; 4198 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4199 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4200 action_cur = &actions_pre; 4201 break; 4202 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4203 raw_encap = actions->conf; 4204 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 4205 action_cur = &actions_pre; 4206 break; 4207 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4208 raw_decap = actions->conf; 4209 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4210 action_cur = &actions_pre; 4211 break; 4212 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4213 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4214 copy_vlan = true; 4215 break; 4216 default: 4217 break; 4218 } 4219 if (!action_cur) 4220 action_cur = &actions_sfx; 4221 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 4222 (*action_cur)++; 4223 } 4224 /* Add end action to the actions. */ 4225 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 4226 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 4227 actions_pre++; 4228 /* Set the tag. */ 4229 set_tag = (void *)actions_pre; 4230 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4231 /* 4232 * Get the id from the qrss_pool to make qrss share the id with meter. 4233 */ 4234 tag_id = flow_qrss_get_id(dev); 4235 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 4236 assert(tag_action); 4237 tag_action->conf = set_tag; 4238 /* Prepare the suffix subflow items. */ 4239 tag_item = sfx_items++; 4240 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4241 int item_type = items->type; 4242 4243 switch (item_type) { 4244 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4245 memcpy(sfx_items, items, sizeof(*sfx_items)); 4246 sfx_items++; 4247 break; 4248 case RTE_FLOW_ITEM_TYPE_VLAN: 4249 if (copy_vlan) { 4250 memcpy(sfx_items, items, sizeof(*sfx_items)); 4251 /* 4252 * Convert to internal match item, it is used 4253 * for vlan push and set vid. 4254 */ 4255 sfx_items->type = (enum rte_flow_item_type) 4256 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 4257 sfx_items++; 4258 } 4259 break; 4260 default: 4261 break; 4262 } 4263 } 4264 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4265 sfx_items++; 4266 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 4267 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 4268 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 4269 tag_mask = tag_spec + 1; 4270 tag_mask->data = 0xffffff00; 4271 tag_item->type = (enum rte_flow_item_type) 4272 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4273 tag_item->spec = tag_spec; 4274 tag_item->last = NULL; 4275 tag_item->mask = tag_mask; 4276 return tag_id; 4277 } 4278 4279 /** 4280 * Split action list having QUEUE/RSS for metadata register copy. 4281 * 4282 * Once Q/RSS action is detected in user's action list, the flow action 4283 * should be split in order to copy metadata registers, which will happen in 4284 * RX_CP_TBL like, 4285 * - CQE->flow_tag := reg_c[1] (MARK) 4286 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4287 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 4288 * This is because the last action of each flow must be a terminal action 4289 * (QUEUE, RSS or DROP). 4290 * 4291 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 4292 * stored and kept in the mlx5_flow structure per each sub_flow. 4293 * 4294 * The Q/RSS action is replaced with, 4295 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 4296 * And the following JUMP action is added at the end, 4297 * - JUMP, to RX_CP_TBL. 4298 * 4299 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 4300 * flow_create_split_metadata() routine. The flow will look like, 4301 * - If flow ID matches (reg_c[2]), perform Q/RSS. 4302 * 4303 * @param dev 4304 * Pointer to Ethernet device. 4305 * @param[out] split_actions 4306 * Pointer to store split actions to jump to CP_TBL. 4307 * @param[in] actions 4308 * Pointer to the list of original flow actions. 4309 * @param[in] qrss 4310 * Pointer to the Q/RSS action. 4311 * @param[in] actions_n 4312 * Number of original actions. 4313 * @param[out] error 4314 * Perform verbose error reporting if not NULL. 4315 * 4316 * @return 4317 * non-zero unique flow_id on success, otherwise 0 and 4318 * error/rte_error are set. 4319 */ 4320 static uint32_t 4321 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 4322 struct rte_flow_action *split_actions, 4323 const struct rte_flow_action *actions, 4324 const struct rte_flow_action *qrss, 4325 int actions_n, struct rte_flow_error *error) 4326 { 4327 struct mlx5_rte_flow_action_set_tag *set_tag; 4328 struct rte_flow_action_jump *jump; 4329 const int qrss_idx = qrss - actions; 4330 uint32_t flow_id = 0; 4331 int ret = 0; 4332 4333 /* 4334 * Given actions will be split 4335 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 4336 * - Add jump to mreg CP_TBL. 4337 * As a result, there will be one more action. 4338 */ 4339 ++actions_n; 4340 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 4341 set_tag = (void *)(split_actions + actions_n); 4342 /* 4343 * If tag action is not set to void(it means we are not the meter 4344 * suffix flow), add the tag action. Since meter suffix flow already 4345 * has the tag added. 4346 */ 4347 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 4348 /* 4349 * Allocate the new subflow ID. This one is unique within 4350 * device and not shared with representors. Otherwise, 4351 * we would have to resolve multi-thread access synch 4352 * issue. Each flow on the shared device is appended 4353 * with source vport identifier, so the resulting 4354 * flows will be unique in the shared (by master and 4355 * representors) domain even if they have coinciding 4356 * IDs. 4357 */ 4358 flow_id = flow_qrss_get_id(dev); 4359 if (!flow_id) 4360 return rte_flow_error_set(error, ENOMEM, 4361 RTE_FLOW_ERROR_TYPE_ACTION, 4362 NULL, "can't allocate id " 4363 "for split Q/RSS subflow"); 4364 /* Internal SET_TAG action to set flow ID. */ 4365 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 4366 .data = flow_id, 4367 }; 4368 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 4369 if (ret < 0) 4370 return ret; 4371 set_tag->id = ret; 4372 /* Construct new actions array. */ 4373 /* Replace QUEUE/RSS action. */ 4374 split_actions[qrss_idx] = (struct rte_flow_action){ 4375 .type = (enum rte_flow_action_type) 4376 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4377 .conf = set_tag, 4378 }; 4379 } 4380 /* JUMP action to jump to mreg copy table (CP_TBL). */ 4381 jump = (void *)(set_tag + 1); 4382 *jump = (struct rte_flow_action_jump){ 4383 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4384 }; 4385 split_actions[actions_n - 2] = (struct rte_flow_action){ 4386 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4387 .conf = jump, 4388 }; 4389 split_actions[actions_n - 1] = (struct rte_flow_action){ 4390 .type = RTE_FLOW_ACTION_TYPE_END, 4391 }; 4392 return flow_id; 4393 } 4394 4395 /** 4396 * Extend the given action list for Tx metadata copy. 4397 * 4398 * Copy the given action list to the ext_actions and add flow metadata register 4399 * copy action in order to copy reg_a set by WQE to reg_c[0]. 4400 * 4401 * @param[out] ext_actions 4402 * Pointer to the extended action list. 4403 * @param[in] actions 4404 * Pointer to the list of actions. 4405 * @param[in] actions_n 4406 * Number of actions in the list. 4407 * @param[out] error 4408 * Perform verbose error reporting if not NULL. 4409 * @param[in] encap_idx 4410 * The encap action inndex. 4411 * 4412 * @return 4413 * 0 on success, negative value otherwise 4414 */ 4415 static int 4416 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 4417 struct rte_flow_action *ext_actions, 4418 const struct rte_flow_action *actions, 4419 int actions_n, struct rte_flow_error *error, 4420 int encap_idx) 4421 { 4422 struct mlx5_flow_action_copy_mreg *cp_mreg = 4423 (struct mlx5_flow_action_copy_mreg *) 4424 (ext_actions + actions_n + 1); 4425 int ret; 4426 4427 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4428 if (ret < 0) 4429 return ret; 4430 cp_mreg->dst = ret; 4431 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 4432 if (ret < 0) 4433 return ret; 4434 cp_mreg->src = ret; 4435 if (encap_idx != 0) 4436 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 4437 if (encap_idx == actions_n - 1) { 4438 ext_actions[actions_n - 1] = (struct rte_flow_action){ 4439 .type = (enum rte_flow_action_type) 4440 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4441 .conf = cp_mreg, 4442 }; 4443 ext_actions[actions_n] = (struct rte_flow_action){ 4444 .type = RTE_FLOW_ACTION_TYPE_END, 4445 }; 4446 } else { 4447 ext_actions[encap_idx] = (struct rte_flow_action){ 4448 .type = (enum rte_flow_action_type) 4449 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4450 .conf = cp_mreg, 4451 }; 4452 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 4453 sizeof(*ext_actions) * (actions_n - encap_idx)); 4454 } 4455 return 0; 4456 } 4457 4458 /** 4459 * Check the match action from the action list. 4460 * 4461 * @param[in] actions 4462 * Pointer to the list of actions. 4463 * @param[in] attr 4464 * Flow rule attributes. 4465 * @param[in] action 4466 * The action to be check if exist. 4467 * @param[out] match_action_pos 4468 * Pointer to the position of the matched action if exists, otherwise is -1. 4469 * @param[out] qrss_action_pos 4470 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 4471 * 4472 * @return 4473 * > 0 the total number of actions. 4474 * 0 if not found match action in action list. 4475 */ 4476 static int 4477 flow_check_match_action(const struct rte_flow_action actions[], 4478 const struct rte_flow_attr *attr, 4479 enum rte_flow_action_type action, 4480 int *match_action_pos, int *qrss_action_pos) 4481 { 4482 const struct rte_flow_action_sample *sample; 4483 int actions_n = 0; 4484 int jump_flag = 0; 4485 uint32_t ratio = 0; 4486 int sub_type = 0; 4487 int flag = 0; 4488 4489 *match_action_pos = -1; 4490 *qrss_action_pos = -1; 4491 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4492 if (actions->type == action) { 4493 flag = 1; 4494 *match_action_pos = actions_n; 4495 } 4496 if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE || 4497 actions->type == RTE_FLOW_ACTION_TYPE_RSS) 4498 *qrss_action_pos = actions_n; 4499 if (actions->type == RTE_FLOW_ACTION_TYPE_JUMP) 4500 jump_flag = 1; 4501 if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE) { 4502 sample = actions->conf; 4503 ratio = sample->ratio; 4504 sub_type = ((const struct rte_flow_action *) 4505 (sample->actions))->type; 4506 } 4507 actions_n++; 4508 } 4509 if (flag && action == RTE_FLOW_ACTION_TYPE_SAMPLE && attr->transfer) { 4510 if (ratio == 1) { 4511 /* JUMP Action not support for Mirroring; 4512 * Mirroring support multi-destination; 4513 */ 4514 if (!jump_flag && sub_type != RTE_FLOW_ACTION_TYPE_END) 4515 flag = 0; 4516 } 4517 } 4518 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4519 return flag ? actions_n + 1 : 0; 4520 } 4521 4522 #define SAMPLE_SUFFIX_ITEM 2 4523 4524 /** 4525 * Split the sample flow. 4526 * 4527 * As sample flow will split to two sub flow, sample flow with 4528 * sample action, the other actions will move to new suffix flow. 4529 * 4530 * Also add unique tag id with tag action in the sample flow, 4531 * the same tag id will be as match in the suffix flow. 4532 * 4533 * @param dev 4534 * Pointer to Ethernet device. 4535 * @param[in] fdb_tx 4536 * FDB egress flow flag. 4537 * @param[out] sfx_items 4538 * Suffix flow match items (list terminated by the END pattern item). 4539 * @param[in] actions 4540 * Associated actions (list terminated by the END action). 4541 * @param[out] actions_sfx 4542 * Suffix flow actions. 4543 * @param[out] actions_pre 4544 * Prefix flow actions. 4545 * @param[in] actions_n 4546 * The total number of actions. 4547 * @param[in] sample_action_pos 4548 * The sample action position. 4549 * @param[in] qrss_action_pos 4550 * The Queue/RSS action position. 4551 * @param[out] error 4552 * Perform verbose error reporting if not NULL. 4553 * 4554 * @return 4555 * 0 on success, or unique flow_id, a negative errno value 4556 * otherwise and rte_errno is set. 4557 */ 4558 static int 4559 flow_sample_split_prep(struct rte_eth_dev *dev, 4560 uint32_t fdb_tx, 4561 struct rte_flow_item sfx_items[], 4562 const struct rte_flow_action actions[], 4563 struct rte_flow_action actions_sfx[], 4564 struct rte_flow_action actions_pre[], 4565 int actions_n, 4566 int sample_action_pos, 4567 int qrss_action_pos, 4568 struct rte_flow_error *error) 4569 { 4570 struct mlx5_rte_flow_action_set_tag *set_tag; 4571 struct mlx5_rte_flow_item_tag *tag_spec; 4572 struct mlx5_rte_flow_item_tag *tag_mask; 4573 uint32_t tag_id = 0; 4574 int index; 4575 int ret; 4576 4577 if (sample_action_pos < 0) 4578 return rte_flow_error_set(error, EINVAL, 4579 RTE_FLOW_ERROR_TYPE_ACTION, 4580 NULL, "invalid position of sample " 4581 "action in list"); 4582 if (!fdb_tx) { 4583 /* Prepare the prefix tag action. */ 4584 set_tag = (void *)(actions_pre + actions_n + 1); 4585 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 4586 if (ret < 0) 4587 return ret; 4588 set_tag->id = ret; 4589 tag_id = flow_qrss_get_id(dev); 4590 set_tag->data = tag_id; 4591 /* Prepare the suffix subflow items. */ 4592 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 4593 tag_spec->data = tag_id; 4594 tag_spec->id = set_tag->id; 4595 tag_mask = tag_spec + 1; 4596 tag_mask->data = UINT32_MAX; 4597 sfx_items[0] = (struct rte_flow_item){ 4598 .type = (enum rte_flow_item_type) 4599 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4600 .spec = tag_spec, 4601 .last = NULL, 4602 .mask = tag_mask, 4603 }; 4604 sfx_items[1] = (struct rte_flow_item){ 4605 .type = (enum rte_flow_item_type) 4606 RTE_FLOW_ITEM_TYPE_END, 4607 }; 4608 } 4609 /* Prepare the actions for prefix and suffix flow. */ 4610 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 4611 index = qrss_action_pos; 4612 /* Put the preceding the Queue/RSS action into prefix flow. */ 4613 if (index != 0) 4614 memcpy(actions_pre, actions, 4615 sizeof(struct rte_flow_action) * index); 4616 /* Put others preceding the sample action into prefix flow. */ 4617 if (sample_action_pos > index + 1) 4618 memcpy(actions_pre + index, actions + index + 1, 4619 sizeof(struct rte_flow_action) * 4620 (sample_action_pos - index - 1)); 4621 index = sample_action_pos - 1; 4622 /* Put Queue/RSS action into Suffix flow. */ 4623 memcpy(actions_sfx, actions + qrss_action_pos, 4624 sizeof(struct rte_flow_action)); 4625 actions_sfx++; 4626 } else { 4627 index = sample_action_pos; 4628 if (index != 0) 4629 memcpy(actions_pre, actions, 4630 sizeof(struct rte_flow_action) * index); 4631 } 4632 /* Add the extra tag action for NIC-RX and E-Switch ingress. */ 4633 if (!fdb_tx) { 4634 actions_pre[index++] = 4635 (struct rte_flow_action){ 4636 .type = (enum rte_flow_action_type) 4637 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 4638 .conf = set_tag, 4639 }; 4640 } 4641 memcpy(actions_pre + index, actions + sample_action_pos, 4642 sizeof(struct rte_flow_action)); 4643 index += 1; 4644 actions_pre[index] = (struct rte_flow_action){ 4645 .type = (enum rte_flow_action_type) 4646 RTE_FLOW_ACTION_TYPE_END, 4647 }; 4648 /* Put the actions after sample into Suffix flow. */ 4649 memcpy(actions_sfx, actions + sample_action_pos + 1, 4650 sizeof(struct rte_flow_action) * 4651 (actions_n - sample_action_pos - 1)); 4652 return tag_id; 4653 } 4654 4655 /** 4656 * The splitting for metadata feature. 4657 * 4658 * - Q/RSS action on NIC Rx should be split in order to pass by 4659 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4660 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4661 * 4662 * - All the actions on NIC Tx should have a mreg copy action to 4663 * copy reg_a from WQE to reg_c[0]. 4664 * 4665 * @param dev 4666 * Pointer to Ethernet device. 4667 * @param[in] flow 4668 * Parent flow structure pointer. 4669 * @param[in] prefix_layers 4670 * Prefix flow layer flags. 4671 * @param[in] prefix_mark 4672 * Prefix subflow mark flag, may be 0. 4673 * @param[in] attr 4674 * Flow rule attributes. 4675 * @param[in] items 4676 * Pattern specification (list terminated by the END pattern item). 4677 * @param[in] actions 4678 * Associated actions (list terminated by the END action). 4679 * @param[in] external 4680 * This flow rule is created by request external to PMD. 4681 * @param[in] flow_idx 4682 * This memory pool index to the flow. 4683 * @param[out] error 4684 * Perform verbose error reporting if not NULL. 4685 * @return 4686 * 0 on success, negative value otherwise 4687 */ 4688 static int 4689 flow_create_split_metadata(struct rte_eth_dev *dev, 4690 struct rte_flow *flow, 4691 uint64_t prefix_layers, 4692 uint32_t prefix_mark, 4693 const struct rte_flow_attr *attr, 4694 const struct rte_flow_item items[], 4695 const struct rte_flow_action actions[], 4696 bool external, uint32_t flow_idx, 4697 struct rte_flow_error *error) 4698 { 4699 struct mlx5_priv *priv = dev->data->dev_private; 4700 struct mlx5_dev_config *config = &priv->config; 4701 const struct rte_flow_action *qrss = NULL; 4702 struct rte_flow_action *ext_actions = NULL; 4703 struct mlx5_flow *dev_flow = NULL; 4704 uint32_t qrss_id = 0; 4705 int mtr_sfx = 0; 4706 size_t act_size; 4707 int actions_n; 4708 int encap_idx; 4709 int ret; 4710 4711 /* Check whether extensive metadata feature is engaged. */ 4712 if (!config->dv_flow_en || 4713 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4714 !mlx5_flow_ext_mreg_supported(dev)) 4715 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 4716 prefix_mark, attr, items, 4717 actions, external, flow_idx, 4718 error); 4719 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 4720 &encap_idx); 4721 if (qrss) { 4722 /* Exclude hairpin flows from splitting. */ 4723 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 4724 const struct rte_flow_action_queue *queue; 4725 4726 queue = qrss->conf; 4727 if (mlx5_rxq_get_type(dev, queue->index) == 4728 MLX5_RXQ_TYPE_HAIRPIN) 4729 qrss = NULL; 4730 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 4731 const struct rte_flow_action_rss *rss; 4732 4733 rss = qrss->conf; 4734 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 4735 MLX5_RXQ_TYPE_HAIRPIN) 4736 qrss = NULL; 4737 } 4738 } 4739 if (qrss) { 4740 /* Check if it is in meter suffix table. */ 4741 mtr_sfx = attr->group == (attr->transfer ? 4742 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4743 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 4744 /* 4745 * Q/RSS action on NIC Rx should be split in order to pass by 4746 * the mreg copy table (RX_CP_TBL) and then it jumps to the 4747 * action table (RX_ACT_TBL) which has the split Q/RSS action. 4748 */ 4749 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4750 sizeof(struct rte_flow_action_set_tag) + 4751 sizeof(struct rte_flow_action_jump); 4752 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4753 SOCKET_ID_ANY); 4754 if (!ext_actions) 4755 return rte_flow_error_set(error, ENOMEM, 4756 RTE_FLOW_ERROR_TYPE_ACTION, 4757 NULL, "no memory to split " 4758 "metadata flow"); 4759 /* 4760 * If we are the suffix flow of meter, tag already exist. 4761 * Set the tag action to void. 4762 */ 4763 if (mtr_sfx) 4764 ext_actions[qrss - actions].type = 4765 RTE_FLOW_ACTION_TYPE_VOID; 4766 else 4767 ext_actions[qrss - actions].type = 4768 (enum rte_flow_action_type) 4769 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4770 /* 4771 * Create the new actions list with removed Q/RSS action 4772 * and appended set tag and jump to register copy table 4773 * (RX_CP_TBL). We should preallocate unique tag ID here 4774 * in advance, because it is needed for set tag action. 4775 */ 4776 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 4777 qrss, actions_n, error); 4778 if (!mtr_sfx && !qrss_id) { 4779 ret = -rte_errno; 4780 goto exit; 4781 } 4782 } else if (attr->egress && !attr->transfer) { 4783 /* 4784 * All the actions on NIC Tx should have a metadata register 4785 * copy action to copy reg_a from WQE to reg_c[meta] 4786 */ 4787 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4788 sizeof(struct mlx5_flow_action_copy_mreg); 4789 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 4790 SOCKET_ID_ANY); 4791 if (!ext_actions) 4792 return rte_flow_error_set(error, ENOMEM, 4793 RTE_FLOW_ERROR_TYPE_ACTION, 4794 NULL, "no memory to split " 4795 "metadata flow"); 4796 /* Create the action list appended with copy register. */ 4797 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 4798 actions_n, error, encap_idx); 4799 if (ret < 0) 4800 goto exit; 4801 } 4802 /* Add the unmodified original or prefix subflow. */ 4803 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, 4804 prefix_mark, attr, 4805 items, ext_actions ? ext_actions : 4806 actions, external, flow_idx, error); 4807 if (ret < 0) 4808 goto exit; 4809 MLX5_ASSERT(dev_flow); 4810 if (qrss) { 4811 const struct rte_flow_attr q_attr = { 4812 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4813 .ingress = 1, 4814 }; 4815 /* Internal PMD action to set register. */ 4816 struct mlx5_rte_flow_item_tag q_tag_spec = { 4817 .data = qrss_id, 4818 .id = REG_NON, 4819 }; 4820 struct rte_flow_item q_items[] = { 4821 { 4822 .type = (enum rte_flow_item_type) 4823 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4824 .spec = &q_tag_spec, 4825 .last = NULL, 4826 .mask = NULL, 4827 }, 4828 { 4829 .type = RTE_FLOW_ITEM_TYPE_END, 4830 }, 4831 }; 4832 struct rte_flow_action q_actions[] = { 4833 { 4834 .type = qrss->type, 4835 .conf = qrss->conf, 4836 }, 4837 { 4838 .type = RTE_FLOW_ACTION_TYPE_END, 4839 }, 4840 }; 4841 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4842 4843 /* 4844 * Configure the tag item only if there is no meter subflow. 4845 * Since tag is already marked in the meter suffix subflow 4846 * we can just use the meter suffix items as is. 4847 */ 4848 if (qrss_id) { 4849 /* Not meter subflow. */ 4850 MLX5_ASSERT(!mtr_sfx); 4851 /* 4852 * Put unique id in prefix flow due to it is destroyed 4853 * after suffix flow and id will be freed after there 4854 * is no actual flows with this id and identifier 4855 * reallocation becomes possible (for example, for 4856 * other flows in other threads). 4857 */ 4858 dev_flow->handle->split_flow_id = qrss_id; 4859 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4860 error); 4861 if (ret < 0) 4862 goto exit; 4863 q_tag_spec.id = ret; 4864 } 4865 dev_flow = NULL; 4866 /* Add suffix subflow to execute Q/RSS. */ 4867 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 0, 4868 &q_attr, mtr_sfx ? items : 4869 q_items, q_actions, 4870 external, flow_idx, error); 4871 if (ret < 0) 4872 goto exit; 4873 /* qrss ID should be freed if failed. */ 4874 qrss_id = 0; 4875 MLX5_ASSERT(dev_flow); 4876 } 4877 4878 exit: 4879 /* 4880 * We do not destroy the partially created sub_flows in case of error. 4881 * These ones are included into parent flow list and will be destroyed 4882 * by flow_drv_destroy. 4883 */ 4884 flow_qrss_free_id(dev, qrss_id); 4885 mlx5_free(ext_actions); 4886 return ret; 4887 } 4888 4889 /** 4890 * The splitting for meter feature. 4891 * 4892 * - The meter flow will be split to two flows as prefix and 4893 * suffix flow. The packets make sense only it pass the prefix 4894 * meter action. 4895 * 4896 * - Reg_C_5 is used for the packet to match betweend prefix and 4897 * suffix flow. 4898 * 4899 * @param dev 4900 * Pointer to Ethernet device. 4901 * @param[in] flow 4902 * Parent flow structure pointer. 4903 * @param[in] prefix_layers 4904 * Prefix subflow layers, may be 0. 4905 * @param[in] prefix_mark 4906 * Prefix subflow mark flag, may be 0. 4907 * @param[in] attr 4908 * Flow rule attributes. 4909 * @param[in] items 4910 * Pattern specification (list terminated by the END pattern item). 4911 * @param[in] actions 4912 * Associated actions (list terminated by the END action). 4913 * @param[in] external 4914 * This flow rule is created by request external to PMD. 4915 * @param[in] flow_idx 4916 * This memory pool index to the flow. 4917 * @param[out] error 4918 * Perform verbose error reporting if not NULL. 4919 * @return 4920 * 0 on success, negative value otherwise 4921 */ 4922 static int 4923 flow_create_split_meter(struct rte_eth_dev *dev, 4924 struct rte_flow *flow, 4925 uint64_t prefix_layers, 4926 uint32_t prefix_mark, 4927 const struct rte_flow_attr *attr, 4928 const struct rte_flow_item items[], 4929 const struct rte_flow_action actions[], 4930 bool external, uint32_t flow_idx, 4931 struct rte_flow_error *error) 4932 { 4933 struct mlx5_priv *priv = dev->data->dev_private; 4934 struct rte_flow_action *sfx_actions = NULL; 4935 struct rte_flow_action *pre_actions = NULL; 4936 struct rte_flow_item *sfx_items = NULL; 4937 struct mlx5_flow *dev_flow = NULL; 4938 struct rte_flow_attr sfx_attr = *attr; 4939 uint32_t mtr = 0; 4940 uint32_t mtr_tag_id = 0; 4941 size_t act_size; 4942 size_t item_size; 4943 int actions_n = 0; 4944 int ret; 4945 4946 if (priv->mtr_en) 4947 actions_n = flow_check_meter_action(actions, &mtr); 4948 if (mtr) { 4949 /* The five prefix actions: meter, decap, encap, tag, end. */ 4950 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4951 sizeof(struct mlx5_rte_flow_action_set_tag); 4952 /* tag, vlan, port id, end. */ 4953 #define METER_SUFFIX_ITEM 4 4954 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4955 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4956 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 4957 0, SOCKET_ID_ANY); 4958 if (!sfx_actions) 4959 return rte_flow_error_set(error, ENOMEM, 4960 RTE_FLOW_ERROR_TYPE_ACTION, 4961 NULL, "no memory to split " 4962 "meter flow"); 4963 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4964 act_size); 4965 pre_actions = sfx_actions + actions_n; 4966 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4967 actions, sfx_actions, 4968 pre_actions); 4969 if (!mtr_tag_id) { 4970 ret = -rte_errno; 4971 goto exit; 4972 } 4973 /* Add the prefix subflow. */ 4974 ret = flow_create_split_inner(dev, flow, &dev_flow, 4975 prefix_layers, 0, 4976 attr, items, 4977 pre_actions, external, 4978 flow_idx, error); 4979 if (ret) { 4980 ret = -rte_errno; 4981 goto exit; 4982 } 4983 dev_flow->handle->split_flow_id = mtr_tag_id; 4984 /* Setting the sfx group atrr. */ 4985 sfx_attr.group = sfx_attr.transfer ? 4986 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4987 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4988 } 4989 /* Add the prefix subflow. */ 4990 ret = flow_create_split_metadata(dev, flow, dev_flow ? 4991 flow_get_prefix_layer_flags(dev_flow) : 4992 prefix_layers, dev_flow ? 4993 dev_flow->handle->mark : prefix_mark, 4994 &sfx_attr, sfx_items ? 4995 sfx_items : items, 4996 sfx_actions ? sfx_actions : actions, 4997 external, flow_idx, error); 4998 exit: 4999 if (sfx_actions) 5000 mlx5_free(sfx_actions); 5001 return ret; 5002 } 5003 5004 /** 5005 * The splitting for sample feature. 5006 * 5007 * Once Sample action is detected in the action list, the flow actions should 5008 * be split into prefix sub flow and suffix sub flow. 5009 * 5010 * The original items remain in the prefix sub flow, all actions preceding the 5011 * sample action and the sample action itself will be copied to the prefix 5012 * sub flow, the actions following the sample action will be copied to the 5013 * suffix sub flow, Queue action always be located in the suffix sub flow. 5014 * 5015 * In order to make the packet from prefix sub flow matches with suffix sub 5016 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 5017 * flow uses tag item with the unique flow id. 5018 * 5019 * @param dev 5020 * Pointer to Ethernet device. 5021 * @param[in] flow 5022 * Parent flow structure pointer. 5023 * @param[in] attr 5024 * Flow rule attributes. 5025 * @param[in] items 5026 * Pattern specification (list terminated by the END pattern item). 5027 * @param[in] actions 5028 * Associated actions (list terminated by the END action). 5029 * @param[in] external 5030 * This flow rule is created by request external to PMD. 5031 * @param[in] flow_idx 5032 * This memory pool index to the flow. 5033 * @param[out] error 5034 * Perform verbose error reporting if not NULL. 5035 * @return 5036 * 0 on success, negative value otherwise 5037 */ 5038 static int 5039 flow_create_split_sample(struct rte_eth_dev *dev, 5040 struct rte_flow *flow, 5041 const struct rte_flow_attr *attr, 5042 const struct rte_flow_item items[], 5043 const struct rte_flow_action actions[], 5044 bool external, uint32_t flow_idx, 5045 struct rte_flow_error *error) 5046 { 5047 struct mlx5_priv *priv = dev->data->dev_private; 5048 struct rte_flow_action *sfx_actions = NULL; 5049 struct rte_flow_action *pre_actions = NULL; 5050 struct rte_flow_item *sfx_items = NULL; 5051 struct mlx5_flow *dev_flow = NULL; 5052 struct rte_flow_attr sfx_attr = *attr; 5053 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5054 struct mlx5_flow_dv_sample_resource *sample_res; 5055 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 5056 struct mlx5_flow_tbl_resource *sfx_tbl; 5057 union mlx5_flow_tbl_key sfx_table_key; 5058 #endif 5059 size_t act_size; 5060 size_t item_size; 5061 uint32_t fdb_tx = 0; 5062 int32_t tag_id = 0; 5063 int actions_n = 0; 5064 int sample_action_pos; 5065 int qrss_action_pos; 5066 int ret = 0; 5067 5068 if (priv->sampler_en) 5069 actions_n = flow_check_match_action(actions, attr, 5070 RTE_FLOW_ACTION_TYPE_SAMPLE, 5071 &sample_action_pos, &qrss_action_pos); 5072 if (actions_n) { 5073 /* The prefix actions must includes sample, tag, end. */ 5074 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 5075 + sizeof(struct mlx5_rte_flow_action_set_tag); 5076 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 5077 sizeof(struct mlx5_rte_flow_item_tag) * 2; 5078 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 5079 item_size), 0, SOCKET_ID_ANY); 5080 if (!sfx_actions) 5081 return rte_flow_error_set(error, ENOMEM, 5082 RTE_FLOW_ERROR_TYPE_ACTION, 5083 NULL, "no memory to split " 5084 "sample flow"); 5085 /* The representor_id is -1 for uplink. */ 5086 fdb_tx = (attr->transfer && priv->representor_id != -1); 5087 if (!fdb_tx) 5088 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 5089 + act_size); 5090 pre_actions = sfx_actions + actions_n; 5091 tag_id = flow_sample_split_prep(dev, fdb_tx, sfx_items, 5092 actions, sfx_actions, 5093 pre_actions, actions_n, 5094 sample_action_pos, 5095 qrss_action_pos, error); 5096 if (tag_id < 0 || (!fdb_tx && !tag_id)) { 5097 ret = -rte_errno; 5098 goto exit; 5099 } 5100 /* Add the prefix subflow. */ 5101 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, 0, attr, 5102 items, pre_actions, external, 5103 flow_idx, error); 5104 if (ret) { 5105 ret = -rte_errno; 5106 goto exit; 5107 } 5108 dev_flow->handle->split_flow_id = tag_id; 5109 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5110 /* Set the sfx group attr. */ 5111 sample_res = (struct mlx5_flow_dv_sample_resource *) 5112 dev_flow->dv.sample_res; 5113 sfx_tbl = (struct mlx5_flow_tbl_resource *) 5114 sample_res->normal_path_tbl; 5115 sfx_tbl_data = container_of(sfx_tbl, 5116 struct mlx5_flow_tbl_data_entry, tbl); 5117 sfx_table_key.v64 = sfx_tbl_data->entry.key; 5118 sfx_attr.group = sfx_attr.transfer ? 5119 (sfx_table_key.table_id - 1) : 5120 sfx_table_key.table_id; 5121 #endif 5122 } 5123 /* Add the suffix subflow. */ 5124 ret = flow_create_split_meter(dev, flow, dev_flow ? 5125 flow_get_prefix_layer_flags(dev_flow) : 0, 5126 dev_flow ? dev_flow->handle->mark : 0, 5127 &sfx_attr, sfx_items ? sfx_items : items, 5128 sfx_actions ? sfx_actions : actions, 5129 external, flow_idx, error); 5130 exit: 5131 if (sfx_actions) 5132 mlx5_free(sfx_actions); 5133 return ret; 5134 } 5135 5136 /** 5137 * Split the flow to subflow set. The splitters might be linked 5138 * in the chain, like this: 5139 * flow_create_split_outer() calls: 5140 * flow_create_split_meter() calls: 5141 * flow_create_split_metadata(meter_subflow_0) calls: 5142 * flow_create_split_inner(metadata_subflow_0) 5143 * flow_create_split_inner(metadata_subflow_1) 5144 * flow_create_split_inner(metadata_subflow_2) 5145 * flow_create_split_metadata(meter_subflow_1) calls: 5146 * flow_create_split_inner(metadata_subflow_0) 5147 * flow_create_split_inner(metadata_subflow_1) 5148 * flow_create_split_inner(metadata_subflow_2) 5149 * 5150 * This provide flexible way to add new levels of flow splitting. 5151 * The all of successfully created subflows are included to the 5152 * parent flow dev_flow list. 5153 * 5154 * @param dev 5155 * Pointer to Ethernet device. 5156 * @param[in] flow 5157 * Parent flow structure pointer. 5158 * @param[in] attr 5159 * Flow rule attributes. 5160 * @param[in] items 5161 * Pattern specification (list terminated by the END pattern item). 5162 * @param[in] actions 5163 * Associated actions (list terminated by the END action). 5164 * @param[in] external 5165 * This flow rule is created by request external to PMD. 5166 * @param[in] flow_idx 5167 * This memory pool index to the flow. 5168 * @param[out] error 5169 * Perform verbose error reporting if not NULL. 5170 * @return 5171 * 0 on success, negative value otherwise 5172 */ 5173 static int 5174 flow_create_split_outer(struct rte_eth_dev *dev, 5175 struct rte_flow *flow, 5176 const struct rte_flow_attr *attr, 5177 const struct rte_flow_item items[], 5178 const struct rte_flow_action actions[], 5179 bool external, uint32_t flow_idx, 5180 struct rte_flow_error *error) 5181 { 5182 int ret; 5183 5184 ret = flow_create_split_sample(dev, flow, attr, items, 5185 actions, external, flow_idx, error); 5186 MLX5_ASSERT(ret <= 0); 5187 return ret; 5188 } 5189 5190 /** 5191 * Create a flow and add it to @p list. 5192 * 5193 * @param dev 5194 * Pointer to Ethernet device. 5195 * @param list 5196 * Pointer to a TAILQ flow list. If this parameter NULL, 5197 * no list insertion occurred, flow is just created, 5198 * this is caller's responsibility to track the 5199 * created flow. 5200 * @param[in] attr 5201 * Flow rule attributes. 5202 * @param[in] items 5203 * Pattern specification (list terminated by the END pattern item). 5204 * @param[in] actions 5205 * Associated actions (list terminated by the END action). 5206 * @param[in] external 5207 * This flow rule is created by request external to PMD. 5208 * @param[out] error 5209 * Perform verbose error reporting if not NULL. 5210 * 5211 * @return 5212 * A flow index on success, 0 otherwise and rte_errno is set. 5213 */ 5214 static uint32_t 5215 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 5216 const struct rte_flow_attr *attr, 5217 const struct rte_flow_item items[], 5218 const struct rte_flow_action original_actions[], 5219 bool external, struct rte_flow_error *error) 5220 { 5221 struct mlx5_priv *priv = dev->data->dev_private; 5222 struct rte_flow *flow = NULL; 5223 struct mlx5_flow *dev_flow; 5224 const struct rte_flow_action_rss *rss; 5225 struct mlx5_translated_shared_action 5226 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5227 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5228 union { 5229 struct mlx5_flow_expand_rss buf; 5230 uint8_t buffer[2048]; 5231 } expand_buffer; 5232 union { 5233 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5234 uint8_t buffer[2048]; 5235 } actions_rx; 5236 union { 5237 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 5238 uint8_t buffer[2048]; 5239 } actions_hairpin_tx; 5240 union { 5241 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 5242 uint8_t buffer[2048]; 5243 } items_tx; 5244 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 5245 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 5246 priv->rss_desc)[!!priv->flow_idx]; 5247 const struct rte_flow_action *p_actions_rx; 5248 uint32_t i; 5249 uint32_t idx = 0; 5250 int hairpin_flow; 5251 uint32_t hairpin_id = 0; 5252 struct rte_flow_attr attr_tx = { .priority = 0 }; 5253 struct rte_flow_attr attr_factor = {0}; 5254 const struct rte_flow_action *actions; 5255 struct rte_flow_action *translated_actions = NULL; 5256 int ret = flow_shared_actions_translate(original_actions, 5257 shared_actions, 5258 &shared_actions_n, 5259 &translated_actions, error); 5260 5261 if (ret < 0) { 5262 MLX5_ASSERT(translated_actions == NULL); 5263 return 0; 5264 } 5265 actions = translated_actions ? translated_actions : original_actions; 5266 memcpy((void *)&attr_factor, (const void *)attr, sizeof(*attr)); 5267 if (external) 5268 attr_factor.group *= MLX5_FLOW_TABLE_FACTOR; 5269 p_actions_rx = actions; 5270 hairpin_flow = flow_check_hairpin_split(dev, &attr_factor, actions); 5271 ret = flow_drv_validate(dev, &attr_factor, items, p_actions_rx, 5272 external, hairpin_flow, error); 5273 if (ret < 0) 5274 goto error_before_hairpin_split; 5275 if (hairpin_flow > 0) { 5276 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 5277 rte_errno = EINVAL; 5278 goto error_before_hairpin_split; 5279 } 5280 flow_hairpin_split(dev, actions, actions_rx.actions, 5281 actions_hairpin_tx.actions, items_tx.items, 5282 &hairpin_id); 5283 p_actions_rx = actions_rx.actions; 5284 } 5285 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 5286 if (!flow) { 5287 rte_errno = ENOMEM; 5288 goto error_before_flow; 5289 } 5290 flow->drv_type = flow_get_drv_type(dev, &attr_factor); 5291 if (hairpin_id != 0) 5292 flow->hairpin_flow_id = hairpin_id; 5293 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 5294 flow->drv_type < MLX5_FLOW_TYPE_MAX); 5295 memset(rss_desc, 0, sizeof(*rss_desc)); 5296 rss = flow_get_rss_action(p_actions_rx); 5297 if (rss) { 5298 /* 5299 * The following information is required by 5300 * mlx5_flow_hashfields_adjust() in advance. 5301 */ 5302 rss_desc->level = rss->level; 5303 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 5304 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 5305 } 5306 flow->dev_handles = 0; 5307 if (rss && rss->types) { 5308 unsigned int graph_root; 5309 5310 graph_root = find_graph_root(items, rss->level); 5311 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 5312 items, rss->types, 5313 mlx5_support_expansion, graph_root); 5314 MLX5_ASSERT(ret > 0 && 5315 (unsigned int)ret < sizeof(expand_buffer.buffer)); 5316 } else { 5317 buf->entries = 1; 5318 buf->entry[0].pattern = (void *)(uintptr_t)items; 5319 } 5320 flow->shared_rss = flow_get_shared_rss_action(shared_actions, 5321 shared_actions_n); 5322 /* 5323 * Record the start index when there is a nested call. All sub-flows 5324 * need to be translated before another calling. 5325 * No need to use ping-pong buffer to save memory here. 5326 */ 5327 if (priv->flow_idx) { 5328 MLX5_ASSERT(!priv->flow_nested_idx); 5329 priv->flow_nested_idx = priv->flow_idx; 5330 } 5331 for (i = 0; i < buf->entries; ++i) { 5332 /* 5333 * The splitter may create multiple dev_flows, 5334 * depending on configuration. In the simplest 5335 * case it just creates unmodified original flow. 5336 */ 5337 ret = flow_create_split_outer(dev, flow, &attr_factor, 5338 buf->entry[i].pattern, 5339 p_actions_rx, external, idx, 5340 error); 5341 if (ret < 0) 5342 goto error; 5343 } 5344 /* Create the tx flow. */ 5345 if (hairpin_flow) { 5346 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 5347 attr_tx.ingress = 0; 5348 attr_tx.egress = 1; 5349 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 5350 actions_hairpin_tx.actions, 5351 idx, error); 5352 if (!dev_flow) 5353 goto error; 5354 dev_flow->flow = flow; 5355 dev_flow->external = 0; 5356 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5357 dev_flow->handle, next); 5358 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 5359 items_tx.items, 5360 actions_hairpin_tx.actions, error); 5361 if (ret < 0) 5362 goto error; 5363 } 5364 /* 5365 * Update the metadata register copy table. If extensive 5366 * metadata feature is enabled and registers are supported 5367 * we might create the extra rte_flow for each unique 5368 * MARK/FLAG action ID. 5369 * 5370 * The table is updated for ingress Flows only, because 5371 * the egress Flows belong to the different device and 5372 * copy table should be updated in peer NIC Rx domain. 5373 */ 5374 if (attr_factor.ingress && 5375 (external || attr_factor.group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 5376 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 5377 if (ret) 5378 goto error; 5379 } 5380 /* 5381 * If the flow is external (from application) OR device is started, then 5382 * the flow will be applied immediately. 5383 */ 5384 if (external || dev->data->dev_started) { 5385 ret = flow_drv_apply(dev, flow, error); 5386 if (ret < 0) 5387 goto error; 5388 } 5389 if (list) 5390 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 5391 flow, next); 5392 flow_rxq_flags_set(dev, flow); 5393 rte_free(translated_actions); 5394 /* Nested flow creation index recovery. */ 5395 priv->flow_idx = priv->flow_nested_idx; 5396 if (priv->flow_nested_idx) 5397 priv->flow_nested_idx = 0; 5398 return idx; 5399 error: 5400 MLX5_ASSERT(flow); 5401 ret = rte_errno; /* Save rte_errno before cleanup. */ 5402 flow_mreg_del_copy_action(dev, flow); 5403 flow_drv_destroy(dev, flow); 5404 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 5405 rte_errno = ret; /* Restore rte_errno. */ 5406 error_before_flow: 5407 ret = rte_errno; 5408 if (hairpin_id) 5409 mlx5_flow_id_release(priv->sh->flow_id_pool, 5410 hairpin_id); 5411 rte_errno = ret; 5412 priv->flow_idx = priv->flow_nested_idx; 5413 if (priv->flow_nested_idx) 5414 priv->flow_nested_idx = 0; 5415 error_before_hairpin_split: 5416 rte_free(translated_actions); 5417 return 0; 5418 } 5419 5420 /** 5421 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 5422 * incoming packets to table 1. 5423 * 5424 * Other flow rules, requested for group n, will be created in 5425 * e-switch table n+1. 5426 * Jump action to e-switch group n will be created to group n+1. 5427 * 5428 * Used when working in switchdev mode, to utilise advantages of table 1 5429 * and above. 5430 * 5431 * @param dev 5432 * Pointer to Ethernet device. 5433 * 5434 * @return 5435 * Pointer to flow on success, NULL otherwise and rte_errno is set. 5436 */ 5437 struct rte_flow * 5438 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 5439 { 5440 const struct rte_flow_attr attr = { 5441 .group = 0, 5442 .priority = 0, 5443 .ingress = 1, 5444 .egress = 0, 5445 .transfer = 1, 5446 }; 5447 const struct rte_flow_item pattern = { 5448 .type = RTE_FLOW_ITEM_TYPE_END, 5449 }; 5450 struct rte_flow_action_jump jump = { 5451 .group = 1, 5452 }; 5453 const struct rte_flow_action actions[] = { 5454 { 5455 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5456 .conf = &jump, 5457 }, 5458 { 5459 .type = RTE_FLOW_ACTION_TYPE_END, 5460 }, 5461 }; 5462 struct mlx5_priv *priv = dev->data->dev_private; 5463 struct rte_flow_error error; 5464 5465 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 5466 &attr, &pattern, 5467 actions, false, &error); 5468 } 5469 5470 /** 5471 * Validate a flow supported by the NIC. 5472 * 5473 * @see rte_flow_validate() 5474 * @see rte_flow_ops 5475 */ 5476 int 5477 mlx5_flow_validate(struct rte_eth_dev *dev, 5478 const struct rte_flow_attr *attr, 5479 const struct rte_flow_item items[], 5480 const struct rte_flow_action original_actions[], 5481 struct rte_flow_error *error) 5482 { 5483 int hairpin_flow; 5484 struct mlx5_translated_shared_action 5485 shared_actions[MLX5_MAX_SHARED_ACTIONS]; 5486 int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; 5487 const struct rte_flow_action *actions; 5488 struct rte_flow_action *translated_actions = NULL; 5489 int ret = flow_shared_actions_translate(original_actions, 5490 shared_actions, 5491 &shared_actions_n, 5492 &translated_actions, error); 5493 5494 if (ret) 5495 return ret; 5496 actions = translated_actions ? translated_actions : original_actions; 5497 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 5498 ret = flow_drv_validate(dev, attr, items, actions, 5499 true, hairpin_flow, error); 5500 rte_free(translated_actions); 5501 return ret; 5502 } 5503 5504 /** 5505 * Create a flow. 5506 * 5507 * @see rte_flow_create() 5508 * @see rte_flow_ops 5509 */ 5510 struct rte_flow * 5511 mlx5_flow_create(struct rte_eth_dev *dev, 5512 const struct rte_flow_attr *attr, 5513 const struct rte_flow_item items[], 5514 const struct rte_flow_action actions[], 5515 struct rte_flow_error *error) 5516 { 5517 struct mlx5_priv *priv = dev->data->dev_private; 5518 5519 /* 5520 * If the device is not started yet, it is not allowed to created a 5521 * flow from application. PMD default flows and traffic control flows 5522 * are not affected. 5523 */ 5524 if (unlikely(!dev->data->dev_started)) { 5525 DRV_LOG(DEBUG, "port %u is not started when " 5526 "inserting a flow", dev->data->port_id); 5527 rte_flow_error_set(error, ENODEV, 5528 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5529 NULL, 5530 "port not started"); 5531 return NULL; 5532 } 5533 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 5534 attr, items, actions, true, error); 5535 } 5536 5537 /** 5538 * Destroy a flow in a list. 5539 * 5540 * @param dev 5541 * Pointer to Ethernet device. 5542 * @param list 5543 * Pointer to the Indexed flow list. If this parameter NULL, 5544 * there is no flow removal from the list. Be noted that as 5545 * flow is add to the indexed list, memory of the indexed 5546 * list points to maybe changed as flow destroyed. 5547 * @param[in] flow_idx 5548 * Index of flow to destroy. 5549 */ 5550 static void 5551 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 5552 uint32_t flow_idx) 5553 { 5554 struct mlx5_priv *priv = dev->data->dev_private; 5555 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5556 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5557 [MLX5_IPOOL_RTE_FLOW], flow_idx); 5558 5559 if (!flow) 5560 return; 5561 /* 5562 * Update RX queue flags only if port is started, otherwise it is 5563 * already clean. 5564 */ 5565 if (dev->data->dev_started) 5566 flow_rxq_flags_trim(dev, flow); 5567 if (flow->hairpin_flow_id) 5568 mlx5_flow_id_release(priv->sh->flow_id_pool, 5569 flow->hairpin_flow_id); 5570 flow_drv_destroy(dev, flow); 5571 if (list) 5572 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 5573 flow_idx, flow, next); 5574 flow_mreg_del_copy_action(dev, flow); 5575 if (flow->fdir) { 5576 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5577 if (priv_fdir_flow->rix_flow == flow_idx) 5578 break; 5579 } 5580 if (priv_fdir_flow) { 5581 LIST_REMOVE(priv_fdir_flow, next); 5582 mlx5_free(priv_fdir_flow->fdir); 5583 mlx5_free(priv_fdir_flow); 5584 } 5585 } 5586 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5587 } 5588 5589 /** 5590 * Destroy all flows. 5591 * 5592 * @param dev 5593 * Pointer to Ethernet device. 5594 * @param list 5595 * Pointer to the Indexed flow list. 5596 * @param active 5597 * If flushing is called avtively. 5598 */ 5599 void 5600 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 5601 { 5602 uint32_t num_flushed = 0; 5603 5604 while (*list) { 5605 flow_list_destroy(dev, list, *list); 5606 num_flushed++; 5607 } 5608 if (active) { 5609 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 5610 dev->data->port_id, num_flushed); 5611 } 5612 } 5613 5614 /** 5615 * Remove all flows. 5616 * 5617 * @param dev 5618 * Pointer to Ethernet device. 5619 * @param list 5620 * Pointer to the Indexed flow list. 5621 */ 5622 void 5623 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list) 5624 { 5625 struct mlx5_priv *priv = dev->data->dev_private; 5626 struct rte_flow *flow = NULL; 5627 uint32_t idx; 5628 5629 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 5630 flow, next) { 5631 flow_drv_remove(dev, flow); 5632 flow_mreg_stop_copy_action(dev, flow); 5633 } 5634 flow_mreg_del_default_copy_action(dev); 5635 flow_rxq_flags_clear(dev); 5636 } 5637 5638 /** 5639 * Add all flows. 5640 * 5641 * @param dev 5642 * Pointer to Ethernet device. 5643 * @param list 5644 * Pointer to the Indexed flow list. 5645 * 5646 * @return 5647 * 0 on success, a negative errno value otherwise and rte_errno is set. 5648 */ 5649 int 5650 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list) 5651 { 5652 struct mlx5_priv *priv = dev->data->dev_private; 5653 struct rte_flow *flow = NULL; 5654 struct rte_flow_error error; 5655 uint32_t idx; 5656 int ret = 0; 5657 5658 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 5659 ret = flow_mreg_add_default_copy_action(dev, &error); 5660 if (ret < 0) 5661 return -rte_errno; 5662 /* Apply Flows created by application. */ 5663 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 5664 flow, next) { 5665 ret = flow_mreg_start_copy_action(dev, flow); 5666 if (ret < 0) 5667 goto error; 5668 ret = flow_drv_apply(dev, flow, &error); 5669 if (ret < 0) 5670 goto error; 5671 flow_rxq_flags_set(dev, flow); 5672 } 5673 return 0; 5674 error: 5675 ret = rte_errno; /* Save rte_errno before cleanup. */ 5676 mlx5_flow_stop(dev, list); 5677 rte_errno = ret; /* Restore rte_errno. */ 5678 return -rte_errno; 5679 } 5680 5681 /** 5682 * Stop all default actions for flows. 5683 * 5684 * @param dev 5685 * Pointer to Ethernet device. 5686 */ 5687 void 5688 mlx5_flow_stop_default(struct rte_eth_dev *dev) 5689 { 5690 flow_mreg_del_default_copy_action(dev); 5691 flow_rxq_flags_clear(dev); 5692 } 5693 5694 /** 5695 * Start all default actions for flows. 5696 * 5697 * @param dev 5698 * Pointer to Ethernet device. 5699 * @return 5700 * 0 on success, a negative errno value otherwise and rte_errno is set. 5701 */ 5702 int 5703 mlx5_flow_start_default(struct rte_eth_dev *dev) 5704 { 5705 struct rte_flow_error error; 5706 5707 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 5708 return flow_mreg_add_default_copy_action(dev, &error); 5709 } 5710 5711 /** 5712 * Allocate intermediate resources for flow creation. 5713 * 5714 * @param dev 5715 * Pointer to Ethernet device. 5716 */ 5717 void 5718 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) 5719 { 5720 struct mlx5_priv *priv = dev->data->dev_private; 5721 5722 if (!priv->inter_flows) { 5723 priv->inter_flows = mlx5_malloc(MLX5_MEM_ZERO, 5724 MLX5_NUM_MAX_DEV_FLOWS * 5725 sizeof(struct mlx5_flow) + 5726 (sizeof(struct mlx5_flow_rss_desc) + 5727 sizeof(uint16_t) * UINT16_MAX) * 2, 0, 5728 SOCKET_ID_ANY); 5729 if (!priv->inter_flows) { 5730 DRV_LOG(ERR, "can't allocate intermediate memory."); 5731 return; 5732 } 5733 } 5734 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows) 5735 [MLX5_NUM_MAX_DEV_FLOWS]; 5736 /* Reset the index. */ 5737 priv->flow_idx = 0; 5738 priv->flow_nested_idx = 0; 5739 } 5740 5741 /** 5742 * Free intermediate resources for flows. 5743 * 5744 * @param dev 5745 * Pointer to Ethernet device. 5746 */ 5747 void 5748 mlx5_flow_free_intermediate(struct rte_eth_dev *dev) 5749 { 5750 struct mlx5_priv *priv = dev->data->dev_private; 5751 5752 mlx5_free(priv->inter_flows); 5753 priv->inter_flows = NULL; 5754 } 5755 5756 /** 5757 * Verify the flow list is empty 5758 * 5759 * @param dev 5760 * Pointer to Ethernet device. 5761 * 5762 * @return the number of flows not released. 5763 */ 5764 int 5765 mlx5_flow_verify(struct rte_eth_dev *dev) 5766 { 5767 struct mlx5_priv *priv = dev->data->dev_private; 5768 struct rte_flow *flow; 5769 uint32_t idx; 5770 int ret = 0; 5771 5772 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 5773 flow, next) { 5774 DRV_LOG(DEBUG, "port %u flow %p still referenced", 5775 dev->data->port_id, (void *)flow); 5776 ++ret; 5777 } 5778 return ret; 5779 } 5780 5781 /** 5782 * Enable default hairpin egress flow. 5783 * 5784 * @param dev 5785 * Pointer to Ethernet device. 5786 * @param queue 5787 * The queue index. 5788 * 5789 * @return 5790 * 0 on success, a negative errno value otherwise and rte_errno is set. 5791 */ 5792 int 5793 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 5794 uint32_t queue) 5795 { 5796 struct mlx5_priv *priv = dev->data->dev_private; 5797 const struct rte_flow_attr attr = { 5798 .egress = 1, 5799 .priority = 0, 5800 }; 5801 struct mlx5_rte_flow_item_tx_queue queue_spec = { 5802 .queue = queue, 5803 }; 5804 struct mlx5_rte_flow_item_tx_queue queue_mask = { 5805 .queue = UINT32_MAX, 5806 }; 5807 struct rte_flow_item items[] = { 5808 { 5809 .type = (enum rte_flow_item_type) 5810 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 5811 .spec = &queue_spec, 5812 .last = NULL, 5813 .mask = &queue_mask, 5814 }, 5815 { 5816 .type = RTE_FLOW_ITEM_TYPE_END, 5817 }, 5818 }; 5819 struct rte_flow_action_jump jump = { 5820 .group = MLX5_HAIRPIN_TX_TABLE, 5821 }; 5822 struct rte_flow_action actions[2]; 5823 uint32_t flow_idx; 5824 struct rte_flow_error error; 5825 5826 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 5827 actions[0].conf = &jump; 5828 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 5829 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5830 &attr, items, actions, false, &error); 5831 if (!flow_idx) { 5832 DRV_LOG(DEBUG, 5833 "Failed to create ctrl flow: rte_errno(%d)," 5834 " type(%d), message(%s)", 5835 rte_errno, error.type, 5836 error.message ? error.message : " (no stated reason)"); 5837 return -rte_errno; 5838 } 5839 return 0; 5840 } 5841 5842 /** 5843 * Enable a control flow configured from the control plane. 5844 * 5845 * @param dev 5846 * Pointer to Ethernet device. 5847 * @param eth_spec 5848 * An Ethernet flow spec to apply. 5849 * @param eth_mask 5850 * An Ethernet flow mask to apply. 5851 * @param vlan_spec 5852 * A VLAN flow spec to apply. 5853 * @param vlan_mask 5854 * A VLAN flow mask to apply. 5855 * 5856 * @return 5857 * 0 on success, a negative errno value otherwise and rte_errno is set. 5858 */ 5859 int 5860 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 5861 struct rte_flow_item_eth *eth_spec, 5862 struct rte_flow_item_eth *eth_mask, 5863 struct rte_flow_item_vlan *vlan_spec, 5864 struct rte_flow_item_vlan *vlan_mask) 5865 { 5866 struct mlx5_priv *priv = dev->data->dev_private; 5867 const struct rte_flow_attr attr = { 5868 .ingress = 1, 5869 .priority = MLX5_FLOW_PRIO_RSVD, 5870 }; 5871 struct rte_flow_item items[] = { 5872 { 5873 .type = RTE_FLOW_ITEM_TYPE_ETH, 5874 .spec = eth_spec, 5875 .last = NULL, 5876 .mask = eth_mask, 5877 }, 5878 { 5879 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 5880 RTE_FLOW_ITEM_TYPE_END, 5881 .spec = vlan_spec, 5882 .last = NULL, 5883 .mask = vlan_mask, 5884 }, 5885 { 5886 .type = RTE_FLOW_ITEM_TYPE_END, 5887 }, 5888 }; 5889 uint16_t queue[priv->reta_idx_n]; 5890 struct rte_flow_action_rss action_rss = { 5891 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 5892 .level = 0, 5893 .types = priv->rss_conf.rss_hf, 5894 .key_len = priv->rss_conf.rss_key_len, 5895 .queue_num = priv->reta_idx_n, 5896 .key = priv->rss_conf.rss_key, 5897 .queue = queue, 5898 }; 5899 struct rte_flow_action actions[] = { 5900 { 5901 .type = RTE_FLOW_ACTION_TYPE_RSS, 5902 .conf = &action_rss, 5903 }, 5904 { 5905 .type = RTE_FLOW_ACTION_TYPE_END, 5906 }, 5907 }; 5908 uint32_t flow_idx; 5909 struct rte_flow_error error; 5910 unsigned int i; 5911 5912 if (!priv->reta_idx_n || !priv->rxqs_n) { 5913 return 0; 5914 } 5915 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 5916 action_rss.types = 0; 5917 for (i = 0; i != priv->reta_idx_n; ++i) 5918 queue[i] = (*priv->reta_idx)[i]; 5919 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5920 &attr, items, actions, false, &error); 5921 if (!flow_idx) 5922 return -rte_errno; 5923 return 0; 5924 } 5925 5926 /** 5927 * Enable a flow control configured from the control plane. 5928 * 5929 * @param dev 5930 * Pointer to Ethernet device. 5931 * @param eth_spec 5932 * An Ethernet flow spec to apply. 5933 * @param eth_mask 5934 * An Ethernet flow mask to apply. 5935 * 5936 * @return 5937 * 0 on success, a negative errno value otherwise and rte_errno is set. 5938 */ 5939 int 5940 mlx5_ctrl_flow(struct rte_eth_dev *dev, 5941 struct rte_flow_item_eth *eth_spec, 5942 struct rte_flow_item_eth *eth_mask) 5943 { 5944 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 5945 } 5946 5947 /** 5948 * Create default miss flow rule matching lacp traffic 5949 * 5950 * @param dev 5951 * Pointer to Ethernet device. 5952 * @param eth_spec 5953 * An Ethernet flow spec to apply. 5954 * 5955 * @return 5956 * 0 on success, a negative errno value otherwise and rte_errno is set. 5957 */ 5958 int 5959 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 5960 { 5961 struct mlx5_priv *priv = dev->data->dev_private; 5962 /* 5963 * The LACP matching is done by only using ether type since using 5964 * a multicast dst mac causes kernel to give low priority to this flow. 5965 */ 5966 static const struct rte_flow_item_eth lacp_spec = { 5967 .type = RTE_BE16(0x8809), 5968 }; 5969 static const struct rte_flow_item_eth lacp_mask = { 5970 .type = 0xffff, 5971 }; 5972 const struct rte_flow_attr attr = { 5973 .ingress = 1, 5974 }; 5975 struct rte_flow_item items[] = { 5976 { 5977 .type = RTE_FLOW_ITEM_TYPE_ETH, 5978 .spec = &lacp_spec, 5979 .mask = &lacp_mask, 5980 }, 5981 { 5982 .type = RTE_FLOW_ITEM_TYPE_END, 5983 }, 5984 }; 5985 struct rte_flow_action actions[] = { 5986 { 5987 .type = (enum rte_flow_action_type) 5988 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 5989 }, 5990 { 5991 .type = RTE_FLOW_ACTION_TYPE_END, 5992 }, 5993 }; 5994 struct rte_flow_error error; 5995 uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows, 5996 &attr, items, actions, false, &error); 5997 5998 if (!flow_idx) 5999 return -rte_errno; 6000 return 0; 6001 } 6002 6003 /** 6004 * Destroy a flow. 6005 * 6006 * @see rte_flow_destroy() 6007 * @see rte_flow_ops 6008 */ 6009 int 6010 mlx5_flow_destroy(struct rte_eth_dev *dev, 6011 struct rte_flow *flow, 6012 struct rte_flow_error *error __rte_unused) 6013 { 6014 struct mlx5_priv *priv = dev->data->dev_private; 6015 6016 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 6017 return 0; 6018 } 6019 6020 /** 6021 * Destroy all flows. 6022 * 6023 * @see rte_flow_flush() 6024 * @see rte_flow_ops 6025 */ 6026 int 6027 mlx5_flow_flush(struct rte_eth_dev *dev, 6028 struct rte_flow_error *error __rte_unused) 6029 { 6030 struct mlx5_priv *priv = dev->data->dev_private; 6031 6032 mlx5_flow_list_flush(dev, &priv->flows, false); 6033 return 0; 6034 } 6035 6036 /** 6037 * Isolated mode. 6038 * 6039 * @see rte_flow_isolate() 6040 * @see rte_flow_ops 6041 */ 6042 int 6043 mlx5_flow_isolate(struct rte_eth_dev *dev, 6044 int enable, 6045 struct rte_flow_error *error) 6046 { 6047 struct mlx5_priv *priv = dev->data->dev_private; 6048 6049 if (dev->data->dev_started) { 6050 rte_flow_error_set(error, EBUSY, 6051 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6052 NULL, 6053 "port must be stopped first"); 6054 return -rte_errno; 6055 } 6056 priv->isolated = !!enable; 6057 if (enable) 6058 dev->dev_ops = &mlx5_os_dev_ops_isolate; 6059 else 6060 dev->dev_ops = &mlx5_os_dev_ops; 6061 6062 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 6063 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 6064 6065 return 0; 6066 } 6067 6068 /** 6069 * Query a flow. 6070 * 6071 * @see rte_flow_query() 6072 * @see rte_flow_ops 6073 */ 6074 static int 6075 flow_drv_query(struct rte_eth_dev *dev, 6076 uint32_t flow_idx, 6077 const struct rte_flow_action *actions, 6078 void *data, 6079 struct rte_flow_error *error) 6080 { 6081 struct mlx5_priv *priv = dev->data->dev_private; 6082 const struct mlx5_flow_driver_ops *fops; 6083 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 6084 [MLX5_IPOOL_RTE_FLOW], 6085 flow_idx); 6086 enum mlx5_flow_drv_type ftype; 6087 6088 if (!flow) { 6089 return rte_flow_error_set(error, ENOENT, 6090 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6091 NULL, 6092 "invalid flow handle"); 6093 } 6094 ftype = flow->drv_type; 6095 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 6096 fops = flow_get_drv_ops(ftype); 6097 6098 return fops->query(dev, flow, actions, data, error); 6099 } 6100 6101 /** 6102 * Query a flow. 6103 * 6104 * @see rte_flow_query() 6105 * @see rte_flow_ops 6106 */ 6107 int 6108 mlx5_flow_query(struct rte_eth_dev *dev, 6109 struct rte_flow *flow, 6110 const struct rte_flow_action *actions, 6111 void *data, 6112 struct rte_flow_error *error) 6113 { 6114 int ret; 6115 6116 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 6117 error); 6118 if (ret < 0) 6119 return ret; 6120 return 0; 6121 } 6122 6123 /** 6124 * Convert a flow director filter to a generic flow. 6125 * 6126 * @param dev 6127 * Pointer to Ethernet device. 6128 * @param fdir_filter 6129 * Flow director filter to add. 6130 * @param attributes 6131 * Generic flow parameters structure. 6132 * 6133 * @return 6134 * 0 on success, a negative errno value otherwise and rte_errno is set. 6135 */ 6136 static int 6137 flow_fdir_filter_convert(struct rte_eth_dev *dev, 6138 const struct rte_eth_fdir_filter *fdir_filter, 6139 struct mlx5_fdir *attributes) 6140 { 6141 struct mlx5_priv *priv = dev->data->dev_private; 6142 const struct rte_eth_fdir_input *input = &fdir_filter->input; 6143 const struct rte_eth_fdir_masks *mask = 6144 &dev->data->dev_conf.fdir_conf.mask; 6145 6146 /* Validate queue number. */ 6147 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 6148 DRV_LOG(ERR, "port %u invalid queue number %d", 6149 dev->data->port_id, fdir_filter->action.rx_queue); 6150 rte_errno = EINVAL; 6151 return -rte_errno; 6152 } 6153 attributes->attr.ingress = 1; 6154 attributes->items[0] = (struct rte_flow_item) { 6155 .type = RTE_FLOW_ITEM_TYPE_ETH, 6156 .spec = &attributes->l2, 6157 .mask = &attributes->l2_mask, 6158 }; 6159 switch (fdir_filter->action.behavior) { 6160 case RTE_ETH_FDIR_ACCEPT: 6161 attributes->actions[0] = (struct rte_flow_action){ 6162 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 6163 .conf = &attributes->queue, 6164 }; 6165 break; 6166 case RTE_ETH_FDIR_REJECT: 6167 attributes->actions[0] = (struct rte_flow_action){ 6168 .type = RTE_FLOW_ACTION_TYPE_DROP, 6169 }; 6170 break; 6171 default: 6172 DRV_LOG(ERR, "port %u invalid behavior %d", 6173 dev->data->port_id, 6174 fdir_filter->action.behavior); 6175 rte_errno = ENOTSUP; 6176 return -rte_errno; 6177 } 6178 attributes->queue.index = fdir_filter->action.rx_queue; 6179 /* Handle L3. */ 6180 switch (fdir_filter->input.flow_type) { 6181 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 6182 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 6183 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 6184 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 6185 .src_addr = input->flow.ip4_flow.src_ip, 6186 .dst_addr = input->flow.ip4_flow.dst_ip, 6187 .time_to_live = input->flow.ip4_flow.ttl, 6188 .type_of_service = input->flow.ip4_flow.tos, 6189 }; 6190 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 6191 .src_addr = mask->ipv4_mask.src_ip, 6192 .dst_addr = mask->ipv4_mask.dst_ip, 6193 .time_to_live = mask->ipv4_mask.ttl, 6194 .type_of_service = mask->ipv4_mask.tos, 6195 .next_proto_id = mask->ipv4_mask.proto, 6196 }; 6197 attributes->items[1] = (struct rte_flow_item){ 6198 .type = RTE_FLOW_ITEM_TYPE_IPV4, 6199 .spec = &attributes->l3, 6200 .mask = &attributes->l3_mask, 6201 }; 6202 break; 6203 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 6204 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 6205 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 6206 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 6207 .hop_limits = input->flow.ipv6_flow.hop_limits, 6208 .proto = input->flow.ipv6_flow.proto, 6209 }; 6210 6211 memcpy(attributes->l3.ipv6.hdr.src_addr, 6212 input->flow.ipv6_flow.src_ip, 6213 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 6214 memcpy(attributes->l3.ipv6.hdr.dst_addr, 6215 input->flow.ipv6_flow.dst_ip, 6216 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 6217 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 6218 mask->ipv6_mask.src_ip, 6219 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 6220 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 6221 mask->ipv6_mask.dst_ip, 6222 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 6223 attributes->items[1] = (struct rte_flow_item){ 6224 .type = RTE_FLOW_ITEM_TYPE_IPV6, 6225 .spec = &attributes->l3, 6226 .mask = &attributes->l3_mask, 6227 }; 6228 break; 6229 default: 6230 DRV_LOG(ERR, "port %u invalid flow type%d", 6231 dev->data->port_id, fdir_filter->input.flow_type); 6232 rte_errno = ENOTSUP; 6233 return -rte_errno; 6234 } 6235 /* Handle L4. */ 6236 switch (fdir_filter->input.flow_type) { 6237 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 6238 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 6239 .src_port = input->flow.udp4_flow.src_port, 6240 .dst_port = input->flow.udp4_flow.dst_port, 6241 }; 6242 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 6243 .src_port = mask->src_port_mask, 6244 .dst_port = mask->dst_port_mask, 6245 }; 6246 attributes->items[2] = (struct rte_flow_item){ 6247 .type = RTE_FLOW_ITEM_TYPE_UDP, 6248 .spec = &attributes->l4, 6249 .mask = &attributes->l4_mask, 6250 }; 6251 break; 6252 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 6253 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 6254 .src_port = input->flow.tcp4_flow.src_port, 6255 .dst_port = input->flow.tcp4_flow.dst_port, 6256 }; 6257 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 6258 .src_port = mask->src_port_mask, 6259 .dst_port = mask->dst_port_mask, 6260 }; 6261 attributes->items[2] = (struct rte_flow_item){ 6262 .type = RTE_FLOW_ITEM_TYPE_TCP, 6263 .spec = &attributes->l4, 6264 .mask = &attributes->l4_mask, 6265 }; 6266 break; 6267 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 6268 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 6269 .src_port = input->flow.udp6_flow.src_port, 6270 .dst_port = input->flow.udp6_flow.dst_port, 6271 }; 6272 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 6273 .src_port = mask->src_port_mask, 6274 .dst_port = mask->dst_port_mask, 6275 }; 6276 attributes->items[2] = (struct rte_flow_item){ 6277 .type = RTE_FLOW_ITEM_TYPE_UDP, 6278 .spec = &attributes->l4, 6279 .mask = &attributes->l4_mask, 6280 }; 6281 break; 6282 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 6283 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 6284 .src_port = input->flow.tcp6_flow.src_port, 6285 .dst_port = input->flow.tcp6_flow.dst_port, 6286 }; 6287 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 6288 .src_port = mask->src_port_mask, 6289 .dst_port = mask->dst_port_mask, 6290 }; 6291 attributes->items[2] = (struct rte_flow_item){ 6292 .type = RTE_FLOW_ITEM_TYPE_TCP, 6293 .spec = &attributes->l4, 6294 .mask = &attributes->l4_mask, 6295 }; 6296 break; 6297 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 6298 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 6299 break; 6300 default: 6301 DRV_LOG(ERR, "port %u invalid flow type%d", 6302 dev->data->port_id, fdir_filter->input.flow_type); 6303 rte_errno = ENOTSUP; 6304 return -rte_errno; 6305 } 6306 return 0; 6307 } 6308 6309 #define FLOW_FDIR_CMP(f1, f2, fld) \ 6310 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 6311 6312 /** 6313 * Compare two FDIR flows. If items and actions are identical, the two flows are 6314 * regarded as same. 6315 * 6316 * @param dev 6317 * Pointer to Ethernet device. 6318 * @param f1 6319 * FDIR flow to compare. 6320 * @param f2 6321 * FDIR flow to compare. 6322 * 6323 * @return 6324 * Zero on match, 1 otherwise. 6325 */ 6326 static int 6327 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 6328 { 6329 if (FLOW_FDIR_CMP(f1, f2, attr) || 6330 FLOW_FDIR_CMP(f1, f2, l2) || 6331 FLOW_FDIR_CMP(f1, f2, l2_mask) || 6332 FLOW_FDIR_CMP(f1, f2, l3) || 6333 FLOW_FDIR_CMP(f1, f2, l3_mask) || 6334 FLOW_FDIR_CMP(f1, f2, l4) || 6335 FLOW_FDIR_CMP(f1, f2, l4_mask) || 6336 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 6337 return 1; 6338 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 6339 FLOW_FDIR_CMP(f1, f2, queue)) 6340 return 1; 6341 return 0; 6342 } 6343 6344 /** 6345 * Search device flow list to find out a matched FDIR flow. 6346 * 6347 * @param dev 6348 * Pointer to Ethernet device. 6349 * @param fdir_flow 6350 * FDIR flow to lookup. 6351 * 6352 * @return 6353 * Index of flow if found, 0 otherwise. 6354 */ 6355 static uint32_t 6356 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 6357 { 6358 struct mlx5_priv *priv = dev->data->dev_private; 6359 uint32_t flow_idx = 0; 6360 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6361 6362 MLX5_ASSERT(fdir_flow); 6363 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 6364 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { 6365 DRV_LOG(DEBUG, "port %u found FDIR flow %u", 6366 dev->data->port_id, flow_idx); 6367 flow_idx = priv_fdir_flow->rix_flow; 6368 break; 6369 } 6370 } 6371 return flow_idx; 6372 } 6373 6374 /** 6375 * Add new flow director filter and store it in list. 6376 * 6377 * @param dev 6378 * Pointer to Ethernet device. 6379 * @param fdir_filter 6380 * Flow director filter to add. 6381 * 6382 * @return 6383 * 0 on success, a negative errno value otherwise and rte_errno is set. 6384 */ 6385 static int 6386 flow_fdir_filter_add(struct rte_eth_dev *dev, 6387 const struct rte_eth_fdir_filter *fdir_filter) 6388 { 6389 struct mlx5_priv *priv = dev->data->dev_private; 6390 struct mlx5_fdir *fdir_flow; 6391 struct rte_flow *flow; 6392 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6393 uint32_t flow_idx; 6394 int ret; 6395 6396 fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*fdir_flow), 0, 6397 SOCKET_ID_ANY); 6398 if (!fdir_flow) { 6399 rte_errno = ENOMEM; 6400 return -rte_errno; 6401 } 6402 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 6403 if (ret) 6404 goto error; 6405 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); 6406 if (flow_idx) { 6407 rte_errno = EEXIST; 6408 goto error; 6409 } 6410 priv_fdir_flow = mlx5_malloc(MLX5_MEM_ZERO, 6411 sizeof(struct mlx5_fdir_flow), 6412 0, SOCKET_ID_ANY); 6413 if (!priv_fdir_flow) { 6414 rte_errno = ENOMEM; 6415 goto error; 6416 } 6417 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 6418 fdir_flow->items, fdir_flow->actions, true, 6419 NULL); 6420 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 6421 if (!flow) 6422 goto error; 6423 flow->fdir = 1; 6424 priv_fdir_flow->fdir = fdir_flow; 6425 priv_fdir_flow->rix_flow = flow_idx; 6426 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); 6427 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 6428 dev->data->port_id, (void *)flow); 6429 return 0; 6430 error: 6431 mlx5_free(priv_fdir_flow); 6432 mlx5_free(fdir_flow); 6433 return -rte_errno; 6434 } 6435 6436 /** 6437 * Delete specific filter. 6438 * 6439 * @param dev 6440 * Pointer to Ethernet device. 6441 * @param fdir_filter 6442 * Filter to be deleted. 6443 * 6444 * @return 6445 * 0 on success, a negative errno value otherwise and rte_errno is set. 6446 */ 6447 static int 6448 flow_fdir_filter_delete(struct rte_eth_dev *dev, 6449 const struct rte_eth_fdir_filter *fdir_filter) 6450 { 6451 struct mlx5_priv *priv = dev->data->dev_private; 6452 uint32_t flow_idx; 6453 struct mlx5_fdir fdir_flow = { 6454 .attr.group = 0, 6455 }; 6456 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6457 int ret; 6458 6459 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 6460 if (ret) 6461 return -rte_errno; 6462 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 6463 /* Find the fdir in priv list */ 6464 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) 6465 break; 6466 } 6467 if (!priv_fdir_flow) 6468 return 0; 6469 LIST_REMOVE(priv_fdir_flow, next); 6470 flow_idx = priv_fdir_flow->rix_flow; 6471 flow_list_destroy(dev, &priv->flows, flow_idx); 6472 mlx5_free(priv_fdir_flow->fdir); 6473 mlx5_free(priv_fdir_flow); 6474 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", 6475 dev->data->port_id, flow_idx); 6476 return 0; 6477 } 6478 6479 /** 6480 * Update queue for specific filter. 6481 * 6482 * @param dev 6483 * Pointer to Ethernet device. 6484 * @param fdir_filter 6485 * Filter to be updated. 6486 * 6487 * @return 6488 * 0 on success, a negative errno value otherwise and rte_errno is set. 6489 */ 6490 static int 6491 flow_fdir_filter_update(struct rte_eth_dev *dev, 6492 const struct rte_eth_fdir_filter *fdir_filter) 6493 { 6494 int ret; 6495 6496 ret = flow_fdir_filter_delete(dev, fdir_filter); 6497 if (ret) 6498 return ret; 6499 return flow_fdir_filter_add(dev, fdir_filter); 6500 } 6501 6502 /** 6503 * Flush all filters. 6504 * 6505 * @param dev 6506 * Pointer to Ethernet device. 6507 */ 6508 static void 6509 flow_fdir_filter_flush(struct rte_eth_dev *dev) 6510 { 6511 struct mlx5_priv *priv = dev->data->dev_private; 6512 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 6513 6514 while (!LIST_EMPTY(&priv->fdir_flows)) { 6515 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); 6516 LIST_REMOVE(priv_fdir_flow, next); 6517 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); 6518 mlx5_free(priv_fdir_flow->fdir); 6519 mlx5_free(priv_fdir_flow); 6520 } 6521 } 6522 6523 /** 6524 * Get flow director information. 6525 * 6526 * @param dev 6527 * Pointer to Ethernet device. 6528 * @param[out] fdir_info 6529 * Resulting flow director information. 6530 */ 6531 static void 6532 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 6533 { 6534 struct rte_eth_fdir_masks *mask = 6535 &dev->data->dev_conf.fdir_conf.mask; 6536 6537 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 6538 fdir_info->guarant_spc = 0; 6539 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 6540 fdir_info->max_flexpayload = 0; 6541 fdir_info->flow_types_mask[0] = 0; 6542 fdir_info->flex_payload_unit = 0; 6543 fdir_info->max_flex_payload_segment_num = 0; 6544 fdir_info->flex_payload_limit = 0; 6545 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 6546 } 6547 6548 /** 6549 * Deal with flow director operations. 6550 * 6551 * @param dev 6552 * Pointer to Ethernet device. 6553 * @param filter_op 6554 * Operation to perform. 6555 * @param arg 6556 * Pointer to operation-specific structure. 6557 * 6558 * @return 6559 * 0 on success, a negative errno value otherwise and rte_errno is set. 6560 */ 6561 static int 6562 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 6563 void *arg) 6564 { 6565 enum rte_fdir_mode fdir_mode = 6566 dev->data->dev_conf.fdir_conf.mode; 6567 6568 if (filter_op == RTE_ETH_FILTER_NOP) 6569 return 0; 6570 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 6571 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 6572 DRV_LOG(ERR, "port %u flow director mode %d not supported", 6573 dev->data->port_id, fdir_mode); 6574 rte_errno = EINVAL; 6575 return -rte_errno; 6576 } 6577 switch (filter_op) { 6578 case RTE_ETH_FILTER_ADD: 6579 return flow_fdir_filter_add(dev, arg); 6580 case RTE_ETH_FILTER_UPDATE: 6581 return flow_fdir_filter_update(dev, arg); 6582 case RTE_ETH_FILTER_DELETE: 6583 return flow_fdir_filter_delete(dev, arg); 6584 case RTE_ETH_FILTER_FLUSH: 6585 flow_fdir_filter_flush(dev); 6586 break; 6587 case RTE_ETH_FILTER_INFO: 6588 flow_fdir_info_get(dev, arg); 6589 break; 6590 default: 6591 DRV_LOG(DEBUG, "port %u unknown operation %u", 6592 dev->data->port_id, filter_op); 6593 rte_errno = EINVAL; 6594 return -rte_errno; 6595 } 6596 return 0; 6597 } 6598 6599 /** 6600 * Manage filter operations. 6601 * 6602 * @param dev 6603 * Pointer to Ethernet device structure. 6604 * @param filter_type 6605 * Filter type. 6606 * @param filter_op 6607 * Operation to perform. 6608 * @param arg 6609 * Pointer to operation-specific structure. 6610 * 6611 * @return 6612 * 0 on success, a negative errno value otherwise and rte_errno is set. 6613 */ 6614 int 6615 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 6616 enum rte_filter_type filter_type, 6617 enum rte_filter_op filter_op, 6618 void *arg) 6619 { 6620 switch (filter_type) { 6621 case RTE_ETH_FILTER_GENERIC: 6622 if (filter_op != RTE_ETH_FILTER_GET) { 6623 rte_errno = EINVAL; 6624 return -rte_errno; 6625 } 6626 *(const void **)arg = &mlx5_flow_ops; 6627 return 0; 6628 case RTE_ETH_FILTER_FDIR: 6629 return flow_fdir_ctrl_func(dev, filter_op, arg); 6630 default: 6631 DRV_LOG(ERR, "port %u filter type (%d) not supported", 6632 dev->data->port_id, filter_type); 6633 rte_errno = ENOTSUP; 6634 return -rte_errno; 6635 } 6636 return 0; 6637 } 6638 6639 /** 6640 * Create the needed meter and suffix tables. 6641 * 6642 * @param[in] dev 6643 * Pointer to Ethernet device. 6644 * @param[in] fm 6645 * Pointer to the flow meter. 6646 * 6647 * @return 6648 * Pointer to table set on success, NULL otherwise. 6649 */ 6650 struct mlx5_meter_domains_infos * 6651 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 6652 const struct mlx5_flow_meter *fm) 6653 { 6654 const struct mlx5_flow_driver_ops *fops; 6655 6656 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6657 return fops->create_mtr_tbls(dev, fm); 6658 } 6659 6660 /** 6661 * Destroy the meter table set. 6662 * 6663 * @param[in] dev 6664 * Pointer to Ethernet device. 6665 * @param[in] tbl 6666 * Pointer to the meter table set. 6667 * 6668 * @return 6669 * 0 on success. 6670 */ 6671 int 6672 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 6673 struct mlx5_meter_domains_infos *tbls) 6674 { 6675 const struct mlx5_flow_driver_ops *fops; 6676 6677 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6678 return fops->destroy_mtr_tbls(dev, tbls); 6679 } 6680 6681 /** 6682 * Create policer rules. 6683 * 6684 * @param[in] dev 6685 * Pointer to Ethernet device. 6686 * @param[in] fm 6687 * Pointer to flow meter structure. 6688 * @param[in] attr 6689 * Pointer to flow attributes. 6690 * 6691 * @return 6692 * 0 on success, -1 otherwise. 6693 */ 6694 int 6695 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 6696 struct mlx5_flow_meter *fm, 6697 const struct rte_flow_attr *attr) 6698 { 6699 const struct mlx5_flow_driver_ops *fops; 6700 6701 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6702 return fops->create_policer_rules(dev, fm, attr); 6703 } 6704 6705 /** 6706 * Destroy policer rules. 6707 * 6708 * @param[in] fm 6709 * Pointer to flow meter structure. 6710 * @param[in] attr 6711 * Pointer to flow attributes. 6712 * 6713 * @return 6714 * 0 on success, -1 otherwise. 6715 */ 6716 int 6717 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 6718 struct mlx5_flow_meter *fm, 6719 const struct rte_flow_attr *attr) 6720 { 6721 const struct mlx5_flow_driver_ops *fops; 6722 6723 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6724 return fops->destroy_policer_rules(dev, fm, attr); 6725 } 6726 6727 /** 6728 * Allocate a counter. 6729 * 6730 * @param[in] dev 6731 * Pointer to Ethernet device structure. 6732 * 6733 * @return 6734 * Index to allocated counter on success, 0 otherwise. 6735 */ 6736 uint32_t 6737 mlx5_counter_alloc(struct rte_eth_dev *dev) 6738 { 6739 const struct mlx5_flow_driver_ops *fops; 6740 struct rte_flow_attr attr = { .transfer = 0 }; 6741 6742 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6743 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6744 return fops->counter_alloc(dev); 6745 } 6746 DRV_LOG(ERR, 6747 "port %u counter allocate is not supported.", 6748 dev->data->port_id); 6749 return 0; 6750 } 6751 6752 /** 6753 * Free a counter. 6754 * 6755 * @param[in] dev 6756 * Pointer to Ethernet device structure. 6757 * @param[in] cnt 6758 * Index to counter to be free. 6759 */ 6760 void 6761 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 6762 { 6763 const struct mlx5_flow_driver_ops *fops; 6764 struct rte_flow_attr attr = { .transfer = 0 }; 6765 6766 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6767 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6768 fops->counter_free(dev, cnt); 6769 return; 6770 } 6771 DRV_LOG(ERR, 6772 "port %u counter free is not supported.", 6773 dev->data->port_id); 6774 } 6775 6776 /** 6777 * Query counter statistics. 6778 * 6779 * @param[in] dev 6780 * Pointer to Ethernet device structure. 6781 * @param[in] cnt 6782 * Index to counter to query. 6783 * @param[in] clear 6784 * Set to clear counter statistics. 6785 * @param[out] pkts 6786 * The counter hits packets number to save. 6787 * @param[out] bytes 6788 * The counter hits bytes number to save. 6789 * 6790 * @return 6791 * 0 on success, a negative errno value otherwise. 6792 */ 6793 int 6794 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 6795 bool clear, uint64_t *pkts, uint64_t *bytes) 6796 { 6797 const struct mlx5_flow_driver_ops *fops; 6798 struct rte_flow_attr attr = { .transfer = 0 }; 6799 6800 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6801 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6802 return fops->counter_query(dev, cnt, clear, pkts, bytes); 6803 } 6804 DRV_LOG(ERR, 6805 "port %u counter query is not supported.", 6806 dev->data->port_id); 6807 return -ENOTSUP; 6808 } 6809 6810 /** 6811 * Allocate a new memory for the counter values wrapped by all the needed 6812 * management. 6813 * 6814 * @param[in] sh 6815 * Pointer to mlx5_dev_ctx_shared object. 6816 * 6817 * @return 6818 * 0 on success, a negative errno value otherwise. 6819 */ 6820 static int 6821 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 6822 { 6823 struct mlx5_devx_mkey_attr mkey_attr; 6824 struct mlx5_counter_stats_mem_mng *mem_mng; 6825 volatile struct flow_counter_stats *raw_data; 6826 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 6827 int size = (sizeof(struct flow_counter_stats) * 6828 MLX5_COUNTERS_PER_POOL + 6829 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 6830 sizeof(struct mlx5_counter_stats_mem_mng); 6831 size_t pgsize = rte_mem_page_size(); 6832 uint8_t *mem; 6833 int i; 6834 6835 if (pgsize == (size_t)-1) { 6836 DRV_LOG(ERR, "Failed to get mem page size"); 6837 rte_errno = ENOMEM; 6838 return -ENOMEM; 6839 } 6840 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 6841 if (!mem) { 6842 rte_errno = ENOMEM; 6843 return -ENOMEM; 6844 } 6845 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 6846 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 6847 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 6848 IBV_ACCESS_LOCAL_WRITE); 6849 if (!mem_mng->umem) { 6850 rte_errno = errno; 6851 mlx5_free(mem); 6852 return -rte_errno; 6853 } 6854 mkey_attr.addr = (uintptr_t)mem; 6855 mkey_attr.size = size; 6856 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 6857 mkey_attr.pd = sh->pdn; 6858 mkey_attr.log_entity_size = 0; 6859 mkey_attr.pg_access = 0; 6860 mkey_attr.klm_array = NULL; 6861 mkey_attr.klm_num = 0; 6862 mkey_attr.relaxed_ordering = sh->cmng.relaxed_ordering; 6863 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 6864 if (!mem_mng->dm) { 6865 mlx5_glue->devx_umem_dereg(mem_mng->umem); 6866 rte_errno = errno; 6867 mlx5_free(mem); 6868 return -rte_errno; 6869 } 6870 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 6871 raw_data = (volatile struct flow_counter_stats *)mem; 6872 for (i = 0; i < raws_n; ++i) { 6873 mem_mng->raws[i].mem_mng = mem_mng; 6874 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 6875 } 6876 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 6877 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 6878 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 6879 next); 6880 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 6881 sh->cmng.mem_mng = mem_mng; 6882 return 0; 6883 } 6884 6885 /** 6886 * Set the statistic memory to the new counter pool. 6887 * 6888 * @param[in] sh 6889 * Pointer to mlx5_dev_ctx_shared object. 6890 * @param[in] pool 6891 * Pointer to the pool to set the statistic memory. 6892 * 6893 * @return 6894 * 0 on success, a negative errno value otherwise. 6895 */ 6896 static int 6897 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 6898 struct mlx5_flow_counter_pool *pool) 6899 { 6900 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6901 /* Resize statistic memory once used out. */ 6902 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 6903 mlx5_flow_create_counter_stat_mem_mng(sh)) { 6904 DRV_LOG(ERR, "Cannot resize counter stat mem."); 6905 return -1; 6906 } 6907 rte_spinlock_lock(&pool->sl); 6908 pool->raw = cmng->mem_mng->raws + pool->index % 6909 MLX5_CNT_CONTAINER_RESIZE; 6910 rte_spinlock_unlock(&pool->sl); 6911 pool->raw_hw = NULL; 6912 return 0; 6913 } 6914 6915 #define MLX5_POOL_QUERY_FREQ_US 1000000 6916 6917 /** 6918 * Set the periodic procedure for triggering asynchronous batch queries for all 6919 * the counter pools. 6920 * 6921 * @param[in] sh 6922 * Pointer to mlx5_dev_ctx_shared object. 6923 */ 6924 void 6925 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 6926 { 6927 uint32_t pools_n, us; 6928 6929 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 6930 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 6931 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 6932 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 6933 sh->cmng.query_thread_on = 0; 6934 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 6935 } else { 6936 sh->cmng.query_thread_on = 1; 6937 } 6938 } 6939 6940 /** 6941 * The periodic procedure for triggering asynchronous batch queries for all the 6942 * counter pools. This function is probably called by the host thread. 6943 * 6944 * @param[in] arg 6945 * The parameter for the alarm process. 6946 */ 6947 void 6948 mlx5_flow_query_alarm(void *arg) 6949 { 6950 struct mlx5_dev_ctx_shared *sh = arg; 6951 int ret; 6952 uint16_t pool_index = sh->cmng.pool_index; 6953 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 6954 struct mlx5_flow_counter_pool *pool; 6955 uint16_t n_valid; 6956 6957 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 6958 goto set_alarm; 6959 rte_spinlock_lock(&cmng->pool_update_sl); 6960 pool = cmng->pools[pool_index]; 6961 n_valid = cmng->n_valid; 6962 rte_spinlock_unlock(&cmng->pool_update_sl); 6963 /* Set the statistic memory to the new created pool. */ 6964 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 6965 goto set_alarm; 6966 if (pool->raw_hw) 6967 /* There is a pool query in progress. */ 6968 goto set_alarm; 6969 pool->raw_hw = 6970 LIST_FIRST(&sh->cmng.free_stat_raws); 6971 if (!pool->raw_hw) 6972 /* No free counter statistics raw memory. */ 6973 goto set_alarm; 6974 /* 6975 * Identify the counters released between query trigger and query 6976 * handle more efficiently. The counter released in this gap period 6977 * should wait for a new round of query as the new arrived packets 6978 * will not be taken into account. 6979 */ 6980 pool->query_gen++; 6981 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 6982 MLX5_COUNTERS_PER_POOL, 6983 NULL, NULL, 6984 pool->raw_hw->mem_mng->dm->id, 6985 (void *)(uintptr_t) 6986 pool->raw_hw->data, 6987 sh->devx_comp, 6988 (uint64_t)(uintptr_t)pool); 6989 if (ret) { 6990 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 6991 " %d", pool->min_dcs->id); 6992 pool->raw_hw = NULL; 6993 goto set_alarm; 6994 } 6995 LIST_REMOVE(pool->raw_hw, next); 6996 sh->cmng.pending_queries++; 6997 pool_index++; 6998 if (pool_index >= n_valid) 6999 pool_index = 0; 7000 set_alarm: 7001 sh->cmng.pool_index = pool_index; 7002 mlx5_set_query_alarm(sh); 7003 } 7004 7005 /** 7006 * Check and callback event for new aged flow in the counter pool 7007 * 7008 * @param[in] sh 7009 * Pointer to mlx5_dev_ctx_shared object. 7010 * @param[in] pool 7011 * Pointer to Current counter pool. 7012 */ 7013 static void 7014 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 7015 struct mlx5_flow_counter_pool *pool) 7016 { 7017 struct mlx5_priv *priv; 7018 struct mlx5_flow_counter *cnt; 7019 struct mlx5_age_info *age_info; 7020 struct mlx5_age_param *age_param; 7021 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 7022 struct mlx5_counter_stats_raw *prev = pool->raw; 7023 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 7024 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 7025 uint16_t expected = AGE_CANDIDATE; 7026 uint32_t i; 7027 7028 pool->time_of_last_age_check = curr_time; 7029 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 7030 cnt = MLX5_POOL_GET_CNT(pool, i); 7031 age_param = MLX5_CNT_TO_AGE(cnt); 7032 if (__atomic_load_n(&age_param->state, 7033 __ATOMIC_RELAXED) != AGE_CANDIDATE) 7034 continue; 7035 if (cur->data[i].hits != prev->data[i].hits) { 7036 __atomic_store_n(&age_param->sec_since_last_hit, 0, 7037 __ATOMIC_RELAXED); 7038 continue; 7039 } 7040 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 7041 time_delta, 7042 __ATOMIC_RELAXED) <= age_param->timeout) 7043 continue; 7044 /** 7045 * Hold the lock first, or if between the 7046 * state AGE_TMOUT and tailq operation the 7047 * release happened, the release procedure 7048 * may delete a non-existent tailq node. 7049 */ 7050 priv = rte_eth_devices[age_param->port_id].data->dev_private; 7051 age_info = GET_PORT_AGE_INFO(priv); 7052 rte_spinlock_lock(&age_info->aged_sl); 7053 if (__atomic_compare_exchange_n(&age_param->state, &expected, 7054 AGE_TMOUT, false, 7055 __ATOMIC_RELAXED, 7056 __ATOMIC_RELAXED)) { 7057 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 7058 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 7059 } 7060 rte_spinlock_unlock(&age_info->aged_sl); 7061 } 7062 for (i = 0; i < sh->max_port; i++) { 7063 age_info = &sh->port[i].age_info; 7064 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 7065 continue; 7066 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) 7067 rte_eth_dev_callback_process 7068 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 7069 RTE_ETH_EVENT_FLOW_AGED, NULL); 7070 age_info->flags = 0; 7071 } 7072 } 7073 7074 /** 7075 * Handler for the HW respond about ready values from an asynchronous batch 7076 * query. This function is probably called by the host thread. 7077 * 7078 * @param[in] sh 7079 * The pointer to the shared device context. 7080 * @param[in] async_id 7081 * The Devx async ID. 7082 * @param[in] status 7083 * The status of the completion. 7084 */ 7085 void 7086 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 7087 uint64_t async_id, int status) 7088 { 7089 struct mlx5_flow_counter_pool *pool = 7090 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 7091 struct mlx5_counter_stats_raw *raw_to_free; 7092 uint8_t query_gen = pool->query_gen ^ 1; 7093 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 7094 enum mlx5_counter_type cnt_type = 7095 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 7096 MLX5_COUNTER_TYPE_ORIGIN; 7097 7098 if (unlikely(status)) { 7099 raw_to_free = pool->raw_hw; 7100 } else { 7101 raw_to_free = pool->raw; 7102 if (pool->is_aged) 7103 mlx5_flow_aging_check(sh, pool); 7104 rte_spinlock_lock(&pool->sl); 7105 pool->raw = pool->raw_hw; 7106 rte_spinlock_unlock(&pool->sl); 7107 /* Be sure the new raw counters data is updated in memory. */ 7108 rte_io_wmb(); 7109 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 7110 rte_spinlock_lock(&cmng->csl[cnt_type]); 7111 TAILQ_CONCAT(&cmng->counters[cnt_type], 7112 &pool->counters[query_gen], next); 7113 rte_spinlock_unlock(&cmng->csl[cnt_type]); 7114 } 7115 } 7116 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 7117 pool->raw_hw = NULL; 7118 sh->cmng.pending_queries--; 7119 } 7120 7121 /** 7122 * Translate the rte_flow group index to HW table value. 7123 * 7124 * @param[in] attributes 7125 * Pointer to flow attributes 7126 * @param[in] external 7127 * Value is part of flow rule created by request external to PMD. 7128 * @param[in] group 7129 * rte_flow group index value. 7130 * @param[out] fdb_def_rule 7131 * Whether fdb jump to table 1 is configured. 7132 * @param[out] table 7133 * HW table value. 7134 * @param[out] error 7135 * Pointer to error structure. 7136 * 7137 * @return 7138 * 0 on success, a negative errno value otherwise and rte_errno is set. 7139 */ 7140 int 7141 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 7142 uint32_t group, bool fdb_def_rule, uint32_t *table, 7143 struct rte_flow_error *error) 7144 { 7145 if (attributes->transfer && external && fdb_def_rule) { 7146 if (group == UINT32_MAX) 7147 return rte_flow_error_set 7148 (error, EINVAL, 7149 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 7150 NULL, 7151 "group index not supported"); 7152 *table = group + 1; 7153 } else { 7154 *table = group; 7155 } 7156 return 0; 7157 } 7158 7159 /** 7160 * Discover availability of metadata reg_c's. 7161 * 7162 * Iteratively use test flows to check availability. 7163 * 7164 * @param[in] dev 7165 * Pointer to the Ethernet device structure. 7166 * 7167 * @return 7168 * 0 on success, a negative errno value otherwise and rte_errno is set. 7169 */ 7170 int 7171 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 7172 { 7173 struct mlx5_priv *priv = dev->data->dev_private; 7174 struct mlx5_dev_config *config = &priv->config; 7175 enum modify_reg idx; 7176 int n = 0; 7177 7178 /* reg_c[0] and reg_c[1] are reserved. */ 7179 config->flow_mreg_c[n++] = REG_C_0; 7180 config->flow_mreg_c[n++] = REG_C_1; 7181 /* Discover availability of other reg_c's. */ 7182 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 7183 struct rte_flow_attr attr = { 7184 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 7185 .priority = MLX5_FLOW_PRIO_RSVD, 7186 .ingress = 1, 7187 }; 7188 struct rte_flow_item items[] = { 7189 [0] = { 7190 .type = RTE_FLOW_ITEM_TYPE_END, 7191 }, 7192 }; 7193 struct rte_flow_action actions[] = { 7194 [0] = { 7195 .type = (enum rte_flow_action_type) 7196 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 7197 .conf = &(struct mlx5_flow_action_copy_mreg){ 7198 .src = REG_C_1, 7199 .dst = idx, 7200 }, 7201 }, 7202 [1] = { 7203 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7204 .conf = &(struct rte_flow_action_jump){ 7205 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 7206 }, 7207 }, 7208 [2] = { 7209 .type = RTE_FLOW_ACTION_TYPE_END, 7210 }, 7211 }; 7212 uint32_t flow_idx; 7213 struct rte_flow *flow; 7214 struct rte_flow_error error; 7215 7216 if (!config->dv_flow_en) 7217 break; 7218 /* Create internal flow, validation skips copy action. */ 7219 flow_idx = flow_list_create(dev, NULL, &attr, items, 7220 actions, false, &error); 7221 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 7222 flow_idx); 7223 if (!flow) 7224 continue; 7225 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 7226 config->flow_mreg_c[n++] = idx; 7227 flow_list_destroy(dev, NULL, flow_idx); 7228 } 7229 for (; n < MLX5_MREG_C_NUM; ++n) 7230 config->flow_mreg_c[n] = REG_NON; 7231 return 0; 7232 } 7233 7234 /** 7235 * Dump flow raw hw data to file 7236 * 7237 * @param[in] dev 7238 * The pointer to Ethernet device. 7239 * @param[in] file 7240 * A pointer to a file for output. 7241 * @param[out] error 7242 * Perform verbose error reporting if not NULL. PMDs initialize this 7243 * structure in case of error only. 7244 * @return 7245 * 0 on success, a nagative value otherwise. 7246 */ 7247 int 7248 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 7249 FILE *file, 7250 struct rte_flow_error *error __rte_unused) 7251 { 7252 struct mlx5_priv *priv = dev->data->dev_private; 7253 struct mlx5_dev_ctx_shared *sh = priv->sh; 7254 7255 if (!priv->config.dv_flow_en) { 7256 if (fputs("device dv flow disabled\n", file) <= 0) 7257 return -errno; 7258 return -ENOTSUP; 7259 } 7260 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 7261 sh->tx_domain, file); 7262 } 7263 7264 /** 7265 * Get aged-out flows. 7266 * 7267 * @param[in] dev 7268 * Pointer to the Ethernet device structure. 7269 * @param[in] context 7270 * The address of an array of pointers to the aged-out flows contexts. 7271 * @param[in] nb_countexts 7272 * The length of context array pointers. 7273 * @param[out] error 7274 * Perform verbose error reporting if not NULL. Initialized in case of 7275 * error only. 7276 * 7277 * @return 7278 * how many contexts get in success, otherwise negative errno value. 7279 * if nb_contexts is 0, return the amount of all aged contexts. 7280 * if nb_contexts is not 0 , return the amount of aged flows reported 7281 * in the context array. 7282 */ 7283 int 7284 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 7285 uint32_t nb_contexts, struct rte_flow_error *error) 7286 { 7287 const struct mlx5_flow_driver_ops *fops; 7288 struct rte_flow_attr attr = { .transfer = 0 }; 7289 7290 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7291 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7292 return fops->get_aged_flows(dev, contexts, nb_contexts, 7293 error); 7294 } 7295 DRV_LOG(ERR, 7296 "port %u get aged flows is not supported.", 7297 dev->data->port_id); 7298 return -ENOTSUP; 7299 } 7300 7301 /* Wrapper for driver action_validate op callback */ 7302 static int 7303 flow_drv_action_validate(struct rte_eth_dev *dev, 7304 const struct rte_flow_shared_action_conf *conf, 7305 const struct rte_flow_action *action, 7306 const struct mlx5_flow_driver_ops *fops, 7307 struct rte_flow_error *error) 7308 { 7309 static const char err_msg[] = "shared action validation unsupported"; 7310 7311 if (!fops->action_validate) { 7312 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7313 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7314 NULL, err_msg); 7315 return -rte_errno; 7316 } 7317 return fops->action_validate(dev, conf, action, error); 7318 } 7319 7320 /** 7321 * Destroys the shared action by handle. 7322 * 7323 * @param dev 7324 * Pointer to Ethernet device structure. 7325 * @param[in] action 7326 * Handle for the shared action to be destroyed. 7327 * @param[out] error 7328 * Perform verbose error reporting if not NULL. PMDs initialize this 7329 * structure in case of error only. 7330 * 7331 * @return 7332 * 0 on success, a negative errno value otherwise and rte_errno is set. 7333 * 7334 * @note: wrapper for driver action_create op callback. 7335 */ 7336 static int 7337 mlx5_shared_action_destroy(struct rte_eth_dev *dev, 7338 struct rte_flow_shared_action *action, 7339 struct rte_flow_error *error) 7340 { 7341 static const char err_msg[] = "shared action destruction unsupported"; 7342 struct rte_flow_attr attr = { .transfer = 0 }; 7343 const struct mlx5_flow_driver_ops *fops = 7344 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7345 7346 if (!fops->action_destroy) { 7347 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7348 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7349 NULL, err_msg); 7350 return -rte_errno; 7351 } 7352 return fops->action_destroy(dev, action, error); 7353 } 7354 7355 /* Wrapper for driver action_destroy op callback */ 7356 static int 7357 flow_drv_action_update(struct rte_eth_dev *dev, 7358 struct rte_flow_shared_action *action, 7359 const void *action_conf, 7360 const struct mlx5_flow_driver_ops *fops, 7361 struct rte_flow_error *error) 7362 { 7363 static const char err_msg[] = "shared action update unsupported"; 7364 7365 if (!fops->action_update) { 7366 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7367 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7368 NULL, err_msg); 7369 return -rte_errno; 7370 } 7371 return fops->action_update(dev, action, action_conf, error); 7372 } 7373 7374 /** 7375 * Create shared action for reuse in multiple flow rules. 7376 * 7377 * @param dev 7378 * Pointer to Ethernet device structure. 7379 * @param[in] action 7380 * Action configuration for shared action creation. 7381 * @param[out] error 7382 * Perform verbose error reporting if not NULL. PMDs initialize this 7383 * structure in case of error only. 7384 * @return 7385 * A valid handle in case of success, NULL otherwise and rte_errno is set. 7386 */ 7387 static struct rte_flow_shared_action * 7388 mlx5_shared_action_create(struct rte_eth_dev *dev, 7389 const struct rte_flow_shared_action_conf *conf, 7390 const struct rte_flow_action *action, 7391 struct rte_flow_error *error) 7392 { 7393 static const char err_msg[] = "shared action creation unsupported"; 7394 struct rte_flow_attr attr = { .transfer = 0 }; 7395 const struct mlx5_flow_driver_ops *fops = 7396 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7397 7398 if (flow_drv_action_validate(dev, conf, action, fops, error)) 7399 return NULL; 7400 if (!fops->action_create) { 7401 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 7402 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 7403 NULL, err_msg); 7404 return NULL; 7405 } 7406 return fops->action_create(dev, conf, action, error); 7407 } 7408 7409 /** 7410 * Updates inplace the shared action configuration pointed by *action* handle 7411 * with the configuration provided as *action* argument. 7412 * The update of the shared action configuration effects all flow rules reusing 7413 * the action via handle. 7414 * 7415 * @param dev 7416 * Pointer to Ethernet device structure. 7417 * @param[in] shared_action 7418 * Handle for the shared action to be updated. 7419 * @param[in] action 7420 * Action specification used to modify the action pointed by handle. 7421 * *action* should be of same type with the action pointed by the *action* 7422 * handle argument, otherwise considered as invalid. 7423 * @param[out] error 7424 * Perform verbose error reporting if not NULL. PMDs initialize this 7425 * structure in case of error only. 7426 * 7427 * @return 7428 * 0 on success, a negative errno value otherwise and rte_errno is set. 7429 */ 7430 static int 7431 mlx5_shared_action_update(struct rte_eth_dev *dev, 7432 struct rte_flow_shared_action *shared_action, 7433 const struct rte_flow_action *action, 7434 struct rte_flow_error *error) 7435 { 7436 struct rte_flow_attr attr = { .transfer = 0 }; 7437 const struct mlx5_flow_driver_ops *fops = 7438 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 7439 int ret; 7440 7441 switch (shared_action->type) { 7442 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 7443 if (action->type != RTE_FLOW_ACTION_TYPE_RSS) { 7444 return rte_flow_error_set(error, EINVAL, 7445 RTE_FLOW_ERROR_TYPE_ACTION, 7446 NULL, 7447 "update action type invalid"); 7448 } 7449 ret = flow_drv_action_validate(dev, NULL, action, fops, error); 7450 if (ret) 7451 return ret; 7452 return flow_drv_action_update(dev, shared_action, action->conf, 7453 fops, error); 7454 default: 7455 return rte_flow_error_set(error, ENOTSUP, 7456 RTE_FLOW_ERROR_TYPE_ACTION, 7457 NULL, 7458 "action type not supported"); 7459 } 7460 } 7461 7462 /** 7463 * Query the shared action by handle. 7464 * 7465 * This function allows retrieving action-specific data such as counters. 7466 * Data is gathered by special action which may be present/referenced in 7467 * more than one flow rule definition. 7468 * 7469 * \see RTE_FLOW_ACTION_TYPE_COUNT 7470 * 7471 * @param dev 7472 * Pointer to Ethernet device structure. 7473 * @param[in] action 7474 * Handle for the shared action to query. 7475 * @param[in, out] data 7476 * Pointer to storage for the associated query data type. 7477 * @param[out] error 7478 * Perform verbose error reporting if not NULL. PMDs initialize this 7479 * structure in case of error only. 7480 * 7481 * @return 7482 * 0 on success, a negative errno value otherwise and rte_errno is set. 7483 */ 7484 static int 7485 mlx5_shared_action_query(struct rte_eth_dev *dev, 7486 const struct rte_flow_shared_action *action, 7487 void *data, 7488 struct rte_flow_error *error) 7489 { 7490 (void)dev; 7491 switch (action->type) { 7492 case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: 7493 __atomic_load(&action->refcnt, (uint32_t *)data, 7494 __ATOMIC_RELAXED); 7495 return 0; 7496 default: 7497 return rte_flow_error_set(error, ENOTSUP, 7498 RTE_FLOW_ERROR_TYPE_ACTION, 7499 NULL, 7500 "action type not supported"); 7501 } 7502 } 7503 7504 /** 7505 * Destroy all shared actions. 7506 * 7507 * @param dev 7508 * Pointer to Ethernet device. 7509 * 7510 * @return 7511 * 0 on success, a negative errno value otherwise and rte_errno is set. 7512 */ 7513 int 7514 mlx5_shared_action_flush(struct rte_eth_dev *dev) 7515 { 7516 struct rte_flow_error error; 7517 struct mlx5_priv *priv = dev->data->dev_private; 7518 struct rte_flow_shared_action *action; 7519 int ret = 0; 7520 7521 while (!LIST_EMPTY(&priv->shared_actions)) { 7522 action = LIST_FIRST(&priv->shared_actions); 7523 ret = mlx5_shared_action_destroy(dev, action, &error); 7524 } 7525 return ret; 7526 } 7527