1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 12 /* Verbs header. */ 13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 14 #ifdef PEDANTIC 15 #pragma GCC diagnostic ignored "-Wpedantic" 16 #endif 17 #include <infiniband/verbs.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_common.h> 23 #include <rte_ether.h> 24 #include <rte_ethdev_driver.h> 25 #include <rte_flow.h> 26 #include <rte_flow_driver.h> 27 #include <rte_malloc.h> 28 #include <rte_ip.h> 29 30 #include "mlx5.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_flow.h" 33 #include "mlx5_glue.h" 34 #include "mlx5_prm.h" 35 #include "mlx5_rxtx.h" 36 37 /* Dev ops structure defined in mlx5.c */ 38 extern const struct eth_dev_ops mlx5_dev_ops; 39 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 40 41 /** Device flow drivers. */ 42 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 43 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 44 #endif 45 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 46 47 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 48 49 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 50 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 51 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 52 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 53 #endif 54 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 55 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 56 }; 57 58 enum mlx5_expansion { 59 MLX5_EXPANSION_ROOT, 60 MLX5_EXPANSION_ROOT_OUTER, 61 MLX5_EXPANSION_ROOT_ETH_VLAN, 62 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 63 MLX5_EXPANSION_OUTER_ETH, 64 MLX5_EXPANSION_OUTER_ETH_VLAN, 65 MLX5_EXPANSION_OUTER_VLAN, 66 MLX5_EXPANSION_OUTER_IPV4, 67 MLX5_EXPANSION_OUTER_IPV4_UDP, 68 MLX5_EXPANSION_OUTER_IPV4_TCP, 69 MLX5_EXPANSION_OUTER_IPV6, 70 MLX5_EXPANSION_OUTER_IPV6_UDP, 71 MLX5_EXPANSION_OUTER_IPV6_TCP, 72 MLX5_EXPANSION_VXLAN, 73 MLX5_EXPANSION_VXLAN_GPE, 74 MLX5_EXPANSION_GRE, 75 MLX5_EXPANSION_MPLS, 76 MLX5_EXPANSION_ETH, 77 MLX5_EXPANSION_ETH_VLAN, 78 MLX5_EXPANSION_VLAN, 79 MLX5_EXPANSION_IPV4, 80 MLX5_EXPANSION_IPV4_UDP, 81 MLX5_EXPANSION_IPV4_TCP, 82 MLX5_EXPANSION_IPV6, 83 MLX5_EXPANSION_IPV6_UDP, 84 MLX5_EXPANSION_IPV6_TCP, 85 }; 86 87 /** Supported expansion of items. */ 88 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 89 [MLX5_EXPANSION_ROOT] = { 90 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 91 MLX5_EXPANSION_IPV4, 92 MLX5_EXPANSION_IPV6), 93 .type = RTE_FLOW_ITEM_TYPE_END, 94 }, 95 [MLX5_EXPANSION_ROOT_OUTER] = { 96 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 97 MLX5_EXPANSION_OUTER_IPV4, 98 MLX5_EXPANSION_OUTER_IPV6), 99 .type = RTE_FLOW_ITEM_TYPE_END, 100 }, 101 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 102 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 103 .type = RTE_FLOW_ITEM_TYPE_END, 104 }, 105 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 107 .type = RTE_FLOW_ITEM_TYPE_END, 108 }, 109 [MLX5_EXPANSION_OUTER_ETH] = { 110 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 111 MLX5_EXPANSION_OUTER_IPV6, 112 MLX5_EXPANSION_MPLS), 113 .type = RTE_FLOW_ITEM_TYPE_ETH, 114 .rss_types = 0, 115 }, 116 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 117 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 118 .type = RTE_FLOW_ITEM_TYPE_ETH, 119 .rss_types = 0, 120 }, 121 [MLX5_EXPANSION_OUTER_VLAN] = { 122 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 123 MLX5_EXPANSION_OUTER_IPV6), 124 .type = RTE_FLOW_ITEM_TYPE_VLAN, 125 }, 126 [MLX5_EXPANSION_OUTER_IPV4] = { 127 .next = RTE_FLOW_EXPAND_RSS_NEXT 128 (MLX5_EXPANSION_OUTER_IPV4_UDP, 129 MLX5_EXPANSION_OUTER_IPV4_TCP, 130 MLX5_EXPANSION_GRE, 131 MLX5_EXPANSION_IPV4, 132 MLX5_EXPANSION_IPV6), 133 .type = RTE_FLOW_ITEM_TYPE_IPV4, 134 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 135 ETH_RSS_NONFRAG_IPV4_OTHER, 136 }, 137 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 138 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 139 MLX5_EXPANSION_VXLAN_GPE), 140 .type = RTE_FLOW_ITEM_TYPE_UDP, 141 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 142 }, 143 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 144 .type = RTE_FLOW_ITEM_TYPE_TCP, 145 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 146 }, 147 [MLX5_EXPANSION_OUTER_IPV6] = { 148 .next = RTE_FLOW_EXPAND_RSS_NEXT 149 (MLX5_EXPANSION_OUTER_IPV6_UDP, 150 MLX5_EXPANSION_OUTER_IPV6_TCP, 151 MLX5_EXPANSION_IPV4, 152 MLX5_EXPANSION_IPV6), 153 .type = RTE_FLOW_ITEM_TYPE_IPV6, 154 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 155 ETH_RSS_NONFRAG_IPV6_OTHER, 156 }, 157 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 158 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 159 MLX5_EXPANSION_VXLAN_GPE), 160 .type = RTE_FLOW_ITEM_TYPE_UDP, 161 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 162 }, 163 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 164 .type = RTE_FLOW_ITEM_TYPE_TCP, 165 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 166 }, 167 [MLX5_EXPANSION_VXLAN] = { 168 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 169 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 170 }, 171 [MLX5_EXPANSION_VXLAN_GPE] = { 172 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 173 MLX5_EXPANSION_IPV4, 174 MLX5_EXPANSION_IPV6), 175 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 176 }, 177 [MLX5_EXPANSION_GRE] = { 178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 179 .type = RTE_FLOW_ITEM_TYPE_GRE, 180 }, 181 [MLX5_EXPANSION_MPLS] = { 182 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 183 MLX5_EXPANSION_IPV6), 184 .type = RTE_FLOW_ITEM_TYPE_MPLS, 185 }, 186 [MLX5_EXPANSION_ETH] = { 187 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 188 MLX5_EXPANSION_IPV6), 189 .type = RTE_FLOW_ITEM_TYPE_ETH, 190 }, 191 [MLX5_EXPANSION_ETH_VLAN] = { 192 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 193 .type = RTE_FLOW_ITEM_TYPE_ETH, 194 }, 195 [MLX5_EXPANSION_VLAN] = { 196 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 197 MLX5_EXPANSION_IPV6), 198 .type = RTE_FLOW_ITEM_TYPE_VLAN, 199 }, 200 [MLX5_EXPANSION_IPV4] = { 201 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 202 MLX5_EXPANSION_IPV4_TCP), 203 .type = RTE_FLOW_ITEM_TYPE_IPV4, 204 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 205 ETH_RSS_NONFRAG_IPV4_OTHER, 206 }, 207 [MLX5_EXPANSION_IPV4_UDP] = { 208 .type = RTE_FLOW_ITEM_TYPE_UDP, 209 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 210 }, 211 [MLX5_EXPANSION_IPV4_TCP] = { 212 .type = RTE_FLOW_ITEM_TYPE_TCP, 213 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 214 }, 215 [MLX5_EXPANSION_IPV6] = { 216 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 217 MLX5_EXPANSION_IPV6_TCP), 218 .type = RTE_FLOW_ITEM_TYPE_IPV6, 219 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 220 ETH_RSS_NONFRAG_IPV6_OTHER, 221 }, 222 [MLX5_EXPANSION_IPV6_UDP] = { 223 .type = RTE_FLOW_ITEM_TYPE_UDP, 224 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 225 }, 226 [MLX5_EXPANSION_IPV6_TCP] = { 227 .type = RTE_FLOW_ITEM_TYPE_TCP, 228 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 229 }, 230 }; 231 232 static const struct rte_flow_ops mlx5_flow_ops = { 233 .validate = mlx5_flow_validate, 234 .create = mlx5_flow_create, 235 .destroy = mlx5_flow_destroy, 236 .flush = mlx5_flow_flush, 237 .isolate = mlx5_flow_isolate, 238 .query = mlx5_flow_query, 239 .dev_dump = mlx5_flow_dev_dump, 240 }; 241 242 /* Convert FDIR request to Generic flow. */ 243 struct mlx5_fdir { 244 struct rte_flow_attr attr; 245 struct rte_flow_item items[4]; 246 struct rte_flow_item_eth l2; 247 struct rte_flow_item_eth l2_mask; 248 union { 249 struct rte_flow_item_ipv4 ipv4; 250 struct rte_flow_item_ipv6 ipv6; 251 } l3; 252 union { 253 struct rte_flow_item_ipv4 ipv4; 254 struct rte_flow_item_ipv6 ipv6; 255 } l3_mask; 256 union { 257 struct rte_flow_item_udp udp; 258 struct rte_flow_item_tcp tcp; 259 } l4; 260 union { 261 struct rte_flow_item_udp udp; 262 struct rte_flow_item_tcp tcp; 263 } l4_mask; 264 struct rte_flow_action actions[2]; 265 struct rte_flow_action_queue queue; 266 }; 267 268 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 269 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 270 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 271 }; 272 273 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 274 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 275 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 276 { 9, 10, 11 }, { 12, 13, 14 }, 277 }; 278 279 /* Tunnel information. */ 280 struct mlx5_flow_tunnel_info { 281 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 282 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 283 }; 284 285 static struct mlx5_flow_tunnel_info tunnels_info[] = { 286 { 287 .tunnel = MLX5_FLOW_LAYER_VXLAN, 288 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 289 }, 290 { 291 .tunnel = MLX5_FLOW_LAYER_GENEVE, 292 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 293 }, 294 { 295 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 296 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 297 }, 298 { 299 .tunnel = MLX5_FLOW_LAYER_GRE, 300 .ptype = RTE_PTYPE_TUNNEL_GRE, 301 }, 302 { 303 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 304 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 305 }, 306 { 307 .tunnel = MLX5_FLOW_LAYER_MPLS, 308 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 309 }, 310 { 311 .tunnel = MLX5_FLOW_LAYER_NVGRE, 312 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 313 }, 314 { 315 .tunnel = MLX5_FLOW_LAYER_IPIP, 316 .ptype = RTE_PTYPE_TUNNEL_IP, 317 }, 318 { 319 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 320 .ptype = RTE_PTYPE_TUNNEL_IP, 321 }, 322 { 323 .tunnel = MLX5_FLOW_LAYER_GTP, 324 .ptype = RTE_PTYPE_TUNNEL_GTPU, 325 }, 326 }; 327 328 /** 329 * Translate tag ID to register. 330 * 331 * @param[in] dev 332 * Pointer to the Ethernet device structure. 333 * @param[in] feature 334 * The feature that request the register. 335 * @param[in] id 336 * The request register ID. 337 * @param[out] error 338 * Error description in case of any. 339 * 340 * @return 341 * The request register on success, a negative errno 342 * value otherwise and rte_errno is set. 343 */ 344 int 345 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 346 enum mlx5_feature_name feature, 347 uint32_t id, 348 struct rte_flow_error *error) 349 { 350 struct mlx5_priv *priv = dev->data->dev_private; 351 struct mlx5_dev_config *config = &priv->config; 352 enum modify_reg start_reg; 353 354 switch (feature) { 355 case MLX5_HAIRPIN_RX: 356 return REG_B; 357 case MLX5_HAIRPIN_TX: 358 return REG_A; 359 case MLX5_METADATA_RX: 360 switch (config->dv_xmeta_en) { 361 case MLX5_XMETA_MODE_LEGACY: 362 return REG_B; 363 case MLX5_XMETA_MODE_META16: 364 return REG_C_0; 365 case MLX5_XMETA_MODE_META32: 366 return REG_C_1; 367 } 368 break; 369 case MLX5_METADATA_TX: 370 return REG_A; 371 case MLX5_METADATA_FDB: 372 switch (config->dv_xmeta_en) { 373 case MLX5_XMETA_MODE_LEGACY: 374 return REG_NONE; 375 case MLX5_XMETA_MODE_META16: 376 return REG_C_0; 377 case MLX5_XMETA_MODE_META32: 378 return REG_C_1; 379 } 380 break; 381 case MLX5_FLOW_MARK: 382 switch (config->dv_xmeta_en) { 383 case MLX5_XMETA_MODE_LEGACY: 384 return REG_NONE; 385 case MLX5_XMETA_MODE_META16: 386 return REG_C_1; 387 case MLX5_XMETA_MODE_META32: 388 return REG_C_0; 389 } 390 break; 391 case MLX5_COPY_MARK: 392 case MLX5_MTR_SFX: 393 /* 394 * Metadata COPY_MARK register using is in meter suffix sub 395 * flow while with meter. It's safe to share the same register. 396 */ 397 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 398 case MLX5_MTR_COLOR: 399 RTE_ASSERT(priv->mtr_color_reg != REG_NONE); 400 return priv->mtr_color_reg; 401 case MLX5_APP_TAG: 402 /* 403 * If meter is enable, it will engage two registers for color 404 * match and flow match. If meter color match is not using the 405 * REG_C_2, need to skip the REG_C_x be used by meter color 406 * match. 407 * If meter is disable, free to use all available registers. 408 */ 409 if (priv->mtr_color_reg != REG_NONE) 410 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_3 : 411 REG_C_4; 412 else 413 start_reg = REG_C_2; 414 if (id > (REG_C_7 - start_reg)) 415 return rte_flow_error_set(error, EINVAL, 416 RTE_FLOW_ERROR_TYPE_ITEM, 417 NULL, "invalid tag id"); 418 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 419 return rte_flow_error_set(error, ENOTSUP, 420 RTE_FLOW_ERROR_TYPE_ITEM, 421 NULL, "unsupported tag id"); 422 /* 423 * This case means meter is using the REG_C_x great than 2. 424 * Take care not to conflict with meter color REG_C_x. 425 * If the available index REG_C_y >= REG_C_x, skip the 426 * color register. 427 */ 428 if (start_reg == REG_C_3 && config->flow_mreg_c 429 [id + REG_C_3 - REG_C_0] >= priv->mtr_color_reg) { 430 if (config->flow_mreg_c[id + 1 + REG_C_3 - REG_C_0] != 431 REG_NONE) 432 return config->flow_mreg_c 433 [id + 1 + REG_C_3 - REG_C_0]; 434 return rte_flow_error_set(error, ENOTSUP, 435 RTE_FLOW_ERROR_TYPE_ITEM, 436 NULL, "unsupported tag id"); 437 } 438 return config->flow_mreg_c[id + start_reg - REG_C_0]; 439 } 440 assert(false); 441 return rte_flow_error_set(error, EINVAL, 442 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 443 NULL, "invalid feature name"); 444 } 445 446 /** 447 * Check extensive flow metadata register support. 448 * 449 * @param dev 450 * Pointer to rte_eth_dev structure. 451 * 452 * @return 453 * True if device supports extensive flow metadata register, otherwise false. 454 */ 455 bool 456 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 457 { 458 struct mlx5_priv *priv = dev->data->dev_private; 459 struct mlx5_dev_config *config = &priv->config; 460 461 /* 462 * Having available reg_c can be regarded inclusively as supporting 463 * extensive flow metadata register, which could mean, 464 * - metadata register copy action by modify header. 465 * - 16 modify header actions is supported. 466 * - reg_c's are preserved across different domain (FDB and NIC) on 467 * packet loopback by flow lookup miss. 468 */ 469 return config->flow_mreg_c[2] != REG_NONE; 470 } 471 472 /** 473 * Discover the maximum number of priority available. 474 * 475 * @param[in] dev 476 * Pointer to the Ethernet device structure. 477 * 478 * @return 479 * number of supported flow priority on success, a negative errno 480 * value otherwise and rte_errno is set. 481 */ 482 int 483 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 484 { 485 struct mlx5_priv *priv = dev->data->dev_private; 486 struct { 487 struct ibv_flow_attr attr; 488 struct ibv_flow_spec_eth eth; 489 struct ibv_flow_spec_action_drop drop; 490 } flow_attr = { 491 .attr = { 492 .num_of_specs = 2, 493 .port = (uint8_t)priv->ibv_port, 494 }, 495 .eth = { 496 .type = IBV_FLOW_SPEC_ETH, 497 .size = sizeof(struct ibv_flow_spec_eth), 498 }, 499 .drop = { 500 .size = sizeof(struct ibv_flow_spec_action_drop), 501 .type = IBV_FLOW_SPEC_ACTION_DROP, 502 }, 503 }; 504 struct ibv_flow *flow; 505 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 506 uint16_t vprio[] = { 8, 16 }; 507 int i; 508 int priority = 0; 509 510 if (!drop) { 511 rte_errno = ENOTSUP; 512 return -rte_errno; 513 } 514 for (i = 0; i != RTE_DIM(vprio); i++) { 515 flow_attr.attr.priority = vprio[i] - 1; 516 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 517 if (!flow) 518 break; 519 claim_zero(mlx5_glue->destroy_flow(flow)); 520 priority = vprio[i]; 521 } 522 mlx5_hrxq_drop_release(dev); 523 switch (priority) { 524 case 8: 525 priority = RTE_DIM(priority_map_3); 526 break; 527 case 16: 528 priority = RTE_DIM(priority_map_5); 529 break; 530 default: 531 rte_errno = ENOTSUP; 532 DRV_LOG(ERR, 533 "port %u verbs maximum priority: %d expected 8/16", 534 dev->data->port_id, priority); 535 return -rte_errno; 536 } 537 DRV_LOG(INFO, "port %u flow maximum priority: %d", 538 dev->data->port_id, priority); 539 return priority; 540 } 541 542 /** 543 * Adjust flow priority based on the highest layer and the request priority. 544 * 545 * @param[in] dev 546 * Pointer to the Ethernet device structure. 547 * @param[in] priority 548 * The rule base priority. 549 * @param[in] subpriority 550 * The priority based on the items. 551 * 552 * @return 553 * The new priority. 554 */ 555 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 556 uint32_t subpriority) 557 { 558 uint32_t res = 0; 559 struct mlx5_priv *priv = dev->data->dev_private; 560 561 switch (priv->config.flow_prio) { 562 case RTE_DIM(priority_map_3): 563 res = priority_map_3[priority][subpriority]; 564 break; 565 case RTE_DIM(priority_map_5): 566 res = priority_map_5[priority][subpriority]; 567 break; 568 } 569 return res; 570 } 571 572 /** 573 * Verify the @p item specifications (spec, last, mask) are compatible with the 574 * NIC capabilities. 575 * 576 * @param[in] item 577 * Item specification. 578 * @param[in] mask 579 * @p item->mask or flow default bit-masks. 580 * @param[in] nic_mask 581 * Bit-masks covering supported fields by the NIC to compare with user mask. 582 * @param[in] size 583 * Bit-masks size in bytes. 584 * @param[out] error 585 * Pointer to error structure. 586 * 587 * @return 588 * 0 on success, a negative errno value otherwise and rte_errno is set. 589 */ 590 int 591 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 592 const uint8_t *mask, 593 const uint8_t *nic_mask, 594 unsigned int size, 595 struct rte_flow_error *error) 596 { 597 unsigned int i; 598 599 assert(nic_mask); 600 for (i = 0; i < size; ++i) 601 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 602 return rte_flow_error_set(error, ENOTSUP, 603 RTE_FLOW_ERROR_TYPE_ITEM, 604 item, 605 "mask enables non supported" 606 " bits"); 607 if (!item->spec && (item->mask || item->last)) 608 return rte_flow_error_set(error, EINVAL, 609 RTE_FLOW_ERROR_TYPE_ITEM, item, 610 "mask/last without a spec is not" 611 " supported"); 612 if (item->spec && item->last) { 613 uint8_t spec[size]; 614 uint8_t last[size]; 615 unsigned int i; 616 int ret; 617 618 for (i = 0; i < size; ++i) { 619 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 620 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 621 } 622 ret = memcmp(spec, last, size); 623 if (ret != 0) 624 return rte_flow_error_set(error, EINVAL, 625 RTE_FLOW_ERROR_TYPE_ITEM, 626 item, 627 "range is not valid"); 628 } 629 return 0; 630 } 631 632 /** 633 * Adjust the hash fields according to the @p flow information. 634 * 635 * @param[in] dev_flow. 636 * Pointer to the mlx5_flow. 637 * @param[in] tunnel 638 * 1 when the hash field is for a tunnel item. 639 * @param[in] layer_types 640 * ETH_RSS_* types. 641 * @param[in] hash_fields 642 * Item hash fields. 643 * 644 * @return 645 * The hash fields that should be used. 646 */ 647 uint64_t 648 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, 649 int tunnel __rte_unused, uint64_t layer_types, 650 uint64_t hash_fields) 651 { 652 struct rte_flow *flow = dev_flow->flow; 653 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 654 int rss_request_inner = flow->rss.level >= 2; 655 656 /* Check RSS hash level for tunnel. */ 657 if (tunnel && rss_request_inner) 658 hash_fields |= IBV_RX_HASH_INNER; 659 else if (tunnel || rss_request_inner) 660 return 0; 661 #endif 662 /* Check if requested layer matches RSS hash fields. */ 663 if (!(flow->rss.types & layer_types)) 664 return 0; 665 return hash_fields; 666 } 667 668 /** 669 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 670 * if several tunnel rules are used on this queue, the tunnel ptype will be 671 * cleared. 672 * 673 * @param rxq_ctrl 674 * Rx queue to update. 675 */ 676 static void 677 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 678 { 679 unsigned int i; 680 uint32_t tunnel_ptype = 0; 681 682 /* Look up for the ptype to use. */ 683 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 684 if (!rxq_ctrl->flow_tunnels_n[i]) 685 continue; 686 if (!tunnel_ptype) { 687 tunnel_ptype = tunnels_info[i].ptype; 688 } else { 689 tunnel_ptype = 0; 690 break; 691 } 692 } 693 rxq_ctrl->rxq.tunnel = tunnel_ptype; 694 } 695 696 /** 697 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 698 * flow. 699 * 700 * @param[in] dev 701 * Pointer to the Ethernet device structure. 702 * @param[in] dev_flow 703 * Pointer to device flow structure. 704 */ 705 static void 706 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 707 { 708 struct mlx5_priv *priv = dev->data->dev_private; 709 struct rte_flow *flow = dev_flow->flow; 710 const int mark = !!(dev_flow->actions & 711 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 712 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 713 unsigned int i; 714 715 for (i = 0; i != flow->rss.queue_num; ++i) { 716 int idx = (*flow->rss.queue)[i]; 717 struct mlx5_rxq_ctrl *rxq_ctrl = 718 container_of((*priv->rxqs)[idx], 719 struct mlx5_rxq_ctrl, rxq); 720 721 /* 722 * To support metadata register copy on Tx loopback, 723 * this must be always enabled (metadata may arive 724 * from other port - not from local flows only. 725 */ 726 if (priv->config.dv_flow_en && 727 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 728 mlx5_flow_ext_mreg_supported(dev)) { 729 rxq_ctrl->rxq.mark = 1; 730 rxq_ctrl->flow_mark_n = 1; 731 } else if (mark) { 732 rxq_ctrl->rxq.mark = 1; 733 rxq_ctrl->flow_mark_n++; 734 } 735 if (tunnel) { 736 unsigned int j; 737 738 /* Increase the counter matching the flow. */ 739 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 740 if ((tunnels_info[j].tunnel & 741 dev_flow->layers) == 742 tunnels_info[j].tunnel) { 743 rxq_ctrl->flow_tunnels_n[j]++; 744 break; 745 } 746 } 747 flow_rxq_tunnel_ptype_update(rxq_ctrl); 748 } 749 } 750 } 751 752 /** 753 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 754 * 755 * @param[in] dev 756 * Pointer to the Ethernet device structure. 757 * @param[in] flow 758 * Pointer to flow structure. 759 */ 760 static void 761 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 762 { 763 struct mlx5_flow *dev_flow; 764 765 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 766 flow_drv_rxq_flags_set(dev, dev_flow); 767 } 768 769 /** 770 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 771 * device flow if no other flow uses it with the same kind of request. 772 * 773 * @param dev 774 * Pointer to Ethernet device. 775 * @param[in] dev_flow 776 * Pointer to the device flow. 777 */ 778 static void 779 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 780 { 781 struct mlx5_priv *priv = dev->data->dev_private; 782 struct rte_flow *flow = dev_flow->flow; 783 const int mark = !!(dev_flow->actions & 784 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 785 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 786 unsigned int i; 787 788 assert(dev->data->dev_started); 789 for (i = 0; i != flow->rss.queue_num; ++i) { 790 int idx = (*flow->rss.queue)[i]; 791 struct mlx5_rxq_ctrl *rxq_ctrl = 792 container_of((*priv->rxqs)[idx], 793 struct mlx5_rxq_ctrl, rxq); 794 795 if (priv->config.dv_flow_en && 796 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 797 mlx5_flow_ext_mreg_supported(dev)) { 798 rxq_ctrl->rxq.mark = 1; 799 rxq_ctrl->flow_mark_n = 1; 800 } else if (mark) { 801 rxq_ctrl->flow_mark_n--; 802 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 803 } 804 if (tunnel) { 805 unsigned int j; 806 807 /* Decrease the counter matching the flow. */ 808 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 809 if ((tunnels_info[j].tunnel & 810 dev_flow->layers) == 811 tunnels_info[j].tunnel) { 812 rxq_ctrl->flow_tunnels_n[j]--; 813 break; 814 } 815 } 816 flow_rxq_tunnel_ptype_update(rxq_ctrl); 817 } 818 } 819 } 820 821 /** 822 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 823 * @p flow if no other flow uses it with the same kind of request. 824 * 825 * @param dev 826 * Pointer to Ethernet device. 827 * @param[in] flow 828 * Pointer to the flow. 829 */ 830 static void 831 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 832 { 833 struct mlx5_flow *dev_flow; 834 835 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 836 flow_drv_rxq_flags_trim(dev, dev_flow); 837 } 838 839 /** 840 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 841 * 842 * @param dev 843 * Pointer to Ethernet device. 844 */ 845 static void 846 flow_rxq_flags_clear(struct rte_eth_dev *dev) 847 { 848 struct mlx5_priv *priv = dev->data->dev_private; 849 unsigned int i; 850 851 for (i = 0; i != priv->rxqs_n; ++i) { 852 struct mlx5_rxq_ctrl *rxq_ctrl; 853 unsigned int j; 854 855 if (!(*priv->rxqs)[i]) 856 continue; 857 rxq_ctrl = container_of((*priv->rxqs)[i], 858 struct mlx5_rxq_ctrl, rxq); 859 rxq_ctrl->flow_mark_n = 0; 860 rxq_ctrl->rxq.mark = 0; 861 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 862 rxq_ctrl->flow_tunnels_n[j] = 0; 863 rxq_ctrl->rxq.tunnel = 0; 864 } 865 } 866 867 /* 868 * return a pointer to the desired action in the list of actions. 869 * 870 * @param[in] actions 871 * The list of actions to search the action in. 872 * @param[in] action 873 * The action to find. 874 * 875 * @return 876 * Pointer to the action in the list, if found. NULL otherwise. 877 */ 878 const struct rte_flow_action * 879 mlx5_flow_find_action(const struct rte_flow_action *actions, 880 enum rte_flow_action_type action) 881 { 882 if (actions == NULL) 883 return NULL; 884 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 885 if (actions->type == action) 886 return actions; 887 return NULL; 888 } 889 890 /* 891 * Validate the flag action. 892 * 893 * @param[in] action_flags 894 * Bit-fields that holds the actions detected until now. 895 * @param[in] attr 896 * Attributes of flow that includes this action. 897 * @param[out] error 898 * Pointer to error structure. 899 * 900 * @return 901 * 0 on success, a negative errno value otherwise and rte_errno is set. 902 */ 903 int 904 mlx5_flow_validate_action_flag(uint64_t action_flags, 905 const struct rte_flow_attr *attr, 906 struct rte_flow_error *error) 907 { 908 if (action_flags & MLX5_FLOW_ACTION_MARK) 909 return rte_flow_error_set(error, EINVAL, 910 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 911 "can't mark and flag in same flow"); 912 if (action_flags & MLX5_FLOW_ACTION_FLAG) 913 return rte_flow_error_set(error, EINVAL, 914 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 915 "can't have 2 flag" 916 " actions in same flow"); 917 if (attr->egress) 918 return rte_flow_error_set(error, ENOTSUP, 919 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 920 "flag action not supported for " 921 "egress"); 922 return 0; 923 } 924 925 /* 926 * Validate the mark action. 927 * 928 * @param[in] action 929 * Pointer to the queue action. 930 * @param[in] action_flags 931 * Bit-fields that holds the actions detected until now. 932 * @param[in] attr 933 * Attributes of flow that includes this action. 934 * @param[out] error 935 * Pointer to error structure. 936 * 937 * @return 938 * 0 on success, a negative errno value otherwise and rte_errno is set. 939 */ 940 int 941 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 942 uint64_t action_flags, 943 const struct rte_flow_attr *attr, 944 struct rte_flow_error *error) 945 { 946 const struct rte_flow_action_mark *mark = action->conf; 947 948 if (!mark) 949 return rte_flow_error_set(error, EINVAL, 950 RTE_FLOW_ERROR_TYPE_ACTION, 951 action, 952 "configuration cannot be null"); 953 if (mark->id >= MLX5_FLOW_MARK_MAX) 954 return rte_flow_error_set(error, EINVAL, 955 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 956 &mark->id, 957 "mark id must in 0 <= id < " 958 RTE_STR(MLX5_FLOW_MARK_MAX)); 959 if (action_flags & MLX5_FLOW_ACTION_FLAG) 960 return rte_flow_error_set(error, EINVAL, 961 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 962 "can't flag and mark in same flow"); 963 if (action_flags & MLX5_FLOW_ACTION_MARK) 964 return rte_flow_error_set(error, EINVAL, 965 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 966 "can't have 2 mark actions in same" 967 " flow"); 968 if (attr->egress) 969 return rte_flow_error_set(error, ENOTSUP, 970 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 971 "mark action not supported for " 972 "egress"); 973 return 0; 974 } 975 976 /* 977 * Validate the drop action. 978 * 979 * @param[in] action_flags 980 * Bit-fields that holds the actions detected until now. 981 * @param[in] attr 982 * Attributes of flow that includes this action. 983 * @param[out] error 984 * Pointer to error structure. 985 * 986 * @return 987 * 0 on success, a negative errno value otherwise and rte_errno is set. 988 */ 989 int 990 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 991 const struct rte_flow_attr *attr, 992 struct rte_flow_error *error) 993 { 994 if (attr->egress) 995 return rte_flow_error_set(error, ENOTSUP, 996 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 997 "drop action not supported for " 998 "egress"); 999 return 0; 1000 } 1001 1002 /* 1003 * Validate the queue action. 1004 * 1005 * @param[in] action 1006 * Pointer to the queue action. 1007 * @param[in] action_flags 1008 * Bit-fields that holds the actions detected until now. 1009 * @param[in] dev 1010 * Pointer to the Ethernet device structure. 1011 * @param[in] attr 1012 * Attributes of flow that includes this action. 1013 * @param[out] error 1014 * Pointer to error structure. 1015 * 1016 * @return 1017 * 0 on success, a negative errno value otherwise and rte_errno is set. 1018 */ 1019 int 1020 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1021 uint64_t action_flags, 1022 struct rte_eth_dev *dev, 1023 const struct rte_flow_attr *attr, 1024 struct rte_flow_error *error) 1025 { 1026 struct mlx5_priv *priv = dev->data->dev_private; 1027 const struct rte_flow_action_queue *queue = action->conf; 1028 1029 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1030 return rte_flow_error_set(error, EINVAL, 1031 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1032 "can't have 2 fate actions in" 1033 " same flow"); 1034 if (!priv->rxqs_n) 1035 return rte_flow_error_set(error, EINVAL, 1036 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1037 NULL, "No Rx queues configured"); 1038 if (queue->index >= priv->rxqs_n) 1039 return rte_flow_error_set(error, EINVAL, 1040 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1041 &queue->index, 1042 "queue index out of range"); 1043 if (!(*priv->rxqs)[queue->index]) 1044 return rte_flow_error_set(error, EINVAL, 1045 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1046 &queue->index, 1047 "queue is not configured"); 1048 if (attr->egress) 1049 return rte_flow_error_set(error, ENOTSUP, 1050 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1051 "queue action not supported for " 1052 "egress"); 1053 return 0; 1054 } 1055 1056 /* 1057 * Validate the rss action. 1058 * 1059 * @param[in] action 1060 * Pointer to the queue action. 1061 * @param[in] action_flags 1062 * Bit-fields that holds the actions detected until now. 1063 * @param[in] dev 1064 * Pointer to the Ethernet device structure. 1065 * @param[in] attr 1066 * Attributes of flow that includes this action. 1067 * @param[in] item_flags 1068 * Items that were detected. 1069 * @param[out] error 1070 * Pointer to error structure. 1071 * 1072 * @return 1073 * 0 on success, a negative errno value otherwise and rte_errno is set. 1074 */ 1075 int 1076 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1077 uint64_t action_flags, 1078 struct rte_eth_dev *dev, 1079 const struct rte_flow_attr *attr, 1080 uint64_t item_flags, 1081 struct rte_flow_error *error) 1082 { 1083 struct mlx5_priv *priv = dev->data->dev_private; 1084 const struct rte_flow_action_rss *rss = action->conf; 1085 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1086 unsigned int i; 1087 1088 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1089 return rte_flow_error_set(error, EINVAL, 1090 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1091 "can't have 2 fate actions" 1092 " in same flow"); 1093 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1094 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1095 return rte_flow_error_set(error, ENOTSUP, 1096 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1097 &rss->func, 1098 "RSS hash function not supported"); 1099 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1100 if (rss->level > 2) 1101 #else 1102 if (rss->level > 1) 1103 #endif 1104 return rte_flow_error_set(error, ENOTSUP, 1105 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1106 &rss->level, 1107 "tunnel RSS is not supported"); 1108 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1109 if (rss->key_len == 0 && rss->key != NULL) 1110 return rte_flow_error_set(error, ENOTSUP, 1111 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1112 &rss->key_len, 1113 "RSS hash key length 0"); 1114 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1115 return rte_flow_error_set(error, ENOTSUP, 1116 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1117 &rss->key_len, 1118 "RSS hash key too small"); 1119 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1120 return rte_flow_error_set(error, ENOTSUP, 1121 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1122 &rss->key_len, 1123 "RSS hash key too large"); 1124 if (rss->queue_num > priv->config.ind_table_max_size) 1125 return rte_flow_error_set(error, ENOTSUP, 1126 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1127 &rss->queue_num, 1128 "number of queues too large"); 1129 if (rss->types & MLX5_RSS_HF_MASK) 1130 return rte_flow_error_set(error, ENOTSUP, 1131 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1132 &rss->types, 1133 "some RSS protocols are not" 1134 " supported"); 1135 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1136 !(rss->types & ETH_RSS_IP)) 1137 return rte_flow_error_set(error, EINVAL, 1138 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1139 "L3 partial RSS requested but L3 RSS" 1140 " type not specified"); 1141 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1142 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1143 return rte_flow_error_set(error, EINVAL, 1144 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1145 "L4 partial RSS requested but L4 RSS" 1146 " type not specified"); 1147 if (!priv->rxqs_n) 1148 return rte_flow_error_set(error, EINVAL, 1149 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1150 NULL, "No Rx queues configured"); 1151 if (!rss->queue_num) 1152 return rte_flow_error_set(error, EINVAL, 1153 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1154 NULL, "No queues configured"); 1155 for (i = 0; i != rss->queue_num; ++i) { 1156 if (rss->queue[i] >= priv->rxqs_n) 1157 return rte_flow_error_set 1158 (error, EINVAL, 1159 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1160 &rss->queue[i], "queue index out of range"); 1161 if (!(*priv->rxqs)[rss->queue[i]]) 1162 return rte_flow_error_set 1163 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1164 &rss->queue[i], "queue is not configured"); 1165 } 1166 if (attr->egress) 1167 return rte_flow_error_set(error, ENOTSUP, 1168 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1169 "rss action not supported for " 1170 "egress"); 1171 if (rss->level > 1 && !tunnel) 1172 return rte_flow_error_set(error, EINVAL, 1173 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1174 "inner RSS is not supported for " 1175 "non-tunnel flows"); 1176 return 0; 1177 } 1178 1179 /* 1180 * Validate the count action. 1181 * 1182 * @param[in] dev 1183 * Pointer to the Ethernet device structure. 1184 * @param[in] attr 1185 * Attributes of flow that includes this action. 1186 * @param[out] error 1187 * Pointer to error structure. 1188 * 1189 * @return 1190 * 0 on success, a negative errno value otherwise and rte_errno is set. 1191 */ 1192 int 1193 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1194 const struct rte_flow_attr *attr, 1195 struct rte_flow_error *error) 1196 { 1197 if (attr->egress) 1198 return rte_flow_error_set(error, ENOTSUP, 1199 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1200 "count action not supported for " 1201 "egress"); 1202 return 0; 1203 } 1204 1205 /** 1206 * Verify the @p attributes will be correctly understood by the NIC and store 1207 * them in the @p flow if everything is correct. 1208 * 1209 * @param[in] dev 1210 * Pointer to the Ethernet device structure. 1211 * @param[in] attributes 1212 * Pointer to flow attributes 1213 * @param[out] error 1214 * Pointer to error structure. 1215 * 1216 * @return 1217 * 0 on success, a negative errno value otherwise and rte_errno is set. 1218 */ 1219 int 1220 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1221 const struct rte_flow_attr *attributes, 1222 struct rte_flow_error *error) 1223 { 1224 struct mlx5_priv *priv = dev->data->dev_private; 1225 uint32_t priority_max = priv->config.flow_prio - 1; 1226 1227 if (attributes->group) 1228 return rte_flow_error_set(error, ENOTSUP, 1229 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1230 NULL, "groups is not supported"); 1231 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1232 attributes->priority >= priority_max) 1233 return rte_flow_error_set(error, ENOTSUP, 1234 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1235 NULL, "priority out of range"); 1236 if (attributes->egress) 1237 return rte_flow_error_set(error, ENOTSUP, 1238 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1239 "egress is not supported"); 1240 if (attributes->transfer && !priv->config.dv_esw_en) 1241 return rte_flow_error_set(error, ENOTSUP, 1242 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1243 NULL, "transfer is not supported"); 1244 if (!attributes->ingress) 1245 return rte_flow_error_set(error, EINVAL, 1246 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1247 NULL, 1248 "ingress attribute is mandatory"); 1249 return 0; 1250 } 1251 1252 /** 1253 * Validate ICMP6 item. 1254 * 1255 * @param[in] item 1256 * Item specification. 1257 * @param[in] item_flags 1258 * Bit-fields that holds the items detected until now. 1259 * @param[out] error 1260 * Pointer to error structure. 1261 * 1262 * @return 1263 * 0 on success, a negative errno value otherwise and rte_errno is set. 1264 */ 1265 int 1266 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1267 uint64_t item_flags, 1268 uint8_t target_protocol, 1269 struct rte_flow_error *error) 1270 { 1271 const struct rte_flow_item_icmp6 *mask = item->mask; 1272 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1273 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1274 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1275 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1276 MLX5_FLOW_LAYER_OUTER_L4; 1277 int ret; 1278 1279 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1280 return rte_flow_error_set(error, EINVAL, 1281 RTE_FLOW_ERROR_TYPE_ITEM, item, 1282 "protocol filtering not compatible" 1283 " with ICMP6 layer"); 1284 if (!(item_flags & l3m)) 1285 return rte_flow_error_set(error, EINVAL, 1286 RTE_FLOW_ERROR_TYPE_ITEM, item, 1287 "IPv6 is mandatory to filter on" 1288 " ICMP6"); 1289 if (item_flags & l4m) 1290 return rte_flow_error_set(error, EINVAL, 1291 RTE_FLOW_ERROR_TYPE_ITEM, item, 1292 "multiple L4 layers not supported"); 1293 if (!mask) 1294 mask = &rte_flow_item_icmp6_mask; 1295 ret = mlx5_flow_item_acceptable 1296 (item, (const uint8_t *)mask, 1297 (const uint8_t *)&rte_flow_item_icmp6_mask, 1298 sizeof(struct rte_flow_item_icmp6), error); 1299 if (ret < 0) 1300 return ret; 1301 return 0; 1302 } 1303 1304 /** 1305 * Validate ICMP item. 1306 * 1307 * @param[in] item 1308 * Item specification. 1309 * @param[in] item_flags 1310 * Bit-fields that holds the items detected until now. 1311 * @param[out] error 1312 * Pointer to error structure. 1313 * 1314 * @return 1315 * 0 on success, a negative errno value otherwise and rte_errno is set. 1316 */ 1317 int 1318 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1319 uint64_t item_flags, 1320 uint8_t target_protocol, 1321 struct rte_flow_error *error) 1322 { 1323 const struct rte_flow_item_icmp *mask = item->mask; 1324 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1325 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1326 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1327 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1328 MLX5_FLOW_LAYER_OUTER_L4; 1329 int ret; 1330 1331 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1332 return rte_flow_error_set(error, EINVAL, 1333 RTE_FLOW_ERROR_TYPE_ITEM, item, 1334 "protocol filtering not compatible" 1335 " with ICMP layer"); 1336 if (!(item_flags & l3m)) 1337 return rte_flow_error_set(error, EINVAL, 1338 RTE_FLOW_ERROR_TYPE_ITEM, item, 1339 "IPv4 is mandatory to filter" 1340 " on ICMP"); 1341 if (item_flags & l4m) 1342 return rte_flow_error_set(error, EINVAL, 1343 RTE_FLOW_ERROR_TYPE_ITEM, item, 1344 "multiple L4 layers not supported"); 1345 if (!mask) 1346 mask = &rte_flow_item_icmp_mask; 1347 ret = mlx5_flow_item_acceptable 1348 (item, (const uint8_t *)mask, 1349 (const uint8_t *)&rte_flow_item_icmp_mask, 1350 sizeof(struct rte_flow_item_icmp), error); 1351 if (ret < 0) 1352 return ret; 1353 return 0; 1354 } 1355 1356 /** 1357 * Validate Ethernet item. 1358 * 1359 * @param[in] item 1360 * Item specification. 1361 * @param[in] item_flags 1362 * Bit-fields that holds the items detected until now. 1363 * @param[out] error 1364 * Pointer to error structure. 1365 * 1366 * @return 1367 * 0 on success, a negative errno value otherwise and rte_errno is set. 1368 */ 1369 int 1370 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1371 uint64_t item_flags, 1372 struct rte_flow_error *error) 1373 { 1374 const struct rte_flow_item_eth *mask = item->mask; 1375 const struct rte_flow_item_eth nic_mask = { 1376 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1377 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1378 .type = RTE_BE16(0xffff), 1379 }; 1380 int ret; 1381 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1382 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1383 MLX5_FLOW_LAYER_OUTER_L2; 1384 1385 if (item_flags & ethm) 1386 return rte_flow_error_set(error, ENOTSUP, 1387 RTE_FLOW_ERROR_TYPE_ITEM, item, 1388 "multiple L2 layers not supported"); 1389 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1390 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1391 return rte_flow_error_set(error, EINVAL, 1392 RTE_FLOW_ERROR_TYPE_ITEM, item, 1393 "L2 layer should not follow " 1394 "L3 layers"); 1395 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1396 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1397 return rte_flow_error_set(error, EINVAL, 1398 RTE_FLOW_ERROR_TYPE_ITEM, item, 1399 "L2 layer should not follow VLAN"); 1400 if (!mask) 1401 mask = &rte_flow_item_eth_mask; 1402 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1403 (const uint8_t *)&nic_mask, 1404 sizeof(struct rte_flow_item_eth), 1405 error); 1406 return ret; 1407 } 1408 1409 /** 1410 * Validate VLAN item. 1411 * 1412 * @param[in] item 1413 * Item specification. 1414 * @param[in] item_flags 1415 * Bit-fields that holds the items detected until now. 1416 * @param[in] dev 1417 * Ethernet device flow is being created on. 1418 * @param[out] error 1419 * Pointer to error structure. 1420 * 1421 * @return 1422 * 0 on success, a negative errno value otherwise and rte_errno is set. 1423 */ 1424 int 1425 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1426 uint64_t item_flags, 1427 struct rte_eth_dev *dev, 1428 struct rte_flow_error *error) 1429 { 1430 const struct rte_flow_item_vlan *spec = item->spec; 1431 const struct rte_flow_item_vlan *mask = item->mask; 1432 const struct rte_flow_item_vlan nic_mask = { 1433 .tci = RTE_BE16(UINT16_MAX), 1434 .inner_type = RTE_BE16(UINT16_MAX), 1435 }; 1436 uint16_t vlan_tag = 0; 1437 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1438 int ret; 1439 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1440 MLX5_FLOW_LAYER_INNER_L4) : 1441 (MLX5_FLOW_LAYER_OUTER_L3 | 1442 MLX5_FLOW_LAYER_OUTER_L4); 1443 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1444 MLX5_FLOW_LAYER_OUTER_VLAN; 1445 1446 if (item_flags & vlanm) 1447 return rte_flow_error_set(error, EINVAL, 1448 RTE_FLOW_ERROR_TYPE_ITEM, item, 1449 "multiple VLAN layers not supported"); 1450 else if ((item_flags & l34m) != 0) 1451 return rte_flow_error_set(error, EINVAL, 1452 RTE_FLOW_ERROR_TYPE_ITEM, item, 1453 "VLAN cannot follow L3/L4 layer"); 1454 if (!mask) 1455 mask = &rte_flow_item_vlan_mask; 1456 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1457 (const uint8_t *)&nic_mask, 1458 sizeof(struct rte_flow_item_vlan), 1459 error); 1460 if (ret) 1461 return ret; 1462 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1463 struct mlx5_priv *priv = dev->data->dev_private; 1464 1465 if (priv->vmwa_context) { 1466 /* 1467 * Non-NULL context means we have a virtual machine 1468 * and SR-IOV enabled, we have to create VLAN interface 1469 * to make hypervisor to setup E-Switch vport 1470 * context correctly. We avoid creating the multiple 1471 * VLAN interfaces, so we cannot support VLAN tag mask. 1472 */ 1473 return rte_flow_error_set(error, EINVAL, 1474 RTE_FLOW_ERROR_TYPE_ITEM, 1475 item, 1476 "VLAN tag mask is not" 1477 " supported in virtual" 1478 " environment"); 1479 } 1480 } 1481 if (spec) { 1482 vlan_tag = spec->tci; 1483 vlan_tag &= mask->tci; 1484 } 1485 /* 1486 * From verbs perspective an empty VLAN is equivalent 1487 * to a packet without VLAN layer. 1488 */ 1489 if (!vlan_tag) 1490 return rte_flow_error_set(error, EINVAL, 1491 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1492 item->spec, 1493 "VLAN cannot be empty"); 1494 return 0; 1495 } 1496 1497 /** 1498 * Validate IPV4 item. 1499 * 1500 * @param[in] item 1501 * Item specification. 1502 * @param[in] item_flags 1503 * Bit-fields that holds the items detected until now. 1504 * @param[in] acc_mask 1505 * Acceptable mask, if NULL default internal default mask 1506 * will be used to check whether item fields are supported. 1507 * @param[out] error 1508 * Pointer to error structure. 1509 * 1510 * @return 1511 * 0 on success, a negative errno value otherwise and rte_errno is set. 1512 */ 1513 int 1514 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1515 uint64_t item_flags, 1516 uint64_t last_item, 1517 uint16_t ether_type, 1518 const struct rte_flow_item_ipv4 *acc_mask, 1519 struct rte_flow_error *error) 1520 { 1521 const struct rte_flow_item_ipv4 *mask = item->mask; 1522 const struct rte_flow_item_ipv4 *spec = item->spec; 1523 const struct rte_flow_item_ipv4 nic_mask = { 1524 .hdr = { 1525 .src_addr = RTE_BE32(0xffffffff), 1526 .dst_addr = RTE_BE32(0xffffffff), 1527 .type_of_service = 0xff, 1528 .next_proto_id = 0xff, 1529 }, 1530 }; 1531 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1532 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1533 MLX5_FLOW_LAYER_OUTER_L3; 1534 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1535 MLX5_FLOW_LAYER_OUTER_L4; 1536 int ret; 1537 uint8_t next_proto = 0xFF; 1538 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1539 MLX5_FLOW_LAYER_OUTER_VLAN | 1540 MLX5_FLOW_LAYER_INNER_VLAN); 1541 1542 if ((last_item & l2_vlan) && ether_type && 1543 ether_type != RTE_ETHER_TYPE_IPV4) 1544 return rte_flow_error_set(error, EINVAL, 1545 RTE_FLOW_ERROR_TYPE_ITEM, item, 1546 "IPv4 cannot follow L2/VLAN layer " 1547 "which ether type is not IPv4"); 1548 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1549 if (mask && spec) 1550 next_proto = mask->hdr.next_proto_id & 1551 spec->hdr.next_proto_id; 1552 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1553 return rte_flow_error_set(error, EINVAL, 1554 RTE_FLOW_ERROR_TYPE_ITEM, 1555 item, 1556 "multiple tunnel " 1557 "not supported"); 1558 } 1559 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1560 return rte_flow_error_set(error, EINVAL, 1561 RTE_FLOW_ERROR_TYPE_ITEM, item, 1562 "wrong tunnel type - IPv6 specified " 1563 "but IPv4 item provided"); 1564 if (item_flags & l3m) 1565 return rte_flow_error_set(error, ENOTSUP, 1566 RTE_FLOW_ERROR_TYPE_ITEM, item, 1567 "multiple L3 layers not supported"); 1568 else if (item_flags & l4m) 1569 return rte_flow_error_set(error, EINVAL, 1570 RTE_FLOW_ERROR_TYPE_ITEM, item, 1571 "L3 cannot follow an L4 layer."); 1572 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1573 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1574 return rte_flow_error_set(error, EINVAL, 1575 RTE_FLOW_ERROR_TYPE_ITEM, item, 1576 "L3 cannot follow an NVGRE layer."); 1577 if (!mask) 1578 mask = &rte_flow_item_ipv4_mask; 1579 else if (mask->hdr.next_proto_id != 0 && 1580 mask->hdr.next_proto_id != 0xff) 1581 return rte_flow_error_set(error, EINVAL, 1582 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1583 "partial mask is not supported" 1584 " for protocol"); 1585 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1586 acc_mask ? (const uint8_t *)acc_mask 1587 : (const uint8_t *)&nic_mask, 1588 sizeof(struct rte_flow_item_ipv4), 1589 error); 1590 if (ret < 0) 1591 return ret; 1592 return 0; 1593 } 1594 1595 /** 1596 * Validate IPV6 item. 1597 * 1598 * @param[in] item 1599 * Item specification. 1600 * @param[in] item_flags 1601 * Bit-fields that holds the items detected until now. 1602 * @param[in] acc_mask 1603 * Acceptable mask, if NULL default internal default mask 1604 * will be used to check whether item fields are supported. 1605 * @param[out] error 1606 * Pointer to error structure. 1607 * 1608 * @return 1609 * 0 on success, a negative errno value otherwise and rte_errno is set. 1610 */ 1611 int 1612 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1613 uint64_t item_flags, 1614 uint64_t last_item, 1615 uint16_t ether_type, 1616 const struct rte_flow_item_ipv6 *acc_mask, 1617 struct rte_flow_error *error) 1618 { 1619 const struct rte_flow_item_ipv6 *mask = item->mask; 1620 const struct rte_flow_item_ipv6 *spec = item->spec; 1621 const struct rte_flow_item_ipv6 nic_mask = { 1622 .hdr = { 1623 .src_addr = 1624 "\xff\xff\xff\xff\xff\xff\xff\xff" 1625 "\xff\xff\xff\xff\xff\xff\xff\xff", 1626 .dst_addr = 1627 "\xff\xff\xff\xff\xff\xff\xff\xff" 1628 "\xff\xff\xff\xff\xff\xff\xff\xff", 1629 .vtc_flow = RTE_BE32(0xffffffff), 1630 .proto = 0xff, 1631 .hop_limits = 0xff, 1632 }, 1633 }; 1634 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1635 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1636 MLX5_FLOW_LAYER_OUTER_L3; 1637 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1638 MLX5_FLOW_LAYER_OUTER_L4; 1639 int ret; 1640 uint8_t next_proto = 0xFF; 1641 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1642 MLX5_FLOW_LAYER_OUTER_VLAN | 1643 MLX5_FLOW_LAYER_INNER_VLAN); 1644 1645 if ((last_item & l2_vlan) && ether_type && 1646 ether_type != RTE_ETHER_TYPE_IPV6) 1647 return rte_flow_error_set(error, EINVAL, 1648 RTE_FLOW_ERROR_TYPE_ITEM, item, 1649 "IPv6 cannot follow L2/VLAN layer " 1650 "which ether type is not IPv6"); 1651 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1652 if (mask && spec) 1653 next_proto = mask->hdr.proto & spec->hdr.proto; 1654 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1655 return rte_flow_error_set(error, EINVAL, 1656 RTE_FLOW_ERROR_TYPE_ITEM, 1657 item, 1658 "multiple tunnel " 1659 "not supported"); 1660 } 1661 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1662 return rte_flow_error_set(error, EINVAL, 1663 RTE_FLOW_ERROR_TYPE_ITEM, item, 1664 "wrong tunnel type - IPv4 specified " 1665 "but IPv6 item provided"); 1666 if (item_flags & l3m) 1667 return rte_flow_error_set(error, ENOTSUP, 1668 RTE_FLOW_ERROR_TYPE_ITEM, item, 1669 "multiple L3 layers not supported"); 1670 else if (item_flags & l4m) 1671 return rte_flow_error_set(error, EINVAL, 1672 RTE_FLOW_ERROR_TYPE_ITEM, item, 1673 "L3 cannot follow an L4 layer."); 1674 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1675 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1676 return rte_flow_error_set(error, EINVAL, 1677 RTE_FLOW_ERROR_TYPE_ITEM, item, 1678 "L3 cannot follow an NVGRE layer."); 1679 if (!mask) 1680 mask = &rte_flow_item_ipv6_mask; 1681 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1682 acc_mask ? (const uint8_t *)acc_mask 1683 : (const uint8_t *)&nic_mask, 1684 sizeof(struct rte_flow_item_ipv6), 1685 error); 1686 if (ret < 0) 1687 return ret; 1688 return 0; 1689 } 1690 1691 /** 1692 * Validate UDP item. 1693 * 1694 * @param[in] item 1695 * Item specification. 1696 * @param[in] item_flags 1697 * Bit-fields that holds the items detected until now. 1698 * @param[in] target_protocol 1699 * The next protocol in the previous item. 1700 * @param[in] flow_mask 1701 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1702 * @param[out] error 1703 * Pointer to error structure. 1704 * 1705 * @return 1706 * 0 on success, a negative errno value otherwise and rte_errno is set. 1707 */ 1708 int 1709 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1710 uint64_t item_flags, 1711 uint8_t target_protocol, 1712 struct rte_flow_error *error) 1713 { 1714 const struct rte_flow_item_udp *mask = item->mask; 1715 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1716 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1717 MLX5_FLOW_LAYER_OUTER_L3; 1718 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1719 MLX5_FLOW_LAYER_OUTER_L4; 1720 int ret; 1721 1722 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1723 return rte_flow_error_set(error, EINVAL, 1724 RTE_FLOW_ERROR_TYPE_ITEM, item, 1725 "protocol filtering not compatible" 1726 " with UDP layer"); 1727 if (!(item_flags & l3m)) 1728 return rte_flow_error_set(error, EINVAL, 1729 RTE_FLOW_ERROR_TYPE_ITEM, item, 1730 "L3 is mandatory to filter on L4"); 1731 if (item_flags & l4m) 1732 return rte_flow_error_set(error, EINVAL, 1733 RTE_FLOW_ERROR_TYPE_ITEM, item, 1734 "multiple L4 layers not supported"); 1735 if (!mask) 1736 mask = &rte_flow_item_udp_mask; 1737 ret = mlx5_flow_item_acceptable 1738 (item, (const uint8_t *)mask, 1739 (const uint8_t *)&rte_flow_item_udp_mask, 1740 sizeof(struct rte_flow_item_udp), error); 1741 if (ret < 0) 1742 return ret; 1743 return 0; 1744 } 1745 1746 /** 1747 * Validate TCP item. 1748 * 1749 * @param[in] item 1750 * Item specification. 1751 * @param[in] item_flags 1752 * Bit-fields that holds the items detected until now. 1753 * @param[in] target_protocol 1754 * The next protocol in the previous item. 1755 * @param[out] error 1756 * Pointer to error structure. 1757 * 1758 * @return 1759 * 0 on success, a negative errno value otherwise and rte_errno is set. 1760 */ 1761 int 1762 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1763 uint64_t item_flags, 1764 uint8_t target_protocol, 1765 const struct rte_flow_item_tcp *flow_mask, 1766 struct rte_flow_error *error) 1767 { 1768 const struct rte_flow_item_tcp *mask = item->mask; 1769 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1770 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1771 MLX5_FLOW_LAYER_OUTER_L3; 1772 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1773 MLX5_FLOW_LAYER_OUTER_L4; 1774 int ret; 1775 1776 assert(flow_mask); 1777 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1778 return rte_flow_error_set(error, EINVAL, 1779 RTE_FLOW_ERROR_TYPE_ITEM, item, 1780 "protocol filtering not compatible" 1781 " with TCP layer"); 1782 if (!(item_flags & l3m)) 1783 return rte_flow_error_set(error, EINVAL, 1784 RTE_FLOW_ERROR_TYPE_ITEM, item, 1785 "L3 is mandatory to filter on L4"); 1786 if (item_flags & l4m) 1787 return rte_flow_error_set(error, EINVAL, 1788 RTE_FLOW_ERROR_TYPE_ITEM, item, 1789 "multiple L4 layers not supported"); 1790 if (!mask) 1791 mask = &rte_flow_item_tcp_mask; 1792 ret = mlx5_flow_item_acceptable 1793 (item, (const uint8_t *)mask, 1794 (const uint8_t *)flow_mask, 1795 sizeof(struct rte_flow_item_tcp), error); 1796 if (ret < 0) 1797 return ret; 1798 return 0; 1799 } 1800 1801 /** 1802 * Validate VXLAN item. 1803 * 1804 * @param[in] item 1805 * Item specification. 1806 * @param[in] item_flags 1807 * Bit-fields that holds the items detected until now. 1808 * @param[in] target_protocol 1809 * The next protocol in the previous item. 1810 * @param[out] error 1811 * Pointer to error structure. 1812 * 1813 * @return 1814 * 0 on success, a negative errno value otherwise and rte_errno is set. 1815 */ 1816 int 1817 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1818 uint64_t item_flags, 1819 struct rte_flow_error *error) 1820 { 1821 const struct rte_flow_item_vxlan *spec = item->spec; 1822 const struct rte_flow_item_vxlan *mask = item->mask; 1823 int ret; 1824 union vni { 1825 uint32_t vlan_id; 1826 uint8_t vni[4]; 1827 } id = { .vlan_id = 0, }; 1828 uint32_t vlan_id = 0; 1829 1830 1831 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1832 return rte_flow_error_set(error, ENOTSUP, 1833 RTE_FLOW_ERROR_TYPE_ITEM, item, 1834 "multiple tunnel layers not" 1835 " supported"); 1836 /* 1837 * Verify only UDPv4 is present as defined in 1838 * https://tools.ietf.org/html/rfc7348 1839 */ 1840 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1841 return rte_flow_error_set(error, EINVAL, 1842 RTE_FLOW_ERROR_TYPE_ITEM, item, 1843 "no outer UDP layer found"); 1844 if (!mask) 1845 mask = &rte_flow_item_vxlan_mask; 1846 ret = mlx5_flow_item_acceptable 1847 (item, (const uint8_t *)mask, 1848 (const uint8_t *)&rte_flow_item_vxlan_mask, 1849 sizeof(struct rte_flow_item_vxlan), 1850 error); 1851 if (ret < 0) 1852 return ret; 1853 if (spec) { 1854 memcpy(&id.vni[1], spec->vni, 3); 1855 vlan_id = id.vlan_id; 1856 memcpy(&id.vni[1], mask->vni, 3); 1857 vlan_id &= id.vlan_id; 1858 } 1859 /* 1860 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if 1861 * only this layer is defined in the Verbs specification it is 1862 * interpreted as wildcard and all packets will match this 1863 * rule, if it follows a full stack layer (ex: eth / ipv4 / 1864 * udp), all packets matching the layers before will also 1865 * match this rule. To avoid such situation, VNI 0 is 1866 * currently refused. 1867 */ 1868 if (!vlan_id) 1869 return rte_flow_error_set(error, ENOTSUP, 1870 RTE_FLOW_ERROR_TYPE_ITEM, item, 1871 "VXLAN vni cannot be 0"); 1872 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1873 return rte_flow_error_set(error, ENOTSUP, 1874 RTE_FLOW_ERROR_TYPE_ITEM, item, 1875 "VXLAN tunnel must be fully defined"); 1876 return 0; 1877 } 1878 1879 /** 1880 * Validate VXLAN_GPE item. 1881 * 1882 * @param[in] item 1883 * Item specification. 1884 * @param[in] item_flags 1885 * Bit-fields that holds the items detected until now. 1886 * @param[in] priv 1887 * Pointer to the private data structure. 1888 * @param[in] target_protocol 1889 * The next protocol in the previous item. 1890 * @param[out] error 1891 * Pointer to error structure. 1892 * 1893 * @return 1894 * 0 on success, a negative errno value otherwise and rte_errno is set. 1895 */ 1896 int 1897 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1898 uint64_t item_flags, 1899 struct rte_eth_dev *dev, 1900 struct rte_flow_error *error) 1901 { 1902 struct mlx5_priv *priv = dev->data->dev_private; 1903 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1904 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1905 int ret; 1906 union vni { 1907 uint32_t vlan_id; 1908 uint8_t vni[4]; 1909 } id = { .vlan_id = 0, }; 1910 uint32_t vlan_id = 0; 1911 1912 if (!priv->config.l3_vxlan_en) 1913 return rte_flow_error_set(error, ENOTSUP, 1914 RTE_FLOW_ERROR_TYPE_ITEM, item, 1915 "L3 VXLAN is not enabled by device" 1916 " parameter and/or not configured in" 1917 " firmware"); 1918 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1919 return rte_flow_error_set(error, ENOTSUP, 1920 RTE_FLOW_ERROR_TYPE_ITEM, item, 1921 "multiple tunnel layers not" 1922 " supported"); 1923 /* 1924 * Verify only UDPv4 is present as defined in 1925 * https://tools.ietf.org/html/rfc7348 1926 */ 1927 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1928 return rte_flow_error_set(error, EINVAL, 1929 RTE_FLOW_ERROR_TYPE_ITEM, item, 1930 "no outer UDP layer found"); 1931 if (!mask) 1932 mask = &rte_flow_item_vxlan_gpe_mask; 1933 ret = mlx5_flow_item_acceptable 1934 (item, (const uint8_t *)mask, 1935 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1936 sizeof(struct rte_flow_item_vxlan_gpe), 1937 error); 1938 if (ret < 0) 1939 return ret; 1940 if (spec) { 1941 if (spec->protocol) 1942 return rte_flow_error_set(error, ENOTSUP, 1943 RTE_FLOW_ERROR_TYPE_ITEM, 1944 item, 1945 "VxLAN-GPE protocol" 1946 " not supported"); 1947 memcpy(&id.vni[1], spec->vni, 3); 1948 vlan_id = id.vlan_id; 1949 memcpy(&id.vni[1], mask->vni, 3); 1950 vlan_id &= id.vlan_id; 1951 } 1952 /* 1953 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1954 * layer is defined in the Verbs specification it is interpreted as 1955 * wildcard and all packets will match this rule, if it follows a full 1956 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1957 * before will also match this rule. To avoid such situation, VNI 0 1958 * is currently refused. 1959 */ 1960 if (!vlan_id) 1961 return rte_flow_error_set(error, ENOTSUP, 1962 RTE_FLOW_ERROR_TYPE_ITEM, item, 1963 "VXLAN-GPE vni cannot be 0"); 1964 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1965 return rte_flow_error_set(error, ENOTSUP, 1966 RTE_FLOW_ERROR_TYPE_ITEM, item, 1967 "VXLAN-GPE tunnel must be fully" 1968 " defined"); 1969 return 0; 1970 } 1971 /** 1972 * Validate GRE Key item. 1973 * 1974 * @param[in] item 1975 * Item specification. 1976 * @param[in] item_flags 1977 * Bit flags to mark detected items. 1978 * @param[in] gre_item 1979 * Pointer to gre_item 1980 * @param[out] error 1981 * Pointer to error structure. 1982 * 1983 * @return 1984 * 0 on success, a negative errno value otherwise and rte_errno is set. 1985 */ 1986 int 1987 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 1988 uint64_t item_flags, 1989 const struct rte_flow_item *gre_item, 1990 struct rte_flow_error *error) 1991 { 1992 const rte_be32_t *mask = item->mask; 1993 int ret = 0; 1994 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 1995 const struct rte_flow_item_gre *gre_spec; 1996 const struct rte_flow_item_gre *gre_mask; 1997 1998 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 1999 return rte_flow_error_set(error, ENOTSUP, 2000 RTE_FLOW_ERROR_TYPE_ITEM, item, 2001 "Multiple GRE key not support"); 2002 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2003 return rte_flow_error_set(error, ENOTSUP, 2004 RTE_FLOW_ERROR_TYPE_ITEM, item, 2005 "No preceding GRE header"); 2006 if (item_flags & MLX5_FLOW_LAYER_INNER) 2007 return rte_flow_error_set(error, ENOTSUP, 2008 RTE_FLOW_ERROR_TYPE_ITEM, item, 2009 "GRE key following a wrong item"); 2010 gre_mask = gre_item->mask; 2011 if (!gre_mask) 2012 gre_mask = &rte_flow_item_gre_mask; 2013 gre_spec = gre_item->spec; 2014 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2015 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2016 return rte_flow_error_set(error, EINVAL, 2017 RTE_FLOW_ERROR_TYPE_ITEM, item, 2018 "Key bit must be on"); 2019 2020 if (!mask) 2021 mask = &gre_key_default_mask; 2022 ret = mlx5_flow_item_acceptable 2023 (item, (const uint8_t *)mask, 2024 (const uint8_t *)&gre_key_default_mask, 2025 sizeof(rte_be32_t), error); 2026 return ret; 2027 } 2028 2029 /** 2030 * Validate GRE item. 2031 * 2032 * @param[in] item 2033 * Item specification. 2034 * @param[in] item_flags 2035 * Bit flags to mark detected items. 2036 * @param[in] target_protocol 2037 * The next protocol in the previous item. 2038 * @param[out] error 2039 * Pointer to error structure. 2040 * 2041 * @return 2042 * 0 on success, a negative errno value otherwise and rte_errno is set. 2043 */ 2044 int 2045 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2046 uint64_t item_flags, 2047 uint8_t target_protocol, 2048 struct rte_flow_error *error) 2049 { 2050 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2051 const struct rte_flow_item_gre *mask = item->mask; 2052 int ret; 2053 const struct rte_flow_item_gre nic_mask = { 2054 .c_rsvd0_ver = RTE_BE16(0xB000), 2055 .protocol = RTE_BE16(UINT16_MAX), 2056 }; 2057 2058 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2059 return rte_flow_error_set(error, EINVAL, 2060 RTE_FLOW_ERROR_TYPE_ITEM, item, 2061 "protocol filtering not compatible" 2062 " with this GRE layer"); 2063 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2064 return rte_flow_error_set(error, ENOTSUP, 2065 RTE_FLOW_ERROR_TYPE_ITEM, item, 2066 "multiple tunnel layers not" 2067 " supported"); 2068 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2069 return rte_flow_error_set(error, ENOTSUP, 2070 RTE_FLOW_ERROR_TYPE_ITEM, item, 2071 "L3 Layer is missing"); 2072 if (!mask) 2073 mask = &rte_flow_item_gre_mask; 2074 ret = mlx5_flow_item_acceptable 2075 (item, (const uint8_t *)mask, 2076 (const uint8_t *)&nic_mask, 2077 sizeof(struct rte_flow_item_gre), error); 2078 if (ret < 0) 2079 return ret; 2080 #ifndef HAVE_MLX5DV_DR 2081 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2082 if (spec && (spec->protocol & mask->protocol)) 2083 return rte_flow_error_set(error, ENOTSUP, 2084 RTE_FLOW_ERROR_TYPE_ITEM, item, 2085 "without MPLS support the" 2086 " specification cannot be used for" 2087 " filtering"); 2088 #endif 2089 #endif 2090 return 0; 2091 } 2092 2093 /** 2094 * Validate Geneve item. 2095 * 2096 * @param[in] item 2097 * Item specification. 2098 * @param[in] itemFlags 2099 * Bit-fields that holds the items detected until now. 2100 * @param[in] enPriv 2101 * Pointer to the private data structure. 2102 * @param[out] error 2103 * Pointer to error structure. 2104 * 2105 * @return 2106 * 0 on success, a negative errno value otherwise and rte_errno is set. 2107 */ 2108 2109 int 2110 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2111 uint64_t item_flags, 2112 struct rte_eth_dev *dev, 2113 struct rte_flow_error *error) 2114 { 2115 struct mlx5_priv *priv = dev->data->dev_private; 2116 const struct rte_flow_item_geneve *spec = item->spec; 2117 const struct rte_flow_item_geneve *mask = item->mask; 2118 int ret; 2119 uint16_t gbhdr; 2120 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2121 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2122 const struct rte_flow_item_geneve nic_mask = { 2123 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2124 .vni = "\xff\xff\xff", 2125 .protocol = RTE_BE16(UINT16_MAX), 2126 }; 2127 2128 if (!(priv->config.hca_attr.flex_parser_protocols & 2129 MLX5_HCA_FLEX_GENEVE_ENABLED) || 2130 !priv->config.hca_attr.tunnel_stateless_geneve_rx) 2131 return rte_flow_error_set(error, ENOTSUP, 2132 RTE_FLOW_ERROR_TYPE_ITEM, item, 2133 "L3 Geneve is not enabled by device" 2134 " parameter and/or not configured in" 2135 " firmware"); 2136 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2137 return rte_flow_error_set(error, ENOTSUP, 2138 RTE_FLOW_ERROR_TYPE_ITEM, item, 2139 "multiple tunnel layers not" 2140 " supported"); 2141 /* 2142 * Verify only UDPv4 is present as defined in 2143 * https://tools.ietf.org/html/rfc7348 2144 */ 2145 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2146 return rte_flow_error_set(error, EINVAL, 2147 RTE_FLOW_ERROR_TYPE_ITEM, item, 2148 "no outer UDP layer found"); 2149 if (!mask) 2150 mask = &rte_flow_item_geneve_mask; 2151 ret = mlx5_flow_item_acceptable 2152 (item, (const uint8_t *)mask, 2153 (const uint8_t *)&nic_mask, 2154 sizeof(struct rte_flow_item_geneve), error); 2155 if (ret) 2156 return ret; 2157 if (spec) { 2158 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2159 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2160 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2161 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2162 return rte_flow_error_set(error, ENOTSUP, 2163 RTE_FLOW_ERROR_TYPE_ITEM, 2164 item, 2165 "Geneve protocol unsupported" 2166 " fields are being used"); 2167 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2168 return rte_flow_error_set 2169 (error, ENOTSUP, 2170 RTE_FLOW_ERROR_TYPE_ITEM, 2171 item, 2172 "Unsupported Geneve options length"); 2173 } 2174 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2175 return rte_flow_error_set 2176 (error, ENOTSUP, 2177 RTE_FLOW_ERROR_TYPE_ITEM, item, 2178 "Geneve tunnel must be fully defined"); 2179 return 0; 2180 } 2181 2182 /** 2183 * Validate MPLS item. 2184 * 2185 * @param[in] dev 2186 * Pointer to the rte_eth_dev structure. 2187 * @param[in] item 2188 * Item specification. 2189 * @param[in] item_flags 2190 * Bit-fields that holds the items detected until now. 2191 * @param[in] prev_layer 2192 * The protocol layer indicated in previous item. 2193 * @param[out] error 2194 * Pointer to error structure. 2195 * 2196 * @return 2197 * 0 on success, a negative errno value otherwise and rte_errno is set. 2198 */ 2199 int 2200 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2201 const struct rte_flow_item *item __rte_unused, 2202 uint64_t item_flags __rte_unused, 2203 uint64_t prev_layer __rte_unused, 2204 struct rte_flow_error *error) 2205 { 2206 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2207 const struct rte_flow_item_mpls *mask = item->mask; 2208 struct mlx5_priv *priv = dev->data->dev_private; 2209 int ret; 2210 2211 if (!priv->config.mpls_en) 2212 return rte_flow_error_set(error, ENOTSUP, 2213 RTE_FLOW_ERROR_TYPE_ITEM, item, 2214 "MPLS not supported or" 2215 " disabled in firmware" 2216 " configuration."); 2217 /* MPLS over IP, UDP, GRE is allowed */ 2218 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2219 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2220 MLX5_FLOW_LAYER_GRE))) 2221 return rte_flow_error_set(error, EINVAL, 2222 RTE_FLOW_ERROR_TYPE_ITEM, item, 2223 "protocol filtering not compatible" 2224 " with MPLS layer"); 2225 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2226 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2227 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2228 return rte_flow_error_set(error, ENOTSUP, 2229 RTE_FLOW_ERROR_TYPE_ITEM, item, 2230 "multiple tunnel layers not" 2231 " supported"); 2232 if (!mask) 2233 mask = &rte_flow_item_mpls_mask; 2234 ret = mlx5_flow_item_acceptable 2235 (item, (const uint8_t *)mask, 2236 (const uint8_t *)&rte_flow_item_mpls_mask, 2237 sizeof(struct rte_flow_item_mpls), error); 2238 if (ret < 0) 2239 return ret; 2240 return 0; 2241 #endif 2242 return rte_flow_error_set(error, ENOTSUP, 2243 RTE_FLOW_ERROR_TYPE_ITEM, item, 2244 "MPLS is not supported by Verbs, please" 2245 " update."); 2246 } 2247 2248 /** 2249 * Validate NVGRE item. 2250 * 2251 * @param[in] item 2252 * Item specification. 2253 * @param[in] item_flags 2254 * Bit flags to mark detected items. 2255 * @param[in] target_protocol 2256 * The next protocol in the previous item. 2257 * @param[out] error 2258 * Pointer to error structure. 2259 * 2260 * @return 2261 * 0 on success, a negative errno value otherwise and rte_errno is set. 2262 */ 2263 int 2264 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2265 uint64_t item_flags, 2266 uint8_t target_protocol, 2267 struct rte_flow_error *error) 2268 { 2269 const struct rte_flow_item_nvgre *mask = item->mask; 2270 int ret; 2271 2272 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2273 return rte_flow_error_set(error, EINVAL, 2274 RTE_FLOW_ERROR_TYPE_ITEM, item, 2275 "protocol filtering not compatible" 2276 " with this GRE layer"); 2277 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2278 return rte_flow_error_set(error, ENOTSUP, 2279 RTE_FLOW_ERROR_TYPE_ITEM, item, 2280 "multiple tunnel layers not" 2281 " supported"); 2282 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2283 return rte_flow_error_set(error, ENOTSUP, 2284 RTE_FLOW_ERROR_TYPE_ITEM, item, 2285 "L3 Layer is missing"); 2286 if (!mask) 2287 mask = &rte_flow_item_nvgre_mask; 2288 ret = mlx5_flow_item_acceptable 2289 (item, (const uint8_t *)mask, 2290 (const uint8_t *)&rte_flow_item_nvgre_mask, 2291 sizeof(struct rte_flow_item_nvgre), error); 2292 if (ret < 0) 2293 return ret; 2294 return 0; 2295 } 2296 2297 /* Allocate unique ID for the split Q/RSS subflows. */ 2298 static uint32_t 2299 flow_qrss_get_id(struct rte_eth_dev *dev) 2300 { 2301 struct mlx5_priv *priv = dev->data->dev_private; 2302 uint32_t qrss_id, ret; 2303 2304 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2305 if (ret) 2306 return 0; 2307 assert(qrss_id); 2308 return qrss_id; 2309 } 2310 2311 /* Free unique ID for the split Q/RSS subflows. */ 2312 static void 2313 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2314 { 2315 struct mlx5_priv *priv = dev->data->dev_private; 2316 2317 if (qrss_id) 2318 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2319 } 2320 2321 /** 2322 * Release resource related QUEUE/RSS action split. 2323 * 2324 * @param dev 2325 * Pointer to Ethernet device. 2326 * @param flow 2327 * Flow to release id's from. 2328 */ 2329 static void 2330 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2331 struct rte_flow *flow) 2332 { 2333 struct mlx5_flow *dev_flow; 2334 2335 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 2336 if (dev_flow->qrss_id) 2337 flow_qrss_free_id(dev, dev_flow->qrss_id); 2338 } 2339 2340 static int 2341 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2342 const struct rte_flow_attr *attr __rte_unused, 2343 const struct rte_flow_item items[] __rte_unused, 2344 const struct rte_flow_action actions[] __rte_unused, 2345 bool external __rte_unused, 2346 struct rte_flow_error *error) 2347 { 2348 return rte_flow_error_set(error, ENOTSUP, 2349 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2350 } 2351 2352 static struct mlx5_flow * 2353 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, 2354 const struct rte_flow_item items[] __rte_unused, 2355 const struct rte_flow_action actions[] __rte_unused, 2356 struct rte_flow_error *error) 2357 { 2358 rte_flow_error_set(error, ENOTSUP, 2359 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2360 return NULL; 2361 } 2362 2363 static int 2364 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2365 struct mlx5_flow *dev_flow __rte_unused, 2366 const struct rte_flow_attr *attr __rte_unused, 2367 const struct rte_flow_item items[] __rte_unused, 2368 const struct rte_flow_action actions[] __rte_unused, 2369 struct rte_flow_error *error) 2370 { 2371 return rte_flow_error_set(error, ENOTSUP, 2372 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2373 } 2374 2375 static int 2376 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2377 struct rte_flow *flow __rte_unused, 2378 struct rte_flow_error *error) 2379 { 2380 return rte_flow_error_set(error, ENOTSUP, 2381 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2382 } 2383 2384 static void 2385 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2386 struct rte_flow *flow __rte_unused) 2387 { 2388 } 2389 2390 static void 2391 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2392 struct rte_flow *flow __rte_unused) 2393 { 2394 } 2395 2396 static int 2397 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2398 struct rte_flow *flow __rte_unused, 2399 const struct rte_flow_action *actions __rte_unused, 2400 void *data __rte_unused, 2401 struct rte_flow_error *error) 2402 { 2403 return rte_flow_error_set(error, ENOTSUP, 2404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2405 } 2406 2407 /* Void driver to protect from null pointer reference. */ 2408 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2409 .validate = flow_null_validate, 2410 .prepare = flow_null_prepare, 2411 .translate = flow_null_translate, 2412 .apply = flow_null_apply, 2413 .remove = flow_null_remove, 2414 .destroy = flow_null_destroy, 2415 .query = flow_null_query, 2416 }; 2417 2418 /** 2419 * Select flow driver type according to flow attributes and device 2420 * configuration. 2421 * 2422 * @param[in] dev 2423 * Pointer to the dev structure. 2424 * @param[in] attr 2425 * Pointer to the flow attributes. 2426 * 2427 * @return 2428 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2429 */ 2430 static enum mlx5_flow_drv_type 2431 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2432 { 2433 struct mlx5_priv *priv = dev->data->dev_private; 2434 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2435 2436 if (attr->transfer && priv->config.dv_esw_en) 2437 type = MLX5_FLOW_TYPE_DV; 2438 if (!attr->transfer) 2439 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2440 MLX5_FLOW_TYPE_VERBS; 2441 return type; 2442 } 2443 2444 #define flow_get_drv_ops(type) flow_drv_ops[type] 2445 2446 /** 2447 * Flow driver validation API. This abstracts calling driver specific functions. 2448 * The type of flow driver is determined according to flow attributes. 2449 * 2450 * @param[in] dev 2451 * Pointer to the dev structure. 2452 * @param[in] attr 2453 * Pointer to the flow attributes. 2454 * @param[in] items 2455 * Pointer to the list of items. 2456 * @param[in] actions 2457 * Pointer to the list of actions. 2458 * @param[in] external 2459 * This flow rule is created by request external to PMD. 2460 * @param[out] error 2461 * Pointer to the error structure. 2462 * 2463 * @return 2464 * 0 on success, a negative errno value otherwise and rte_errno is set. 2465 */ 2466 static inline int 2467 flow_drv_validate(struct rte_eth_dev *dev, 2468 const struct rte_flow_attr *attr, 2469 const struct rte_flow_item items[], 2470 const struct rte_flow_action actions[], 2471 bool external, struct rte_flow_error *error) 2472 { 2473 const struct mlx5_flow_driver_ops *fops; 2474 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2475 2476 fops = flow_get_drv_ops(type); 2477 return fops->validate(dev, attr, items, actions, external, error); 2478 } 2479 2480 /** 2481 * Flow driver preparation API. This abstracts calling driver specific 2482 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2483 * calculates the size of memory required for device flow, allocates the memory, 2484 * initializes the device flow and returns the pointer. 2485 * 2486 * @note 2487 * This function initializes device flow structure such as dv or verbs in 2488 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2489 * rest. For example, adding returning device flow to flow->dev_flow list and 2490 * setting backward reference to the flow should be done out of this function. 2491 * layers field is not filled either. 2492 * 2493 * @param[in] attr 2494 * Pointer to the flow attributes. 2495 * @param[in] items 2496 * Pointer to the list of items. 2497 * @param[in] actions 2498 * Pointer to the list of actions. 2499 * @param[out] error 2500 * Pointer to the error structure. 2501 * 2502 * @return 2503 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2504 */ 2505 static inline struct mlx5_flow * 2506 flow_drv_prepare(const struct rte_flow *flow, 2507 const struct rte_flow_attr *attr, 2508 const struct rte_flow_item items[], 2509 const struct rte_flow_action actions[], 2510 struct rte_flow_error *error) 2511 { 2512 const struct mlx5_flow_driver_ops *fops; 2513 enum mlx5_flow_drv_type type = flow->drv_type; 2514 2515 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2516 fops = flow_get_drv_ops(type); 2517 return fops->prepare(attr, items, actions, error); 2518 } 2519 2520 /** 2521 * Flow driver translation API. This abstracts calling driver specific 2522 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2523 * translates a generic flow into a driver flow. flow_drv_prepare() must 2524 * precede. 2525 * 2526 * @note 2527 * dev_flow->layers could be filled as a result of parsing during translation 2528 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2529 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2530 * flow->actions could be overwritten even though all the expanded dev_flows 2531 * have the same actions. 2532 * 2533 * @param[in] dev 2534 * Pointer to the rte dev structure. 2535 * @param[in, out] dev_flow 2536 * Pointer to the mlx5 flow. 2537 * @param[in] attr 2538 * Pointer to the flow attributes. 2539 * @param[in] items 2540 * Pointer to the list of items. 2541 * @param[in] actions 2542 * Pointer to the list of actions. 2543 * @param[out] error 2544 * Pointer to the error structure. 2545 * 2546 * @return 2547 * 0 on success, a negative errno value otherwise and rte_errno is set. 2548 */ 2549 static inline int 2550 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2551 const struct rte_flow_attr *attr, 2552 const struct rte_flow_item items[], 2553 const struct rte_flow_action actions[], 2554 struct rte_flow_error *error) 2555 { 2556 const struct mlx5_flow_driver_ops *fops; 2557 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2558 2559 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2560 fops = flow_get_drv_ops(type); 2561 return fops->translate(dev, dev_flow, attr, items, actions, error); 2562 } 2563 2564 /** 2565 * Flow driver apply API. This abstracts calling driver specific functions. 2566 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2567 * translated driver flows on to device. flow_drv_translate() must precede. 2568 * 2569 * @param[in] dev 2570 * Pointer to Ethernet device structure. 2571 * @param[in, out] flow 2572 * Pointer to flow structure. 2573 * @param[out] error 2574 * Pointer to error structure. 2575 * 2576 * @return 2577 * 0 on success, a negative errno value otherwise and rte_errno is set. 2578 */ 2579 static inline int 2580 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2581 struct rte_flow_error *error) 2582 { 2583 const struct mlx5_flow_driver_ops *fops; 2584 enum mlx5_flow_drv_type type = flow->drv_type; 2585 2586 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2587 fops = flow_get_drv_ops(type); 2588 return fops->apply(dev, flow, error); 2589 } 2590 2591 /** 2592 * Flow driver remove API. This abstracts calling driver specific functions. 2593 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2594 * on device. All the resources of the flow should be freed by calling 2595 * flow_drv_destroy(). 2596 * 2597 * @param[in] dev 2598 * Pointer to Ethernet device. 2599 * @param[in, out] flow 2600 * Pointer to flow structure. 2601 */ 2602 static inline void 2603 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2604 { 2605 const struct mlx5_flow_driver_ops *fops; 2606 enum mlx5_flow_drv_type type = flow->drv_type; 2607 2608 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2609 fops = flow_get_drv_ops(type); 2610 fops->remove(dev, flow); 2611 } 2612 2613 /** 2614 * Flow driver destroy API. This abstracts calling driver specific functions. 2615 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2616 * on device and releases resources of the flow. 2617 * 2618 * @param[in] dev 2619 * Pointer to Ethernet device. 2620 * @param[in, out] flow 2621 * Pointer to flow structure. 2622 */ 2623 static inline void 2624 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2625 { 2626 const struct mlx5_flow_driver_ops *fops; 2627 enum mlx5_flow_drv_type type = flow->drv_type; 2628 2629 flow_mreg_split_qrss_release(dev, flow); 2630 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2631 fops = flow_get_drv_ops(type); 2632 fops->destroy(dev, flow); 2633 } 2634 2635 /** 2636 * Validate a flow supported by the NIC. 2637 * 2638 * @see rte_flow_validate() 2639 * @see rte_flow_ops 2640 */ 2641 int 2642 mlx5_flow_validate(struct rte_eth_dev *dev, 2643 const struct rte_flow_attr *attr, 2644 const struct rte_flow_item items[], 2645 const struct rte_flow_action actions[], 2646 struct rte_flow_error *error) 2647 { 2648 int ret; 2649 2650 ret = flow_drv_validate(dev, attr, items, actions, true, error); 2651 if (ret < 0) 2652 return ret; 2653 return 0; 2654 } 2655 2656 /** 2657 * Get port id item from the item list. 2658 * 2659 * @param[in] item 2660 * Pointer to the list of items. 2661 * 2662 * @return 2663 * Pointer to the port id item if exist, else return NULL. 2664 */ 2665 static const struct rte_flow_item * 2666 find_port_id_item(const struct rte_flow_item *item) 2667 { 2668 assert(item); 2669 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2670 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) 2671 return item; 2672 } 2673 return NULL; 2674 } 2675 2676 /** 2677 * Get RSS action from the action list. 2678 * 2679 * @param[in] actions 2680 * Pointer to the list of actions. 2681 * 2682 * @return 2683 * Pointer to the RSS action if exist, else return NULL. 2684 */ 2685 static const struct rte_flow_action_rss* 2686 flow_get_rss_action(const struct rte_flow_action actions[]) 2687 { 2688 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2689 switch (actions->type) { 2690 case RTE_FLOW_ACTION_TYPE_RSS: 2691 return (const struct rte_flow_action_rss *) 2692 actions->conf; 2693 default: 2694 break; 2695 } 2696 } 2697 return NULL; 2698 } 2699 2700 static unsigned int 2701 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2702 { 2703 const struct rte_flow_item *item; 2704 unsigned int has_vlan = 0; 2705 2706 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2707 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2708 has_vlan = 1; 2709 break; 2710 } 2711 } 2712 if (has_vlan) 2713 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2714 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2715 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2716 MLX5_EXPANSION_ROOT_OUTER; 2717 } 2718 2719 /** 2720 * Get QUEUE/RSS action from the action list. 2721 * 2722 * @param[in] actions 2723 * Pointer to the list of actions. 2724 * @param[out] qrss 2725 * Pointer to the return pointer. 2726 * @param[out] qrss_type 2727 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2728 * if no QUEUE/RSS is found. 2729 * 2730 * @return 2731 * Total number of actions. 2732 */ 2733 static int 2734 flow_parse_qrss_action(const struct rte_flow_action actions[], 2735 const struct rte_flow_action **qrss) 2736 { 2737 int actions_n = 0; 2738 2739 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2740 switch (actions->type) { 2741 case RTE_FLOW_ACTION_TYPE_QUEUE: 2742 case RTE_FLOW_ACTION_TYPE_RSS: 2743 *qrss = actions; 2744 break; 2745 default: 2746 break; 2747 } 2748 actions_n++; 2749 } 2750 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2751 return actions_n + 1; 2752 } 2753 2754 /** 2755 * Check meter action from the action list. 2756 * 2757 * @param[in] actions 2758 * Pointer to the list of actions. 2759 * @param[out] mtr 2760 * Pointer to the meter exist flag. 2761 * 2762 * @return 2763 * Total number of actions. 2764 */ 2765 static int 2766 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2767 { 2768 int actions_n = 0; 2769 2770 assert(mtr); 2771 *mtr = 0; 2772 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2773 switch (actions->type) { 2774 case RTE_FLOW_ACTION_TYPE_METER: 2775 *mtr = 1; 2776 break; 2777 default: 2778 break; 2779 } 2780 actions_n++; 2781 } 2782 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2783 return actions_n + 1; 2784 } 2785 2786 /** 2787 * Check if the flow should be splited due to hairpin. 2788 * The reason for the split is that in current HW we can't 2789 * support encap on Rx, so if a flow have encap we move it 2790 * to Tx. 2791 * 2792 * @param dev 2793 * Pointer to Ethernet device. 2794 * @param[in] attr 2795 * Flow rule attributes. 2796 * @param[in] actions 2797 * Associated actions (list terminated by the END action). 2798 * 2799 * @return 2800 * > 0 the number of actions and the flow should be split, 2801 * 0 when no split required. 2802 */ 2803 static int 2804 flow_check_hairpin_split(struct rte_eth_dev *dev, 2805 const struct rte_flow_attr *attr, 2806 const struct rte_flow_action actions[]) 2807 { 2808 int queue_action = 0; 2809 int action_n = 0; 2810 int encap = 0; 2811 const struct rte_flow_action_queue *queue; 2812 const struct rte_flow_action_rss *rss; 2813 const struct rte_flow_action_raw_encap *raw_encap; 2814 2815 if (!attr->ingress) 2816 return 0; 2817 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2818 switch (actions->type) { 2819 case RTE_FLOW_ACTION_TYPE_QUEUE: 2820 queue = actions->conf; 2821 if (queue == NULL) 2822 return 0; 2823 if (mlx5_rxq_get_type(dev, queue->index) != 2824 MLX5_RXQ_TYPE_HAIRPIN) 2825 return 0; 2826 queue_action = 1; 2827 action_n++; 2828 break; 2829 case RTE_FLOW_ACTION_TYPE_RSS: 2830 rss = actions->conf; 2831 if (rss == NULL || rss->queue_num == 0) 2832 return 0; 2833 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2834 MLX5_RXQ_TYPE_HAIRPIN) 2835 return 0; 2836 queue_action = 1; 2837 action_n++; 2838 break; 2839 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2840 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2841 encap = 1; 2842 action_n++; 2843 break; 2844 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2845 raw_encap = actions->conf; 2846 if (raw_encap->size > 2847 (sizeof(struct rte_flow_item_eth) + 2848 sizeof(struct rte_flow_item_ipv4))) 2849 encap = 1; 2850 action_n++; 2851 break; 2852 default: 2853 action_n++; 2854 break; 2855 } 2856 } 2857 if (encap == 1 && queue_action) 2858 return action_n; 2859 return 0; 2860 } 2861 2862 /* Declare flow create/destroy prototype in advance. */ 2863 static struct rte_flow * 2864 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 2865 const struct rte_flow_attr *attr, 2866 const struct rte_flow_item items[], 2867 const struct rte_flow_action actions[], 2868 bool external, struct rte_flow_error *error); 2869 2870 static void 2871 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2872 struct rte_flow *flow); 2873 2874 /** 2875 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2876 * 2877 * As mark_id is unique, if there's already a registered flow for the mark_id, 2878 * return by increasing the reference counter of the resource. Otherwise, create 2879 * the resource (mcp_res) and flow. 2880 * 2881 * Flow looks like, 2882 * - If ingress port is ANY and reg_c[1] is mark_id, 2883 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2884 * 2885 * For default flow (zero mark_id), flow is like, 2886 * - If ingress port is ANY, 2887 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2888 * 2889 * @param dev 2890 * Pointer to Ethernet device. 2891 * @param mark_id 2892 * ID of MARK action, zero means default flow for META. 2893 * @param[out] error 2894 * Perform verbose error reporting if not NULL. 2895 * 2896 * @return 2897 * Associated resource on success, NULL otherwise and rte_errno is set. 2898 */ 2899 static struct mlx5_flow_mreg_copy_resource * 2900 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2901 struct rte_flow_error *error) 2902 { 2903 struct mlx5_priv *priv = dev->data->dev_private; 2904 struct rte_flow_attr attr = { 2905 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2906 .ingress = 1, 2907 }; 2908 struct mlx5_rte_flow_item_tag tag_spec = { 2909 .data = mark_id, 2910 }; 2911 struct rte_flow_item items[] = { 2912 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2913 }; 2914 struct rte_flow_action_mark ftag = { 2915 .id = mark_id, 2916 }; 2917 struct mlx5_flow_action_copy_mreg cp_mreg = { 2918 .dst = REG_B, 2919 .src = 0, 2920 }; 2921 struct rte_flow_action_jump jump = { 2922 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2923 }; 2924 struct rte_flow_action actions[] = { 2925 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2926 }; 2927 struct mlx5_flow_mreg_copy_resource *mcp_res; 2928 int ret; 2929 2930 /* Fill the register fileds in the flow. */ 2931 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2932 if (ret < 0) 2933 return NULL; 2934 tag_spec.id = ret; 2935 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 2936 if (ret < 0) 2937 return NULL; 2938 cp_mreg.src = ret; 2939 /* Check if already registered. */ 2940 assert(priv->mreg_cp_tbl); 2941 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 2942 if (mcp_res) { 2943 /* For non-default rule. */ 2944 if (mark_id != MLX5_DEFAULT_COPY_ID) 2945 mcp_res->refcnt++; 2946 assert(mark_id != MLX5_DEFAULT_COPY_ID || mcp_res->refcnt == 1); 2947 return mcp_res; 2948 } 2949 /* Provide the full width of FLAG specific value. */ 2950 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 2951 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 2952 /* Build a new flow. */ 2953 if (mark_id != MLX5_DEFAULT_COPY_ID) { 2954 items[0] = (struct rte_flow_item){ 2955 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 2956 .spec = &tag_spec, 2957 }; 2958 items[1] = (struct rte_flow_item){ 2959 .type = RTE_FLOW_ITEM_TYPE_END, 2960 }; 2961 actions[0] = (struct rte_flow_action){ 2962 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK, 2963 .conf = &ftag, 2964 }; 2965 actions[1] = (struct rte_flow_action){ 2966 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2967 .conf = &cp_mreg, 2968 }; 2969 actions[2] = (struct rte_flow_action){ 2970 .type = RTE_FLOW_ACTION_TYPE_JUMP, 2971 .conf = &jump, 2972 }; 2973 actions[3] = (struct rte_flow_action){ 2974 .type = RTE_FLOW_ACTION_TYPE_END, 2975 }; 2976 } else { 2977 /* Default rule, wildcard match. */ 2978 attr.priority = MLX5_FLOW_PRIO_RSVD; 2979 items[0] = (struct rte_flow_item){ 2980 .type = RTE_FLOW_ITEM_TYPE_END, 2981 }; 2982 actions[0] = (struct rte_flow_action){ 2983 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2984 .conf = &cp_mreg, 2985 }; 2986 actions[1] = (struct rte_flow_action){ 2987 .type = RTE_FLOW_ACTION_TYPE_JUMP, 2988 .conf = &jump, 2989 }; 2990 actions[2] = (struct rte_flow_action){ 2991 .type = RTE_FLOW_ACTION_TYPE_END, 2992 }; 2993 } 2994 /* Build a new entry. */ 2995 mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0); 2996 if (!mcp_res) { 2997 rte_errno = ENOMEM; 2998 return NULL; 2999 } 3000 /* 3001 * The copy Flows are not included in any list. There 3002 * ones are referenced from other Flows and can not 3003 * be applied, removed, deleted in ardbitrary order 3004 * by list traversing. 3005 */ 3006 mcp_res->flow = flow_list_create(dev, NULL, &attr, items, 3007 actions, false, error); 3008 if (!mcp_res->flow) 3009 goto error; 3010 mcp_res->refcnt++; 3011 mcp_res->hlist_ent.key = mark_id; 3012 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3013 &mcp_res->hlist_ent); 3014 assert(!ret); 3015 if (ret) 3016 goto error; 3017 return mcp_res; 3018 error: 3019 if (mcp_res->flow) 3020 flow_list_destroy(dev, NULL, mcp_res->flow); 3021 rte_free(mcp_res); 3022 return NULL; 3023 } 3024 3025 /** 3026 * Release flow in RX_CP_TBL. 3027 * 3028 * @param dev 3029 * Pointer to Ethernet device. 3030 * @flow 3031 * Parent flow for wich copying is provided. 3032 */ 3033 static void 3034 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3035 struct rte_flow *flow) 3036 { 3037 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3038 struct mlx5_priv *priv = dev->data->dev_private; 3039 3040 if (!mcp_res || !priv->mreg_cp_tbl) 3041 return; 3042 if (flow->copy_applied) { 3043 assert(mcp_res->appcnt); 3044 flow->copy_applied = 0; 3045 --mcp_res->appcnt; 3046 if (!mcp_res->appcnt) 3047 flow_drv_remove(dev, mcp_res->flow); 3048 } 3049 /* 3050 * We do not check availability of metadata registers here, 3051 * because copy resources are not allocated in this case. 3052 */ 3053 if (--mcp_res->refcnt) 3054 return; 3055 assert(mcp_res->flow); 3056 flow_list_destroy(dev, NULL, mcp_res->flow); 3057 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3058 rte_free(mcp_res); 3059 flow->mreg_copy = NULL; 3060 } 3061 3062 /** 3063 * Start flow in RX_CP_TBL. 3064 * 3065 * @param dev 3066 * Pointer to Ethernet device. 3067 * @flow 3068 * Parent flow for wich copying is provided. 3069 * 3070 * @return 3071 * 0 on success, a negative errno value otherwise and rte_errno is set. 3072 */ 3073 static int 3074 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3075 struct rte_flow *flow) 3076 { 3077 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3078 int ret; 3079 3080 if (!mcp_res || flow->copy_applied) 3081 return 0; 3082 if (!mcp_res->appcnt) { 3083 ret = flow_drv_apply(dev, mcp_res->flow, NULL); 3084 if (ret) 3085 return ret; 3086 } 3087 ++mcp_res->appcnt; 3088 flow->copy_applied = 1; 3089 return 0; 3090 } 3091 3092 /** 3093 * Stop flow in RX_CP_TBL. 3094 * 3095 * @param dev 3096 * Pointer to Ethernet device. 3097 * @flow 3098 * Parent flow for wich copying is provided. 3099 */ 3100 static void 3101 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3102 struct rte_flow *flow) 3103 { 3104 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3105 3106 if (!mcp_res || !flow->copy_applied) 3107 return; 3108 assert(mcp_res->appcnt); 3109 --mcp_res->appcnt; 3110 flow->copy_applied = 0; 3111 if (!mcp_res->appcnt) 3112 flow_drv_remove(dev, mcp_res->flow); 3113 } 3114 3115 /** 3116 * Remove the default copy action from RX_CP_TBL. 3117 * 3118 * @param dev 3119 * Pointer to Ethernet device. 3120 */ 3121 static void 3122 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3123 { 3124 struct mlx5_flow_mreg_copy_resource *mcp_res; 3125 struct mlx5_priv *priv = dev->data->dev_private; 3126 3127 /* Check if default flow is registered. */ 3128 if (!priv->mreg_cp_tbl) 3129 return; 3130 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3131 MLX5_DEFAULT_COPY_ID); 3132 if (!mcp_res) 3133 return; 3134 assert(mcp_res->flow); 3135 flow_list_destroy(dev, NULL, mcp_res->flow); 3136 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3137 rte_free(mcp_res); 3138 } 3139 3140 /** 3141 * Add the default copy action in in RX_CP_TBL. 3142 * 3143 * @param dev 3144 * Pointer to Ethernet device. 3145 * @param[out] error 3146 * Perform verbose error reporting if not NULL. 3147 * 3148 * @return 3149 * 0 for success, negative value otherwise and rte_errno is set. 3150 */ 3151 static int 3152 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3153 struct rte_flow_error *error) 3154 { 3155 struct mlx5_priv *priv = dev->data->dev_private; 3156 struct mlx5_flow_mreg_copy_resource *mcp_res; 3157 3158 /* Check whether extensive metadata feature is engaged. */ 3159 if (!priv->config.dv_flow_en || 3160 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3161 !mlx5_flow_ext_mreg_supported(dev) || 3162 !priv->sh->dv_regc0_mask) 3163 return 0; 3164 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3165 if (!mcp_res) 3166 return -rte_errno; 3167 return 0; 3168 } 3169 3170 /** 3171 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3172 * 3173 * All the flow having Q/RSS action should be split by 3174 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3175 * performs the following, 3176 * - CQE->flow_tag := reg_c[1] (MARK) 3177 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3178 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3179 * but there should be a flow per each MARK ID set by MARK action. 3180 * 3181 * For the aforementioned reason, if there's a MARK action in flow's action 3182 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3183 * the MARK ID to CQE's flow_tag like, 3184 * - If reg_c[1] is mark_id, 3185 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3186 * 3187 * For SET_META action which stores value in reg_c[0], as the destination is 3188 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3189 * MARK ID means the default flow. The default flow looks like, 3190 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3191 * 3192 * @param dev 3193 * Pointer to Ethernet device. 3194 * @param flow 3195 * Pointer to flow structure. 3196 * @param[in] actions 3197 * Pointer to the list of actions. 3198 * @param[out] error 3199 * Perform verbose error reporting if not NULL. 3200 * 3201 * @return 3202 * 0 on success, negative value otherwise and rte_errno is set. 3203 */ 3204 static int 3205 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3206 struct rte_flow *flow, 3207 const struct rte_flow_action *actions, 3208 struct rte_flow_error *error) 3209 { 3210 struct mlx5_priv *priv = dev->data->dev_private; 3211 struct mlx5_dev_config *config = &priv->config; 3212 struct mlx5_flow_mreg_copy_resource *mcp_res; 3213 const struct rte_flow_action_mark *mark; 3214 3215 /* Check whether extensive metadata feature is engaged. */ 3216 if (!config->dv_flow_en || 3217 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3218 !mlx5_flow_ext_mreg_supported(dev) || 3219 !priv->sh->dv_regc0_mask) 3220 return 0; 3221 /* Find MARK action. */ 3222 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3223 switch (actions->type) { 3224 case RTE_FLOW_ACTION_TYPE_FLAG: 3225 mcp_res = flow_mreg_add_copy_action 3226 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3227 if (!mcp_res) 3228 return -rte_errno; 3229 flow->mreg_copy = mcp_res; 3230 if (dev->data->dev_started) { 3231 mcp_res->appcnt++; 3232 flow->copy_applied = 1; 3233 } 3234 return 0; 3235 case RTE_FLOW_ACTION_TYPE_MARK: 3236 mark = (const struct rte_flow_action_mark *) 3237 actions->conf; 3238 mcp_res = 3239 flow_mreg_add_copy_action(dev, mark->id, error); 3240 if (!mcp_res) 3241 return -rte_errno; 3242 flow->mreg_copy = mcp_res; 3243 if (dev->data->dev_started) { 3244 mcp_res->appcnt++; 3245 flow->copy_applied = 1; 3246 } 3247 return 0; 3248 default: 3249 break; 3250 } 3251 } 3252 return 0; 3253 } 3254 3255 #define MLX5_MAX_SPLIT_ACTIONS 24 3256 #define MLX5_MAX_SPLIT_ITEMS 24 3257 3258 /** 3259 * Split the hairpin flow. 3260 * Since HW can't support encap on Rx we move the encap to Tx. 3261 * If the count action is after the encap then we also 3262 * move the count action. in this case the count will also measure 3263 * the outer bytes. 3264 * 3265 * @param dev 3266 * Pointer to Ethernet device. 3267 * @param[in] actions 3268 * Associated actions (list terminated by the END action). 3269 * @param[out] actions_rx 3270 * Rx flow actions. 3271 * @param[out] actions_tx 3272 * Tx flow actions.. 3273 * @param[out] pattern_tx 3274 * The pattern items for the Tx flow. 3275 * @param[out] flow_id 3276 * The flow ID connected to this flow. 3277 * 3278 * @return 3279 * 0 on success. 3280 */ 3281 static int 3282 flow_hairpin_split(struct rte_eth_dev *dev, 3283 const struct rte_flow_action actions[], 3284 struct rte_flow_action actions_rx[], 3285 struct rte_flow_action actions_tx[], 3286 struct rte_flow_item pattern_tx[], 3287 uint32_t *flow_id) 3288 { 3289 struct mlx5_priv *priv = dev->data->dev_private; 3290 const struct rte_flow_action_raw_encap *raw_encap; 3291 const struct rte_flow_action_raw_decap *raw_decap; 3292 struct mlx5_rte_flow_action_set_tag *set_tag; 3293 struct rte_flow_action *tag_action; 3294 struct mlx5_rte_flow_item_tag *tag_item; 3295 struct rte_flow_item *item; 3296 char *addr; 3297 int encap = 0; 3298 3299 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3300 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3301 switch (actions->type) { 3302 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3303 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3304 rte_memcpy(actions_tx, actions, 3305 sizeof(struct rte_flow_action)); 3306 actions_tx++; 3307 break; 3308 case RTE_FLOW_ACTION_TYPE_COUNT: 3309 if (encap) { 3310 rte_memcpy(actions_tx, actions, 3311 sizeof(struct rte_flow_action)); 3312 actions_tx++; 3313 } else { 3314 rte_memcpy(actions_rx, actions, 3315 sizeof(struct rte_flow_action)); 3316 actions_rx++; 3317 } 3318 break; 3319 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3320 raw_encap = actions->conf; 3321 if (raw_encap->size > 3322 (sizeof(struct rte_flow_item_eth) + 3323 sizeof(struct rte_flow_item_ipv4))) { 3324 memcpy(actions_tx, actions, 3325 sizeof(struct rte_flow_action)); 3326 actions_tx++; 3327 encap = 1; 3328 } else { 3329 rte_memcpy(actions_rx, actions, 3330 sizeof(struct rte_flow_action)); 3331 actions_rx++; 3332 } 3333 break; 3334 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3335 raw_decap = actions->conf; 3336 if (raw_decap->size < 3337 (sizeof(struct rte_flow_item_eth) + 3338 sizeof(struct rte_flow_item_ipv4))) { 3339 memcpy(actions_tx, actions, 3340 sizeof(struct rte_flow_action)); 3341 actions_tx++; 3342 } else { 3343 rte_memcpy(actions_rx, actions, 3344 sizeof(struct rte_flow_action)); 3345 actions_rx++; 3346 } 3347 break; 3348 default: 3349 rte_memcpy(actions_rx, actions, 3350 sizeof(struct rte_flow_action)); 3351 actions_rx++; 3352 break; 3353 } 3354 } 3355 /* Add set meta action and end action for the Rx flow. */ 3356 tag_action = actions_rx; 3357 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3358 actions_rx++; 3359 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3360 actions_rx++; 3361 set_tag = (void *)actions_rx; 3362 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3363 assert(set_tag->id > REG_NONE); 3364 set_tag->data = *flow_id; 3365 tag_action->conf = set_tag; 3366 /* Create Tx item list. */ 3367 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3368 addr = (void *)&pattern_tx[2]; 3369 item = pattern_tx; 3370 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3371 tag_item = (void *)addr; 3372 tag_item->data = *flow_id; 3373 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3374 assert(set_tag->id > REG_NONE); 3375 item->spec = tag_item; 3376 addr += sizeof(struct mlx5_rte_flow_item_tag); 3377 tag_item = (void *)addr; 3378 tag_item->data = UINT32_MAX; 3379 tag_item->id = UINT16_MAX; 3380 item->mask = tag_item; 3381 addr += sizeof(struct mlx5_rte_flow_item_tag); 3382 item->last = NULL; 3383 item++; 3384 item->type = RTE_FLOW_ITEM_TYPE_END; 3385 return 0; 3386 } 3387 3388 /** 3389 * The last stage of splitting chain, just creates the subflow 3390 * without any modification. 3391 * 3392 * @param dev 3393 * Pointer to Ethernet device. 3394 * @param[in] flow 3395 * Parent flow structure pointer. 3396 * @param[in, out] sub_flow 3397 * Pointer to return the created subflow, may be NULL. 3398 * @param[in] attr 3399 * Flow rule attributes. 3400 * @param[in] items 3401 * Pattern specification (list terminated by the END pattern item). 3402 * @param[in] actions 3403 * Associated actions (list terminated by the END action). 3404 * @param[in] external 3405 * This flow rule is created by request external to PMD. 3406 * @param[out] error 3407 * Perform verbose error reporting if not NULL. 3408 * @return 3409 * 0 on success, negative value otherwise 3410 */ 3411 static int 3412 flow_create_split_inner(struct rte_eth_dev *dev, 3413 struct rte_flow *flow, 3414 struct mlx5_flow **sub_flow, 3415 const struct rte_flow_attr *attr, 3416 const struct rte_flow_item items[], 3417 const struct rte_flow_action actions[], 3418 bool external, struct rte_flow_error *error) 3419 { 3420 struct mlx5_flow *dev_flow; 3421 3422 dev_flow = flow_drv_prepare(flow, attr, items, actions, error); 3423 if (!dev_flow) 3424 return -rte_errno; 3425 dev_flow->flow = flow; 3426 dev_flow->external = external; 3427 /* Subflow object was created, we must include one in the list. */ 3428 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 3429 if (sub_flow) 3430 *sub_flow = dev_flow; 3431 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3432 } 3433 3434 /** 3435 * Split the meter flow. 3436 * 3437 * As meter flow will split to three sub flow, other than meter 3438 * action, the other actions make sense to only meter accepts 3439 * the packet. If it need to be dropped, no other additional 3440 * actions should be take. 3441 * 3442 * One kind of special action which decapsulates the L3 tunnel 3443 * header will be in the prefix sub flow, as not to take the 3444 * L3 tunnel header into account. 3445 * 3446 * @param dev 3447 * Pointer to Ethernet device. 3448 * @param[in] actions 3449 * Associated actions (list terminated by the END action). 3450 * @param[out] actions_sfx 3451 * Suffix flow actions. 3452 * @param[out] actions_pre 3453 * Prefix flow actions. 3454 * @param[out] pattern_sfx 3455 * The pattern items for the suffix flow. 3456 * @param[out] tag_sfx 3457 * Pointer to suffix flow tag. 3458 * 3459 * @return 3460 * 0 on success. 3461 */ 3462 static int 3463 flow_meter_split_prep(struct rte_eth_dev *dev, 3464 const struct rte_flow_action actions[], 3465 struct rte_flow_action actions_sfx[], 3466 struct rte_flow_action actions_pre[]) 3467 { 3468 struct rte_flow_action *tag_action; 3469 struct mlx5_rte_flow_action_set_tag *set_tag; 3470 struct rte_flow_error error; 3471 const struct rte_flow_action_raw_encap *raw_encap; 3472 const struct rte_flow_action_raw_decap *raw_decap; 3473 uint32_t tag_id; 3474 3475 /* Add the extra tag action first. */ 3476 tag_action = actions_pre; 3477 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3478 actions_pre++; 3479 /* Prepare the actions for prefix and suffix flow. */ 3480 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3481 switch (actions->type) { 3482 case RTE_FLOW_ACTION_TYPE_METER: 3483 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3484 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3485 memcpy(actions_pre, actions, 3486 sizeof(struct rte_flow_action)); 3487 actions_pre++; 3488 break; 3489 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3490 raw_encap = actions->conf; 3491 if (raw_encap->size > 3492 (sizeof(struct rte_flow_item_eth) + 3493 sizeof(struct rte_flow_item_ipv4))) { 3494 memcpy(actions_sfx, actions, 3495 sizeof(struct rte_flow_action)); 3496 actions_sfx++; 3497 } else { 3498 rte_memcpy(actions_pre, actions, 3499 sizeof(struct rte_flow_action)); 3500 actions_pre++; 3501 } 3502 break; 3503 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3504 raw_decap = actions->conf; 3505 /* Size 0 decap means 50 bytes as vxlan decap. */ 3506 if (raw_decap->size && (raw_decap->size < 3507 (sizeof(struct rte_flow_item_eth) + 3508 sizeof(struct rte_flow_item_ipv4)))) { 3509 memcpy(actions_sfx, actions, 3510 sizeof(struct rte_flow_action)); 3511 actions_sfx++; 3512 } else { 3513 rte_memcpy(actions_pre, actions, 3514 sizeof(struct rte_flow_action)); 3515 actions_pre++; 3516 } 3517 break; 3518 default: 3519 memcpy(actions_sfx, actions, 3520 sizeof(struct rte_flow_action)); 3521 actions_sfx++; 3522 break; 3523 } 3524 } 3525 /* Add end action to the actions. */ 3526 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3527 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3528 actions_pre++; 3529 /* Set the tag. */ 3530 set_tag = (void *)actions_pre; 3531 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3532 /* 3533 * Get the id from the qrss_pool to make qrss share the id with meter. 3534 */ 3535 tag_id = flow_qrss_get_id(dev); 3536 set_tag->data = rte_cpu_to_be_32(tag_id); 3537 tag_action->conf = set_tag; 3538 return tag_id; 3539 } 3540 3541 /** 3542 * Split action list having QUEUE/RSS for metadata register copy. 3543 * 3544 * Once Q/RSS action is detected in user's action list, the flow action 3545 * should be split in order to copy metadata registers, which will happen in 3546 * RX_CP_TBL like, 3547 * - CQE->flow_tag := reg_c[1] (MARK) 3548 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3549 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3550 * This is because the last action of each flow must be a terminal action 3551 * (QUEUE, RSS or DROP). 3552 * 3553 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3554 * stored and kept in the mlx5_flow structure per each sub_flow. 3555 * 3556 * The Q/RSS action is replaced with, 3557 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3558 * And the following JUMP action is added at the end, 3559 * - JUMP, to RX_CP_TBL. 3560 * 3561 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3562 * flow_create_split_metadata() routine. The flow will look like, 3563 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3564 * 3565 * @param dev 3566 * Pointer to Ethernet device. 3567 * @param[out] split_actions 3568 * Pointer to store split actions to jump to CP_TBL. 3569 * @param[in] actions 3570 * Pointer to the list of original flow actions. 3571 * @param[in] qrss 3572 * Pointer to the Q/RSS action. 3573 * @param[in] actions_n 3574 * Number of original actions. 3575 * @param[out] error 3576 * Perform verbose error reporting if not NULL. 3577 * 3578 * @return 3579 * non-zero unique flow_id on success, otherwise 0 and 3580 * error/rte_error are set. 3581 */ 3582 static uint32_t 3583 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3584 struct rte_flow_action *split_actions, 3585 const struct rte_flow_action *actions, 3586 const struct rte_flow_action *qrss, 3587 int actions_n, struct rte_flow_error *error) 3588 { 3589 struct mlx5_rte_flow_action_set_tag *set_tag; 3590 struct rte_flow_action_jump *jump; 3591 const int qrss_idx = qrss - actions; 3592 uint32_t flow_id = 0; 3593 int ret = 0; 3594 3595 /* 3596 * Given actions will be split 3597 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3598 * - Add jump to mreg CP_TBL. 3599 * As a result, there will be one more action. 3600 */ 3601 ++actions_n; 3602 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3603 set_tag = (void *)(split_actions + actions_n); 3604 /* 3605 * If tag action is not set to void(it means we are not the meter 3606 * suffix flow), add the tag action. Since meter suffix flow already 3607 * has the tag added. 3608 */ 3609 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3610 /* 3611 * Allocate the new subflow ID. This one is unique within 3612 * device and not shared with representors. Otherwise, 3613 * we would have to resolve multi-thread access synch 3614 * issue. Each flow on the shared device is appended 3615 * with source vport identifier, so the resulting 3616 * flows will be unique in the shared (by master and 3617 * representors) domain even if they have coinciding 3618 * IDs. 3619 */ 3620 flow_id = flow_qrss_get_id(dev); 3621 if (!flow_id) 3622 return rte_flow_error_set(error, ENOMEM, 3623 RTE_FLOW_ERROR_TYPE_ACTION, 3624 NULL, "can't allocate id " 3625 "for split Q/RSS subflow"); 3626 /* Internal SET_TAG action to set flow ID. */ 3627 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3628 .data = flow_id, 3629 }; 3630 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3631 if (ret < 0) 3632 return ret; 3633 set_tag->id = ret; 3634 /* Construct new actions array. */ 3635 /* Replace QUEUE/RSS action. */ 3636 split_actions[qrss_idx] = (struct rte_flow_action){ 3637 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3638 .conf = set_tag, 3639 }; 3640 } 3641 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3642 jump = (void *)(set_tag + 1); 3643 *jump = (struct rte_flow_action_jump){ 3644 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3645 }; 3646 split_actions[actions_n - 2] = (struct rte_flow_action){ 3647 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3648 .conf = jump, 3649 }; 3650 split_actions[actions_n - 1] = (struct rte_flow_action){ 3651 .type = RTE_FLOW_ACTION_TYPE_END, 3652 }; 3653 return flow_id; 3654 } 3655 3656 /** 3657 * Extend the given action list for Tx metadata copy. 3658 * 3659 * Copy the given action list to the ext_actions and add flow metadata register 3660 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3661 * 3662 * @param[out] ext_actions 3663 * Pointer to the extended action list. 3664 * @param[in] actions 3665 * Pointer to the list of actions. 3666 * @param[in] actions_n 3667 * Number of actions in the list. 3668 * @param[out] error 3669 * Perform verbose error reporting if not NULL. 3670 * 3671 * @return 3672 * 0 on success, negative value otherwise 3673 */ 3674 static int 3675 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3676 struct rte_flow_action *ext_actions, 3677 const struct rte_flow_action *actions, 3678 int actions_n, struct rte_flow_error *error) 3679 { 3680 struct mlx5_flow_action_copy_mreg *cp_mreg = 3681 (struct mlx5_flow_action_copy_mreg *) 3682 (ext_actions + actions_n + 1); 3683 int ret; 3684 3685 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3686 if (ret < 0) 3687 return ret; 3688 cp_mreg->dst = ret; 3689 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3690 if (ret < 0) 3691 return ret; 3692 cp_mreg->src = ret; 3693 memcpy(ext_actions, actions, 3694 sizeof(*ext_actions) * actions_n); 3695 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3696 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3697 .conf = cp_mreg, 3698 }; 3699 ext_actions[actions_n] = (struct rte_flow_action){ 3700 .type = RTE_FLOW_ACTION_TYPE_END, 3701 }; 3702 return 0; 3703 } 3704 3705 /** 3706 * The splitting for metadata feature. 3707 * 3708 * - Q/RSS action on NIC Rx should be split in order to pass by 3709 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3710 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3711 * 3712 * - All the actions on NIC Tx should have a mreg copy action to 3713 * copy reg_a from WQE to reg_c[0]. 3714 * 3715 * @param dev 3716 * Pointer to Ethernet device. 3717 * @param[in] flow 3718 * Parent flow structure pointer. 3719 * @param[in] attr 3720 * Flow rule attributes. 3721 * @param[in] items 3722 * Pattern specification (list terminated by the END pattern item). 3723 * @param[in] actions 3724 * Associated actions (list terminated by the END action). 3725 * @param[in] external 3726 * This flow rule is created by request external to PMD. 3727 * @param[out] error 3728 * Perform verbose error reporting if not NULL. 3729 * @return 3730 * 0 on success, negative value otherwise 3731 */ 3732 static int 3733 flow_create_split_metadata(struct rte_eth_dev *dev, 3734 struct rte_flow *flow, 3735 const struct rte_flow_attr *attr, 3736 const struct rte_flow_item items[], 3737 const struct rte_flow_action actions[], 3738 bool external, struct rte_flow_error *error) 3739 { 3740 struct mlx5_priv *priv = dev->data->dev_private; 3741 struct mlx5_dev_config *config = &priv->config; 3742 const struct rte_flow_action *qrss = NULL; 3743 struct rte_flow_action *ext_actions = NULL; 3744 struct mlx5_flow *dev_flow = NULL; 3745 uint32_t qrss_id = 0; 3746 int mtr_sfx = 0; 3747 size_t act_size; 3748 int actions_n; 3749 int ret; 3750 3751 /* Check whether extensive metadata feature is engaged. */ 3752 if (!config->dv_flow_en || 3753 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3754 !mlx5_flow_ext_mreg_supported(dev)) 3755 return flow_create_split_inner(dev, flow, NULL, attr, items, 3756 actions, external, error); 3757 actions_n = flow_parse_qrss_action(actions, &qrss); 3758 if (qrss) { 3759 /* Exclude hairpin flows from splitting. */ 3760 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3761 const struct rte_flow_action_queue *queue; 3762 3763 queue = qrss->conf; 3764 if (mlx5_rxq_get_type(dev, queue->index) == 3765 MLX5_RXQ_TYPE_HAIRPIN) 3766 qrss = NULL; 3767 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3768 const struct rte_flow_action_rss *rss; 3769 3770 rss = qrss->conf; 3771 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3772 MLX5_RXQ_TYPE_HAIRPIN) 3773 qrss = NULL; 3774 } 3775 } 3776 if (qrss) { 3777 /* Check if it is in meter suffix table. */ 3778 mtr_sfx = attr->group == (attr->transfer ? 3779 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3780 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3781 /* 3782 * Q/RSS action on NIC Rx should be split in order to pass by 3783 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3784 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3785 */ 3786 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3787 sizeof(struct rte_flow_action_set_tag) + 3788 sizeof(struct rte_flow_action_jump); 3789 ext_actions = rte_zmalloc(__func__, act_size, 0); 3790 if (!ext_actions) 3791 return rte_flow_error_set(error, ENOMEM, 3792 RTE_FLOW_ERROR_TYPE_ACTION, 3793 NULL, "no memory to split " 3794 "metadata flow"); 3795 /* 3796 * If we are the suffix flow of meter, tag already exist. 3797 * Set the tag action to void. 3798 */ 3799 if (mtr_sfx) 3800 ext_actions[qrss - actions].type = 3801 RTE_FLOW_ACTION_TYPE_VOID; 3802 else 3803 ext_actions[qrss - actions].type = 3804 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3805 /* 3806 * Create the new actions list with removed Q/RSS action 3807 * and appended set tag and jump to register copy table 3808 * (RX_CP_TBL). We should preallocate unique tag ID here 3809 * in advance, because it is needed for set tag action. 3810 */ 3811 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3812 qrss, actions_n, error); 3813 if (!mtr_sfx && !qrss_id) { 3814 ret = -rte_errno; 3815 goto exit; 3816 } 3817 } else if (attr->egress && !attr->transfer) { 3818 /* 3819 * All the actions on NIC Tx should have a metadata register 3820 * copy action to copy reg_a from WQE to reg_c[meta] 3821 */ 3822 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3823 sizeof(struct mlx5_flow_action_copy_mreg); 3824 ext_actions = rte_zmalloc(__func__, act_size, 0); 3825 if (!ext_actions) 3826 return rte_flow_error_set(error, ENOMEM, 3827 RTE_FLOW_ERROR_TYPE_ACTION, 3828 NULL, "no memory to split " 3829 "metadata flow"); 3830 /* Create the action list appended with copy register. */ 3831 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 3832 actions_n, error); 3833 if (ret < 0) 3834 goto exit; 3835 } 3836 /* Add the unmodified original or prefix subflow. */ 3837 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, 3838 ext_actions ? ext_actions : actions, 3839 external, error); 3840 if (ret < 0) 3841 goto exit; 3842 assert(dev_flow); 3843 if (qrss) { 3844 const struct rte_flow_attr q_attr = { 3845 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3846 .ingress = 1, 3847 }; 3848 /* Internal PMD action to set register. */ 3849 struct mlx5_rte_flow_item_tag q_tag_spec = { 3850 .data = qrss_id, 3851 .id = 0, 3852 }; 3853 struct rte_flow_item q_items[] = { 3854 { 3855 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3856 .spec = &q_tag_spec, 3857 .last = NULL, 3858 .mask = NULL, 3859 }, 3860 { 3861 .type = RTE_FLOW_ITEM_TYPE_END, 3862 }, 3863 }; 3864 struct rte_flow_action q_actions[] = { 3865 { 3866 .type = qrss->type, 3867 .conf = qrss->conf, 3868 }, 3869 { 3870 .type = RTE_FLOW_ACTION_TYPE_END, 3871 }, 3872 }; 3873 uint64_t hash_fields = dev_flow->hash_fields; 3874 3875 /* 3876 * Configure the tag item only if there is no meter subflow. 3877 * Since tag is already marked in the meter suffix subflow 3878 * we can just use the meter suffix items as is. 3879 */ 3880 if (qrss_id) { 3881 /* Not meter subflow. */ 3882 assert(!mtr_sfx); 3883 /* 3884 * Put unique id in prefix flow due to it is destroyed 3885 * after suffix flow and id will be freed after there 3886 * is no actual flows with this id and identifier 3887 * reallocation becomes possible (for example, for 3888 * other flows in other threads). 3889 */ 3890 dev_flow->qrss_id = qrss_id; 3891 qrss_id = 0; 3892 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 3893 error); 3894 if (ret < 0) 3895 goto exit; 3896 q_tag_spec.id = ret; 3897 } 3898 dev_flow = NULL; 3899 /* Add suffix subflow to execute Q/RSS. */ 3900 ret = flow_create_split_inner(dev, flow, &dev_flow, 3901 &q_attr, mtr_sfx ? items : 3902 q_items, q_actions, 3903 external, error); 3904 if (ret < 0) 3905 goto exit; 3906 assert(dev_flow); 3907 dev_flow->hash_fields = hash_fields; 3908 } 3909 3910 exit: 3911 /* 3912 * We do not destroy the partially created sub_flows in case of error. 3913 * These ones are included into parent flow list and will be destroyed 3914 * by flow_drv_destroy. 3915 */ 3916 flow_qrss_free_id(dev, qrss_id); 3917 rte_free(ext_actions); 3918 return ret; 3919 } 3920 3921 /** 3922 * The splitting for meter feature. 3923 * 3924 * - The meter flow will be split to two flows as prefix and 3925 * suffix flow. The packets make sense only it pass the prefix 3926 * meter action. 3927 * 3928 * - Reg_C_5 is used for the packet to match betweend prefix and 3929 * suffix flow. 3930 * 3931 * @param dev 3932 * Pointer to Ethernet device. 3933 * @param[in] flow 3934 * Parent flow structure pointer. 3935 * @param[in] attr 3936 * Flow rule attributes. 3937 * @param[in] items 3938 * Pattern specification (list terminated by the END pattern item). 3939 * @param[in] actions 3940 * Associated actions (list terminated by the END action). 3941 * @param[in] external 3942 * This flow rule is created by request external to PMD. 3943 * @param[out] error 3944 * Perform verbose error reporting if not NULL. 3945 * @return 3946 * 0 on success, negative value otherwise 3947 */ 3948 static int 3949 flow_create_split_meter(struct rte_eth_dev *dev, 3950 struct rte_flow *flow, 3951 const struct rte_flow_attr *attr, 3952 const struct rte_flow_item items[], 3953 const struct rte_flow_action actions[], 3954 bool external, struct rte_flow_error *error) 3955 { 3956 struct mlx5_priv *priv = dev->data->dev_private; 3957 struct rte_flow_action *sfx_actions = NULL; 3958 struct rte_flow_action *pre_actions = NULL; 3959 struct rte_flow_item *sfx_items = NULL; 3960 const struct rte_flow_item *sfx_port_id_item; 3961 struct mlx5_flow *dev_flow = NULL; 3962 struct rte_flow_attr sfx_attr = *attr; 3963 uint32_t mtr = 0; 3964 uint32_t mtr_tag_id = 0; 3965 size_t act_size; 3966 size_t item_size; 3967 int actions_n = 0; 3968 int ret; 3969 3970 if (priv->mtr_en) 3971 actions_n = flow_check_meter_action(actions, &mtr); 3972 if (mtr) { 3973 struct mlx5_rte_flow_item_tag *tag_spec; 3974 /* The five prefix actions: meter, decap, encap, tag, end. */ 3975 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 3976 sizeof(struct rte_flow_action_set_tag); 3977 /* tag, end. */ 3978 #define METER_SUFFIX_ITEM 3 3979 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 3980 sizeof(struct mlx5_rte_flow_item_tag); 3981 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 3982 if (!sfx_actions) 3983 return rte_flow_error_set(error, ENOMEM, 3984 RTE_FLOW_ERROR_TYPE_ACTION, 3985 NULL, "no memory to split " 3986 "meter flow"); 3987 pre_actions = sfx_actions + actions_n; 3988 mtr_tag_id = flow_meter_split_prep(dev, actions, sfx_actions, 3989 pre_actions); 3990 if (!mtr_tag_id) { 3991 ret = -rte_errno; 3992 goto exit; 3993 } 3994 /* Add the prefix subflow. */ 3995 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, 3996 pre_actions, external, error); 3997 if (ret) { 3998 ret = -rte_errno; 3999 goto exit; 4000 } 4001 dev_flow->mtr_flow_id = mtr_tag_id; 4002 /* Prepare the suffix flow match pattern. */ 4003 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4004 act_size); 4005 tag_spec = (struct mlx5_rte_flow_item_tag *)(sfx_items + 4006 METER_SUFFIX_ITEM); 4007 tag_spec->data = rte_cpu_to_be_32(dev_flow->mtr_flow_id); 4008 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, 4009 error); 4010 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4011 sfx_items->spec = tag_spec; 4012 sfx_items->last = NULL; 4013 sfx_items->mask = NULL; 4014 sfx_items++; 4015 sfx_port_id_item = find_port_id_item(items); 4016 if (sfx_port_id_item) { 4017 memcpy(sfx_items, sfx_port_id_item, 4018 sizeof(*sfx_items)); 4019 sfx_items++; 4020 } 4021 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4022 sfx_items -= sfx_port_id_item ? 2 : 1; 4023 /* Setting the sfx group atrr. */ 4024 sfx_attr.group = sfx_attr.transfer ? 4025 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4026 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4027 } 4028 /* Add the prefix subflow. */ 4029 ret = flow_create_split_metadata(dev, flow, &sfx_attr, 4030 sfx_items ? sfx_items : items, 4031 sfx_actions ? sfx_actions : actions, 4032 external, error); 4033 exit: 4034 if (sfx_actions) 4035 rte_free(sfx_actions); 4036 return ret; 4037 } 4038 4039 /** 4040 * Split the flow to subflow set. The splitters might be linked 4041 * in the chain, like this: 4042 * flow_create_split_outer() calls: 4043 * flow_create_split_meter() calls: 4044 * flow_create_split_metadata(meter_subflow_0) calls: 4045 * flow_create_split_inner(metadata_subflow_0) 4046 * flow_create_split_inner(metadata_subflow_1) 4047 * flow_create_split_inner(metadata_subflow_2) 4048 * flow_create_split_metadata(meter_subflow_1) calls: 4049 * flow_create_split_inner(metadata_subflow_0) 4050 * flow_create_split_inner(metadata_subflow_1) 4051 * flow_create_split_inner(metadata_subflow_2) 4052 * 4053 * This provide flexible way to add new levels of flow splitting. 4054 * The all of successfully created subflows are included to the 4055 * parent flow dev_flow list. 4056 * 4057 * @param dev 4058 * Pointer to Ethernet device. 4059 * @param[in] flow 4060 * Parent flow structure pointer. 4061 * @param[in] attr 4062 * Flow rule attributes. 4063 * @param[in] items 4064 * Pattern specification (list terminated by the END pattern item). 4065 * @param[in] actions 4066 * Associated actions (list terminated by the END action). 4067 * @param[in] external 4068 * This flow rule is created by request external to PMD. 4069 * @param[out] error 4070 * Perform verbose error reporting if not NULL. 4071 * @return 4072 * 0 on success, negative value otherwise 4073 */ 4074 static int 4075 flow_create_split_outer(struct rte_eth_dev *dev, 4076 struct rte_flow *flow, 4077 const struct rte_flow_attr *attr, 4078 const struct rte_flow_item items[], 4079 const struct rte_flow_action actions[], 4080 bool external, struct rte_flow_error *error) 4081 { 4082 int ret; 4083 4084 ret = flow_create_split_meter(dev, flow, attr, items, 4085 actions, external, error); 4086 assert(ret <= 0); 4087 return ret; 4088 } 4089 4090 /** 4091 * Create a flow and add it to @p list. 4092 * 4093 * @param dev 4094 * Pointer to Ethernet device. 4095 * @param list 4096 * Pointer to a TAILQ flow list. If this parameter NULL, 4097 * no list insertion occurred, flow is just created, 4098 * this is caller's responsibility to track the 4099 * created flow. 4100 * @param[in] attr 4101 * Flow rule attributes. 4102 * @param[in] items 4103 * Pattern specification (list terminated by the END pattern item). 4104 * @param[in] actions 4105 * Associated actions (list terminated by the END action). 4106 * @param[in] external 4107 * This flow rule is created by request external to PMD. 4108 * @param[out] error 4109 * Perform verbose error reporting if not NULL. 4110 * 4111 * @return 4112 * A flow on success, NULL otherwise and rte_errno is set. 4113 */ 4114 static struct rte_flow * 4115 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 4116 const struct rte_flow_attr *attr, 4117 const struct rte_flow_item items[], 4118 const struct rte_flow_action actions[], 4119 bool external, struct rte_flow_error *error) 4120 { 4121 struct mlx5_priv *priv = dev->data->dev_private; 4122 struct rte_flow *flow = NULL; 4123 struct mlx5_flow *dev_flow; 4124 const struct rte_flow_action_rss *rss; 4125 union { 4126 struct rte_flow_expand_rss buf; 4127 uint8_t buffer[2048]; 4128 } expand_buffer; 4129 union { 4130 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4131 uint8_t buffer[2048]; 4132 } actions_rx; 4133 union { 4134 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4135 uint8_t buffer[2048]; 4136 } actions_hairpin_tx; 4137 union { 4138 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4139 uint8_t buffer[2048]; 4140 } items_tx; 4141 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4142 const struct rte_flow_action *p_actions_rx = actions; 4143 int ret; 4144 uint32_t i; 4145 uint32_t flow_size; 4146 int hairpin_flow = 0; 4147 uint32_t hairpin_id = 0; 4148 struct rte_flow_attr attr_tx = { .priority = 0 }; 4149 4150 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4151 if (hairpin_flow > 0) { 4152 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4153 rte_errno = EINVAL; 4154 return NULL; 4155 } 4156 flow_hairpin_split(dev, actions, actions_rx.actions, 4157 actions_hairpin_tx.actions, items_tx.items, 4158 &hairpin_id); 4159 p_actions_rx = actions_rx.actions; 4160 } 4161 ret = flow_drv_validate(dev, attr, items, p_actions_rx, external, 4162 error); 4163 if (ret < 0) 4164 goto error_before_flow; 4165 flow_size = sizeof(struct rte_flow); 4166 rss = flow_get_rss_action(p_actions_rx); 4167 if (rss) 4168 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t), 4169 sizeof(void *)); 4170 else 4171 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *)); 4172 flow = rte_calloc(__func__, 1, flow_size, 0); 4173 if (!flow) { 4174 rte_errno = ENOMEM; 4175 goto error_before_flow; 4176 } 4177 flow->drv_type = flow_get_drv_type(dev, attr); 4178 if (hairpin_id != 0) 4179 flow->hairpin_flow_id = hairpin_id; 4180 assert(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4181 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4182 flow->rss.queue = (void *)(flow + 1); 4183 if (rss) { 4184 /* 4185 * The following information is required by 4186 * mlx5_flow_hashfields_adjust() in advance. 4187 */ 4188 flow->rss.level = rss->level; 4189 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4190 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; 4191 } 4192 LIST_INIT(&flow->dev_flows); 4193 if (rss && rss->types) { 4194 unsigned int graph_root; 4195 4196 graph_root = find_graph_root(items, rss->level); 4197 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4198 items, rss->types, 4199 mlx5_support_expansion, 4200 graph_root); 4201 assert(ret > 0 && 4202 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4203 } else { 4204 buf->entries = 1; 4205 buf->entry[0].pattern = (void *)(uintptr_t)items; 4206 } 4207 for (i = 0; i < buf->entries; ++i) { 4208 /* 4209 * The splitter may create multiple dev_flows, 4210 * depending on configuration. In the simplest 4211 * case it just creates unmodified original flow. 4212 */ 4213 ret = flow_create_split_outer(dev, flow, attr, 4214 buf->entry[i].pattern, 4215 p_actions_rx, external, 4216 error); 4217 if (ret < 0) 4218 goto error; 4219 } 4220 /* Create the tx flow. */ 4221 if (hairpin_flow) { 4222 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4223 attr_tx.ingress = 0; 4224 attr_tx.egress = 1; 4225 dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items, 4226 actions_hairpin_tx.actions, error); 4227 if (!dev_flow) 4228 goto error; 4229 dev_flow->flow = flow; 4230 dev_flow->external = 0; 4231 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 4232 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4233 items_tx.items, 4234 actions_hairpin_tx.actions, error); 4235 if (ret < 0) 4236 goto error; 4237 } 4238 /* 4239 * Update the metadata register copy table. If extensive 4240 * metadata feature is enabled and registers are supported 4241 * we might create the extra rte_flow for each unique 4242 * MARK/FLAG action ID. 4243 * 4244 * The table is updated for ingress Flows only, because 4245 * the egress Flows belong to the different device and 4246 * copy table should be updated in peer NIC Rx domain. 4247 */ 4248 if (attr->ingress && 4249 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4250 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4251 if (ret) 4252 goto error; 4253 } 4254 if (dev->data->dev_started) { 4255 ret = flow_drv_apply(dev, flow, error); 4256 if (ret < 0) 4257 goto error; 4258 } 4259 if (list) 4260 TAILQ_INSERT_TAIL(list, flow, next); 4261 flow_rxq_flags_set(dev, flow); 4262 return flow; 4263 error_before_flow: 4264 if (hairpin_id) 4265 mlx5_flow_id_release(priv->sh->flow_id_pool, 4266 hairpin_id); 4267 return NULL; 4268 error: 4269 assert(flow); 4270 flow_mreg_del_copy_action(dev, flow); 4271 ret = rte_errno; /* Save rte_errno before cleanup. */ 4272 if (flow->hairpin_flow_id) 4273 mlx5_flow_id_release(priv->sh->flow_id_pool, 4274 flow->hairpin_flow_id); 4275 assert(flow); 4276 flow_drv_destroy(dev, flow); 4277 rte_free(flow); 4278 rte_errno = ret; /* Restore rte_errno. */ 4279 return NULL; 4280 } 4281 4282 /** 4283 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4284 * incoming packets to table 1. 4285 * 4286 * Other flow rules, requested for group n, will be created in 4287 * e-switch table n+1. 4288 * Jump action to e-switch group n will be created to group n+1. 4289 * 4290 * Used when working in switchdev mode, to utilise advantages of table 1 4291 * and above. 4292 * 4293 * @param dev 4294 * Pointer to Ethernet device. 4295 * 4296 * @return 4297 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4298 */ 4299 struct rte_flow * 4300 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4301 { 4302 const struct rte_flow_attr attr = { 4303 .group = 0, 4304 .priority = 0, 4305 .ingress = 1, 4306 .egress = 0, 4307 .transfer = 1, 4308 }; 4309 const struct rte_flow_item pattern = { 4310 .type = RTE_FLOW_ITEM_TYPE_END, 4311 }; 4312 struct rte_flow_action_jump jump = { 4313 .group = 1, 4314 }; 4315 const struct rte_flow_action actions[] = { 4316 { 4317 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4318 .conf = &jump, 4319 }, 4320 { 4321 .type = RTE_FLOW_ACTION_TYPE_END, 4322 }, 4323 }; 4324 struct mlx5_priv *priv = dev->data->dev_private; 4325 struct rte_flow_error error; 4326 4327 return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern, 4328 actions, false, &error); 4329 } 4330 4331 /** 4332 * Create a flow. 4333 * 4334 * @see rte_flow_create() 4335 * @see rte_flow_ops 4336 */ 4337 struct rte_flow * 4338 mlx5_flow_create(struct rte_eth_dev *dev, 4339 const struct rte_flow_attr *attr, 4340 const struct rte_flow_item items[], 4341 const struct rte_flow_action actions[], 4342 struct rte_flow_error *error) 4343 { 4344 struct mlx5_priv *priv = dev->data->dev_private; 4345 4346 return flow_list_create(dev, &priv->flows, 4347 attr, items, actions, true, error); 4348 } 4349 4350 /** 4351 * Destroy a flow in a list. 4352 * 4353 * @param dev 4354 * Pointer to Ethernet device. 4355 * @param list 4356 * Pointer to a TAILQ flow list. If this parameter NULL, 4357 * there is no flow removal from the list. 4358 * @param[in] flow 4359 * Flow to destroy. 4360 */ 4361 static void 4362 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 4363 struct rte_flow *flow) 4364 { 4365 struct mlx5_priv *priv = dev->data->dev_private; 4366 4367 /* 4368 * Update RX queue flags only if port is started, otherwise it is 4369 * already clean. 4370 */ 4371 if (dev->data->dev_started) 4372 flow_rxq_flags_trim(dev, flow); 4373 if (flow->hairpin_flow_id) 4374 mlx5_flow_id_release(priv->sh->flow_id_pool, 4375 flow->hairpin_flow_id); 4376 flow_drv_destroy(dev, flow); 4377 if (list) 4378 TAILQ_REMOVE(list, flow, next); 4379 flow_mreg_del_copy_action(dev, flow); 4380 rte_free(flow->fdir); 4381 rte_free(flow); 4382 } 4383 4384 /** 4385 * Destroy all flows. 4386 * 4387 * @param dev 4388 * Pointer to Ethernet device. 4389 * @param list 4390 * Pointer to a TAILQ flow list. 4391 */ 4392 void 4393 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 4394 { 4395 while (!TAILQ_EMPTY(list)) { 4396 struct rte_flow *flow; 4397 4398 flow = TAILQ_FIRST(list); 4399 flow_list_destroy(dev, list, flow); 4400 } 4401 } 4402 4403 /** 4404 * Remove all flows. 4405 * 4406 * @param dev 4407 * Pointer to Ethernet device. 4408 * @param list 4409 * Pointer to a TAILQ flow list. 4410 */ 4411 void 4412 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 4413 { 4414 struct rte_flow *flow; 4415 4416 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 4417 flow_drv_remove(dev, flow); 4418 flow_mreg_stop_copy_action(dev, flow); 4419 } 4420 flow_mreg_del_default_copy_action(dev); 4421 flow_rxq_flags_clear(dev); 4422 } 4423 4424 /** 4425 * Add all flows. 4426 * 4427 * @param dev 4428 * Pointer to Ethernet device. 4429 * @param list 4430 * Pointer to a TAILQ flow list. 4431 * 4432 * @return 4433 * 0 on success, a negative errno value otherwise and rte_errno is set. 4434 */ 4435 int 4436 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 4437 { 4438 struct rte_flow *flow; 4439 struct rte_flow_error error; 4440 int ret = 0; 4441 4442 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4443 ret = flow_mreg_add_default_copy_action(dev, &error); 4444 if (ret < 0) 4445 return -rte_errno; 4446 /* Apply Flows created by application. */ 4447 TAILQ_FOREACH(flow, list, next) { 4448 ret = flow_mreg_start_copy_action(dev, flow); 4449 if (ret < 0) 4450 goto error; 4451 ret = flow_drv_apply(dev, flow, &error); 4452 if (ret < 0) 4453 goto error; 4454 flow_rxq_flags_set(dev, flow); 4455 } 4456 return 0; 4457 error: 4458 ret = rte_errno; /* Save rte_errno before cleanup. */ 4459 mlx5_flow_stop(dev, list); 4460 rte_errno = ret; /* Restore rte_errno. */ 4461 return -rte_errno; 4462 } 4463 4464 /** 4465 * Verify the flow list is empty 4466 * 4467 * @param dev 4468 * Pointer to Ethernet device. 4469 * 4470 * @return the number of flows not released. 4471 */ 4472 int 4473 mlx5_flow_verify(struct rte_eth_dev *dev) 4474 { 4475 struct mlx5_priv *priv = dev->data->dev_private; 4476 struct rte_flow *flow; 4477 int ret = 0; 4478 4479 TAILQ_FOREACH(flow, &priv->flows, next) { 4480 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4481 dev->data->port_id, (void *)flow); 4482 ++ret; 4483 } 4484 return ret; 4485 } 4486 4487 /** 4488 * Enable default hairpin egress flow. 4489 * 4490 * @param dev 4491 * Pointer to Ethernet device. 4492 * @param queue 4493 * The queue index. 4494 * 4495 * @return 4496 * 0 on success, a negative errno value otherwise and rte_errno is set. 4497 */ 4498 int 4499 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4500 uint32_t queue) 4501 { 4502 struct mlx5_priv *priv = dev->data->dev_private; 4503 const struct rte_flow_attr attr = { 4504 .egress = 1, 4505 .priority = 0, 4506 }; 4507 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4508 .queue = queue, 4509 }; 4510 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4511 .queue = UINT32_MAX, 4512 }; 4513 struct rte_flow_item items[] = { 4514 { 4515 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4516 .spec = &queue_spec, 4517 .last = NULL, 4518 .mask = &queue_mask, 4519 }, 4520 { 4521 .type = RTE_FLOW_ITEM_TYPE_END, 4522 }, 4523 }; 4524 struct rte_flow_action_jump jump = { 4525 .group = MLX5_HAIRPIN_TX_TABLE, 4526 }; 4527 struct rte_flow_action actions[2]; 4528 struct rte_flow *flow; 4529 struct rte_flow_error error; 4530 4531 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4532 actions[0].conf = &jump; 4533 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4534 flow = flow_list_create(dev, &priv->ctrl_flows, 4535 &attr, items, actions, false, &error); 4536 if (!flow) { 4537 DRV_LOG(DEBUG, 4538 "Failed to create ctrl flow: rte_errno(%d)," 4539 " type(%d), message(%s)", 4540 rte_errno, error.type, 4541 error.message ? error.message : " (no stated reason)"); 4542 return -rte_errno; 4543 } 4544 return 0; 4545 } 4546 4547 /** 4548 * Enable a control flow configured from the control plane. 4549 * 4550 * @param dev 4551 * Pointer to Ethernet device. 4552 * @param eth_spec 4553 * An Ethernet flow spec to apply. 4554 * @param eth_mask 4555 * An Ethernet flow mask to apply. 4556 * @param vlan_spec 4557 * A VLAN flow spec to apply. 4558 * @param vlan_mask 4559 * A VLAN flow mask to apply. 4560 * 4561 * @return 4562 * 0 on success, a negative errno value otherwise and rte_errno is set. 4563 */ 4564 int 4565 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4566 struct rte_flow_item_eth *eth_spec, 4567 struct rte_flow_item_eth *eth_mask, 4568 struct rte_flow_item_vlan *vlan_spec, 4569 struct rte_flow_item_vlan *vlan_mask) 4570 { 4571 struct mlx5_priv *priv = dev->data->dev_private; 4572 const struct rte_flow_attr attr = { 4573 .ingress = 1, 4574 .priority = MLX5_FLOW_PRIO_RSVD, 4575 }; 4576 struct rte_flow_item items[] = { 4577 { 4578 .type = RTE_FLOW_ITEM_TYPE_ETH, 4579 .spec = eth_spec, 4580 .last = NULL, 4581 .mask = eth_mask, 4582 }, 4583 { 4584 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4585 RTE_FLOW_ITEM_TYPE_END, 4586 .spec = vlan_spec, 4587 .last = NULL, 4588 .mask = vlan_mask, 4589 }, 4590 { 4591 .type = RTE_FLOW_ITEM_TYPE_END, 4592 }, 4593 }; 4594 uint16_t queue[priv->reta_idx_n]; 4595 struct rte_flow_action_rss action_rss = { 4596 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4597 .level = 0, 4598 .types = priv->rss_conf.rss_hf, 4599 .key_len = priv->rss_conf.rss_key_len, 4600 .queue_num = priv->reta_idx_n, 4601 .key = priv->rss_conf.rss_key, 4602 .queue = queue, 4603 }; 4604 struct rte_flow_action actions[] = { 4605 { 4606 .type = RTE_FLOW_ACTION_TYPE_RSS, 4607 .conf = &action_rss, 4608 }, 4609 { 4610 .type = RTE_FLOW_ACTION_TYPE_END, 4611 }, 4612 }; 4613 struct rte_flow *flow; 4614 struct rte_flow_error error; 4615 unsigned int i; 4616 4617 if (!priv->reta_idx_n || !priv->rxqs_n) { 4618 return 0; 4619 } 4620 for (i = 0; i != priv->reta_idx_n; ++i) 4621 queue[i] = (*priv->reta_idx)[i]; 4622 flow = flow_list_create(dev, &priv->ctrl_flows, 4623 &attr, items, actions, false, &error); 4624 if (!flow) 4625 return -rte_errno; 4626 return 0; 4627 } 4628 4629 /** 4630 * Enable a flow control configured from the control plane. 4631 * 4632 * @param dev 4633 * Pointer to Ethernet device. 4634 * @param eth_spec 4635 * An Ethernet flow spec to apply. 4636 * @param eth_mask 4637 * An Ethernet flow mask to apply. 4638 * 4639 * @return 4640 * 0 on success, a negative errno value otherwise and rte_errno is set. 4641 */ 4642 int 4643 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4644 struct rte_flow_item_eth *eth_spec, 4645 struct rte_flow_item_eth *eth_mask) 4646 { 4647 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4648 } 4649 4650 /** 4651 * Destroy a flow. 4652 * 4653 * @see rte_flow_destroy() 4654 * @see rte_flow_ops 4655 */ 4656 int 4657 mlx5_flow_destroy(struct rte_eth_dev *dev, 4658 struct rte_flow *flow, 4659 struct rte_flow_error *error __rte_unused) 4660 { 4661 struct mlx5_priv *priv = dev->data->dev_private; 4662 4663 flow_list_destroy(dev, &priv->flows, flow); 4664 return 0; 4665 } 4666 4667 /** 4668 * Destroy all flows. 4669 * 4670 * @see rte_flow_flush() 4671 * @see rte_flow_ops 4672 */ 4673 int 4674 mlx5_flow_flush(struct rte_eth_dev *dev, 4675 struct rte_flow_error *error __rte_unused) 4676 { 4677 struct mlx5_priv *priv = dev->data->dev_private; 4678 4679 mlx5_flow_list_flush(dev, &priv->flows); 4680 return 0; 4681 } 4682 4683 /** 4684 * Isolated mode. 4685 * 4686 * @see rte_flow_isolate() 4687 * @see rte_flow_ops 4688 */ 4689 int 4690 mlx5_flow_isolate(struct rte_eth_dev *dev, 4691 int enable, 4692 struct rte_flow_error *error) 4693 { 4694 struct mlx5_priv *priv = dev->data->dev_private; 4695 4696 if (dev->data->dev_started) { 4697 rte_flow_error_set(error, EBUSY, 4698 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4699 NULL, 4700 "port must be stopped first"); 4701 return -rte_errno; 4702 } 4703 priv->isolated = !!enable; 4704 if (enable) 4705 dev->dev_ops = &mlx5_dev_ops_isolate; 4706 else 4707 dev->dev_ops = &mlx5_dev_ops; 4708 return 0; 4709 } 4710 4711 /** 4712 * Query a flow. 4713 * 4714 * @see rte_flow_query() 4715 * @see rte_flow_ops 4716 */ 4717 static int 4718 flow_drv_query(struct rte_eth_dev *dev, 4719 struct rte_flow *flow, 4720 const struct rte_flow_action *actions, 4721 void *data, 4722 struct rte_flow_error *error) 4723 { 4724 const struct mlx5_flow_driver_ops *fops; 4725 enum mlx5_flow_drv_type ftype = flow->drv_type; 4726 4727 assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 4728 fops = flow_get_drv_ops(ftype); 4729 4730 return fops->query(dev, flow, actions, data, error); 4731 } 4732 4733 /** 4734 * Query a flow. 4735 * 4736 * @see rte_flow_query() 4737 * @see rte_flow_ops 4738 */ 4739 int 4740 mlx5_flow_query(struct rte_eth_dev *dev, 4741 struct rte_flow *flow, 4742 const struct rte_flow_action *actions, 4743 void *data, 4744 struct rte_flow_error *error) 4745 { 4746 int ret; 4747 4748 ret = flow_drv_query(dev, flow, actions, data, error); 4749 if (ret < 0) 4750 return ret; 4751 return 0; 4752 } 4753 4754 /** 4755 * Convert a flow director filter to a generic flow. 4756 * 4757 * @param dev 4758 * Pointer to Ethernet device. 4759 * @param fdir_filter 4760 * Flow director filter to add. 4761 * @param attributes 4762 * Generic flow parameters structure. 4763 * 4764 * @return 4765 * 0 on success, a negative errno value otherwise and rte_errno is set. 4766 */ 4767 static int 4768 flow_fdir_filter_convert(struct rte_eth_dev *dev, 4769 const struct rte_eth_fdir_filter *fdir_filter, 4770 struct mlx5_fdir *attributes) 4771 { 4772 struct mlx5_priv *priv = dev->data->dev_private; 4773 const struct rte_eth_fdir_input *input = &fdir_filter->input; 4774 const struct rte_eth_fdir_masks *mask = 4775 &dev->data->dev_conf.fdir_conf.mask; 4776 4777 /* Validate queue number. */ 4778 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 4779 DRV_LOG(ERR, "port %u invalid queue number %d", 4780 dev->data->port_id, fdir_filter->action.rx_queue); 4781 rte_errno = EINVAL; 4782 return -rte_errno; 4783 } 4784 attributes->attr.ingress = 1; 4785 attributes->items[0] = (struct rte_flow_item) { 4786 .type = RTE_FLOW_ITEM_TYPE_ETH, 4787 .spec = &attributes->l2, 4788 .mask = &attributes->l2_mask, 4789 }; 4790 switch (fdir_filter->action.behavior) { 4791 case RTE_ETH_FDIR_ACCEPT: 4792 attributes->actions[0] = (struct rte_flow_action){ 4793 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 4794 .conf = &attributes->queue, 4795 }; 4796 break; 4797 case RTE_ETH_FDIR_REJECT: 4798 attributes->actions[0] = (struct rte_flow_action){ 4799 .type = RTE_FLOW_ACTION_TYPE_DROP, 4800 }; 4801 break; 4802 default: 4803 DRV_LOG(ERR, "port %u invalid behavior %d", 4804 dev->data->port_id, 4805 fdir_filter->action.behavior); 4806 rte_errno = ENOTSUP; 4807 return -rte_errno; 4808 } 4809 attributes->queue.index = fdir_filter->action.rx_queue; 4810 /* Handle L3. */ 4811 switch (fdir_filter->input.flow_type) { 4812 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4813 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4814 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4815 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 4816 .src_addr = input->flow.ip4_flow.src_ip, 4817 .dst_addr = input->flow.ip4_flow.dst_ip, 4818 .time_to_live = input->flow.ip4_flow.ttl, 4819 .type_of_service = input->flow.ip4_flow.tos, 4820 }; 4821 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 4822 .src_addr = mask->ipv4_mask.src_ip, 4823 .dst_addr = mask->ipv4_mask.dst_ip, 4824 .time_to_live = mask->ipv4_mask.ttl, 4825 .type_of_service = mask->ipv4_mask.tos, 4826 .next_proto_id = mask->ipv4_mask.proto, 4827 }; 4828 attributes->items[1] = (struct rte_flow_item){ 4829 .type = RTE_FLOW_ITEM_TYPE_IPV4, 4830 .spec = &attributes->l3, 4831 .mask = &attributes->l3_mask, 4832 }; 4833 break; 4834 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4835 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4836 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4837 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 4838 .hop_limits = input->flow.ipv6_flow.hop_limits, 4839 .proto = input->flow.ipv6_flow.proto, 4840 }; 4841 4842 memcpy(attributes->l3.ipv6.hdr.src_addr, 4843 input->flow.ipv6_flow.src_ip, 4844 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4845 memcpy(attributes->l3.ipv6.hdr.dst_addr, 4846 input->flow.ipv6_flow.dst_ip, 4847 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4848 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 4849 mask->ipv6_mask.src_ip, 4850 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4851 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 4852 mask->ipv6_mask.dst_ip, 4853 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4854 attributes->items[1] = (struct rte_flow_item){ 4855 .type = RTE_FLOW_ITEM_TYPE_IPV6, 4856 .spec = &attributes->l3, 4857 .mask = &attributes->l3_mask, 4858 }; 4859 break; 4860 default: 4861 DRV_LOG(ERR, "port %u invalid flow type%d", 4862 dev->data->port_id, fdir_filter->input.flow_type); 4863 rte_errno = ENOTSUP; 4864 return -rte_errno; 4865 } 4866 /* Handle L4. */ 4867 switch (fdir_filter->input.flow_type) { 4868 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4869 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4870 .src_port = input->flow.udp4_flow.src_port, 4871 .dst_port = input->flow.udp4_flow.dst_port, 4872 }; 4873 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4874 .src_port = mask->src_port_mask, 4875 .dst_port = mask->dst_port_mask, 4876 }; 4877 attributes->items[2] = (struct rte_flow_item){ 4878 .type = RTE_FLOW_ITEM_TYPE_UDP, 4879 .spec = &attributes->l4, 4880 .mask = &attributes->l4_mask, 4881 }; 4882 break; 4883 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4884 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4885 .src_port = input->flow.tcp4_flow.src_port, 4886 .dst_port = input->flow.tcp4_flow.dst_port, 4887 }; 4888 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4889 .src_port = mask->src_port_mask, 4890 .dst_port = mask->dst_port_mask, 4891 }; 4892 attributes->items[2] = (struct rte_flow_item){ 4893 .type = RTE_FLOW_ITEM_TYPE_TCP, 4894 .spec = &attributes->l4, 4895 .mask = &attributes->l4_mask, 4896 }; 4897 break; 4898 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4899 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4900 .src_port = input->flow.udp6_flow.src_port, 4901 .dst_port = input->flow.udp6_flow.dst_port, 4902 }; 4903 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4904 .src_port = mask->src_port_mask, 4905 .dst_port = mask->dst_port_mask, 4906 }; 4907 attributes->items[2] = (struct rte_flow_item){ 4908 .type = RTE_FLOW_ITEM_TYPE_UDP, 4909 .spec = &attributes->l4, 4910 .mask = &attributes->l4_mask, 4911 }; 4912 break; 4913 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4914 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4915 .src_port = input->flow.tcp6_flow.src_port, 4916 .dst_port = input->flow.tcp6_flow.dst_port, 4917 }; 4918 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4919 .src_port = mask->src_port_mask, 4920 .dst_port = mask->dst_port_mask, 4921 }; 4922 attributes->items[2] = (struct rte_flow_item){ 4923 .type = RTE_FLOW_ITEM_TYPE_TCP, 4924 .spec = &attributes->l4, 4925 .mask = &attributes->l4_mask, 4926 }; 4927 break; 4928 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4929 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4930 break; 4931 default: 4932 DRV_LOG(ERR, "port %u invalid flow type%d", 4933 dev->data->port_id, fdir_filter->input.flow_type); 4934 rte_errno = ENOTSUP; 4935 return -rte_errno; 4936 } 4937 return 0; 4938 } 4939 4940 #define FLOW_FDIR_CMP(f1, f2, fld) \ 4941 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 4942 4943 /** 4944 * Compare two FDIR flows. If items and actions are identical, the two flows are 4945 * regarded as same. 4946 * 4947 * @param dev 4948 * Pointer to Ethernet device. 4949 * @param f1 4950 * FDIR flow to compare. 4951 * @param f2 4952 * FDIR flow to compare. 4953 * 4954 * @return 4955 * Zero on match, 1 otherwise. 4956 */ 4957 static int 4958 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 4959 { 4960 if (FLOW_FDIR_CMP(f1, f2, attr) || 4961 FLOW_FDIR_CMP(f1, f2, l2) || 4962 FLOW_FDIR_CMP(f1, f2, l2_mask) || 4963 FLOW_FDIR_CMP(f1, f2, l3) || 4964 FLOW_FDIR_CMP(f1, f2, l3_mask) || 4965 FLOW_FDIR_CMP(f1, f2, l4) || 4966 FLOW_FDIR_CMP(f1, f2, l4_mask) || 4967 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 4968 return 1; 4969 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 4970 FLOW_FDIR_CMP(f1, f2, queue)) 4971 return 1; 4972 return 0; 4973 } 4974 4975 /** 4976 * Search device flow list to find out a matched FDIR flow. 4977 * 4978 * @param dev 4979 * Pointer to Ethernet device. 4980 * @param fdir_flow 4981 * FDIR flow to lookup. 4982 * 4983 * @return 4984 * Pointer of flow if found, NULL otherwise. 4985 */ 4986 static struct rte_flow * 4987 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 4988 { 4989 struct mlx5_priv *priv = dev->data->dev_private; 4990 struct rte_flow *flow = NULL; 4991 4992 assert(fdir_flow); 4993 TAILQ_FOREACH(flow, &priv->flows, next) { 4994 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) { 4995 DRV_LOG(DEBUG, "port %u found FDIR flow %p", 4996 dev->data->port_id, (void *)flow); 4997 break; 4998 } 4999 } 5000 return flow; 5001 } 5002 5003 /** 5004 * Add new flow director filter and store it in list. 5005 * 5006 * @param dev 5007 * Pointer to Ethernet device. 5008 * @param fdir_filter 5009 * Flow director filter to add. 5010 * 5011 * @return 5012 * 0 on success, a negative errno value otherwise and rte_errno is set. 5013 */ 5014 static int 5015 flow_fdir_filter_add(struct rte_eth_dev *dev, 5016 const struct rte_eth_fdir_filter *fdir_filter) 5017 { 5018 struct mlx5_priv *priv = dev->data->dev_private; 5019 struct mlx5_fdir *fdir_flow; 5020 struct rte_flow *flow; 5021 int ret; 5022 5023 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5024 if (!fdir_flow) { 5025 rte_errno = ENOMEM; 5026 return -rte_errno; 5027 } 5028 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5029 if (ret) 5030 goto error; 5031 flow = flow_fdir_filter_lookup(dev, fdir_flow); 5032 if (flow) { 5033 rte_errno = EEXIST; 5034 goto error; 5035 } 5036 flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5037 fdir_flow->items, fdir_flow->actions, true, 5038 NULL); 5039 if (!flow) 5040 goto error; 5041 assert(!flow->fdir); 5042 flow->fdir = fdir_flow; 5043 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5044 dev->data->port_id, (void *)flow); 5045 return 0; 5046 error: 5047 rte_free(fdir_flow); 5048 return -rte_errno; 5049 } 5050 5051 /** 5052 * Delete specific filter. 5053 * 5054 * @param dev 5055 * Pointer to Ethernet device. 5056 * @param fdir_filter 5057 * Filter to be deleted. 5058 * 5059 * @return 5060 * 0 on success, a negative errno value otherwise and rte_errno is set. 5061 */ 5062 static int 5063 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5064 const struct rte_eth_fdir_filter *fdir_filter) 5065 { 5066 struct mlx5_priv *priv = dev->data->dev_private; 5067 struct rte_flow *flow; 5068 struct mlx5_fdir fdir_flow = { 5069 .attr.group = 0, 5070 }; 5071 int ret; 5072 5073 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5074 if (ret) 5075 return -rte_errno; 5076 flow = flow_fdir_filter_lookup(dev, &fdir_flow); 5077 if (!flow) { 5078 rte_errno = ENOENT; 5079 return -rte_errno; 5080 } 5081 flow_list_destroy(dev, &priv->flows, flow); 5082 DRV_LOG(DEBUG, "port %u deleted FDIR flow %p", 5083 dev->data->port_id, (void *)flow); 5084 return 0; 5085 } 5086 5087 /** 5088 * Update queue for specific filter. 5089 * 5090 * @param dev 5091 * Pointer to Ethernet device. 5092 * @param fdir_filter 5093 * Filter to be updated. 5094 * 5095 * @return 5096 * 0 on success, a negative errno value otherwise and rte_errno is set. 5097 */ 5098 static int 5099 flow_fdir_filter_update(struct rte_eth_dev *dev, 5100 const struct rte_eth_fdir_filter *fdir_filter) 5101 { 5102 int ret; 5103 5104 ret = flow_fdir_filter_delete(dev, fdir_filter); 5105 if (ret) 5106 return ret; 5107 return flow_fdir_filter_add(dev, fdir_filter); 5108 } 5109 5110 /** 5111 * Flush all filters. 5112 * 5113 * @param dev 5114 * Pointer to Ethernet device. 5115 */ 5116 static void 5117 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5118 { 5119 struct mlx5_priv *priv = dev->data->dev_private; 5120 5121 mlx5_flow_list_flush(dev, &priv->flows); 5122 } 5123 5124 /** 5125 * Get flow director information. 5126 * 5127 * @param dev 5128 * Pointer to Ethernet device. 5129 * @param[out] fdir_info 5130 * Resulting flow director information. 5131 */ 5132 static void 5133 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5134 { 5135 struct rte_eth_fdir_masks *mask = 5136 &dev->data->dev_conf.fdir_conf.mask; 5137 5138 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5139 fdir_info->guarant_spc = 0; 5140 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5141 fdir_info->max_flexpayload = 0; 5142 fdir_info->flow_types_mask[0] = 0; 5143 fdir_info->flex_payload_unit = 0; 5144 fdir_info->max_flex_payload_segment_num = 0; 5145 fdir_info->flex_payload_limit = 0; 5146 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5147 } 5148 5149 /** 5150 * Deal with flow director operations. 5151 * 5152 * @param dev 5153 * Pointer to Ethernet device. 5154 * @param filter_op 5155 * Operation to perform. 5156 * @param arg 5157 * Pointer to operation-specific structure. 5158 * 5159 * @return 5160 * 0 on success, a negative errno value otherwise and rte_errno is set. 5161 */ 5162 static int 5163 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5164 void *arg) 5165 { 5166 enum rte_fdir_mode fdir_mode = 5167 dev->data->dev_conf.fdir_conf.mode; 5168 5169 if (filter_op == RTE_ETH_FILTER_NOP) 5170 return 0; 5171 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5172 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5173 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5174 dev->data->port_id, fdir_mode); 5175 rte_errno = EINVAL; 5176 return -rte_errno; 5177 } 5178 switch (filter_op) { 5179 case RTE_ETH_FILTER_ADD: 5180 return flow_fdir_filter_add(dev, arg); 5181 case RTE_ETH_FILTER_UPDATE: 5182 return flow_fdir_filter_update(dev, arg); 5183 case RTE_ETH_FILTER_DELETE: 5184 return flow_fdir_filter_delete(dev, arg); 5185 case RTE_ETH_FILTER_FLUSH: 5186 flow_fdir_filter_flush(dev); 5187 break; 5188 case RTE_ETH_FILTER_INFO: 5189 flow_fdir_info_get(dev, arg); 5190 break; 5191 default: 5192 DRV_LOG(DEBUG, "port %u unknown operation %u", 5193 dev->data->port_id, filter_op); 5194 rte_errno = EINVAL; 5195 return -rte_errno; 5196 } 5197 return 0; 5198 } 5199 5200 /** 5201 * Manage filter operations. 5202 * 5203 * @param dev 5204 * Pointer to Ethernet device structure. 5205 * @param filter_type 5206 * Filter type. 5207 * @param filter_op 5208 * Operation to perform. 5209 * @param arg 5210 * Pointer to operation-specific structure. 5211 * 5212 * @return 5213 * 0 on success, a negative errno value otherwise and rte_errno is set. 5214 */ 5215 int 5216 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5217 enum rte_filter_type filter_type, 5218 enum rte_filter_op filter_op, 5219 void *arg) 5220 { 5221 switch (filter_type) { 5222 case RTE_ETH_FILTER_GENERIC: 5223 if (filter_op != RTE_ETH_FILTER_GET) { 5224 rte_errno = EINVAL; 5225 return -rte_errno; 5226 } 5227 *(const void **)arg = &mlx5_flow_ops; 5228 return 0; 5229 case RTE_ETH_FILTER_FDIR: 5230 return flow_fdir_ctrl_func(dev, filter_op, arg); 5231 default: 5232 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5233 dev->data->port_id, filter_type); 5234 rte_errno = ENOTSUP; 5235 return -rte_errno; 5236 } 5237 return 0; 5238 } 5239 5240 /** 5241 * Create the needed meter and suffix tables. 5242 * 5243 * @param[in] dev 5244 * Pointer to Ethernet device. 5245 * @param[in] fm 5246 * Pointer to the flow meter. 5247 * 5248 * @return 5249 * Pointer to table set on success, NULL otherwise. 5250 */ 5251 struct mlx5_meter_domains_infos * 5252 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5253 const struct mlx5_flow_meter *fm) 5254 { 5255 const struct mlx5_flow_driver_ops *fops; 5256 5257 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5258 return fops->create_mtr_tbls(dev, fm); 5259 } 5260 5261 /** 5262 * Destroy the meter table set. 5263 * 5264 * @param[in] dev 5265 * Pointer to Ethernet device. 5266 * @param[in] tbl 5267 * Pointer to the meter table set. 5268 * 5269 * @return 5270 * 0 on success. 5271 */ 5272 int 5273 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5274 struct mlx5_meter_domains_infos *tbls) 5275 { 5276 const struct mlx5_flow_driver_ops *fops; 5277 5278 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5279 return fops->destroy_mtr_tbls(dev, tbls); 5280 } 5281 5282 /** 5283 * Create policer rules. 5284 * 5285 * @param[in] dev 5286 * Pointer to Ethernet device. 5287 * @param[in] fm 5288 * Pointer to flow meter structure. 5289 * @param[in] attr 5290 * Pointer to flow attributes. 5291 * 5292 * @return 5293 * 0 on success, -1 otherwise. 5294 */ 5295 int 5296 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5297 struct mlx5_flow_meter *fm, 5298 const struct rte_flow_attr *attr) 5299 { 5300 const struct mlx5_flow_driver_ops *fops; 5301 5302 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5303 return fops->create_policer_rules(dev, fm, attr); 5304 } 5305 5306 /** 5307 * Destroy policer rules. 5308 * 5309 * @param[in] fm 5310 * Pointer to flow meter structure. 5311 * @param[in] attr 5312 * Pointer to flow attributes. 5313 * 5314 * @return 5315 * 0 on success, -1 otherwise. 5316 */ 5317 int 5318 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5319 struct mlx5_flow_meter *fm, 5320 const struct rte_flow_attr *attr) 5321 { 5322 const struct mlx5_flow_driver_ops *fops; 5323 5324 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5325 return fops->destroy_policer_rules(dev, fm, attr); 5326 } 5327 5328 /** 5329 * Allocate a counter. 5330 * 5331 * @param[in] dev 5332 * Pointer to Ethernet device structure. 5333 * 5334 * @return 5335 * Pointer to allocated counter on success, NULL otherwise. 5336 */ 5337 struct mlx5_flow_counter * 5338 mlx5_counter_alloc(struct rte_eth_dev *dev) 5339 { 5340 const struct mlx5_flow_driver_ops *fops; 5341 struct rte_flow_attr attr = { .transfer = 0 }; 5342 5343 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5344 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5345 return fops->counter_alloc(dev); 5346 } 5347 DRV_LOG(ERR, 5348 "port %u counter allocate is not supported.", 5349 dev->data->port_id); 5350 return NULL; 5351 } 5352 5353 /** 5354 * Free a counter. 5355 * 5356 * @param[in] dev 5357 * Pointer to Ethernet device structure. 5358 * @param[in] cnt 5359 * Pointer to counter to be free. 5360 */ 5361 void 5362 mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) 5363 { 5364 const struct mlx5_flow_driver_ops *fops; 5365 struct rte_flow_attr attr = { .transfer = 0 }; 5366 5367 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5368 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5369 fops->counter_free(dev, cnt); 5370 return; 5371 } 5372 DRV_LOG(ERR, 5373 "port %u counter free is not supported.", 5374 dev->data->port_id); 5375 } 5376 5377 /** 5378 * Query counter statistics. 5379 * 5380 * @param[in] dev 5381 * Pointer to Ethernet device structure. 5382 * @param[in] cnt 5383 * Pointer to counter to query. 5384 * @param[in] clear 5385 * Set to clear counter statistics. 5386 * @param[out] pkts 5387 * The counter hits packets number to save. 5388 * @param[out] bytes 5389 * The counter hits bytes number to save. 5390 * 5391 * @return 5392 * 0 on success, a negative errno value otherwise. 5393 */ 5394 int 5395 mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt, 5396 bool clear, uint64_t *pkts, uint64_t *bytes) 5397 { 5398 const struct mlx5_flow_driver_ops *fops; 5399 struct rte_flow_attr attr = { .transfer = 0 }; 5400 5401 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5402 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5403 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5404 } 5405 DRV_LOG(ERR, 5406 "port %u counter query is not supported.", 5407 dev->data->port_id); 5408 return -ENOTSUP; 5409 } 5410 5411 #define MLX5_POOL_QUERY_FREQ_US 1000000 5412 5413 /** 5414 * Set the periodic procedure for triggering asynchronous batch queries for all 5415 * the counter pools. 5416 * 5417 * @param[in] sh 5418 * Pointer to mlx5_ibv_shared object. 5419 */ 5420 void 5421 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5422 { 5423 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0); 5424 uint32_t pools_n = rte_atomic16_read(&cont->n_valid); 5425 uint32_t us; 5426 5427 cont = MLX5_CNT_CONTAINER(sh, 1, 0); 5428 pools_n += rte_atomic16_read(&cont->n_valid); 5429 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5430 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5431 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5432 sh->cmng.query_thread_on = 0; 5433 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5434 } else { 5435 sh->cmng.query_thread_on = 1; 5436 } 5437 } 5438 5439 /** 5440 * The periodic procedure for triggering asynchronous batch queries for all the 5441 * counter pools. This function is probably called by the host thread. 5442 * 5443 * @param[in] arg 5444 * The parameter for the alarm process. 5445 */ 5446 void 5447 mlx5_flow_query_alarm(void *arg) 5448 { 5449 struct mlx5_ibv_shared *sh = arg; 5450 struct mlx5_devx_obj *dcs; 5451 uint16_t offset; 5452 int ret; 5453 uint8_t batch = sh->cmng.batch; 5454 uint16_t pool_index = sh->cmng.pool_index; 5455 struct mlx5_pools_container *cont; 5456 struct mlx5_pools_container *mcont; 5457 struct mlx5_flow_counter_pool *pool; 5458 5459 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5460 goto set_alarm; 5461 next_container: 5462 cont = MLX5_CNT_CONTAINER(sh, batch, 1); 5463 mcont = MLX5_CNT_CONTAINER(sh, batch, 0); 5464 /* Check if resize was done and need to flip a container. */ 5465 if (cont != mcont) { 5466 if (cont->pools) { 5467 /* Clean the old container. */ 5468 rte_free(cont->pools); 5469 memset(cont, 0, sizeof(*cont)); 5470 } 5471 rte_cio_wmb(); 5472 /* Flip the host container. */ 5473 sh->cmng.mhi[batch] ^= (uint8_t)2; 5474 cont = mcont; 5475 } 5476 if (!cont->pools) { 5477 /* 2 empty containers case is unexpected. */ 5478 if (unlikely(batch != sh->cmng.batch)) 5479 goto set_alarm; 5480 batch ^= 0x1; 5481 pool_index = 0; 5482 goto next_container; 5483 } 5484 pool = cont->pools[pool_index]; 5485 if (pool->raw_hw) 5486 /* There is a pool query in progress. */ 5487 goto set_alarm; 5488 pool->raw_hw = 5489 LIST_FIRST(&sh->cmng.free_stat_raws); 5490 if (!pool->raw_hw) 5491 /* No free counter statistics raw memory. */ 5492 goto set_alarm; 5493 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5494 (&pool->a64_dcs); 5495 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5496 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5497 offset, NULL, NULL, 5498 pool->raw_hw->mem_mng->dm->id, 5499 (void *)(uintptr_t) 5500 (pool->raw_hw->data + offset), 5501 sh->devx_comp, 5502 (uint64_t)(uintptr_t)pool); 5503 if (ret) { 5504 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5505 " %d", pool->min_dcs->id); 5506 pool->raw_hw = NULL; 5507 goto set_alarm; 5508 } 5509 pool->raw_hw->min_dcs_id = dcs->id; 5510 LIST_REMOVE(pool->raw_hw, next); 5511 sh->cmng.pending_queries++; 5512 pool_index++; 5513 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5514 batch ^= 0x1; 5515 pool_index = 0; 5516 } 5517 set_alarm: 5518 sh->cmng.batch = batch; 5519 sh->cmng.pool_index = pool_index; 5520 mlx5_set_query_alarm(sh); 5521 } 5522 5523 /** 5524 * Handler for the HW respond about ready values from an asynchronous batch 5525 * query. This function is probably called by the host thread. 5526 * 5527 * @param[in] sh 5528 * The pointer to the shared IB device context. 5529 * @param[in] async_id 5530 * The Devx async ID. 5531 * @param[in] status 5532 * The status of the completion. 5533 */ 5534 void 5535 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 5536 uint64_t async_id, int status) 5537 { 5538 struct mlx5_flow_counter_pool *pool = 5539 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 5540 struct mlx5_counter_stats_raw *raw_to_free; 5541 5542 if (unlikely(status)) { 5543 raw_to_free = pool->raw_hw; 5544 } else { 5545 raw_to_free = pool->raw; 5546 rte_spinlock_lock(&pool->sl); 5547 pool->raw = pool->raw_hw; 5548 rte_spinlock_unlock(&pool->sl); 5549 rte_atomic64_add(&pool->query_gen, 1); 5550 /* Be sure the new raw counters data is updated in memory. */ 5551 rte_cio_wmb(); 5552 } 5553 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 5554 pool->raw_hw = NULL; 5555 sh->cmng.pending_queries--; 5556 } 5557 5558 /** 5559 * Translate the rte_flow group index to HW table value. 5560 * 5561 * @param[in] attributes 5562 * Pointer to flow attributes 5563 * @param[in] external 5564 * Value is part of flow rule created by request external to PMD. 5565 * @param[in] group 5566 * rte_flow group index value. 5567 * @param[out] table 5568 * HW table value. 5569 * @param[out] error 5570 * Pointer to error structure. 5571 * 5572 * @return 5573 * 0 on success, a negative errno value otherwise and rte_errno is set. 5574 */ 5575 int 5576 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 5577 uint32_t group, uint32_t *table, 5578 struct rte_flow_error *error) 5579 { 5580 if (attributes->transfer && external) { 5581 if (group == UINT32_MAX) 5582 return rte_flow_error_set 5583 (error, EINVAL, 5584 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 5585 NULL, 5586 "group index not supported"); 5587 *table = group + 1; 5588 } else { 5589 *table = group; 5590 } 5591 return 0; 5592 } 5593 5594 /** 5595 * Discover availability of metadata reg_c's. 5596 * 5597 * Iteratively use test flows to check availability. 5598 * 5599 * @param[in] dev 5600 * Pointer to the Ethernet device structure. 5601 * 5602 * @return 5603 * 0 on success, a negative errno value otherwise and rte_errno is set. 5604 */ 5605 int 5606 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 5607 { 5608 struct mlx5_priv *priv = dev->data->dev_private; 5609 struct mlx5_dev_config *config = &priv->config; 5610 enum modify_reg idx; 5611 int n = 0; 5612 5613 /* reg_c[0] and reg_c[1] are reserved. */ 5614 config->flow_mreg_c[n++] = REG_C_0; 5615 config->flow_mreg_c[n++] = REG_C_1; 5616 /* Discover availability of other reg_c's. */ 5617 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 5618 struct rte_flow_attr attr = { 5619 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5620 .priority = MLX5_FLOW_PRIO_RSVD, 5621 .ingress = 1, 5622 }; 5623 struct rte_flow_item items[] = { 5624 [0] = { 5625 .type = RTE_FLOW_ITEM_TYPE_END, 5626 }, 5627 }; 5628 struct rte_flow_action actions[] = { 5629 [0] = { 5630 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5631 .conf = &(struct mlx5_flow_action_copy_mreg){ 5632 .src = REG_C_1, 5633 .dst = idx, 5634 }, 5635 }, 5636 [1] = { 5637 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5638 .conf = &(struct rte_flow_action_jump){ 5639 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5640 }, 5641 }, 5642 [2] = { 5643 .type = RTE_FLOW_ACTION_TYPE_END, 5644 }, 5645 }; 5646 struct rte_flow *flow; 5647 struct rte_flow_error error; 5648 5649 if (!config->dv_flow_en) 5650 break; 5651 /* Create internal flow, validation skips copy action. */ 5652 flow = flow_list_create(dev, NULL, &attr, items, 5653 actions, false, &error); 5654 if (!flow) 5655 continue; 5656 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 5657 config->flow_mreg_c[n++] = idx; 5658 flow_list_destroy(dev, NULL, flow); 5659 } 5660 for (; n < MLX5_MREG_C_NUM; ++n) 5661 config->flow_mreg_c[n] = REG_NONE; 5662 return 0; 5663 } 5664 5665 /** 5666 * Dump flow raw hw data to file 5667 * 5668 * @param[in] dev 5669 * The pointer to Ethernet device. 5670 * @param[in] file 5671 * A pointer to a file for output. 5672 * @param[out] error 5673 * Perform verbose error reporting if not NULL. PMDs initialize this 5674 * structure in case of error only. 5675 * @return 5676 * 0 on success, a nagative value otherwise. 5677 */ 5678 int 5679 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 5680 FILE *file, 5681 struct rte_flow_error *error __rte_unused) 5682 { 5683 struct mlx5_priv *priv = dev->data->dev_private; 5684 5685 return mlx5_devx_cmd_flow_dump(priv->sh, file); 5686 } 5687