1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 /* Verbs header. */ 14 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 15 #ifdef PEDANTIC 16 #pragma GCC diagnostic ignored "-Wpedantic" 17 #endif 18 #include <infiniband/verbs.h> 19 #ifdef PEDANTIC 20 #pragma GCC diagnostic error "-Wpedantic" 21 #endif 22 23 #include <rte_common.h> 24 #include <rte_ether.h> 25 #include <rte_ethdev_driver.h> 26 #include <rte_flow.h> 27 #include <rte_cycles.h> 28 #include <rte_flow_driver.h> 29 #include <rte_malloc.h> 30 #include <rte_ip.h> 31 32 #include <mlx5_glue.h> 33 #include <mlx5_devx_cmds.h> 34 #include <mlx5_prm.h> 35 36 #include "mlx5_defs.h" 37 #include "mlx5.h" 38 #include "mlx5_flow.h" 39 #include "mlx5_rxtx.h" 40 41 /* Dev ops structure defined in mlx5.c */ 42 extern const struct eth_dev_ops mlx5_dev_ops; 43 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 44 45 /** Device flow drivers. */ 46 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 47 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 48 #endif 49 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 50 51 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 52 53 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 54 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 55 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 56 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 57 #endif 58 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 59 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 60 }; 61 62 enum mlx5_expansion { 63 MLX5_EXPANSION_ROOT, 64 MLX5_EXPANSION_ROOT_OUTER, 65 MLX5_EXPANSION_ROOT_ETH_VLAN, 66 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 67 MLX5_EXPANSION_OUTER_ETH, 68 MLX5_EXPANSION_OUTER_ETH_VLAN, 69 MLX5_EXPANSION_OUTER_VLAN, 70 MLX5_EXPANSION_OUTER_IPV4, 71 MLX5_EXPANSION_OUTER_IPV4_UDP, 72 MLX5_EXPANSION_OUTER_IPV4_TCP, 73 MLX5_EXPANSION_OUTER_IPV6, 74 MLX5_EXPANSION_OUTER_IPV6_UDP, 75 MLX5_EXPANSION_OUTER_IPV6_TCP, 76 MLX5_EXPANSION_VXLAN, 77 MLX5_EXPANSION_VXLAN_GPE, 78 MLX5_EXPANSION_GRE, 79 MLX5_EXPANSION_MPLS, 80 MLX5_EXPANSION_ETH, 81 MLX5_EXPANSION_ETH_VLAN, 82 MLX5_EXPANSION_VLAN, 83 MLX5_EXPANSION_IPV4, 84 MLX5_EXPANSION_IPV4_UDP, 85 MLX5_EXPANSION_IPV4_TCP, 86 MLX5_EXPANSION_IPV6, 87 MLX5_EXPANSION_IPV6_UDP, 88 MLX5_EXPANSION_IPV6_TCP, 89 }; 90 91 /** Supported expansion of items. */ 92 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 93 [MLX5_EXPANSION_ROOT] = { 94 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 95 MLX5_EXPANSION_IPV4, 96 MLX5_EXPANSION_IPV6), 97 .type = RTE_FLOW_ITEM_TYPE_END, 98 }, 99 [MLX5_EXPANSION_ROOT_OUTER] = { 100 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 101 MLX5_EXPANSION_OUTER_IPV4, 102 MLX5_EXPANSION_OUTER_IPV6), 103 .type = RTE_FLOW_ITEM_TYPE_END, 104 }, 105 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 107 .type = RTE_FLOW_ITEM_TYPE_END, 108 }, 109 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 110 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 111 .type = RTE_FLOW_ITEM_TYPE_END, 112 }, 113 [MLX5_EXPANSION_OUTER_ETH] = { 114 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 115 MLX5_EXPANSION_OUTER_IPV6, 116 MLX5_EXPANSION_MPLS), 117 .type = RTE_FLOW_ITEM_TYPE_ETH, 118 .rss_types = 0, 119 }, 120 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 121 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 122 .type = RTE_FLOW_ITEM_TYPE_ETH, 123 .rss_types = 0, 124 }, 125 [MLX5_EXPANSION_OUTER_VLAN] = { 126 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 127 MLX5_EXPANSION_OUTER_IPV6), 128 .type = RTE_FLOW_ITEM_TYPE_VLAN, 129 }, 130 [MLX5_EXPANSION_OUTER_IPV4] = { 131 .next = RTE_FLOW_EXPAND_RSS_NEXT 132 (MLX5_EXPANSION_OUTER_IPV4_UDP, 133 MLX5_EXPANSION_OUTER_IPV4_TCP, 134 MLX5_EXPANSION_GRE, 135 MLX5_EXPANSION_IPV4, 136 MLX5_EXPANSION_IPV6), 137 .type = RTE_FLOW_ITEM_TYPE_IPV4, 138 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 139 ETH_RSS_NONFRAG_IPV4_OTHER, 140 }, 141 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 142 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 143 MLX5_EXPANSION_VXLAN_GPE), 144 .type = RTE_FLOW_ITEM_TYPE_UDP, 145 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 146 }, 147 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 148 .type = RTE_FLOW_ITEM_TYPE_TCP, 149 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 150 }, 151 [MLX5_EXPANSION_OUTER_IPV6] = { 152 .next = RTE_FLOW_EXPAND_RSS_NEXT 153 (MLX5_EXPANSION_OUTER_IPV6_UDP, 154 MLX5_EXPANSION_OUTER_IPV6_TCP, 155 MLX5_EXPANSION_IPV4, 156 MLX5_EXPANSION_IPV6), 157 .type = RTE_FLOW_ITEM_TYPE_IPV6, 158 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 159 ETH_RSS_NONFRAG_IPV6_OTHER, 160 }, 161 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 162 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 163 MLX5_EXPANSION_VXLAN_GPE), 164 .type = RTE_FLOW_ITEM_TYPE_UDP, 165 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 166 }, 167 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 168 .type = RTE_FLOW_ITEM_TYPE_TCP, 169 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 170 }, 171 [MLX5_EXPANSION_VXLAN] = { 172 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 173 MLX5_EXPANSION_IPV4, 174 MLX5_EXPANSION_IPV6), 175 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 176 }, 177 [MLX5_EXPANSION_VXLAN_GPE] = { 178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 179 MLX5_EXPANSION_IPV4, 180 MLX5_EXPANSION_IPV6), 181 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 182 }, 183 [MLX5_EXPANSION_GRE] = { 184 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 185 .type = RTE_FLOW_ITEM_TYPE_GRE, 186 }, 187 [MLX5_EXPANSION_MPLS] = { 188 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 189 MLX5_EXPANSION_IPV6), 190 .type = RTE_FLOW_ITEM_TYPE_MPLS, 191 }, 192 [MLX5_EXPANSION_ETH] = { 193 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 194 MLX5_EXPANSION_IPV6), 195 .type = RTE_FLOW_ITEM_TYPE_ETH, 196 }, 197 [MLX5_EXPANSION_ETH_VLAN] = { 198 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 199 .type = RTE_FLOW_ITEM_TYPE_ETH, 200 }, 201 [MLX5_EXPANSION_VLAN] = { 202 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 203 MLX5_EXPANSION_IPV6), 204 .type = RTE_FLOW_ITEM_TYPE_VLAN, 205 }, 206 [MLX5_EXPANSION_IPV4] = { 207 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 208 MLX5_EXPANSION_IPV4_TCP), 209 .type = RTE_FLOW_ITEM_TYPE_IPV4, 210 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 211 ETH_RSS_NONFRAG_IPV4_OTHER, 212 }, 213 [MLX5_EXPANSION_IPV4_UDP] = { 214 .type = RTE_FLOW_ITEM_TYPE_UDP, 215 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 216 }, 217 [MLX5_EXPANSION_IPV4_TCP] = { 218 .type = RTE_FLOW_ITEM_TYPE_TCP, 219 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 220 }, 221 [MLX5_EXPANSION_IPV6] = { 222 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 223 MLX5_EXPANSION_IPV6_TCP), 224 .type = RTE_FLOW_ITEM_TYPE_IPV6, 225 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 226 ETH_RSS_NONFRAG_IPV6_OTHER, 227 }, 228 [MLX5_EXPANSION_IPV6_UDP] = { 229 .type = RTE_FLOW_ITEM_TYPE_UDP, 230 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 231 }, 232 [MLX5_EXPANSION_IPV6_TCP] = { 233 .type = RTE_FLOW_ITEM_TYPE_TCP, 234 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 235 }, 236 }; 237 238 static const struct rte_flow_ops mlx5_flow_ops = { 239 .validate = mlx5_flow_validate, 240 .create = mlx5_flow_create, 241 .destroy = mlx5_flow_destroy, 242 .flush = mlx5_flow_flush, 243 .isolate = mlx5_flow_isolate, 244 .query = mlx5_flow_query, 245 .dev_dump = mlx5_flow_dev_dump, 246 .get_aged_flows = mlx5_flow_get_aged_flows, 247 }; 248 249 /* Convert FDIR request to Generic flow. */ 250 struct mlx5_fdir { 251 struct rte_flow_attr attr; 252 struct rte_flow_item items[4]; 253 struct rte_flow_item_eth l2; 254 struct rte_flow_item_eth l2_mask; 255 union { 256 struct rte_flow_item_ipv4 ipv4; 257 struct rte_flow_item_ipv6 ipv6; 258 } l3; 259 union { 260 struct rte_flow_item_ipv4 ipv4; 261 struct rte_flow_item_ipv6 ipv6; 262 } l3_mask; 263 union { 264 struct rte_flow_item_udp udp; 265 struct rte_flow_item_tcp tcp; 266 } l4; 267 union { 268 struct rte_flow_item_udp udp; 269 struct rte_flow_item_tcp tcp; 270 } l4_mask; 271 struct rte_flow_action actions[2]; 272 struct rte_flow_action_queue queue; 273 }; 274 275 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 276 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 277 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 278 }; 279 280 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 281 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 282 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 283 { 9, 10, 11 }, { 12, 13, 14 }, 284 }; 285 286 /* Tunnel information. */ 287 struct mlx5_flow_tunnel_info { 288 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 289 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 290 }; 291 292 static struct mlx5_flow_tunnel_info tunnels_info[] = { 293 { 294 .tunnel = MLX5_FLOW_LAYER_VXLAN, 295 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 296 }, 297 { 298 .tunnel = MLX5_FLOW_LAYER_GENEVE, 299 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 300 }, 301 { 302 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 303 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 304 }, 305 { 306 .tunnel = MLX5_FLOW_LAYER_GRE, 307 .ptype = RTE_PTYPE_TUNNEL_GRE, 308 }, 309 { 310 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 311 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 312 }, 313 { 314 .tunnel = MLX5_FLOW_LAYER_MPLS, 315 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 316 }, 317 { 318 .tunnel = MLX5_FLOW_LAYER_NVGRE, 319 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 320 }, 321 { 322 .tunnel = MLX5_FLOW_LAYER_IPIP, 323 .ptype = RTE_PTYPE_TUNNEL_IP, 324 }, 325 { 326 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 327 .ptype = RTE_PTYPE_TUNNEL_IP, 328 }, 329 { 330 .tunnel = MLX5_FLOW_LAYER_GTP, 331 .ptype = RTE_PTYPE_TUNNEL_GTPU, 332 }, 333 }; 334 335 /** 336 * Translate tag ID to register. 337 * 338 * @param[in] dev 339 * Pointer to the Ethernet device structure. 340 * @param[in] feature 341 * The feature that request the register. 342 * @param[in] id 343 * The request register ID. 344 * @param[out] error 345 * Error description in case of any. 346 * 347 * @return 348 * The request register on success, a negative errno 349 * value otherwise and rte_errno is set. 350 */ 351 int 352 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 353 enum mlx5_feature_name feature, 354 uint32_t id, 355 struct rte_flow_error *error) 356 { 357 struct mlx5_priv *priv = dev->data->dev_private; 358 struct mlx5_dev_config *config = &priv->config; 359 enum modify_reg start_reg; 360 bool skip_mtr_reg = false; 361 362 switch (feature) { 363 case MLX5_HAIRPIN_RX: 364 return REG_B; 365 case MLX5_HAIRPIN_TX: 366 return REG_A; 367 case MLX5_METADATA_RX: 368 switch (config->dv_xmeta_en) { 369 case MLX5_XMETA_MODE_LEGACY: 370 return REG_B; 371 case MLX5_XMETA_MODE_META16: 372 return REG_C_0; 373 case MLX5_XMETA_MODE_META32: 374 return REG_C_1; 375 } 376 break; 377 case MLX5_METADATA_TX: 378 return REG_A; 379 case MLX5_METADATA_FDB: 380 switch (config->dv_xmeta_en) { 381 case MLX5_XMETA_MODE_LEGACY: 382 return REG_NONE; 383 case MLX5_XMETA_MODE_META16: 384 return REG_C_0; 385 case MLX5_XMETA_MODE_META32: 386 return REG_C_1; 387 } 388 break; 389 case MLX5_FLOW_MARK: 390 switch (config->dv_xmeta_en) { 391 case MLX5_XMETA_MODE_LEGACY: 392 return REG_NONE; 393 case MLX5_XMETA_MODE_META16: 394 return REG_C_1; 395 case MLX5_XMETA_MODE_META32: 396 return REG_C_0; 397 } 398 break; 399 case MLX5_MTR_SFX: 400 /* 401 * If meter color and flow match share one register, flow match 402 * should use the meter color register for match. 403 */ 404 if (priv->mtr_reg_share) 405 return priv->mtr_color_reg; 406 else 407 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 408 REG_C_3; 409 case MLX5_MTR_COLOR: 410 MLX5_ASSERT(priv->mtr_color_reg != REG_NONE); 411 return priv->mtr_color_reg; 412 case MLX5_COPY_MARK: 413 /* 414 * Metadata COPY_MARK register using is in meter suffix sub 415 * flow while with meter. It's safe to share the same register. 416 */ 417 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 418 case MLX5_APP_TAG: 419 /* 420 * If meter is enable, it will engage the register for color 421 * match and flow match. If meter color match is not using the 422 * REG_C_2, need to skip the REG_C_x be used by meter color 423 * match. 424 * If meter is disable, free to use all available registers. 425 */ 426 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 427 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 428 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 429 if (id > (REG_C_7 - start_reg)) 430 return rte_flow_error_set(error, EINVAL, 431 RTE_FLOW_ERROR_TYPE_ITEM, 432 NULL, "invalid tag id"); 433 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 434 return rte_flow_error_set(error, ENOTSUP, 435 RTE_FLOW_ERROR_TYPE_ITEM, 436 NULL, "unsupported tag id"); 437 /* 438 * This case means meter is using the REG_C_x great than 2. 439 * Take care not to conflict with meter color REG_C_x. 440 * If the available index REG_C_y >= REG_C_x, skip the 441 * color register. 442 */ 443 if (skip_mtr_reg && config->flow_mreg_c 444 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 445 if (config->flow_mreg_c 446 [id + 1 + start_reg - REG_C_0] != REG_NONE) 447 return config->flow_mreg_c 448 [id + 1 + start_reg - REG_C_0]; 449 return rte_flow_error_set(error, ENOTSUP, 450 RTE_FLOW_ERROR_TYPE_ITEM, 451 NULL, "unsupported tag id"); 452 } 453 return config->flow_mreg_c[id + start_reg - REG_C_0]; 454 } 455 MLX5_ASSERT(false); 456 return rte_flow_error_set(error, EINVAL, 457 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 458 NULL, "invalid feature name"); 459 } 460 461 /** 462 * Check extensive flow metadata register support. 463 * 464 * @param dev 465 * Pointer to rte_eth_dev structure. 466 * 467 * @return 468 * True if device supports extensive flow metadata register, otherwise false. 469 */ 470 bool 471 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 472 { 473 struct mlx5_priv *priv = dev->data->dev_private; 474 struct mlx5_dev_config *config = &priv->config; 475 476 /* 477 * Having available reg_c can be regarded inclusively as supporting 478 * extensive flow metadata register, which could mean, 479 * - metadata register copy action by modify header. 480 * - 16 modify header actions is supported. 481 * - reg_c's are preserved across different domain (FDB and NIC) on 482 * packet loopback by flow lookup miss. 483 */ 484 return config->flow_mreg_c[2] != REG_NONE; 485 } 486 487 /** 488 * Discover the maximum number of priority available. 489 * 490 * @param[in] dev 491 * Pointer to the Ethernet device structure. 492 * 493 * @return 494 * number of supported flow priority on success, a negative errno 495 * value otherwise and rte_errno is set. 496 */ 497 int 498 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 499 { 500 struct mlx5_priv *priv = dev->data->dev_private; 501 struct { 502 struct ibv_flow_attr attr; 503 struct ibv_flow_spec_eth eth; 504 struct ibv_flow_spec_action_drop drop; 505 } flow_attr = { 506 .attr = { 507 .num_of_specs = 2, 508 .port = (uint8_t)priv->ibv_port, 509 }, 510 .eth = { 511 .type = IBV_FLOW_SPEC_ETH, 512 .size = sizeof(struct ibv_flow_spec_eth), 513 }, 514 .drop = { 515 .size = sizeof(struct ibv_flow_spec_action_drop), 516 .type = IBV_FLOW_SPEC_ACTION_DROP, 517 }, 518 }; 519 struct ibv_flow *flow; 520 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 521 uint16_t vprio[] = { 8, 16 }; 522 int i; 523 int priority = 0; 524 525 if (!drop) { 526 rte_errno = ENOTSUP; 527 return -rte_errno; 528 } 529 for (i = 0; i != RTE_DIM(vprio); i++) { 530 flow_attr.attr.priority = vprio[i] - 1; 531 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 532 if (!flow) 533 break; 534 claim_zero(mlx5_glue->destroy_flow(flow)); 535 priority = vprio[i]; 536 } 537 mlx5_hrxq_drop_release(dev); 538 switch (priority) { 539 case 8: 540 priority = RTE_DIM(priority_map_3); 541 break; 542 case 16: 543 priority = RTE_DIM(priority_map_5); 544 break; 545 default: 546 rte_errno = ENOTSUP; 547 DRV_LOG(ERR, 548 "port %u verbs maximum priority: %d expected 8/16", 549 dev->data->port_id, priority); 550 return -rte_errno; 551 } 552 DRV_LOG(INFO, "port %u flow maximum priority: %d", 553 dev->data->port_id, priority); 554 return priority; 555 } 556 557 /** 558 * Adjust flow priority based on the highest layer and the request priority. 559 * 560 * @param[in] dev 561 * Pointer to the Ethernet device structure. 562 * @param[in] priority 563 * The rule base priority. 564 * @param[in] subpriority 565 * The priority based on the items. 566 * 567 * @return 568 * The new priority. 569 */ 570 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 571 uint32_t subpriority) 572 { 573 uint32_t res = 0; 574 struct mlx5_priv *priv = dev->data->dev_private; 575 576 switch (priv->config.flow_prio) { 577 case RTE_DIM(priority_map_3): 578 res = priority_map_3[priority][subpriority]; 579 break; 580 case RTE_DIM(priority_map_5): 581 res = priority_map_5[priority][subpriority]; 582 break; 583 } 584 return res; 585 } 586 587 /** 588 * Verify the @p item specifications (spec, last, mask) are compatible with the 589 * NIC capabilities. 590 * 591 * @param[in] item 592 * Item specification. 593 * @param[in] mask 594 * @p item->mask or flow default bit-masks. 595 * @param[in] nic_mask 596 * Bit-masks covering supported fields by the NIC to compare with user mask. 597 * @param[in] size 598 * Bit-masks size in bytes. 599 * @param[out] error 600 * Pointer to error structure. 601 * 602 * @return 603 * 0 on success, a negative errno value otherwise and rte_errno is set. 604 */ 605 int 606 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 607 const uint8_t *mask, 608 const uint8_t *nic_mask, 609 unsigned int size, 610 struct rte_flow_error *error) 611 { 612 unsigned int i; 613 614 MLX5_ASSERT(nic_mask); 615 for (i = 0; i < size; ++i) 616 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 617 return rte_flow_error_set(error, ENOTSUP, 618 RTE_FLOW_ERROR_TYPE_ITEM, 619 item, 620 "mask enables non supported" 621 " bits"); 622 if (!item->spec && (item->mask || item->last)) 623 return rte_flow_error_set(error, EINVAL, 624 RTE_FLOW_ERROR_TYPE_ITEM, item, 625 "mask/last without a spec is not" 626 " supported"); 627 if (item->spec && item->last) { 628 uint8_t spec[size]; 629 uint8_t last[size]; 630 unsigned int i; 631 int ret; 632 633 for (i = 0; i < size; ++i) { 634 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 635 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 636 } 637 ret = memcmp(spec, last, size); 638 if (ret != 0) 639 return rte_flow_error_set(error, EINVAL, 640 RTE_FLOW_ERROR_TYPE_ITEM, 641 item, 642 "range is not valid"); 643 } 644 return 0; 645 } 646 647 /** 648 * Adjust the hash fields according to the @p flow information. 649 * 650 * @param[in] dev_flow. 651 * Pointer to the mlx5_flow. 652 * @param[in] tunnel 653 * 1 when the hash field is for a tunnel item. 654 * @param[in] layer_types 655 * ETH_RSS_* types. 656 * @param[in] hash_fields 657 * Item hash fields. 658 * 659 * @return 660 * The hash fields that should be used. 661 */ 662 uint64_t 663 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 664 int tunnel __rte_unused, uint64_t layer_types, 665 uint64_t hash_fields) 666 { 667 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 668 int rss_request_inner = rss_desc->level >= 2; 669 670 /* Check RSS hash level for tunnel. */ 671 if (tunnel && rss_request_inner) 672 hash_fields |= IBV_RX_HASH_INNER; 673 else if (tunnel || rss_request_inner) 674 return 0; 675 #endif 676 /* Check if requested layer matches RSS hash fields. */ 677 if (!(rss_desc->types & layer_types)) 678 return 0; 679 return hash_fields; 680 } 681 682 /** 683 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 684 * if several tunnel rules are used on this queue, the tunnel ptype will be 685 * cleared. 686 * 687 * @param rxq_ctrl 688 * Rx queue to update. 689 */ 690 static void 691 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 692 { 693 unsigned int i; 694 uint32_t tunnel_ptype = 0; 695 696 /* Look up for the ptype to use. */ 697 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 698 if (!rxq_ctrl->flow_tunnels_n[i]) 699 continue; 700 if (!tunnel_ptype) { 701 tunnel_ptype = tunnels_info[i].ptype; 702 } else { 703 tunnel_ptype = 0; 704 break; 705 } 706 } 707 rxq_ctrl->rxq.tunnel = tunnel_ptype; 708 } 709 710 /** 711 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 712 * flow. 713 * 714 * @param[in] dev 715 * Pointer to the Ethernet device structure. 716 * @param[in] dev_handle 717 * Pointer to device flow handle structure. 718 */ 719 static void 720 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 721 struct mlx5_flow_handle *dev_handle) 722 { 723 struct mlx5_priv *priv = dev->data->dev_private; 724 const int mark = dev_handle->mark; 725 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 726 struct mlx5_hrxq *hrxq; 727 unsigned int i; 728 729 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 730 return; 731 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 732 dev_handle->rix_hrxq); 733 if (!hrxq) 734 return; 735 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 736 int idx = hrxq->ind_table->queues[i]; 737 struct mlx5_rxq_ctrl *rxq_ctrl = 738 container_of((*priv->rxqs)[idx], 739 struct mlx5_rxq_ctrl, rxq); 740 741 /* 742 * To support metadata register copy on Tx loopback, 743 * this must be always enabled (metadata may arive 744 * from other port - not from local flows only. 745 */ 746 if (priv->config.dv_flow_en && 747 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 748 mlx5_flow_ext_mreg_supported(dev)) { 749 rxq_ctrl->rxq.mark = 1; 750 rxq_ctrl->flow_mark_n = 1; 751 } else if (mark) { 752 rxq_ctrl->rxq.mark = 1; 753 rxq_ctrl->flow_mark_n++; 754 } 755 if (tunnel) { 756 unsigned int j; 757 758 /* Increase the counter matching the flow. */ 759 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 760 if ((tunnels_info[j].tunnel & 761 dev_handle->layers) == 762 tunnels_info[j].tunnel) { 763 rxq_ctrl->flow_tunnels_n[j]++; 764 break; 765 } 766 } 767 flow_rxq_tunnel_ptype_update(rxq_ctrl); 768 } 769 } 770 } 771 772 /** 773 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 774 * 775 * @param[in] dev 776 * Pointer to the Ethernet device structure. 777 * @param[in] flow 778 * Pointer to flow structure. 779 */ 780 static void 781 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 782 { 783 struct mlx5_priv *priv = dev->data->dev_private; 784 uint32_t handle_idx; 785 struct mlx5_flow_handle *dev_handle; 786 787 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 788 handle_idx, dev_handle, next) 789 flow_drv_rxq_flags_set(dev, dev_handle); 790 } 791 792 /** 793 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 794 * device flow if no other flow uses it with the same kind of request. 795 * 796 * @param dev 797 * Pointer to Ethernet device. 798 * @param[in] dev_handle 799 * Pointer to the device flow handle structure. 800 */ 801 static void 802 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 803 struct mlx5_flow_handle *dev_handle) 804 { 805 struct mlx5_priv *priv = dev->data->dev_private; 806 const int mark = dev_handle->mark; 807 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 808 struct mlx5_hrxq *hrxq; 809 unsigned int i; 810 811 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 812 return; 813 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 814 dev_handle->rix_hrxq); 815 if (!hrxq) 816 return; 817 MLX5_ASSERT(dev->data->dev_started); 818 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 819 int idx = hrxq->ind_table->queues[i]; 820 struct mlx5_rxq_ctrl *rxq_ctrl = 821 container_of((*priv->rxqs)[idx], 822 struct mlx5_rxq_ctrl, rxq); 823 824 if (priv->config.dv_flow_en && 825 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 826 mlx5_flow_ext_mreg_supported(dev)) { 827 rxq_ctrl->rxq.mark = 1; 828 rxq_ctrl->flow_mark_n = 1; 829 } else if (mark) { 830 rxq_ctrl->flow_mark_n--; 831 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 832 } 833 if (tunnel) { 834 unsigned int j; 835 836 /* Decrease the counter matching the flow. */ 837 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 838 if ((tunnels_info[j].tunnel & 839 dev_handle->layers) == 840 tunnels_info[j].tunnel) { 841 rxq_ctrl->flow_tunnels_n[j]--; 842 break; 843 } 844 } 845 flow_rxq_tunnel_ptype_update(rxq_ctrl); 846 } 847 } 848 } 849 850 /** 851 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 852 * @p flow if no other flow uses it with the same kind of request. 853 * 854 * @param dev 855 * Pointer to Ethernet device. 856 * @param[in] flow 857 * Pointer to the flow. 858 */ 859 static void 860 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 861 { 862 struct mlx5_priv *priv = dev->data->dev_private; 863 uint32_t handle_idx; 864 struct mlx5_flow_handle *dev_handle; 865 866 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 867 handle_idx, dev_handle, next) 868 flow_drv_rxq_flags_trim(dev, dev_handle); 869 } 870 871 /** 872 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 873 * 874 * @param dev 875 * Pointer to Ethernet device. 876 */ 877 static void 878 flow_rxq_flags_clear(struct rte_eth_dev *dev) 879 { 880 struct mlx5_priv *priv = dev->data->dev_private; 881 unsigned int i; 882 883 for (i = 0; i != priv->rxqs_n; ++i) { 884 struct mlx5_rxq_ctrl *rxq_ctrl; 885 unsigned int j; 886 887 if (!(*priv->rxqs)[i]) 888 continue; 889 rxq_ctrl = container_of((*priv->rxqs)[i], 890 struct mlx5_rxq_ctrl, rxq); 891 rxq_ctrl->flow_mark_n = 0; 892 rxq_ctrl->rxq.mark = 0; 893 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 894 rxq_ctrl->flow_tunnels_n[j] = 0; 895 rxq_ctrl->rxq.tunnel = 0; 896 } 897 } 898 899 /** 900 * Set the Rx queue dynamic metadata (mask and offset) for a flow 901 * 902 * @param[in] dev 903 * Pointer to the Ethernet device structure. 904 */ 905 void 906 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 907 { 908 struct mlx5_priv *priv = dev->data->dev_private; 909 struct mlx5_rxq_data *data; 910 unsigned int i; 911 912 for (i = 0; i != priv->rxqs_n; ++i) { 913 if (!(*priv->rxqs)[i]) 914 continue; 915 data = (*priv->rxqs)[i]; 916 if (!rte_flow_dynf_metadata_avail()) { 917 data->dynf_meta = 0; 918 data->flow_meta_mask = 0; 919 data->flow_meta_offset = -1; 920 } else { 921 data->dynf_meta = 1; 922 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 923 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 924 } 925 } 926 } 927 928 /* 929 * return a pointer to the desired action in the list of actions. 930 * 931 * @param[in] actions 932 * The list of actions to search the action in. 933 * @param[in] action 934 * The action to find. 935 * 936 * @return 937 * Pointer to the action in the list, if found. NULL otherwise. 938 */ 939 const struct rte_flow_action * 940 mlx5_flow_find_action(const struct rte_flow_action *actions, 941 enum rte_flow_action_type action) 942 { 943 if (actions == NULL) 944 return NULL; 945 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 946 if (actions->type == action) 947 return actions; 948 return NULL; 949 } 950 951 /* 952 * Validate the flag action. 953 * 954 * @param[in] action_flags 955 * Bit-fields that holds the actions detected until now. 956 * @param[in] attr 957 * Attributes of flow that includes this action. 958 * @param[out] error 959 * Pointer to error structure. 960 * 961 * @return 962 * 0 on success, a negative errno value otherwise and rte_errno is set. 963 */ 964 int 965 mlx5_flow_validate_action_flag(uint64_t action_flags, 966 const struct rte_flow_attr *attr, 967 struct rte_flow_error *error) 968 { 969 if (action_flags & MLX5_FLOW_ACTION_MARK) 970 return rte_flow_error_set(error, EINVAL, 971 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 972 "can't mark and flag in same flow"); 973 if (action_flags & MLX5_FLOW_ACTION_FLAG) 974 return rte_flow_error_set(error, EINVAL, 975 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 976 "can't have 2 flag" 977 " actions in same flow"); 978 if (attr->egress) 979 return rte_flow_error_set(error, ENOTSUP, 980 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 981 "flag action not supported for " 982 "egress"); 983 return 0; 984 } 985 986 /* 987 * Validate the mark action. 988 * 989 * @param[in] action 990 * Pointer to the queue action. 991 * @param[in] action_flags 992 * Bit-fields that holds the actions detected until now. 993 * @param[in] attr 994 * Attributes of flow that includes this action. 995 * @param[out] error 996 * Pointer to error structure. 997 * 998 * @return 999 * 0 on success, a negative errno value otherwise and rte_errno is set. 1000 */ 1001 int 1002 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1003 uint64_t action_flags, 1004 const struct rte_flow_attr *attr, 1005 struct rte_flow_error *error) 1006 { 1007 const struct rte_flow_action_mark *mark = action->conf; 1008 1009 if (!mark) 1010 return rte_flow_error_set(error, EINVAL, 1011 RTE_FLOW_ERROR_TYPE_ACTION, 1012 action, 1013 "configuration cannot be null"); 1014 if (mark->id >= MLX5_FLOW_MARK_MAX) 1015 return rte_flow_error_set(error, EINVAL, 1016 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1017 &mark->id, 1018 "mark id must in 0 <= id < " 1019 RTE_STR(MLX5_FLOW_MARK_MAX)); 1020 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1021 return rte_flow_error_set(error, EINVAL, 1022 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1023 "can't flag and mark in same flow"); 1024 if (action_flags & MLX5_FLOW_ACTION_MARK) 1025 return rte_flow_error_set(error, EINVAL, 1026 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1027 "can't have 2 mark actions in same" 1028 " flow"); 1029 if (attr->egress) 1030 return rte_flow_error_set(error, ENOTSUP, 1031 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1032 "mark action not supported for " 1033 "egress"); 1034 return 0; 1035 } 1036 1037 /* 1038 * Validate the drop action. 1039 * 1040 * @param[in] action_flags 1041 * Bit-fields that holds the actions detected until now. 1042 * @param[in] attr 1043 * Attributes of flow that includes this action. 1044 * @param[out] error 1045 * Pointer to error structure. 1046 * 1047 * @return 1048 * 0 on success, a negative errno value otherwise and rte_errno is set. 1049 */ 1050 int 1051 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1052 const struct rte_flow_attr *attr, 1053 struct rte_flow_error *error) 1054 { 1055 if (attr->egress) 1056 return rte_flow_error_set(error, ENOTSUP, 1057 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1058 "drop action not supported for " 1059 "egress"); 1060 return 0; 1061 } 1062 1063 /* 1064 * Validate the queue action. 1065 * 1066 * @param[in] action 1067 * Pointer to the queue action. 1068 * @param[in] action_flags 1069 * Bit-fields that holds the actions detected until now. 1070 * @param[in] dev 1071 * Pointer to the Ethernet device structure. 1072 * @param[in] attr 1073 * Attributes of flow that includes this action. 1074 * @param[out] error 1075 * Pointer to error structure. 1076 * 1077 * @return 1078 * 0 on success, a negative errno value otherwise and rte_errno is set. 1079 */ 1080 int 1081 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1082 uint64_t action_flags, 1083 struct rte_eth_dev *dev, 1084 const struct rte_flow_attr *attr, 1085 struct rte_flow_error *error) 1086 { 1087 struct mlx5_priv *priv = dev->data->dev_private; 1088 const struct rte_flow_action_queue *queue = action->conf; 1089 1090 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1091 return rte_flow_error_set(error, EINVAL, 1092 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1093 "can't have 2 fate actions in" 1094 " same flow"); 1095 if (!priv->rxqs_n) 1096 return rte_flow_error_set(error, EINVAL, 1097 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1098 NULL, "No Rx queues configured"); 1099 if (queue->index >= priv->rxqs_n) 1100 return rte_flow_error_set(error, EINVAL, 1101 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1102 &queue->index, 1103 "queue index out of range"); 1104 if (!(*priv->rxqs)[queue->index]) 1105 return rte_flow_error_set(error, EINVAL, 1106 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1107 &queue->index, 1108 "queue is not configured"); 1109 if (attr->egress) 1110 return rte_flow_error_set(error, ENOTSUP, 1111 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1112 "queue action not supported for " 1113 "egress"); 1114 return 0; 1115 } 1116 1117 /* 1118 * Validate the rss action. 1119 * 1120 * @param[in] action 1121 * Pointer to the queue action. 1122 * @param[in] action_flags 1123 * Bit-fields that holds the actions detected until now. 1124 * @param[in] dev 1125 * Pointer to the Ethernet device structure. 1126 * @param[in] attr 1127 * Attributes of flow that includes this action. 1128 * @param[in] item_flags 1129 * Items that were detected. 1130 * @param[out] error 1131 * Pointer to error structure. 1132 * 1133 * @return 1134 * 0 on success, a negative errno value otherwise and rte_errno is set. 1135 */ 1136 int 1137 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1138 uint64_t action_flags, 1139 struct rte_eth_dev *dev, 1140 const struct rte_flow_attr *attr, 1141 uint64_t item_flags, 1142 struct rte_flow_error *error) 1143 { 1144 struct mlx5_priv *priv = dev->data->dev_private; 1145 const struct rte_flow_action_rss *rss = action->conf; 1146 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1147 unsigned int i; 1148 1149 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1150 return rte_flow_error_set(error, EINVAL, 1151 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1152 "can't have 2 fate actions" 1153 " in same flow"); 1154 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1155 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1156 return rte_flow_error_set(error, ENOTSUP, 1157 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1158 &rss->func, 1159 "RSS hash function not supported"); 1160 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1161 if (rss->level > 2) 1162 #else 1163 if (rss->level > 1) 1164 #endif 1165 return rte_flow_error_set(error, ENOTSUP, 1166 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1167 &rss->level, 1168 "tunnel RSS is not supported"); 1169 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1170 if (rss->key_len == 0 && rss->key != NULL) 1171 return rte_flow_error_set(error, ENOTSUP, 1172 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1173 &rss->key_len, 1174 "RSS hash key length 0"); 1175 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1176 return rte_flow_error_set(error, ENOTSUP, 1177 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1178 &rss->key_len, 1179 "RSS hash key too small"); 1180 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1181 return rte_flow_error_set(error, ENOTSUP, 1182 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1183 &rss->key_len, 1184 "RSS hash key too large"); 1185 if (rss->queue_num > priv->config.ind_table_max_size) 1186 return rte_flow_error_set(error, ENOTSUP, 1187 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1188 &rss->queue_num, 1189 "number of queues too large"); 1190 if (rss->types & MLX5_RSS_HF_MASK) 1191 return rte_flow_error_set(error, ENOTSUP, 1192 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1193 &rss->types, 1194 "some RSS protocols are not" 1195 " supported"); 1196 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1197 !(rss->types & ETH_RSS_IP)) 1198 return rte_flow_error_set(error, EINVAL, 1199 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1200 "L3 partial RSS requested but L3 RSS" 1201 " type not specified"); 1202 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1203 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1204 return rte_flow_error_set(error, EINVAL, 1205 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1206 "L4 partial RSS requested but L4 RSS" 1207 " type not specified"); 1208 if (!priv->rxqs_n) 1209 return rte_flow_error_set(error, EINVAL, 1210 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1211 NULL, "No Rx queues configured"); 1212 if (!rss->queue_num) 1213 return rte_flow_error_set(error, EINVAL, 1214 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1215 NULL, "No queues configured"); 1216 for (i = 0; i != rss->queue_num; ++i) { 1217 if (rss->queue[i] >= priv->rxqs_n) 1218 return rte_flow_error_set 1219 (error, EINVAL, 1220 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1221 &rss->queue[i], "queue index out of range"); 1222 if (!(*priv->rxqs)[rss->queue[i]]) 1223 return rte_flow_error_set 1224 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1225 &rss->queue[i], "queue is not configured"); 1226 } 1227 if (attr->egress) 1228 return rte_flow_error_set(error, ENOTSUP, 1229 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1230 "rss action not supported for " 1231 "egress"); 1232 if (rss->level > 1 && !tunnel) 1233 return rte_flow_error_set(error, EINVAL, 1234 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1235 "inner RSS is not supported for " 1236 "non-tunnel flows"); 1237 return 0; 1238 } 1239 1240 /* 1241 * Validate the count action. 1242 * 1243 * @param[in] dev 1244 * Pointer to the Ethernet device structure. 1245 * @param[in] attr 1246 * Attributes of flow that includes this action. 1247 * @param[out] error 1248 * Pointer to error structure. 1249 * 1250 * @return 1251 * 0 on success, a negative errno value otherwise and rte_errno is set. 1252 */ 1253 int 1254 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1255 const struct rte_flow_attr *attr, 1256 struct rte_flow_error *error) 1257 { 1258 if (attr->egress) 1259 return rte_flow_error_set(error, ENOTSUP, 1260 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1261 "count action not supported for " 1262 "egress"); 1263 return 0; 1264 } 1265 1266 /** 1267 * Verify the @p attributes will be correctly understood by the NIC and store 1268 * them in the @p flow if everything is correct. 1269 * 1270 * @param[in] dev 1271 * Pointer to the Ethernet device structure. 1272 * @param[in] attributes 1273 * Pointer to flow attributes 1274 * @param[out] error 1275 * Pointer to error structure. 1276 * 1277 * @return 1278 * 0 on success, a negative errno value otherwise and rte_errno is set. 1279 */ 1280 int 1281 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1282 const struct rte_flow_attr *attributes, 1283 struct rte_flow_error *error) 1284 { 1285 struct mlx5_priv *priv = dev->data->dev_private; 1286 uint32_t priority_max = priv->config.flow_prio - 1; 1287 1288 if (attributes->group) 1289 return rte_flow_error_set(error, ENOTSUP, 1290 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1291 NULL, "groups is not supported"); 1292 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1293 attributes->priority >= priority_max) 1294 return rte_flow_error_set(error, ENOTSUP, 1295 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1296 NULL, "priority out of range"); 1297 if (attributes->egress) 1298 return rte_flow_error_set(error, ENOTSUP, 1299 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1300 "egress is not supported"); 1301 if (attributes->transfer && !priv->config.dv_esw_en) 1302 return rte_flow_error_set(error, ENOTSUP, 1303 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1304 NULL, "transfer is not supported"); 1305 if (!attributes->ingress) 1306 return rte_flow_error_set(error, EINVAL, 1307 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1308 NULL, 1309 "ingress attribute is mandatory"); 1310 return 0; 1311 } 1312 1313 /** 1314 * Validate ICMP6 item. 1315 * 1316 * @param[in] item 1317 * Item specification. 1318 * @param[in] item_flags 1319 * Bit-fields that holds the items detected until now. 1320 * @param[out] error 1321 * Pointer to error structure. 1322 * 1323 * @return 1324 * 0 on success, a negative errno value otherwise and rte_errno is set. 1325 */ 1326 int 1327 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1328 uint64_t item_flags, 1329 uint8_t target_protocol, 1330 struct rte_flow_error *error) 1331 { 1332 const struct rte_flow_item_icmp6 *mask = item->mask; 1333 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1334 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1335 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1336 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1337 MLX5_FLOW_LAYER_OUTER_L4; 1338 int ret; 1339 1340 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1341 return rte_flow_error_set(error, EINVAL, 1342 RTE_FLOW_ERROR_TYPE_ITEM, item, 1343 "protocol filtering not compatible" 1344 " with ICMP6 layer"); 1345 if (!(item_flags & l3m)) 1346 return rte_flow_error_set(error, EINVAL, 1347 RTE_FLOW_ERROR_TYPE_ITEM, item, 1348 "IPv6 is mandatory to filter on" 1349 " ICMP6"); 1350 if (item_flags & l4m) 1351 return rte_flow_error_set(error, EINVAL, 1352 RTE_FLOW_ERROR_TYPE_ITEM, item, 1353 "multiple L4 layers not supported"); 1354 if (!mask) 1355 mask = &rte_flow_item_icmp6_mask; 1356 ret = mlx5_flow_item_acceptable 1357 (item, (const uint8_t *)mask, 1358 (const uint8_t *)&rte_flow_item_icmp6_mask, 1359 sizeof(struct rte_flow_item_icmp6), error); 1360 if (ret < 0) 1361 return ret; 1362 return 0; 1363 } 1364 1365 /** 1366 * Validate ICMP item. 1367 * 1368 * @param[in] item 1369 * Item specification. 1370 * @param[in] item_flags 1371 * Bit-fields that holds the items detected until now. 1372 * @param[out] error 1373 * Pointer to error structure. 1374 * 1375 * @return 1376 * 0 on success, a negative errno value otherwise and rte_errno is set. 1377 */ 1378 int 1379 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1380 uint64_t item_flags, 1381 uint8_t target_protocol, 1382 struct rte_flow_error *error) 1383 { 1384 const struct rte_flow_item_icmp *mask = item->mask; 1385 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1386 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1387 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1388 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1389 MLX5_FLOW_LAYER_OUTER_L4; 1390 int ret; 1391 1392 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1393 return rte_flow_error_set(error, EINVAL, 1394 RTE_FLOW_ERROR_TYPE_ITEM, item, 1395 "protocol filtering not compatible" 1396 " with ICMP layer"); 1397 if (!(item_flags & l3m)) 1398 return rte_flow_error_set(error, EINVAL, 1399 RTE_FLOW_ERROR_TYPE_ITEM, item, 1400 "IPv4 is mandatory to filter" 1401 " on ICMP"); 1402 if (item_flags & l4m) 1403 return rte_flow_error_set(error, EINVAL, 1404 RTE_FLOW_ERROR_TYPE_ITEM, item, 1405 "multiple L4 layers not supported"); 1406 if (!mask) 1407 mask = &rte_flow_item_icmp_mask; 1408 ret = mlx5_flow_item_acceptable 1409 (item, (const uint8_t *)mask, 1410 (const uint8_t *)&rte_flow_item_icmp_mask, 1411 sizeof(struct rte_flow_item_icmp), error); 1412 if (ret < 0) 1413 return ret; 1414 return 0; 1415 } 1416 1417 /** 1418 * Validate Ethernet item. 1419 * 1420 * @param[in] item 1421 * Item specification. 1422 * @param[in] item_flags 1423 * Bit-fields that holds the items detected until now. 1424 * @param[out] error 1425 * Pointer to error structure. 1426 * 1427 * @return 1428 * 0 on success, a negative errno value otherwise and rte_errno is set. 1429 */ 1430 int 1431 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1432 uint64_t item_flags, 1433 struct rte_flow_error *error) 1434 { 1435 const struct rte_flow_item_eth *mask = item->mask; 1436 const struct rte_flow_item_eth nic_mask = { 1437 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1438 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1439 .type = RTE_BE16(0xffff), 1440 }; 1441 int ret; 1442 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1443 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1444 MLX5_FLOW_LAYER_OUTER_L2; 1445 1446 if (item_flags & ethm) 1447 return rte_flow_error_set(error, ENOTSUP, 1448 RTE_FLOW_ERROR_TYPE_ITEM, item, 1449 "multiple L2 layers not supported"); 1450 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1451 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1452 return rte_flow_error_set(error, EINVAL, 1453 RTE_FLOW_ERROR_TYPE_ITEM, item, 1454 "L2 layer should not follow " 1455 "L3 layers"); 1456 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1457 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1458 return rte_flow_error_set(error, EINVAL, 1459 RTE_FLOW_ERROR_TYPE_ITEM, item, 1460 "L2 layer should not follow VLAN"); 1461 if (!mask) 1462 mask = &rte_flow_item_eth_mask; 1463 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1464 (const uint8_t *)&nic_mask, 1465 sizeof(struct rte_flow_item_eth), 1466 error); 1467 return ret; 1468 } 1469 1470 /** 1471 * Validate VLAN item. 1472 * 1473 * @param[in] item 1474 * Item specification. 1475 * @param[in] item_flags 1476 * Bit-fields that holds the items detected until now. 1477 * @param[in] dev 1478 * Ethernet device flow is being created on. 1479 * @param[out] error 1480 * Pointer to error structure. 1481 * 1482 * @return 1483 * 0 on success, a negative errno value otherwise and rte_errno is set. 1484 */ 1485 int 1486 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1487 uint64_t item_flags, 1488 struct rte_eth_dev *dev, 1489 struct rte_flow_error *error) 1490 { 1491 const struct rte_flow_item_vlan *spec = item->spec; 1492 const struct rte_flow_item_vlan *mask = item->mask; 1493 const struct rte_flow_item_vlan nic_mask = { 1494 .tci = RTE_BE16(UINT16_MAX), 1495 .inner_type = RTE_BE16(UINT16_MAX), 1496 }; 1497 uint16_t vlan_tag = 0; 1498 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1499 int ret; 1500 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1501 MLX5_FLOW_LAYER_INNER_L4) : 1502 (MLX5_FLOW_LAYER_OUTER_L3 | 1503 MLX5_FLOW_LAYER_OUTER_L4); 1504 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1505 MLX5_FLOW_LAYER_OUTER_VLAN; 1506 1507 if (item_flags & vlanm) 1508 return rte_flow_error_set(error, EINVAL, 1509 RTE_FLOW_ERROR_TYPE_ITEM, item, 1510 "multiple VLAN layers not supported"); 1511 else if ((item_flags & l34m) != 0) 1512 return rte_flow_error_set(error, EINVAL, 1513 RTE_FLOW_ERROR_TYPE_ITEM, item, 1514 "VLAN cannot follow L3/L4 layer"); 1515 if (!mask) 1516 mask = &rte_flow_item_vlan_mask; 1517 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1518 (const uint8_t *)&nic_mask, 1519 sizeof(struct rte_flow_item_vlan), 1520 error); 1521 if (ret) 1522 return ret; 1523 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1524 struct mlx5_priv *priv = dev->data->dev_private; 1525 1526 if (priv->vmwa_context) { 1527 /* 1528 * Non-NULL context means we have a virtual machine 1529 * and SR-IOV enabled, we have to create VLAN interface 1530 * to make hypervisor to setup E-Switch vport 1531 * context correctly. We avoid creating the multiple 1532 * VLAN interfaces, so we cannot support VLAN tag mask. 1533 */ 1534 return rte_flow_error_set(error, EINVAL, 1535 RTE_FLOW_ERROR_TYPE_ITEM, 1536 item, 1537 "VLAN tag mask is not" 1538 " supported in virtual" 1539 " environment"); 1540 } 1541 } 1542 if (spec) { 1543 vlan_tag = spec->tci; 1544 vlan_tag &= mask->tci; 1545 } 1546 /* 1547 * From verbs perspective an empty VLAN is equivalent 1548 * to a packet without VLAN layer. 1549 */ 1550 if (!vlan_tag) 1551 return rte_flow_error_set(error, EINVAL, 1552 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1553 item->spec, 1554 "VLAN cannot be empty"); 1555 return 0; 1556 } 1557 1558 /** 1559 * Validate IPV4 item. 1560 * 1561 * @param[in] item 1562 * Item specification. 1563 * @param[in] item_flags 1564 * Bit-fields that holds the items detected until now. 1565 * @param[in] acc_mask 1566 * Acceptable mask, if NULL default internal default mask 1567 * will be used to check whether item fields are supported. 1568 * @param[out] error 1569 * Pointer to error structure. 1570 * 1571 * @return 1572 * 0 on success, a negative errno value otherwise and rte_errno is set. 1573 */ 1574 int 1575 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1576 uint64_t item_flags, 1577 uint64_t last_item, 1578 uint16_t ether_type, 1579 const struct rte_flow_item_ipv4 *acc_mask, 1580 struct rte_flow_error *error) 1581 { 1582 const struct rte_flow_item_ipv4 *mask = item->mask; 1583 const struct rte_flow_item_ipv4 *spec = item->spec; 1584 const struct rte_flow_item_ipv4 nic_mask = { 1585 .hdr = { 1586 .src_addr = RTE_BE32(0xffffffff), 1587 .dst_addr = RTE_BE32(0xffffffff), 1588 .type_of_service = 0xff, 1589 .next_proto_id = 0xff, 1590 }, 1591 }; 1592 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1593 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1594 MLX5_FLOW_LAYER_OUTER_L3; 1595 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1596 MLX5_FLOW_LAYER_OUTER_L4; 1597 int ret; 1598 uint8_t next_proto = 0xFF; 1599 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1600 MLX5_FLOW_LAYER_OUTER_VLAN | 1601 MLX5_FLOW_LAYER_INNER_VLAN); 1602 1603 if ((last_item & l2_vlan) && ether_type && 1604 ether_type != RTE_ETHER_TYPE_IPV4) 1605 return rte_flow_error_set(error, EINVAL, 1606 RTE_FLOW_ERROR_TYPE_ITEM, item, 1607 "IPv4 cannot follow L2/VLAN layer " 1608 "which ether type is not IPv4"); 1609 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1610 if (mask && spec) 1611 next_proto = mask->hdr.next_proto_id & 1612 spec->hdr.next_proto_id; 1613 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1614 return rte_flow_error_set(error, EINVAL, 1615 RTE_FLOW_ERROR_TYPE_ITEM, 1616 item, 1617 "multiple tunnel " 1618 "not supported"); 1619 } 1620 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1621 return rte_flow_error_set(error, EINVAL, 1622 RTE_FLOW_ERROR_TYPE_ITEM, item, 1623 "wrong tunnel type - IPv6 specified " 1624 "but IPv4 item provided"); 1625 if (item_flags & l3m) 1626 return rte_flow_error_set(error, ENOTSUP, 1627 RTE_FLOW_ERROR_TYPE_ITEM, item, 1628 "multiple L3 layers not supported"); 1629 else if (item_flags & l4m) 1630 return rte_flow_error_set(error, EINVAL, 1631 RTE_FLOW_ERROR_TYPE_ITEM, item, 1632 "L3 cannot follow an L4 layer."); 1633 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1634 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1635 return rte_flow_error_set(error, EINVAL, 1636 RTE_FLOW_ERROR_TYPE_ITEM, item, 1637 "L3 cannot follow an NVGRE layer."); 1638 if (!mask) 1639 mask = &rte_flow_item_ipv4_mask; 1640 else if (mask->hdr.next_proto_id != 0 && 1641 mask->hdr.next_proto_id != 0xff) 1642 return rte_flow_error_set(error, EINVAL, 1643 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1644 "partial mask is not supported" 1645 " for protocol"); 1646 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1647 acc_mask ? (const uint8_t *)acc_mask 1648 : (const uint8_t *)&nic_mask, 1649 sizeof(struct rte_flow_item_ipv4), 1650 error); 1651 if (ret < 0) 1652 return ret; 1653 return 0; 1654 } 1655 1656 /** 1657 * Validate IPV6 item. 1658 * 1659 * @param[in] item 1660 * Item specification. 1661 * @param[in] item_flags 1662 * Bit-fields that holds the items detected until now. 1663 * @param[in] acc_mask 1664 * Acceptable mask, if NULL default internal default mask 1665 * will be used to check whether item fields are supported. 1666 * @param[out] error 1667 * Pointer to error structure. 1668 * 1669 * @return 1670 * 0 on success, a negative errno value otherwise and rte_errno is set. 1671 */ 1672 int 1673 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1674 uint64_t item_flags, 1675 uint64_t last_item, 1676 uint16_t ether_type, 1677 const struct rte_flow_item_ipv6 *acc_mask, 1678 struct rte_flow_error *error) 1679 { 1680 const struct rte_flow_item_ipv6 *mask = item->mask; 1681 const struct rte_flow_item_ipv6 *spec = item->spec; 1682 const struct rte_flow_item_ipv6 nic_mask = { 1683 .hdr = { 1684 .src_addr = 1685 "\xff\xff\xff\xff\xff\xff\xff\xff" 1686 "\xff\xff\xff\xff\xff\xff\xff\xff", 1687 .dst_addr = 1688 "\xff\xff\xff\xff\xff\xff\xff\xff" 1689 "\xff\xff\xff\xff\xff\xff\xff\xff", 1690 .vtc_flow = RTE_BE32(0xffffffff), 1691 .proto = 0xff, 1692 }, 1693 }; 1694 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1695 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1696 MLX5_FLOW_LAYER_OUTER_L3; 1697 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1698 MLX5_FLOW_LAYER_OUTER_L4; 1699 int ret; 1700 uint8_t next_proto = 0xFF; 1701 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1702 MLX5_FLOW_LAYER_OUTER_VLAN | 1703 MLX5_FLOW_LAYER_INNER_VLAN); 1704 1705 if ((last_item & l2_vlan) && ether_type && 1706 ether_type != RTE_ETHER_TYPE_IPV6) 1707 return rte_flow_error_set(error, EINVAL, 1708 RTE_FLOW_ERROR_TYPE_ITEM, item, 1709 "IPv6 cannot follow L2/VLAN layer " 1710 "which ether type is not IPv6"); 1711 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1712 if (mask && spec) 1713 next_proto = mask->hdr.proto & spec->hdr.proto; 1714 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1715 return rte_flow_error_set(error, EINVAL, 1716 RTE_FLOW_ERROR_TYPE_ITEM, 1717 item, 1718 "multiple tunnel " 1719 "not supported"); 1720 } 1721 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1722 return rte_flow_error_set(error, EINVAL, 1723 RTE_FLOW_ERROR_TYPE_ITEM, item, 1724 "wrong tunnel type - IPv4 specified " 1725 "but IPv6 item provided"); 1726 if (item_flags & l3m) 1727 return rte_flow_error_set(error, ENOTSUP, 1728 RTE_FLOW_ERROR_TYPE_ITEM, item, 1729 "multiple L3 layers not supported"); 1730 else if (item_flags & l4m) 1731 return rte_flow_error_set(error, EINVAL, 1732 RTE_FLOW_ERROR_TYPE_ITEM, item, 1733 "L3 cannot follow an L4 layer."); 1734 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1735 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1736 return rte_flow_error_set(error, EINVAL, 1737 RTE_FLOW_ERROR_TYPE_ITEM, item, 1738 "L3 cannot follow an NVGRE layer."); 1739 if (!mask) 1740 mask = &rte_flow_item_ipv6_mask; 1741 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1742 acc_mask ? (const uint8_t *)acc_mask 1743 : (const uint8_t *)&nic_mask, 1744 sizeof(struct rte_flow_item_ipv6), 1745 error); 1746 if (ret < 0) 1747 return ret; 1748 return 0; 1749 } 1750 1751 /** 1752 * Validate UDP item. 1753 * 1754 * @param[in] item 1755 * Item specification. 1756 * @param[in] item_flags 1757 * Bit-fields that holds the items detected until now. 1758 * @param[in] target_protocol 1759 * The next protocol in the previous item. 1760 * @param[in] flow_mask 1761 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1762 * @param[out] error 1763 * Pointer to error structure. 1764 * 1765 * @return 1766 * 0 on success, a negative errno value otherwise and rte_errno is set. 1767 */ 1768 int 1769 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1770 uint64_t item_flags, 1771 uint8_t target_protocol, 1772 struct rte_flow_error *error) 1773 { 1774 const struct rte_flow_item_udp *mask = item->mask; 1775 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1776 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1777 MLX5_FLOW_LAYER_OUTER_L3; 1778 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1779 MLX5_FLOW_LAYER_OUTER_L4; 1780 int ret; 1781 1782 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1783 return rte_flow_error_set(error, EINVAL, 1784 RTE_FLOW_ERROR_TYPE_ITEM, item, 1785 "protocol filtering not compatible" 1786 " with UDP layer"); 1787 if (!(item_flags & l3m)) 1788 return rte_flow_error_set(error, EINVAL, 1789 RTE_FLOW_ERROR_TYPE_ITEM, item, 1790 "L3 is mandatory to filter on L4"); 1791 if (item_flags & l4m) 1792 return rte_flow_error_set(error, EINVAL, 1793 RTE_FLOW_ERROR_TYPE_ITEM, item, 1794 "multiple L4 layers not supported"); 1795 if (!mask) 1796 mask = &rte_flow_item_udp_mask; 1797 ret = mlx5_flow_item_acceptable 1798 (item, (const uint8_t *)mask, 1799 (const uint8_t *)&rte_flow_item_udp_mask, 1800 sizeof(struct rte_flow_item_udp), error); 1801 if (ret < 0) 1802 return ret; 1803 return 0; 1804 } 1805 1806 /** 1807 * Validate TCP item. 1808 * 1809 * @param[in] item 1810 * Item specification. 1811 * @param[in] item_flags 1812 * Bit-fields that holds the items detected until now. 1813 * @param[in] target_protocol 1814 * The next protocol in the previous item. 1815 * @param[out] error 1816 * Pointer to error structure. 1817 * 1818 * @return 1819 * 0 on success, a negative errno value otherwise and rte_errno is set. 1820 */ 1821 int 1822 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1823 uint64_t item_flags, 1824 uint8_t target_protocol, 1825 const struct rte_flow_item_tcp *flow_mask, 1826 struct rte_flow_error *error) 1827 { 1828 const struct rte_flow_item_tcp *mask = item->mask; 1829 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1830 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1831 MLX5_FLOW_LAYER_OUTER_L3; 1832 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1833 MLX5_FLOW_LAYER_OUTER_L4; 1834 int ret; 1835 1836 MLX5_ASSERT(flow_mask); 1837 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1838 return rte_flow_error_set(error, EINVAL, 1839 RTE_FLOW_ERROR_TYPE_ITEM, item, 1840 "protocol filtering not compatible" 1841 " with TCP layer"); 1842 if (!(item_flags & l3m)) 1843 return rte_flow_error_set(error, EINVAL, 1844 RTE_FLOW_ERROR_TYPE_ITEM, item, 1845 "L3 is mandatory to filter on L4"); 1846 if (item_flags & l4m) 1847 return rte_flow_error_set(error, EINVAL, 1848 RTE_FLOW_ERROR_TYPE_ITEM, item, 1849 "multiple L4 layers not supported"); 1850 if (!mask) 1851 mask = &rte_flow_item_tcp_mask; 1852 ret = mlx5_flow_item_acceptable 1853 (item, (const uint8_t *)mask, 1854 (const uint8_t *)flow_mask, 1855 sizeof(struct rte_flow_item_tcp), error); 1856 if (ret < 0) 1857 return ret; 1858 return 0; 1859 } 1860 1861 /** 1862 * Validate VXLAN item. 1863 * 1864 * @param[in] item 1865 * Item specification. 1866 * @param[in] item_flags 1867 * Bit-fields that holds the items detected until now. 1868 * @param[in] target_protocol 1869 * The next protocol in the previous item. 1870 * @param[out] error 1871 * Pointer to error structure. 1872 * 1873 * @return 1874 * 0 on success, a negative errno value otherwise and rte_errno is set. 1875 */ 1876 int 1877 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1878 uint64_t item_flags, 1879 struct rte_flow_error *error) 1880 { 1881 const struct rte_flow_item_vxlan *spec = item->spec; 1882 const struct rte_flow_item_vxlan *mask = item->mask; 1883 int ret; 1884 union vni { 1885 uint32_t vlan_id; 1886 uint8_t vni[4]; 1887 } id = { .vlan_id = 0, }; 1888 1889 1890 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1891 return rte_flow_error_set(error, ENOTSUP, 1892 RTE_FLOW_ERROR_TYPE_ITEM, item, 1893 "multiple tunnel layers not" 1894 " supported"); 1895 /* 1896 * Verify only UDPv4 is present as defined in 1897 * https://tools.ietf.org/html/rfc7348 1898 */ 1899 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1900 return rte_flow_error_set(error, EINVAL, 1901 RTE_FLOW_ERROR_TYPE_ITEM, item, 1902 "no outer UDP layer found"); 1903 if (!mask) 1904 mask = &rte_flow_item_vxlan_mask; 1905 ret = mlx5_flow_item_acceptable 1906 (item, (const uint8_t *)mask, 1907 (const uint8_t *)&rte_flow_item_vxlan_mask, 1908 sizeof(struct rte_flow_item_vxlan), 1909 error); 1910 if (ret < 0) 1911 return ret; 1912 if (spec) { 1913 memcpy(&id.vni[1], spec->vni, 3); 1914 memcpy(&id.vni[1], mask->vni, 3); 1915 } 1916 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1917 return rte_flow_error_set(error, ENOTSUP, 1918 RTE_FLOW_ERROR_TYPE_ITEM, item, 1919 "VXLAN tunnel must be fully defined"); 1920 return 0; 1921 } 1922 1923 /** 1924 * Validate VXLAN_GPE item. 1925 * 1926 * @param[in] item 1927 * Item specification. 1928 * @param[in] item_flags 1929 * Bit-fields that holds the items detected until now. 1930 * @param[in] priv 1931 * Pointer to the private data structure. 1932 * @param[in] target_protocol 1933 * The next protocol in the previous item. 1934 * @param[out] error 1935 * Pointer to error structure. 1936 * 1937 * @return 1938 * 0 on success, a negative errno value otherwise and rte_errno is set. 1939 */ 1940 int 1941 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1942 uint64_t item_flags, 1943 struct rte_eth_dev *dev, 1944 struct rte_flow_error *error) 1945 { 1946 struct mlx5_priv *priv = dev->data->dev_private; 1947 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1948 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1949 int ret; 1950 union vni { 1951 uint32_t vlan_id; 1952 uint8_t vni[4]; 1953 } id = { .vlan_id = 0, }; 1954 1955 if (!priv->config.l3_vxlan_en) 1956 return rte_flow_error_set(error, ENOTSUP, 1957 RTE_FLOW_ERROR_TYPE_ITEM, item, 1958 "L3 VXLAN is not enabled by device" 1959 " parameter and/or not configured in" 1960 " firmware"); 1961 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1962 return rte_flow_error_set(error, ENOTSUP, 1963 RTE_FLOW_ERROR_TYPE_ITEM, item, 1964 "multiple tunnel layers not" 1965 " supported"); 1966 /* 1967 * Verify only UDPv4 is present as defined in 1968 * https://tools.ietf.org/html/rfc7348 1969 */ 1970 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1971 return rte_flow_error_set(error, EINVAL, 1972 RTE_FLOW_ERROR_TYPE_ITEM, item, 1973 "no outer UDP layer found"); 1974 if (!mask) 1975 mask = &rte_flow_item_vxlan_gpe_mask; 1976 ret = mlx5_flow_item_acceptable 1977 (item, (const uint8_t *)mask, 1978 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1979 sizeof(struct rte_flow_item_vxlan_gpe), 1980 error); 1981 if (ret < 0) 1982 return ret; 1983 if (spec) { 1984 if (spec->protocol) 1985 return rte_flow_error_set(error, ENOTSUP, 1986 RTE_FLOW_ERROR_TYPE_ITEM, 1987 item, 1988 "VxLAN-GPE protocol" 1989 " not supported"); 1990 memcpy(&id.vni[1], spec->vni, 3); 1991 memcpy(&id.vni[1], mask->vni, 3); 1992 } 1993 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1994 return rte_flow_error_set(error, ENOTSUP, 1995 RTE_FLOW_ERROR_TYPE_ITEM, item, 1996 "VXLAN-GPE tunnel must be fully" 1997 " defined"); 1998 return 0; 1999 } 2000 /** 2001 * Validate GRE Key item. 2002 * 2003 * @param[in] item 2004 * Item specification. 2005 * @param[in] item_flags 2006 * Bit flags to mark detected items. 2007 * @param[in] gre_item 2008 * Pointer to gre_item 2009 * @param[out] error 2010 * Pointer to error structure. 2011 * 2012 * @return 2013 * 0 on success, a negative errno value otherwise and rte_errno is set. 2014 */ 2015 int 2016 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2017 uint64_t item_flags, 2018 const struct rte_flow_item *gre_item, 2019 struct rte_flow_error *error) 2020 { 2021 const rte_be32_t *mask = item->mask; 2022 int ret = 0; 2023 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2024 const struct rte_flow_item_gre *gre_spec; 2025 const struct rte_flow_item_gre *gre_mask; 2026 2027 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2028 return rte_flow_error_set(error, ENOTSUP, 2029 RTE_FLOW_ERROR_TYPE_ITEM, item, 2030 "Multiple GRE key not support"); 2031 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2032 return rte_flow_error_set(error, ENOTSUP, 2033 RTE_FLOW_ERROR_TYPE_ITEM, item, 2034 "No preceding GRE header"); 2035 if (item_flags & MLX5_FLOW_LAYER_INNER) 2036 return rte_flow_error_set(error, ENOTSUP, 2037 RTE_FLOW_ERROR_TYPE_ITEM, item, 2038 "GRE key following a wrong item"); 2039 gre_mask = gre_item->mask; 2040 if (!gre_mask) 2041 gre_mask = &rte_flow_item_gre_mask; 2042 gre_spec = gre_item->spec; 2043 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2044 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2045 return rte_flow_error_set(error, EINVAL, 2046 RTE_FLOW_ERROR_TYPE_ITEM, item, 2047 "Key bit must be on"); 2048 2049 if (!mask) 2050 mask = &gre_key_default_mask; 2051 ret = mlx5_flow_item_acceptable 2052 (item, (const uint8_t *)mask, 2053 (const uint8_t *)&gre_key_default_mask, 2054 sizeof(rte_be32_t), error); 2055 return ret; 2056 } 2057 2058 /** 2059 * Validate GRE item. 2060 * 2061 * @param[in] item 2062 * Item specification. 2063 * @param[in] item_flags 2064 * Bit flags to mark detected items. 2065 * @param[in] target_protocol 2066 * The next protocol in the previous item. 2067 * @param[out] error 2068 * Pointer to error structure. 2069 * 2070 * @return 2071 * 0 on success, a negative errno value otherwise and rte_errno is set. 2072 */ 2073 int 2074 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2075 uint64_t item_flags, 2076 uint8_t target_protocol, 2077 struct rte_flow_error *error) 2078 { 2079 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2080 const struct rte_flow_item_gre *mask = item->mask; 2081 int ret; 2082 const struct rte_flow_item_gre nic_mask = { 2083 .c_rsvd0_ver = RTE_BE16(0xB000), 2084 .protocol = RTE_BE16(UINT16_MAX), 2085 }; 2086 2087 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2088 return rte_flow_error_set(error, EINVAL, 2089 RTE_FLOW_ERROR_TYPE_ITEM, item, 2090 "protocol filtering not compatible" 2091 " with this GRE layer"); 2092 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2093 return rte_flow_error_set(error, ENOTSUP, 2094 RTE_FLOW_ERROR_TYPE_ITEM, item, 2095 "multiple tunnel layers not" 2096 " supported"); 2097 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2098 return rte_flow_error_set(error, ENOTSUP, 2099 RTE_FLOW_ERROR_TYPE_ITEM, item, 2100 "L3 Layer is missing"); 2101 if (!mask) 2102 mask = &rte_flow_item_gre_mask; 2103 ret = mlx5_flow_item_acceptable 2104 (item, (const uint8_t *)mask, 2105 (const uint8_t *)&nic_mask, 2106 sizeof(struct rte_flow_item_gre), error); 2107 if (ret < 0) 2108 return ret; 2109 #ifndef HAVE_MLX5DV_DR 2110 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2111 if (spec && (spec->protocol & mask->protocol)) 2112 return rte_flow_error_set(error, ENOTSUP, 2113 RTE_FLOW_ERROR_TYPE_ITEM, item, 2114 "without MPLS support the" 2115 " specification cannot be used for" 2116 " filtering"); 2117 #endif 2118 #endif 2119 return 0; 2120 } 2121 2122 /** 2123 * Validate Geneve item. 2124 * 2125 * @param[in] item 2126 * Item specification. 2127 * @param[in] itemFlags 2128 * Bit-fields that holds the items detected until now. 2129 * @param[in] enPriv 2130 * Pointer to the private data structure. 2131 * @param[out] error 2132 * Pointer to error structure. 2133 * 2134 * @return 2135 * 0 on success, a negative errno value otherwise and rte_errno is set. 2136 */ 2137 2138 int 2139 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2140 uint64_t item_flags, 2141 struct rte_eth_dev *dev, 2142 struct rte_flow_error *error) 2143 { 2144 struct mlx5_priv *priv = dev->data->dev_private; 2145 const struct rte_flow_item_geneve *spec = item->spec; 2146 const struct rte_flow_item_geneve *mask = item->mask; 2147 int ret; 2148 uint16_t gbhdr; 2149 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2150 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2151 const struct rte_flow_item_geneve nic_mask = { 2152 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2153 .vni = "\xff\xff\xff", 2154 .protocol = RTE_BE16(UINT16_MAX), 2155 }; 2156 2157 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2158 return rte_flow_error_set(error, ENOTSUP, 2159 RTE_FLOW_ERROR_TYPE_ITEM, item, 2160 "L3 Geneve is not enabled by device" 2161 " parameter and/or not configured in" 2162 " firmware"); 2163 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2164 return rte_flow_error_set(error, ENOTSUP, 2165 RTE_FLOW_ERROR_TYPE_ITEM, item, 2166 "multiple tunnel layers not" 2167 " supported"); 2168 /* 2169 * Verify only UDPv4 is present as defined in 2170 * https://tools.ietf.org/html/rfc7348 2171 */ 2172 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2173 return rte_flow_error_set(error, EINVAL, 2174 RTE_FLOW_ERROR_TYPE_ITEM, item, 2175 "no outer UDP layer found"); 2176 if (!mask) 2177 mask = &rte_flow_item_geneve_mask; 2178 ret = mlx5_flow_item_acceptable 2179 (item, (const uint8_t *)mask, 2180 (const uint8_t *)&nic_mask, 2181 sizeof(struct rte_flow_item_geneve), error); 2182 if (ret) 2183 return ret; 2184 if (spec) { 2185 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2186 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2187 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2188 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2189 return rte_flow_error_set(error, ENOTSUP, 2190 RTE_FLOW_ERROR_TYPE_ITEM, 2191 item, 2192 "Geneve protocol unsupported" 2193 " fields are being used"); 2194 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2195 return rte_flow_error_set 2196 (error, ENOTSUP, 2197 RTE_FLOW_ERROR_TYPE_ITEM, 2198 item, 2199 "Unsupported Geneve options length"); 2200 } 2201 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2202 return rte_flow_error_set 2203 (error, ENOTSUP, 2204 RTE_FLOW_ERROR_TYPE_ITEM, item, 2205 "Geneve tunnel must be fully defined"); 2206 return 0; 2207 } 2208 2209 /** 2210 * Validate MPLS item. 2211 * 2212 * @param[in] dev 2213 * Pointer to the rte_eth_dev structure. 2214 * @param[in] item 2215 * Item specification. 2216 * @param[in] item_flags 2217 * Bit-fields that holds the items detected until now. 2218 * @param[in] prev_layer 2219 * The protocol layer indicated in previous item. 2220 * @param[out] error 2221 * Pointer to error structure. 2222 * 2223 * @return 2224 * 0 on success, a negative errno value otherwise and rte_errno is set. 2225 */ 2226 int 2227 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2228 const struct rte_flow_item *item __rte_unused, 2229 uint64_t item_flags __rte_unused, 2230 uint64_t prev_layer __rte_unused, 2231 struct rte_flow_error *error) 2232 { 2233 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2234 const struct rte_flow_item_mpls *mask = item->mask; 2235 struct mlx5_priv *priv = dev->data->dev_private; 2236 int ret; 2237 2238 if (!priv->config.mpls_en) 2239 return rte_flow_error_set(error, ENOTSUP, 2240 RTE_FLOW_ERROR_TYPE_ITEM, item, 2241 "MPLS not supported or" 2242 " disabled in firmware" 2243 " configuration."); 2244 /* MPLS over IP, UDP, GRE is allowed */ 2245 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2246 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2247 MLX5_FLOW_LAYER_GRE))) 2248 return rte_flow_error_set(error, EINVAL, 2249 RTE_FLOW_ERROR_TYPE_ITEM, item, 2250 "protocol filtering not compatible" 2251 " with MPLS layer"); 2252 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2253 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2254 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2255 return rte_flow_error_set(error, ENOTSUP, 2256 RTE_FLOW_ERROR_TYPE_ITEM, item, 2257 "multiple tunnel layers not" 2258 " supported"); 2259 if (!mask) 2260 mask = &rte_flow_item_mpls_mask; 2261 ret = mlx5_flow_item_acceptable 2262 (item, (const uint8_t *)mask, 2263 (const uint8_t *)&rte_flow_item_mpls_mask, 2264 sizeof(struct rte_flow_item_mpls), error); 2265 if (ret < 0) 2266 return ret; 2267 return 0; 2268 #endif 2269 return rte_flow_error_set(error, ENOTSUP, 2270 RTE_FLOW_ERROR_TYPE_ITEM, item, 2271 "MPLS is not supported by Verbs, please" 2272 " update."); 2273 } 2274 2275 /** 2276 * Validate NVGRE item. 2277 * 2278 * @param[in] item 2279 * Item specification. 2280 * @param[in] item_flags 2281 * Bit flags to mark detected items. 2282 * @param[in] target_protocol 2283 * The next protocol in the previous item. 2284 * @param[out] error 2285 * Pointer to error structure. 2286 * 2287 * @return 2288 * 0 on success, a negative errno value otherwise and rte_errno is set. 2289 */ 2290 int 2291 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2292 uint64_t item_flags, 2293 uint8_t target_protocol, 2294 struct rte_flow_error *error) 2295 { 2296 const struct rte_flow_item_nvgre *mask = item->mask; 2297 int ret; 2298 2299 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2300 return rte_flow_error_set(error, EINVAL, 2301 RTE_FLOW_ERROR_TYPE_ITEM, item, 2302 "protocol filtering not compatible" 2303 " with this GRE layer"); 2304 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2305 return rte_flow_error_set(error, ENOTSUP, 2306 RTE_FLOW_ERROR_TYPE_ITEM, item, 2307 "multiple tunnel layers not" 2308 " supported"); 2309 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2310 return rte_flow_error_set(error, ENOTSUP, 2311 RTE_FLOW_ERROR_TYPE_ITEM, item, 2312 "L3 Layer is missing"); 2313 if (!mask) 2314 mask = &rte_flow_item_nvgre_mask; 2315 ret = mlx5_flow_item_acceptable 2316 (item, (const uint8_t *)mask, 2317 (const uint8_t *)&rte_flow_item_nvgre_mask, 2318 sizeof(struct rte_flow_item_nvgre), error); 2319 if (ret < 0) 2320 return ret; 2321 return 0; 2322 } 2323 2324 /* Allocate unique ID for the split Q/RSS subflows. */ 2325 static uint32_t 2326 flow_qrss_get_id(struct rte_eth_dev *dev) 2327 { 2328 struct mlx5_priv *priv = dev->data->dev_private; 2329 uint32_t qrss_id, ret; 2330 2331 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2332 if (ret) 2333 return 0; 2334 MLX5_ASSERT(qrss_id); 2335 return qrss_id; 2336 } 2337 2338 /* Free unique ID for the split Q/RSS subflows. */ 2339 static void 2340 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2341 { 2342 struct mlx5_priv *priv = dev->data->dev_private; 2343 2344 if (qrss_id) 2345 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2346 } 2347 2348 /** 2349 * Release resource related QUEUE/RSS action split. 2350 * 2351 * @param dev 2352 * Pointer to Ethernet device. 2353 * @param flow 2354 * Flow to release id's from. 2355 */ 2356 static void 2357 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2358 struct rte_flow *flow) 2359 { 2360 struct mlx5_priv *priv = dev->data->dev_private; 2361 uint32_t handle_idx; 2362 struct mlx5_flow_handle *dev_handle; 2363 2364 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2365 handle_idx, dev_handle, next) 2366 if (dev_handle->split_flow_id) 2367 flow_qrss_free_id(dev, dev_handle->split_flow_id); 2368 } 2369 2370 static int 2371 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2372 const struct rte_flow_attr *attr __rte_unused, 2373 const struct rte_flow_item items[] __rte_unused, 2374 const struct rte_flow_action actions[] __rte_unused, 2375 bool external __rte_unused, 2376 int hairpin __rte_unused, 2377 struct rte_flow_error *error) 2378 { 2379 return rte_flow_error_set(error, ENOTSUP, 2380 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2381 } 2382 2383 static struct mlx5_flow * 2384 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2385 const struct rte_flow_attr *attr __rte_unused, 2386 const struct rte_flow_item items[] __rte_unused, 2387 const struct rte_flow_action actions[] __rte_unused, 2388 struct rte_flow_error *error) 2389 { 2390 rte_flow_error_set(error, ENOTSUP, 2391 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2392 return NULL; 2393 } 2394 2395 static int 2396 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2397 struct mlx5_flow *dev_flow __rte_unused, 2398 const struct rte_flow_attr *attr __rte_unused, 2399 const struct rte_flow_item items[] __rte_unused, 2400 const struct rte_flow_action actions[] __rte_unused, 2401 struct rte_flow_error *error) 2402 { 2403 return rte_flow_error_set(error, ENOTSUP, 2404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2405 } 2406 2407 static int 2408 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2409 struct rte_flow *flow __rte_unused, 2410 struct rte_flow_error *error) 2411 { 2412 return rte_flow_error_set(error, ENOTSUP, 2413 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2414 } 2415 2416 static void 2417 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2418 struct rte_flow *flow __rte_unused) 2419 { 2420 } 2421 2422 static void 2423 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2424 struct rte_flow *flow __rte_unused) 2425 { 2426 } 2427 2428 static int 2429 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2430 struct rte_flow *flow __rte_unused, 2431 const struct rte_flow_action *actions __rte_unused, 2432 void *data __rte_unused, 2433 struct rte_flow_error *error) 2434 { 2435 return rte_flow_error_set(error, ENOTSUP, 2436 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2437 } 2438 2439 /* Void driver to protect from null pointer reference. */ 2440 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2441 .validate = flow_null_validate, 2442 .prepare = flow_null_prepare, 2443 .translate = flow_null_translate, 2444 .apply = flow_null_apply, 2445 .remove = flow_null_remove, 2446 .destroy = flow_null_destroy, 2447 .query = flow_null_query, 2448 }; 2449 2450 /** 2451 * Select flow driver type according to flow attributes and device 2452 * configuration. 2453 * 2454 * @param[in] dev 2455 * Pointer to the dev structure. 2456 * @param[in] attr 2457 * Pointer to the flow attributes. 2458 * 2459 * @return 2460 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2461 */ 2462 static enum mlx5_flow_drv_type 2463 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2464 { 2465 struct mlx5_priv *priv = dev->data->dev_private; 2466 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2467 2468 if (attr->transfer && priv->config.dv_esw_en) 2469 type = MLX5_FLOW_TYPE_DV; 2470 if (!attr->transfer) 2471 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2472 MLX5_FLOW_TYPE_VERBS; 2473 return type; 2474 } 2475 2476 #define flow_get_drv_ops(type) flow_drv_ops[type] 2477 2478 /** 2479 * Flow driver validation API. This abstracts calling driver specific functions. 2480 * The type of flow driver is determined according to flow attributes. 2481 * 2482 * @param[in] dev 2483 * Pointer to the dev structure. 2484 * @param[in] attr 2485 * Pointer to the flow attributes. 2486 * @param[in] items 2487 * Pointer to the list of items. 2488 * @param[in] actions 2489 * Pointer to the list of actions. 2490 * @param[in] external 2491 * This flow rule is created by request external to PMD. 2492 * @param[in] hairpin 2493 * Number of hairpin TX actions, 0 means classic flow. 2494 * @param[out] error 2495 * Pointer to the error structure. 2496 * 2497 * @return 2498 * 0 on success, a negative errno value otherwise and rte_errno is set. 2499 */ 2500 static inline int 2501 flow_drv_validate(struct rte_eth_dev *dev, 2502 const struct rte_flow_attr *attr, 2503 const struct rte_flow_item items[], 2504 const struct rte_flow_action actions[], 2505 bool external, int hairpin, struct rte_flow_error *error) 2506 { 2507 const struct mlx5_flow_driver_ops *fops; 2508 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2509 2510 fops = flow_get_drv_ops(type); 2511 return fops->validate(dev, attr, items, actions, external, 2512 hairpin, error); 2513 } 2514 2515 /** 2516 * Flow driver preparation API. This abstracts calling driver specific 2517 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2518 * calculates the size of memory required for device flow, allocates the memory, 2519 * initializes the device flow and returns the pointer. 2520 * 2521 * @note 2522 * This function initializes device flow structure such as dv or verbs in 2523 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2524 * rest. For example, adding returning device flow to flow->dev_flow list and 2525 * setting backward reference to the flow should be done out of this function. 2526 * layers field is not filled either. 2527 * 2528 * @param[in] dev 2529 * Pointer to the dev structure. 2530 * @param[in] attr 2531 * Pointer to the flow attributes. 2532 * @param[in] items 2533 * Pointer to the list of items. 2534 * @param[in] actions 2535 * Pointer to the list of actions. 2536 * @param[in] flow_idx 2537 * This memory pool index to the flow. 2538 * @param[out] error 2539 * Pointer to the error structure. 2540 * 2541 * @return 2542 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2543 */ 2544 static inline struct mlx5_flow * 2545 flow_drv_prepare(struct rte_eth_dev *dev, 2546 const struct rte_flow *flow, 2547 const struct rte_flow_attr *attr, 2548 const struct rte_flow_item items[], 2549 const struct rte_flow_action actions[], 2550 uint32_t flow_idx, 2551 struct rte_flow_error *error) 2552 { 2553 const struct mlx5_flow_driver_ops *fops; 2554 enum mlx5_flow_drv_type type = flow->drv_type; 2555 struct mlx5_flow *mlx5_flow = NULL; 2556 2557 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2558 fops = flow_get_drv_ops(type); 2559 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 2560 if (mlx5_flow) 2561 mlx5_flow->flow_idx = flow_idx; 2562 return mlx5_flow; 2563 } 2564 2565 /** 2566 * Flow driver translation API. This abstracts calling driver specific 2567 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2568 * translates a generic flow into a driver flow. flow_drv_prepare() must 2569 * precede. 2570 * 2571 * @note 2572 * dev_flow->layers could be filled as a result of parsing during translation 2573 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2574 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2575 * flow->actions could be overwritten even though all the expanded dev_flows 2576 * have the same actions. 2577 * 2578 * @param[in] dev 2579 * Pointer to the rte dev structure. 2580 * @param[in, out] dev_flow 2581 * Pointer to the mlx5 flow. 2582 * @param[in] attr 2583 * Pointer to the flow attributes. 2584 * @param[in] items 2585 * Pointer to the list of items. 2586 * @param[in] actions 2587 * Pointer to the list of actions. 2588 * @param[out] error 2589 * Pointer to the error structure. 2590 * 2591 * @return 2592 * 0 on success, a negative errno value otherwise and rte_errno is set. 2593 */ 2594 static inline int 2595 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2596 const struct rte_flow_attr *attr, 2597 const struct rte_flow_item items[], 2598 const struct rte_flow_action actions[], 2599 struct rte_flow_error *error) 2600 { 2601 const struct mlx5_flow_driver_ops *fops; 2602 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2603 2604 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2605 fops = flow_get_drv_ops(type); 2606 return fops->translate(dev, dev_flow, attr, items, actions, error); 2607 } 2608 2609 /** 2610 * Flow driver apply API. This abstracts calling driver specific functions. 2611 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2612 * translated driver flows on to device. flow_drv_translate() must precede. 2613 * 2614 * @param[in] dev 2615 * Pointer to Ethernet device structure. 2616 * @param[in, out] flow 2617 * Pointer to flow structure. 2618 * @param[out] error 2619 * Pointer to error structure. 2620 * 2621 * @return 2622 * 0 on success, a negative errno value otherwise and rte_errno is set. 2623 */ 2624 static inline int 2625 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2626 struct rte_flow_error *error) 2627 { 2628 const struct mlx5_flow_driver_ops *fops; 2629 enum mlx5_flow_drv_type type = flow->drv_type; 2630 2631 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2632 fops = flow_get_drv_ops(type); 2633 return fops->apply(dev, flow, error); 2634 } 2635 2636 /** 2637 * Flow driver remove API. This abstracts calling driver specific functions. 2638 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2639 * on device. All the resources of the flow should be freed by calling 2640 * flow_drv_destroy(). 2641 * 2642 * @param[in] dev 2643 * Pointer to Ethernet device. 2644 * @param[in, out] flow 2645 * Pointer to flow structure. 2646 */ 2647 static inline void 2648 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2649 { 2650 const struct mlx5_flow_driver_ops *fops; 2651 enum mlx5_flow_drv_type type = flow->drv_type; 2652 2653 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2654 fops = flow_get_drv_ops(type); 2655 fops->remove(dev, flow); 2656 } 2657 2658 /** 2659 * Flow driver destroy API. This abstracts calling driver specific functions. 2660 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2661 * on device and releases resources of the flow. 2662 * 2663 * @param[in] dev 2664 * Pointer to Ethernet device. 2665 * @param[in, out] flow 2666 * Pointer to flow structure. 2667 */ 2668 static inline void 2669 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2670 { 2671 const struct mlx5_flow_driver_ops *fops; 2672 enum mlx5_flow_drv_type type = flow->drv_type; 2673 2674 flow_mreg_split_qrss_release(dev, flow); 2675 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2676 fops = flow_get_drv_ops(type); 2677 fops->destroy(dev, flow); 2678 } 2679 2680 /** 2681 * Get RSS action from the action list. 2682 * 2683 * @param[in] actions 2684 * Pointer to the list of actions. 2685 * 2686 * @return 2687 * Pointer to the RSS action if exist, else return NULL. 2688 */ 2689 static const struct rte_flow_action_rss* 2690 flow_get_rss_action(const struct rte_flow_action actions[]) 2691 { 2692 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2693 switch (actions->type) { 2694 case RTE_FLOW_ACTION_TYPE_RSS: 2695 return (const struct rte_flow_action_rss *) 2696 actions->conf; 2697 default: 2698 break; 2699 } 2700 } 2701 return NULL; 2702 } 2703 2704 static unsigned int 2705 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2706 { 2707 const struct rte_flow_item *item; 2708 unsigned int has_vlan = 0; 2709 2710 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2711 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2712 has_vlan = 1; 2713 break; 2714 } 2715 } 2716 if (has_vlan) 2717 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2718 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2719 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2720 MLX5_EXPANSION_ROOT_OUTER; 2721 } 2722 2723 /** 2724 * Get layer flags from the prefix flow. 2725 * 2726 * Some flows may be split to several subflows, the prefix subflow gets the 2727 * match items and the suffix sub flow gets the actions. 2728 * Some actions need the user defined match item flags to get the detail for 2729 * the action. 2730 * This function helps the suffix flow to get the item layer flags from prefix 2731 * subflow. 2732 * 2733 * @param[in] dev_flow 2734 * Pointer the created preifx subflow. 2735 * 2736 * @return 2737 * The layers get from prefix subflow. 2738 */ 2739 static inline uint64_t 2740 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 2741 { 2742 uint64_t layers = 0; 2743 2744 /* 2745 * Layers bits could be localization, but usually the compiler will 2746 * help to do the optimization work for source code. 2747 * If no decap actions, use the layers directly. 2748 */ 2749 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 2750 return dev_flow->handle->layers; 2751 /* Convert L3 layers with decap action. */ 2752 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 2753 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2754 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 2755 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2756 /* Convert L4 layers with decap action. */ 2757 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 2758 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 2759 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 2760 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 2761 return layers; 2762 } 2763 2764 /** 2765 * Get metadata split action information. 2766 * 2767 * @param[in] actions 2768 * Pointer to the list of actions. 2769 * @param[out] qrss 2770 * Pointer to the return pointer. 2771 * @param[out] qrss_type 2772 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2773 * if no QUEUE/RSS is found. 2774 * @param[out] encap_idx 2775 * Pointer to the index of the encap action if exists, otherwise the last 2776 * action index. 2777 * 2778 * @return 2779 * Total number of actions. 2780 */ 2781 static int 2782 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 2783 const struct rte_flow_action **qrss, 2784 int *encap_idx) 2785 { 2786 const struct rte_flow_action_raw_encap *raw_encap; 2787 int actions_n = 0; 2788 int raw_decap_idx = -1; 2789 2790 *encap_idx = -1; 2791 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2792 switch (actions->type) { 2793 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2794 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2795 *encap_idx = actions_n; 2796 break; 2797 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 2798 raw_decap_idx = actions_n; 2799 break; 2800 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2801 raw_encap = actions->conf; 2802 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2803 *encap_idx = raw_decap_idx != -1 ? 2804 raw_decap_idx : actions_n; 2805 break; 2806 case RTE_FLOW_ACTION_TYPE_QUEUE: 2807 case RTE_FLOW_ACTION_TYPE_RSS: 2808 *qrss = actions; 2809 break; 2810 default: 2811 break; 2812 } 2813 actions_n++; 2814 } 2815 if (*encap_idx == -1) 2816 *encap_idx = actions_n; 2817 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2818 return actions_n + 1; 2819 } 2820 2821 /** 2822 * Check meter action from the action list. 2823 * 2824 * @param[in] actions 2825 * Pointer to the list of actions. 2826 * @param[out] mtr 2827 * Pointer to the meter exist flag. 2828 * 2829 * @return 2830 * Total number of actions. 2831 */ 2832 static int 2833 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2834 { 2835 int actions_n = 0; 2836 2837 MLX5_ASSERT(mtr); 2838 *mtr = 0; 2839 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2840 switch (actions->type) { 2841 case RTE_FLOW_ACTION_TYPE_METER: 2842 *mtr = 1; 2843 break; 2844 default: 2845 break; 2846 } 2847 actions_n++; 2848 } 2849 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2850 return actions_n + 1; 2851 } 2852 2853 /** 2854 * Check if the flow should be splited due to hairpin. 2855 * The reason for the split is that in current HW we can't 2856 * support encap on Rx, so if a flow have encap we move it 2857 * to Tx. 2858 * 2859 * @param dev 2860 * Pointer to Ethernet device. 2861 * @param[in] attr 2862 * Flow rule attributes. 2863 * @param[in] actions 2864 * Associated actions (list terminated by the END action). 2865 * 2866 * @return 2867 * > 0 the number of actions and the flow should be split, 2868 * 0 when no split required. 2869 */ 2870 static int 2871 flow_check_hairpin_split(struct rte_eth_dev *dev, 2872 const struct rte_flow_attr *attr, 2873 const struct rte_flow_action actions[]) 2874 { 2875 int queue_action = 0; 2876 int action_n = 0; 2877 int encap = 0; 2878 const struct rte_flow_action_queue *queue; 2879 const struct rte_flow_action_rss *rss; 2880 const struct rte_flow_action_raw_encap *raw_encap; 2881 2882 if (!attr->ingress) 2883 return 0; 2884 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2885 switch (actions->type) { 2886 case RTE_FLOW_ACTION_TYPE_QUEUE: 2887 queue = actions->conf; 2888 if (queue == NULL) 2889 return 0; 2890 if (mlx5_rxq_get_type(dev, queue->index) != 2891 MLX5_RXQ_TYPE_HAIRPIN) 2892 return 0; 2893 queue_action = 1; 2894 action_n++; 2895 break; 2896 case RTE_FLOW_ACTION_TYPE_RSS: 2897 rss = actions->conf; 2898 if (rss == NULL || rss->queue_num == 0) 2899 return 0; 2900 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2901 MLX5_RXQ_TYPE_HAIRPIN) 2902 return 0; 2903 queue_action = 1; 2904 action_n++; 2905 break; 2906 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2907 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2908 encap = 1; 2909 action_n++; 2910 break; 2911 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2912 raw_encap = actions->conf; 2913 if (raw_encap->size > 2914 (sizeof(struct rte_flow_item_eth) + 2915 sizeof(struct rte_flow_item_ipv4))) 2916 encap = 1; 2917 action_n++; 2918 break; 2919 default: 2920 action_n++; 2921 break; 2922 } 2923 } 2924 if (encap == 1 && queue_action) 2925 return action_n; 2926 return 0; 2927 } 2928 2929 /* Declare flow create/destroy prototype in advance. */ 2930 static uint32_t 2931 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 2932 const struct rte_flow_attr *attr, 2933 const struct rte_flow_item items[], 2934 const struct rte_flow_action actions[], 2935 bool external, struct rte_flow_error *error); 2936 2937 static void 2938 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 2939 uint32_t flow_idx); 2940 2941 /** 2942 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2943 * 2944 * As mark_id is unique, if there's already a registered flow for the mark_id, 2945 * return by increasing the reference counter of the resource. Otherwise, create 2946 * the resource (mcp_res) and flow. 2947 * 2948 * Flow looks like, 2949 * - If ingress port is ANY and reg_c[1] is mark_id, 2950 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2951 * 2952 * For default flow (zero mark_id), flow is like, 2953 * - If ingress port is ANY, 2954 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2955 * 2956 * @param dev 2957 * Pointer to Ethernet device. 2958 * @param mark_id 2959 * ID of MARK action, zero means default flow for META. 2960 * @param[out] error 2961 * Perform verbose error reporting if not NULL. 2962 * 2963 * @return 2964 * Associated resource on success, NULL otherwise and rte_errno is set. 2965 */ 2966 static struct mlx5_flow_mreg_copy_resource * 2967 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2968 struct rte_flow_error *error) 2969 { 2970 struct mlx5_priv *priv = dev->data->dev_private; 2971 struct rte_flow_attr attr = { 2972 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2973 .ingress = 1, 2974 }; 2975 struct mlx5_rte_flow_item_tag tag_spec = { 2976 .data = mark_id, 2977 }; 2978 struct rte_flow_item items[] = { 2979 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2980 }; 2981 struct rte_flow_action_mark ftag = { 2982 .id = mark_id, 2983 }; 2984 struct mlx5_flow_action_copy_mreg cp_mreg = { 2985 .dst = REG_B, 2986 .src = 0, 2987 }; 2988 struct rte_flow_action_jump jump = { 2989 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2990 }; 2991 struct rte_flow_action actions[] = { 2992 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2993 }; 2994 struct mlx5_flow_mreg_copy_resource *mcp_res; 2995 uint32_t idx = 0; 2996 int ret; 2997 2998 /* Fill the register fileds in the flow. */ 2999 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3000 if (ret < 0) 3001 return NULL; 3002 tag_spec.id = ret; 3003 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3004 if (ret < 0) 3005 return NULL; 3006 cp_mreg.src = ret; 3007 /* Check if already registered. */ 3008 MLX5_ASSERT(priv->mreg_cp_tbl); 3009 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 3010 if (mcp_res) { 3011 /* For non-default rule. */ 3012 if (mark_id != MLX5_DEFAULT_COPY_ID) 3013 mcp_res->refcnt++; 3014 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 3015 mcp_res->refcnt == 1); 3016 return mcp_res; 3017 } 3018 /* Provide the full width of FLAG specific value. */ 3019 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3020 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3021 /* Build a new flow. */ 3022 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3023 items[0] = (struct rte_flow_item){ 3024 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3025 .spec = &tag_spec, 3026 }; 3027 items[1] = (struct rte_flow_item){ 3028 .type = RTE_FLOW_ITEM_TYPE_END, 3029 }; 3030 actions[0] = (struct rte_flow_action){ 3031 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3032 .conf = &ftag, 3033 }; 3034 actions[1] = (struct rte_flow_action){ 3035 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3036 .conf = &cp_mreg, 3037 }; 3038 actions[2] = (struct rte_flow_action){ 3039 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3040 .conf = &jump, 3041 }; 3042 actions[3] = (struct rte_flow_action){ 3043 .type = RTE_FLOW_ACTION_TYPE_END, 3044 }; 3045 } else { 3046 /* Default rule, wildcard match. */ 3047 attr.priority = MLX5_FLOW_PRIO_RSVD; 3048 items[0] = (struct rte_flow_item){ 3049 .type = RTE_FLOW_ITEM_TYPE_END, 3050 }; 3051 actions[0] = (struct rte_flow_action){ 3052 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3053 .conf = &cp_mreg, 3054 }; 3055 actions[1] = (struct rte_flow_action){ 3056 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3057 .conf = &jump, 3058 }; 3059 actions[2] = (struct rte_flow_action){ 3060 .type = RTE_FLOW_ACTION_TYPE_END, 3061 }; 3062 } 3063 /* Build a new entry. */ 3064 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3065 if (!mcp_res) { 3066 rte_errno = ENOMEM; 3067 return NULL; 3068 } 3069 mcp_res->idx = idx; 3070 /* 3071 * The copy Flows are not included in any list. There 3072 * ones are referenced from other Flows and can not 3073 * be applied, removed, deleted in ardbitrary order 3074 * by list traversing. 3075 */ 3076 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3077 actions, false, error); 3078 if (!mcp_res->rix_flow) 3079 goto error; 3080 mcp_res->refcnt++; 3081 mcp_res->hlist_ent.key = mark_id; 3082 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3083 &mcp_res->hlist_ent); 3084 MLX5_ASSERT(!ret); 3085 if (ret) 3086 goto error; 3087 return mcp_res; 3088 error: 3089 if (mcp_res->rix_flow) 3090 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3091 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3092 return NULL; 3093 } 3094 3095 /** 3096 * Release flow in RX_CP_TBL. 3097 * 3098 * @param dev 3099 * Pointer to Ethernet device. 3100 * @flow 3101 * Parent flow for wich copying is provided. 3102 */ 3103 static void 3104 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3105 struct rte_flow *flow) 3106 { 3107 struct mlx5_flow_mreg_copy_resource *mcp_res; 3108 struct mlx5_priv *priv = dev->data->dev_private; 3109 3110 if (!flow->rix_mreg_copy) 3111 return; 3112 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3113 flow->rix_mreg_copy); 3114 if (!mcp_res || !priv->mreg_cp_tbl) 3115 return; 3116 if (flow->copy_applied) { 3117 MLX5_ASSERT(mcp_res->appcnt); 3118 flow->copy_applied = 0; 3119 --mcp_res->appcnt; 3120 if (!mcp_res->appcnt) { 3121 struct rte_flow *mcp_flow = mlx5_ipool_get 3122 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3123 mcp_res->rix_flow); 3124 3125 if (mcp_flow) 3126 flow_drv_remove(dev, mcp_flow); 3127 } 3128 } 3129 /* 3130 * We do not check availability of metadata registers here, 3131 * because copy resources are not allocated in this case. 3132 */ 3133 if (--mcp_res->refcnt) 3134 return; 3135 MLX5_ASSERT(mcp_res->rix_flow); 3136 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3137 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3138 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3139 flow->rix_mreg_copy = 0; 3140 } 3141 3142 /** 3143 * Start flow in RX_CP_TBL. 3144 * 3145 * @param dev 3146 * Pointer to Ethernet device. 3147 * @flow 3148 * Parent flow for wich copying is provided. 3149 * 3150 * @return 3151 * 0 on success, a negative errno value otherwise and rte_errno is set. 3152 */ 3153 static int 3154 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3155 struct rte_flow *flow) 3156 { 3157 struct mlx5_flow_mreg_copy_resource *mcp_res; 3158 struct mlx5_priv *priv = dev->data->dev_private; 3159 int ret; 3160 3161 if (!flow->rix_mreg_copy || flow->copy_applied) 3162 return 0; 3163 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3164 flow->rix_mreg_copy); 3165 if (!mcp_res) 3166 return 0; 3167 if (!mcp_res->appcnt) { 3168 struct rte_flow *mcp_flow = mlx5_ipool_get 3169 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3170 mcp_res->rix_flow); 3171 3172 if (mcp_flow) { 3173 ret = flow_drv_apply(dev, mcp_flow, NULL); 3174 if (ret) 3175 return ret; 3176 } 3177 } 3178 ++mcp_res->appcnt; 3179 flow->copy_applied = 1; 3180 return 0; 3181 } 3182 3183 /** 3184 * Stop flow in RX_CP_TBL. 3185 * 3186 * @param dev 3187 * Pointer to Ethernet device. 3188 * @flow 3189 * Parent flow for wich copying is provided. 3190 */ 3191 static void 3192 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3193 struct rte_flow *flow) 3194 { 3195 struct mlx5_flow_mreg_copy_resource *mcp_res; 3196 struct mlx5_priv *priv = dev->data->dev_private; 3197 3198 if (!flow->rix_mreg_copy || !flow->copy_applied) 3199 return; 3200 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3201 flow->rix_mreg_copy); 3202 if (!mcp_res) 3203 return; 3204 MLX5_ASSERT(mcp_res->appcnt); 3205 --mcp_res->appcnt; 3206 flow->copy_applied = 0; 3207 if (!mcp_res->appcnt) { 3208 struct rte_flow *mcp_flow = mlx5_ipool_get 3209 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3210 mcp_res->rix_flow); 3211 3212 if (mcp_flow) 3213 flow_drv_remove(dev, mcp_flow); 3214 } 3215 } 3216 3217 /** 3218 * Remove the default copy action from RX_CP_TBL. 3219 * 3220 * @param dev 3221 * Pointer to Ethernet device. 3222 */ 3223 static void 3224 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3225 { 3226 struct mlx5_flow_mreg_copy_resource *mcp_res; 3227 struct mlx5_priv *priv = dev->data->dev_private; 3228 3229 /* Check if default flow is registered. */ 3230 if (!priv->mreg_cp_tbl) 3231 return; 3232 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3233 MLX5_DEFAULT_COPY_ID); 3234 if (!mcp_res) 3235 return; 3236 MLX5_ASSERT(mcp_res->rix_flow); 3237 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3238 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3239 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3240 } 3241 3242 /** 3243 * Add the default copy action in in RX_CP_TBL. 3244 * 3245 * @param dev 3246 * Pointer to Ethernet device. 3247 * @param[out] error 3248 * Perform verbose error reporting if not NULL. 3249 * 3250 * @return 3251 * 0 for success, negative value otherwise and rte_errno is set. 3252 */ 3253 static int 3254 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3255 struct rte_flow_error *error) 3256 { 3257 struct mlx5_priv *priv = dev->data->dev_private; 3258 struct mlx5_flow_mreg_copy_resource *mcp_res; 3259 3260 /* Check whether extensive metadata feature is engaged. */ 3261 if (!priv->config.dv_flow_en || 3262 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3263 !mlx5_flow_ext_mreg_supported(dev) || 3264 !priv->sh->dv_regc0_mask) 3265 return 0; 3266 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3267 if (!mcp_res) 3268 return -rte_errno; 3269 return 0; 3270 } 3271 3272 /** 3273 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3274 * 3275 * All the flow having Q/RSS action should be split by 3276 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3277 * performs the following, 3278 * - CQE->flow_tag := reg_c[1] (MARK) 3279 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3280 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3281 * but there should be a flow per each MARK ID set by MARK action. 3282 * 3283 * For the aforementioned reason, if there's a MARK action in flow's action 3284 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3285 * the MARK ID to CQE's flow_tag like, 3286 * - If reg_c[1] is mark_id, 3287 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3288 * 3289 * For SET_META action which stores value in reg_c[0], as the destination is 3290 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3291 * MARK ID means the default flow. The default flow looks like, 3292 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3293 * 3294 * @param dev 3295 * Pointer to Ethernet device. 3296 * @param flow 3297 * Pointer to flow structure. 3298 * @param[in] actions 3299 * Pointer to the list of actions. 3300 * @param[out] error 3301 * Perform verbose error reporting if not NULL. 3302 * 3303 * @return 3304 * 0 on success, negative value otherwise and rte_errno is set. 3305 */ 3306 static int 3307 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3308 struct rte_flow *flow, 3309 const struct rte_flow_action *actions, 3310 struct rte_flow_error *error) 3311 { 3312 struct mlx5_priv *priv = dev->data->dev_private; 3313 struct mlx5_dev_config *config = &priv->config; 3314 struct mlx5_flow_mreg_copy_resource *mcp_res; 3315 const struct rte_flow_action_mark *mark; 3316 3317 /* Check whether extensive metadata feature is engaged. */ 3318 if (!config->dv_flow_en || 3319 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3320 !mlx5_flow_ext_mreg_supported(dev) || 3321 !priv->sh->dv_regc0_mask) 3322 return 0; 3323 /* Find MARK action. */ 3324 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3325 switch (actions->type) { 3326 case RTE_FLOW_ACTION_TYPE_FLAG: 3327 mcp_res = flow_mreg_add_copy_action 3328 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3329 if (!mcp_res) 3330 return -rte_errno; 3331 flow->rix_mreg_copy = mcp_res->idx; 3332 if (dev->data->dev_started) { 3333 mcp_res->appcnt++; 3334 flow->copy_applied = 1; 3335 } 3336 return 0; 3337 case RTE_FLOW_ACTION_TYPE_MARK: 3338 mark = (const struct rte_flow_action_mark *) 3339 actions->conf; 3340 mcp_res = 3341 flow_mreg_add_copy_action(dev, mark->id, error); 3342 if (!mcp_res) 3343 return -rte_errno; 3344 flow->rix_mreg_copy = mcp_res->idx; 3345 if (dev->data->dev_started) { 3346 mcp_res->appcnt++; 3347 flow->copy_applied = 1; 3348 } 3349 return 0; 3350 default: 3351 break; 3352 } 3353 } 3354 return 0; 3355 } 3356 3357 #define MLX5_MAX_SPLIT_ACTIONS 24 3358 #define MLX5_MAX_SPLIT_ITEMS 24 3359 3360 /** 3361 * Split the hairpin flow. 3362 * Since HW can't support encap on Rx we move the encap to Tx. 3363 * If the count action is after the encap then we also 3364 * move the count action. in this case the count will also measure 3365 * the outer bytes. 3366 * 3367 * @param dev 3368 * Pointer to Ethernet device. 3369 * @param[in] actions 3370 * Associated actions (list terminated by the END action). 3371 * @param[out] actions_rx 3372 * Rx flow actions. 3373 * @param[out] actions_tx 3374 * Tx flow actions.. 3375 * @param[out] pattern_tx 3376 * The pattern items for the Tx flow. 3377 * @param[out] flow_id 3378 * The flow ID connected to this flow. 3379 * 3380 * @return 3381 * 0 on success. 3382 */ 3383 static int 3384 flow_hairpin_split(struct rte_eth_dev *dev, 3385 const struct rte_flow_action actions[], 3386 struct rte_flow_action actions_rx[], 3387 struct rte_flow_action actions_tx[], 3388 struct rte_flow_item pattern_tx[], 3389 uint32_t *flow_id) 3390 { 3391 struct mlx5_priv *priv = dev->data->dev_private; 3392 const struct rte_flow_action_raw_encap *raw_encap; 3393 const struct rte_flow_action_raw_decap *raw_decap; 3394 struct mlx5_rte_flow_action_set_tag *set_tag; 3395 struct rte_flow_action *tag_action; 3396 struct mlx5_rte_flow_item_tag *tag_item; 3397 struct rte_flow_item *item; 3398 char *addr; 3399 int encap = 0; 3400 3401 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3402 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3403 switch (actions->type) { 3404 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3405 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3406 rte_memcpy(actions_tx, actions, 3407 sizeof(struct rte_flow_action)); 3408 actions_tx++; 3409 break; 3410 case RTE_FLOW_ACTION_TYPE_COUNT: 3411 if (encap) { 3412 rte_memcpy(actions_tx, actions, 3413 sizeof(struct rte_flow_action)); 3414 actions_tx++; 3415 } else { 3416 rte_memcpy(actions_rx, actions, 3417 sizeof(struct rte_flow_action)); 3418 actions_rx++; 3419 } 3420 break; 3421 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3422 raw_encap = actions->conf; 3423 if (raw_encap->size > 3424 (sizeof(struct rte_flow_item_eth) + 3425 sizeof(struct rte_flow_item_ipv4))) { 3426 memcpy(actions_tx, actions, 3427 sizeof(struct rte_flow_action)); 3428 actions_tx++; 3429 encap = 1; 3430 } else { 3431 rte_memcpy(actions_rx, actions, 3432 sizeof(struct rte_flow_action)); 3433 actions_rx++; 3434 } 3435 break; 3436 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3437 raw_decap = actions->conf; 3438 if (raw_decap->size < 3439 (sizeof(struct rte_flow_item_eth) + 3440 sizeof(struct rte_flow_item_ipv4))) { 3441 memcpy(actions_tx, actions, 3442 sizeof(struct rte_flow_action)); 3443 actions_tx++; 3444 } else { 3445 rte_memcpy(actions_rx, actions, 3446 sizeof(struct rte_flow_action)); 3447 actions_rx++; 3448 } 3449 break; 3450 default: 3451 rte_memcpy(actions_rx, actions, 3452 sizeof(struct rte_flow_action)); 3453 actions_rx++; 3454 break; 3455 } 3456 } 3457 /* Add set meta action and end action for the Rx flow. */ 3458 tag_action = actions_rx; 3459 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3460 actions_rx++; 3461 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3462 actions_rx++; 3463 set_tag = (void *)actions_rx; 3464 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3465 MLX5_ASSERT(set_tag->id > REG_NONE); 3466 set_tag->data = *flow_id; 3467 tag_action->conf = set_tag; 3468 /* Create Tx item list. */ 3469 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3470 addr = (void *)&pattern_tx[2]; 3471 item = pattern_tx; 3472 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3473 tag_item = (void *)addr; 3474 tag_item->data = *flow_id; 3475 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3476 MLX5_ASSERT(set_tag->id > REG_NONE); 3477 item->spec = tag_item; 3478 addr += sizeof(struct mlx5_rte_flow_item_tag); 3479 tag_item = (void *)addr; 3480 tag_item->data = UINT32_MAX; 3481 tag_item->id = UINT16_MAX; 3482 item->mask = tag_item; 3483 addr += sizeof(struct mlx5_rte_flow_item_tag); 3484 item->last = NULL; 3485 item++; 3486 item->type = RTE_FLOW_ITEM_TYPE_END; 3487 return 0; 3488 } 3489 3490 /** 3491 * The last stage of splitting chain, just creates the subflow 3492 * without any modification. 3493 * 3494 * @param[in] dev 3495 * Pointer to Ethernet device. 3496 * @param[in] flow 3497 * Parent flow structure pointer. 3498 * @param[in, out] sub_flow 3499 * Pointer to return the created subflow, may be NULL. 3500 * @param[in] prefix_layers 3501 * Prefix subflow layers, may be 0. 3502 * @param[in] attr 3503 * Flow rule attributes. 3504 * @param[in] items 3505 * Pattern specification (list terminated by the END pattern item). 3506 * @param[in] actions 3507 * Associated actions (list terminated by the END action). 3508 * @param[in] external 3509 * This flow rule is created by request external to PMD. 3510 * @param[in] flow_idx 3511 * This memory pool index to the flow. 3512 * @param[out] error 3513 * Perform verbose error reporting if not NULL. 3514 * @return 3515 * 0 on success, negative value otherwise 3516 */ 3517 static int 3518 flow_create_split_inner(struct rte_eth_dev *dev, 3519 struct rte_flow *flow, 3520 struct mlx5_flow **sub_flow, 3521 uint64_t prefix_layers, 3522 const struct rte_flow_attr *attr, 3523 const struct rte_flow_item items[], 3524 const struct rte_flow_action actions[], 3525 bool external, uint32_t flow_idx, 3526 struct rte_flow_error *error) 3527 { 3528 struct mlx5_flow *dev_flow; 3529 3530 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 3531 flow_idx, error); 3532 if (!dev_flow) 3533 return -rte_errno; 3534 dev_flow->flow = flow; 3535 dev_flow->external = external; 3536 /* Subflow object was created, we must include one in the list. */ 3537 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 3538 dev_flow->handle, next); 3539 /* 3540 * If dev_flow is as one of the suffix flow, some actions in suffix 3541 * flow may need some user defined item layer flags. 3542 */ 3543 if (prefix_layers) 3544 dev_flow->handle->layers = prefix_layers; 3545 if (sub_flow) 3546 *sub_flow = dev_flow; 3547 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3548 } 3549 3550 /** 3551 * Split the meter flow. 3552 * 3553 * As meter flow will split to three sub flow, other than meter 3554 * action, the other actions make sense to only meter accepts 3555 * the packet. If it need to be dropped, no other additional 3556 * actions should be take. 3557 * 3558 * One kind of special action which decapsulates the L3 tunnel 3559 * header will be in the prefix sub flow, as not to take the 3560 * L3 tunnel header into account. 3561 * 3562 * @param dev 3563 * Pointer to Ethernet device. 3564 * @param[in] items 3565 * Pattern specification (list terminated by the END pattern item). 3566 * @param[out] sfx_items 3567 * Suffix flow match items (list terminated by the END pattern item). 3568 * @param[in] actions 3569 * Associated actions (list terminated by the END action). 3570 * @param[out] actions_sfx 3571 * Suffix flow actions. 3572 * @param[out] actions_pre 3573 * Prefix flow actions. 3574 * @param[out] pattern_sfx 3575 * The pattern items for the suffix flow. 3576 * @param[out] tag_sfx 3577 * Pointer to suffix flow tag. 3578 * 3579 * @return 3580 * 0 on success. 3581 */ 3582 static int 3583 flow_meter_split_prep(struct rte_eth_dev *dev, 3584 const struct rte_flow_item items[], 3585 struct rte_flow_item sfx_items[], 3586 const struct rte_flow_action actions[], 3587 struct rte_flow_action actions_sfx[], 3588 struct rte_flow_action actions_pre[]) 3589 { 3590 struct rte_flow_action *tag_action = NULL; 3591 struct rte_flow_item *tag_item; 3592 struct mlx5_rte_flow_action_set_tag *set_tag; 3593 struct rte_flow_error error; 3594 const struct rte_flow_action_raw_encap *raw_encap; 3595 const struct rte_flow_action_raw_decap *raw_decap; 3596 struct mlx5_rte_flow_item_tag *tag_spec; 3597 struct mlx5_rte_flow_item_tag *tag_mask; 3598 uint32_t tag_id; 3599 bool copy_vlan = false; 3600 3601 /* Prepare the actions for prefix and suffix flow. */ 3602 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3603 struct rte_flow_action **action_cur = NULL; 3604 3605 switch (actions->type) { 3606 case RTE_FLOW_ACTION_TYPE_METER: 3607 /* Add the extra tag action first. */ 3608 tag_action = actions_pre; 3609 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3610 actions_pre++; 3611 action_cur = &actions_pre; 3612 break; 3613 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3614 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3615 action_cur = &actions_pre; 3616 break; 3617 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3618 raw_encap = actions->conf; 3619 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 3620 action_cur = &actions_pre; 3621 break; 3622 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3623 raw_decap = actions->conf; 3624 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3625 action_cur = &actions_pre; 3626 break; 3627 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3628 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3629 copy_vlan = true; 3630 break; 3631 default: 3632 break; 3633 } 3634 if (!action_cur) 3635 action_cur = &actions_sfx; 3636 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 3637 (*action_cur)++; 3638 } 3639 /* Add end action to the actions. */ 3640 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3641 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3642 actions_pre++; 3643 /* Set the tag. */ 3644 set_tag = (void *)actions_pre; 3645 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3646 /* 3647 * Get the id from the qrss_pool to make qrss share the id with meter. 3648 */ 3649 tag_id = flow_qrss_get_id(dev); 3650 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 3651 assert(tag_action); 3652 tag_action->conf = set_tag; 3653 /* Prepare the suffix subflow items. */ 3654 tag_item = sfx_items++; 3655 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 3656 int item_type = items->type; 3657 3658 switch (item_type) { 3659 case RTE_FLOW_ITEM_TYPE_PORT_ID: 3660 memcpy(sfx_items, items, sizeof(*sfx_items)); 3661 sfx_items++; 3662 break; 3663 case RTE_FLOW_ITEM_TYPE_VLAN: 3664 if (copy_vlan) { 3665 memcpy(sfx_items, items, sizeof(*sfx_items)); 3666 /* 3667 * Convert to internal match item, it is used 3668 * for vlan push and set vid. 3669 */ 3670 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 3671 sfx_items++; 3672 } 3673 break; 3674 default: 3675 break; 3676 } 3677 } 3678 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 3679 sfx_items++; 3680 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 3681 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 3682 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3683 tag_mask = tag_spec + 1; 3684 tag_mask->data = 0xffffff00; 3685 tag_item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3686 tag_item->spec = tag_spec; 3687 tag_item->last = NULL; 3688 tag_item->mask = tag_mask; 3689 return tag_id; 3690 } 3691 3692 /** 3693 * Split action list having QUEUE/RSS for metadata register copy. 3694 * 3695 * Once Q/RSS action is detected in user's action list, the flow action 3696 * should be split in order to copy metadata registers, which will happen in 3697 * RX_CP_TBL like, 3698 * - CQE->flow_tag := reg_c[1] (MARK) 3699 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3700 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3701 * This is because the last action of each flow must be a terminal action 3702 * (QUEUE, RSS or DROP). 3703 * 3704 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3705 * stored and kept in the mlx5_flow structure per each sub_flow. 3706 * 3707 * The Q/RSS action is replaced with, 3708 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3709 * And the following JUMP action is added at the end, 3710 * - JUMP, to RX_CP_TBL. 3711 * 3712 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3713 * flow_create_split_metadata() routine. The flow will look like, 3714 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3715 * 3716 * @param dev 3717 * Pointer to Ethernet device. 3718 * @param[out] split_actions 3719 * Pointer to store split actions to jump to CP_TBL. 3720 * @param[in] actions 3721 * Pointer to the list of original flow actions. 3722 * @param[in] qrss 3723 * Pointer to the Q/RSS action. 3724 * @param[in] actions_n 3725 * Number of original actions. 3726 * @param[out] error 3727 * Perform verbose error reporting if not NULL. 3728 * 3729 * @return 3730 * non-zero unique flow_id on success, otherwise 0 and 3731 * error/rte_error are set. 3732 */ 3733 static uint32_t 3734 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3735 struct rte_flow_action *split_actions, 3736 const struct rte_flow_action *actions, 3737 const struct rte_flow_action *qrss, 3738 int actions_n, struct rte_flow_error *error) 3739 { 3740 struct mlx5_rte_flow_action_set_tag *set_tag; 3741 struct rte_flow_action_jump *jump; 3742 const int qrss_idx = qrss - actions; 3743 uint32_t flow_id = 0; 3744 int ret = 0; 3745 3746 /* 3747 * Given actions will be split 3748 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3749 * - Add jump to mreg CP_TBL. 3750 * As a result, there will be one more action. 3751 */ 3752 ++actions_n; 3753 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3754 set_tag = (void *)(split_actions + actions_n); 3755 /* 3756 * If tag action is not set to void(it means we are not the meter 3757 * suffix flow), add the tag action. Since meter suffix flow already 3758 * has the tag added. 3759 */ 3760 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3761 /* 3762 * Allocate the new subflow ID. This one is unique within 3763 * device and not shared with representors. Otherwise, 3764 * we would have to resolve multi-thread access synch 3765 * issue. Each flow on the shared device is appended 3766 * with source vport identifier, so the resulting 3767 * flows will be unique in the shared (by master and 3768 * representors) domain even if they have coinciding 3769 * IDs. 3770 */ 3771 flow_id = flow_qrss_get_id(dev); 3772 if (!flow_id) 3773 return rte_flow_error_set(error, ENOMEM, 3774 RTE_FLOW_ERROR_TYPE_ACTION, 3775 NULL, "can't allocate id " 3776 "for split Q/RSS subflow"); 3777 /* Internal SET_TAG action to set flow ID. */ 3778 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3779 .data = flow_id, 3780 }; 3781 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3782 if (ret < 0) 3783 return ret; 3784 set_tag->id = ret; 3785 /* Construct new actions array. */ 3786 /* Replace QUEUE/RSS action. */ 3787 split_actions[qrss_idx] = (struct rte_flow_action){ 3788 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3789 .conf = set_tag, 3790 }; 3791 } 3792 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3793 jump = (void *)(set_tag + 1); 3794 *jump = (struct rte_flow_action_jump){ 3795 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3796 }; 3797 split_actions[actions_n - 2] = (struct rte_flow_action){ 3798 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3799 .conf = jump, 3800 }; 3801 split_actions[actions_n - 1] = (struct rte_flow_action){ 3802 .type = RTE_FLOW_ACTION_TYPE_END, 3803 }; 3804 return flow_id; 3805 } 3806 3807 /** 3808 * Extend the given action list for Tx metadata copy. 3809 * 3810 * Copy the given action list to the ext_actions and add flow metadata register 3811 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3812 * 3813 * @param[out] ext_actions 3814 * Pointer to the extended action list. 3815 * @param[in] actions 3816 * Pointer to the list of actions. 3817 * @param[in] actions_n 3818 * Number of actions in the list. 3819 * @param[out] error 3820 * Perform verbose error reporting if not NULL. 3821 * @param[in] encap_idx 3822 * The encap action inndex. 3823 * 3824 * @return 3825 * 0 on success, negative value otherwise 3826 */ 3827 static int 3828 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3829 struct rte_flow_action *ext_actions, 3830 const struct rte_flow_action *actions, 3831 int actions_n, struct rte_flow_error *error, 3832 int encap_idx) 3833 { 3834 struct mlx5_flow_action_copy_mreg *cp_mreg = 3835 (struct mlx5_flow_action_copy_mreg *) 3836 (ext_actions + actions_n + 1); 3837 int ret; 3838 3839 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3840 if (ret < 0) 3841 return ret; 3842 cp_mreg->dst = ret; 3843 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3844 if (ret < 0) 3845 return ret; 3846 cp_mreg->src = ret; 3847 if (encap_idx != 0) 3848 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 3849 if (encap_idx == actions_n - 1) { 3850 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3851 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3852 .conf = cp_mreg, 3853 }; 3854 ext_actions[actions_n] = (struct rte_flow_action){ 3855 .type = RTE_FLOW_ACTION_TYPE_END, 3856 }; 3857 } else { 3858 ext_actions[encap_idx] = (struct rte_flow_action){ 3859 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3860 .conf = cp_mreg, 3861 }; 3862 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 3863 sizeof(*ext_actions) * (actions_n - encap_idx)); 3864 } 3865 return 0; 3866 } 3867 3868 /** 3869 * The splitting for metadata feature. 3870 * 3871 * - Q/RSS action on NIC Rx should be split in order to pass by 3872 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3873 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3874 * 3875 * - All the actions on NIC Tx should have a mreg copy action to 3876 * copy reg_a from WQE to reg_c[0]. 3877 * 3878 * @param dev 3879 * Pointer to Ethernet device. 3880 * @param[in] flow 3881 * Parent flow structure pointer. 3882 * @param[in] prefix_layers 3883 * Prefix flow layer flags. 3884 * @param[in] attr 3885 * Flow rule attributes. 3886 * @param[in] items 3887 * Pattern specification (list terminated by the END pattern item). 3888 * @param[in] actions 3889 * Associated actions (list terminated by the END action). 3890 * @param[in] external 3891 * This flow rule is created by request external to PMD. 3892 * @param[in] flow_idx 3893 * This memory pool index to the flow. 3894 * @param[out] error 3895 * Perform verbose error reporting if not NULL. 3896 * @return 3897 * 0 on success, negative value otherwise 3898 */ 3899 static int 3900 flow_create_split_metadata(struct rte_eth_dev *dev, 3901 struct rte_flow *flow, 3902 uint64_t prefix_layers, 3903 const struct rte_flow_attr *attr, 3904 const struct rte_flow_item items[], 3905 const struct rte_flow_action actions[], 3906 bool external, uint32_t flow_idx, 3907 struct rte_flow_error *error) 3908 { 3909 struct mlx5_priv *priv = dev->data->dev_private; 3910 struct mlx5_dev_config *config = &priv->config; 3911 const struct rte_flow_action *qrss = NULL; 3912 struct rte_flow_action *ext_actions = NULL; 3913 struct mlx5_flow *dev_flow = NULL; 3914 uint32_t qrss_id = 0; 3915 int mtr_sfx = 0; 3916 size_t act_size; 3917 int actions_n; 3918 int encap_idx; 3919 int ret; 3920 3921 /* Check whether extensive metadata feature is engaged. */ 3922 if (!config->dv_flow_en || 3923 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3924 !mlx5_flow_ext_mreg_supported(dev)) 3925 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 3926 attr, items, actions, external, 3927 flow_idx, error); 3928 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 3929 &encap_idx); 3930 if (qrss) { 3931 /* Exclude hairpin flows from splitting. */ 3932 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3933 const struct rte_flow_action_queue *queue; 3934 3935 queue = qrss->conf; 3936 if (mlx5_rxq_get_type(dev, queue->index) == 3937 MLX5_RXQ_TYPE_HAIRPIN) 3938 qrss = NULL; 3939 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3940 const struct rte_flow_action_rss *rss; 3941 3942 rss = qrss->conf; 3943 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3944 MLX5_RXQ_TYPE_HAIRPIN) 3945 qrss = NULL; 3946 } 3947 } 3948 if (qrss) { 3949 /* Check if it is in meter suffix table. */ 3950 mtr_sfx = attr->group == (attr->transfer ? 3951 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3952 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3953 /* 3954 * Q/RSS action on NIC Rx should be split in order to pass by 3955 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3956 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3957 */ 3958 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3959 sizeof(struct rte_flow_action_set_tag) + 3960 sizeof(struct rte_flow_action_jump); 3961 ext_actions = rte_zmalloc(__func__, act_size, 0); 3962 if (!ext_actions) 3963 return rte_flow_error_set(error, ENOMEM, 3964 RTE_FLOW_ERROR_TYPE_ACTION, 3965 NULL, "no memory to split " 3966 "metadata flow"); 3967 /* 3968 * If we are the suffix flow of meter, tag already exist. 3969 * Set the tag action to void. 3970 */ 3971 if (mtr_sfx) 3972 ext_actions[qrss - actions].type = 3973 RTE_FLOW_ACTION_TYPE_VOID; 3974 else 3975 ext_actions[qrss - actions].type = 3976 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3977 /* 3978 * Create the new actions list with removed Q/RSS action 3979 * and appended set tag and jump to register copy table 3980 * (RX_CP_TBL). We should preallocate unique tag ID here 3981 * in advance, because it is needed for set tag action. 3982 */ 3983 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3984 qrss, actions_n, error); 3985 if (!mtr_sfx && !qrss_id) { 3986 ret = -rte_errno; 3987 goto exit; 3988 } 3989 } else if (attr->egress && !attr->transfer) { 3990 /* 3991 * All the actions on NIC Tx should have a metadata register 3992 * copy action to copy reg_a from WQE to reg_c[meta] 3993 */ 3994 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3995 sizeof(struct mlx5_flow_action_copy_mreg); 3996 ext_actions = rte_zmalloc(__func__, act_size, 0); 3997 if (!ext_actions) 3998 return rte_flow_error_set(error, ENOMEM, 3999 RTE_FLOW_ERROR_TYPE_ACTION, 4000 NULL, "no memory to split " 4001 "metadata flow"); 4002 /* Create the action list appended with copy register. */ 4003 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 4004 actions_n, error, encap_idx); 4005 if (ret < 0) 4006 goto exit; 4007 } 4008 /* Add the unmodified original or prefix subflow. */ 4009 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr, 4010 items, ext_actions ? ext_actions : 4011 actions, external, flow_idx, error); 4012 if (ret < 0) 4013 goto exit; 4014 MLX5_ASSERT(dev_flow); 4015 if (qrss) { 4016 const struct rte_flow_attr q_attr = { 4017 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4018 .ingress = 1, 4019 }; 4020 /* Internal PMD action to set register. */ 4021 struct mlx5_rte_flow_item_tag q_tag_spec = { 4022 .data = qrss_id, 4023 .id = 0, 4024 }; 4025 struct rte_flow_item q_items[] = { 4026 { 4027 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4028 .spec = &q_tag_spec, 4029 .last = NULL, 4030 .mask = NULL, 4031 }, 4032 { 4033 .type = RTE_FLOW_ITEM_TYPE_END, 4034 }, 4035 }; 4036 struct rte_flow_action q_actions[] = { 4037 { 4038 .type = qrss->type, 4039 .conf = qrss->conf, 4040 }, 4041 { 4042 .type = RTE_FLOW_ACTION_TYPE_END, 4043 }, 4044 }; 4045 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4046 4047 /* 4048 * Configure the tag item only if there is no meter subflow. 4049 * Since tag is already marked in the meter suffix subflow 4050 * we can just use the meter suffix items as is. 4051 */ 4052 if (qrss_id) { 4053 /* Not meter subflow. */ 4054 MLX5_ASSERT(!mtr_sfx); 4055 /* 4056 * Put unique id in prefix flow due to it is destroyed 4057 * after suffix flow and id will be freed after there 4058 * is no actual flows with this id and identifier 4059 * reallocation becomes possible (for example, for 4060 * other flows in other threads). 4061 */ 4062 dev_flow->handle->split_flow_id = qrss_id; 4063 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4064 error); 4065 if (ret < 0) 4066 goto exit; 4067 q_tag_spec.id = ret; 4068 } 4069 dev_flow = NULL; 4070 /* Add suffix subflow to execute Q/RSS. */ 4071 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 4072 &q_attr, mtr_sfx ? items : 4073 q_items, q_actions, 4074 external, flow_idx, error); 4075 if (ret < 0) 4076 goto exit; 4077 /* qrss ID should be freed if failed. */ 4078 qrss_id = 0; 4079 MLX5_ASSERT(dev_flow); 4080 } 4081 4082 exit: 4083 /* 4084 * We do not destroy the partially created sub_flows in case of error. 4085 * These ones are included into parent flow list and will be destroyed 4086 * by flow_drv_destroy. 4087 */ 4088 flow_qrss_free_id(dev, qrss_id); 4089 rte_free(ext_actions); 4090 return ret; 4091 } 4092 4093 /** 4094 * The splitting for meter feature. 4095 * 4096 * - The meter flow will be split to two flows as prefix and 4097 * suffix flow. The packets make sense only it pass the prefix 4098 * meter action. 4099 * 4100 * - Reg_C_5 is used for the packet to match betweend prefix and 4101 * suffix flow. 4102 * 4103 * @param dev 4104 * Pointer to Ethernet device. 4105 * @param[in] flow 4106 * Parent flow structure pointer. 4107 * @param[in] attr 4108 * Flow rule attributes. 4109 * @param[in] items 4110 * Pattern specification (list terminated by the END pattern item). 4111 * @param[in] actions 4112 * Associated actions (list terminated by the END action). 4113 * @param[in] external 4114 * This flow rule is created by request external to PMD. 4115 * @param[in] flow_idx 4116 * This memory pool index to the flow. 4117 * @param[out] error 4118 * Perform verbose error reporting if not NULL. 4119 * @return 4120 * 0 on success, negative value otherwise 4121 */ 4122 static int 4123 flow_create_split_meter(struct rte_eth_dev *dev, 4124 struct rte_flow *flow, 4125 const struct rte_flow_attr *attr, 4126 const struct rte_flow_item items[], 4127 const struct rte_flow_action actions[], 4128 bool external, uint32_t flow_idx, 4129 struct rte_flow_error *error) 4130 { 4131 struct mlx5_priv *priv = dev->data->dev_private; 4132 struct rte_flow_action *sfx_actions = NULL; 4133 struct rte_flow_action *pre_actions = NULL; 4134 struct rte_flow_item *sfx_items = NULL; 4135 struct mlx5_flow *dev_flow = NULL; 4136 struct rte_flow_attr sfx_attr = *attr; 4137 uint32_t mtr = 0; 4138 uint32_t mtr_tag_id = 0; 4139 size_t act_size; 4140 size_t item_size; 4141 int actions_n = 0; 4142 int ret; 4143 4144 if (priv->mtr_en) 4145 actions_n = flow_check_meter_action(actions, &mtr); 4146 if (mtr) { 4147 /* The five prefix actions: meter, decap, encap, tag, end. */ 4148 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4149 sizeof(struct mlx5_rte_flow_action_set_tag); 4150 /* tag, vlan, port id, end. */ 4151 #define METER_SUFFIX_ITEM 4 4152 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4153 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4154 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 4155 if (!sfx_actions) 4156 return rte_flow_error_set(error, ENOMEM, 4157 RTE_FLOW_ERROR_TYPE_ACTION, 4158 NULL, "no memory to split " 4159 "meter flow"); 4160 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4161 act_size); 4162 pre_actions = sfx_actions + actions_n; 4163 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4164 actions, sfx_actions, 4165 pre_actions); 4166 if (!mtr_tag_id) { 4167 ret = -rte_errno; 4168 goto exit; 4169 } 4170 /* Add the prefix subflow. */ 4171 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr, 4172 items, pre_actions, external, 4173 flow_idx, error); 4174 if (ret) { 4175 ret = -rte_errno; 4176 goto exit; 4177 } 4178 dev_flow->handle->split_flow_id = mtr_tag_id; 4179 /* Setting the sfx group atrr. */ 4180 sfx_attr.group = sfx_attr.transfer ? 4181 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4182 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4183 } 4184 /* Add the prefix subflow. */ 4185 ret = flow_create_split_metadata(dev, flow, dev_flow ? 4186 flow_get_prefix_layer_flags(dev_flow) : 4187 0, &sfx_attr, 4188 sfx_items ? sfx_items : items, 4189 sfx_actions ? sfx_actions : actions, 4190 external, flow_idx, error); 4191 exit: 4192 if (sfx_actions) 4193 rte_free(sfx_actions); 4194 return ret; 4195 } 4196 4197 /** 4198 * Split the flow to subflow set. The splitters might be linked 4199 * in the chain, like this: 4200 * flow_create_split_outer() calls: 4201 * flow_create_split_meter() calls: 4202 * flow_create_split_metadata(meter_subflow_0) calls: 4203 * flow_create_split_inner(metadata_subflow_0) 4204 * flow_create_split_inner(metadata_subflow_1) 4205 * flow_create_split_inner(metadata_subflow_2) 4206 * flow_create_split_metadata(meter_subflow_1) calls: 4207 * flow_create_split_inner(metadata_subflow_0) 4208 * flow_create_split_inner(metadata_subflow_1) 4209 * flow_create_split_inner(metadata_subflow_2) 4210 * 4211 * This provide flexible way to add new levels of flow splitting. 4212 * The all of successfully created subflows are included to the 4213 * parent flow dev_flow list. 4214 * 4215 * @param dev 4216 * Pointer to Ethernet device. 4217 * @param[in] flow 4218 * Parent flow structure pointer. 4219 * @param[in] attr 4220 * Flow rule attributes. 4221 * @param[in] items 4222 * Pattern specification (list terminated by the END pattern item). 4223 * @param[in] actions 4224 * Associated actions (list terminated by the END action). 4225 * @param[in] external 4226 * This flow rule is created by request external to PMD. 4227 * @param[in] flow_idx 4228 * This memory pool index to the flow. 4229 * @param[out] error 4230 * Perform verbose error reporting if not NULL. 4231 * @return 4232 * 0 on success, negative value otherwise 4233 */ 4234 static int 4235 flow_create_split_outer(struct rte_eth_dev *dev, 4236 struct rte_flow *flow, 4237 const struct rte_flow_attr *attr, 4238 const struct rte_flow_item items[], 4239 const struct rte_flow_action actions[], 4240 bool external, uint32_t flow_idx, 4241 struct rte_flow_error *error) 4242 { 4243 int ret; 4244 4245 ret = flow_create_split_meter(dev, flow, attr, items, 4246 actions, external, flow_idx, error); 4247 MLX5_ASSERT(ret <= 0); 4248 return ret; 4249 } 4250 4251 /** 4252 * Create a flow and add it to @p list. 4253 * 4254 * @param dev 4255 * Pointer to Ethernet device. 4256 * @param list 4257 * Pointer to a TAILQ flow list. If this parameter NULL, 4258 * no list insertion occurred, flow is just created, 4259 * this is caller's responsibility to track the 4260 * created flow. 4261 * @param[in] attr 4262 * Flow rule attributes. 4263 * @param[in] items 4264 * Pattern specification (list terminated by the END pattern item). 4265 * @param[in] actions 4266 * Associated actions (list terminated by the END action). 4267 * @param[in] external 4268 * This flow rule is created by request external to PMD. 4269 * @param[out] error 4270 * Perform verbose error reporting if not NULL. 4271 * 4272 * @return 4273 * A flow index on success, 0 otherwise and rte_errno is set. 4274 */ 4275 static uint32_t 4276 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 4277 const struct rte_flow_attr *attr, 4278 const struct rte_flow_item items[], 4279 const struct rte_flow_action actions[], 4280 bool external, struct rte_flow_error *error) 4281 { 4282 struct mlx5_priv *priv = dev->data->dev_private; 4283 struct rte_flow *flow = NULL; 4284 struct mlx5_flow *dev_flow; 4285 const struct rte_flow_action_rss *rss; 4286 union { 4287 struct rte_flow_expand_rss buf; 4288 uint8_t buffer[2048]; 4289 } expand_buffer; 4290 union { 4291 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4292 uint8_t buffer[2048]; 4293 } actions_rx; 4294 union { 4295 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4296 uint8_t buffer[2048]; 4297 } actions_hairpin_tx; 4298 union { 4299 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4300 uint8_t buffer[2048]; 4301 } items_tx; 4302 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4303 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 4304 priv->rss_desc)[!!priv->flow_idx]; 4305 const struct rte_flow_action *p_actions_rx = actions; 4306 uint32_t i; 4307 uint32_t idx = 0; 4308 int hairpin_flow; 4309 uint32_t hairpin_id = 0; 4310 struct rte_flow_attr attr_tx = { .priority = 0 }; 4311 int ret; 4312 4313 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4314 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 4315 external, hairpin_flow, error); 4316 if (ret < 0) 4317 return 0; 4318 if (hairpin_flow > 0) { 4319 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4320 rte_errno = EINVAL; 4321 return 0; 4322 } 4323 flow_hairpin_split(dev, actions, actions_rx.actions, 4324 actions_hairpin_tx.actions, items_tx.items, 4325 &hairpin_id); 4326 p_actions_rx = actions_rx.actions; 4327 } 4328 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 4329 if (!flow) { 4330 rte_errno = ENOMEM; 4331 goto error_before_flow; 4332 } 4333 flow->drv_type = flow_get_drv_type(dev, attr); 4334 if (hairpin_id != 0) 4335 flow->hairpin_flow_id = hairpin_id; 4336 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4337 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4338 memset(rss_desc, 0, sizeof(*rss_desc)); 4339 rss = flow_get_rss_action(p_actions_rx); 4340 if (rss) { 4341 /* 4342 * The following information is required by 4343 * mlx5_flow_hashfields_adjust() in advance. 4344 */ 4345 rss_desc->level = rss->level; 4346 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4347 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 4348 } 4349 flow->dev_handles = 0; 4350 if (rss && rss->types) { 4351 unsigned int graph_root; 4352 4353 graph_root = find_graph_root(items, rss->level); 4354 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4355 items, rss->types, 4356 mlx5_support_expansion, 4357 graph_root); 4358 MLX5_ASSERT(ret > 0 && 4359 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4360 } else { 4361 buf->entries = 1; 4362 buf->entry[0].pattern = (void *)(uintptr_t)items; 4363 } 4364 /* 4365 * Record the start index when there is a nested call. All sub-flows 4366 * need to be translated before another calling. 4367 * No need to use ping-pong buffer to save memory here. 4368 */ 4369 if (priv->flow_idx) { 4370 MLX5_ASSERT(!priv->flow_nested_idx); 4371 priv->flow_nested_idx = priv->flow_idx; 4372 } 4373 for (i = 0; i < buf->entries; ++i) { 4374 /* 4375 * The splitter may create multiple dev_flows, 4376 * depending on configuration. In the simplest 4377 * case it just creates unmodified original flow. 4378 */ 4379 ret = flow_create_split_outer(dev, flow, attr, 4380 buf->entry[i].pattern, 4381 p_actions_rx, external, idx, 4382 error); 4383 if (ret < 0) 4384 goto error; 4385 } 4386 /* Create the tx flow. */ 4387 if (hairpin_flow) { 4388 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4389 attr_tx.ingress = 0; 4390 attr_tx.egress = 1; 4391 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 4392 actions_hairpin_tx.actions, 4393 idx, error); 4394 if (!dev_flow) 4395 goto error; 4396 dev_flow->flow = flow; 4397 dev_flow->external = 0; 4398 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4399 dev_flow->handle, next); 4400 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4401 items_tx.items, 4402 actions_hairpin_tx.actions, error); 4403 if (ret < 0) 4404 goto error; 4405 } 4406 /* 4407 * Update the metadata register copy table. If extensive 4408 * metadata feature is enabled and registers are supported 4409 * we might create the extra rte_flow for each unique 4410 * MARK/FLAG action ID. 4411 * 4412 * The table is updated for ingress Flows only, because 4413 * the egress Flows belong to the different device and 4414 * copy table should be updated in peer NIC Rx domain. 4415 */ 4416 if (attr->ingress && 4417 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4418 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4419 if (ret) 4420 goto error; 4421 } 4422 /* 4423 * If the flow is external (from application) OR device is started, then 4424 * the flow will be applied immediately. 4425 */ 4426 if (external || dev->data->dev_started) { 4427 ret = flow_drv_apply(dev, flow, error); 4428 if (ret < 0) 4429 goto error; 4430 } 4431 if (list) 4432 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 4433 flow, next); 4434 flow_rxq_flags_set(dev, flow); 4435 /* Nested flow creation index recovery. */ 4436 priv->flow_idx = priv->flow_nested_idx; 4437 if (priv->flow_nested_idx) 4438 priv->flow_nested_idx = 0; 4439 return idx; 4440 error: 4441 MLX5_ASSERT(flow); 4442 ret = rte_errno; /* Save rte_errno before cleanup. */ 4443 flow_mreg_del_copy_action(dev, flow); 4444 flow_drv_destroy(dev, flow); 4445 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 4446 rte_errno = ret; /* Restore rte_errno. */ 4447 error_before_flow: 4448 ret = rte_errno; 4449 if (hairpin_id) 4450 mlx5_flow_id_release(priv->sh->flow_id_pool, 4451 hairpin_id); 4452 rte_errno = ret; 4453 priv->flow_idx = priv->flow_nested_idx; 4454 if (priv->flow_nested_idx) 4455 priv->flow_nested_idx = 0; 4456 return 0; 4457 } 4458 4459 /** 4460 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4461 * incoming packets to table 1. 4462 * 4463 * Other flow rules, requested for group n, will be created in 4464 * e-switch table n+1. 4465 * Jump action to e-switch group n will be created to group n+1. 4466 * 4467 * Used when working in switchdev mode, to utilise advantages of table 1 4468 * and above. 4469 * 4470 * @param dev 4471 * Pointer to Ethernet device. 4472 * 4473 * @return 4474 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4475 */ 4476 struct rte_flow * 4477 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4478 { 4479 const struct rte_flow_attr attr = { 4480 .group = 0, 4481 .priority = 0, 4482 .ingress = 1, 4483 .egress = 0, 4484 .transfer = 1, 4485 }; 4486 const struct rte_flow_item pattern = { 4487 .type = RTE_FLOW_ITEM_TYPE_END, 4488 }; 4489 struct rte_flow_action_jump jump = { 4490 .group = 1, 4491 }; 4492 const struct rte_flow_action actions[] = { 4493 { 4494 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4495 .conf = &jump, 4496 }, 4497 { 4498 .type = RTE_FLOW_ACTION_TYPE_END, 4499 }, 4500 }; 4501 struct mlx5_priv *priv = dev->data->dev_private; 4502 struct rte_flow_error error; 4503 4504 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 4505 &attr, &pattern, 4506 actions, false, &error); 4507 } 4508 4509 /** 4510 * Validate a flow supported by the NIC. 4511 * 4512 * @see rte_flow_validate() 4513 * @see rte_flow_ops 4514 */ 4515 int 4516 mlx5_flow_validate(struct rte_eth_dev *dev, 4517 const struct rte_flow_attr *attr, 4518 const struct rte_flow_item items[], 4519 const struct rte_flow_action actions[], 4520 struct rte_flow_error *error) 4521 { 4522 int hairpin_flow; 4523 4524 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4525 return flow_drv_validate(dev, attr, items, actions, 4526 true, hairpin_flow, error); 4527 } 4528 4529 /** 4530 * Create a flow. 4531 * 4532 * @see rte_flow_create() 4533 * @see rte_flow_ops 4534 */ 4535 struct rte_flow * 4536 mlx5_flow_create(struct rte_eth_dev *dev, 4537 const struct rte_flow_attr *attr, 4538 const struct rte_flow_item items[], 4539 const struct rte_flow_action actions[], 4540 struct rte_flow_error *error) 4541 { 4542 struct mlx5_priv *priv = dev->data->dev_private; 4543 4544 /* 4545 * If the device is not started yet, it is not allowed to created a 4546 * flow from application. PMD default flows and traffic control flows 4547 * are not affected. 4548 */ 4549 if (unlikely(!dev->data->dev_started)) { 4550 DRV_LOG(DEBUG, "port %u is not started when " 4551 "inserting a flow", dev->data->port_id); 4552 rte_flow_error_set(error, ENODEV, 4553 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4554 NULL, 4555 "port not started"); 4556 return NULL; 4557 } 4558 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 4559 attr, items, actions, true, error); 4560 } 4561 4562 /** 4563 * Destroy a flow in a list. 4564 * 4565 * @param dev 4566 * Pointer to Ethernet device. 4567 * @param list 4568 * Pointer to the Indexed flow list. If this parameter NULL, 4569 * there is no flow removal from the list. Be noted that as 4570 * flow is add to the indexed list, memory of the indexed 4571 * list points to maybe changed as flow destroyed. 4572 * @param[in] flow_idx 4573 * Index of flow to destroy. 4574 */ 4575 static void 4576 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 4577 uint32_t flow_idx) 4578 { 4579 struct mlx5_priv *priv = dev->data->dev_private; 4580 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 4581 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 4582 [MLX5_IPOOL_RTE_FLOW], flow_idx); 4583 4584 if (!flow) 4585 return; 4586 /* 4587 * Update RX queue flags only if port is started, otherwise it is 4588 * already clean. 4589 */ 4590 if (dev->data->dev_started) 4591 flow_rxq_flags_trim(dev, flow); 4592 if (flow->hairpin_flow_id) 4593 mlx5_flow_id_release(priv->sh->flow_id_pool, 4594 flow->hairpin_flow_id); 4595 flow_drv_destroy(dev, flow); 4596 if (list) 4597 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 4598 flow_idx, flow, next); 4599 flow_mreg_del_copy_action(dev, flow); 4600 if (flow->fdir) { 4601 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 4602 if (priv_fdir_flow->rix_flow == flow_idx) 4603 break; 4604 } 4605 if (priv_fdir_flow) { 4606 LIST_REMOVE(priv_fdir_flow, next); 4607 rte_free(priv_fdir_flow->fdir); 4608 rte_free(priv_fdir_flow); 4609 } 4610 } 4611 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 4612 } 4613 4614 /** 4615 * Destroy all flows. 4616 * 4617 * @param dev 4618 * Pointer to Ethernet device. 4619 * @param list 4620 * Pointer to the Indexed flow list. 4621 * @param active 4622 * If flushing is called avtively. 4623 */ 4624 void 4625 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 4626 { 4627 uint32_t num_flushed = 0; 4628 4629 while (*list) { 4630 flow_list_destroy(dev, list, *list); 4631 num_flushed++; 4632 } 4633 if (active) { 4634 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 4635 dev->data->port_id, num_flushed); 4636 } 4637 } 4638 4639 /** 4640 * Remove all flows. 4641 * 4642 * @param dev 4643 * Pointer to Ethernet device. 4644 * @param list 4645 * Pointer to the Indexed flow list. 4646 */ 4647 void 4648 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list) 4649 { 4650 struct mlx5_priv *priv = dev->data->dev_private; 4651 struct rte_flow *flow = NULL; 4652 uint32_t idx; 4653 4654 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4655 flow, next) { 4656 flow_drv_remove(dev, flow); 4657 flow_mreg_stop_copy_action(dev, flow); 4658 } 4659 flow_mreg_del_default_copy_action(dev); 4660 flow_rxq_flags_clear(dev); 4661 } 4662 4663 /** 4664 * Add all flows. 4665 * 4666 * @param dev 4667 * Pointer to Ethernet device. 4668 * @param list 4669 * Pointer to the Indexed flow list. 4670 * 4671 * @return 4672 * 0 on success, a negative errno value otherwise and rte_errno is set. 4673 */ 4674 int 4675 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list) 4676 { 4677 struct mlx5_priv *priv = dev->data->dev_private; 4678 struct rte_flow *flow = NULL; 4679 struct rte_flow_error error; 4680 uint32_t idx; 4681 int ret = 0; 4682 4683 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4684 ret = flow_mreg_add_default_copy_action(dev, &error); 4685 if (ret < 0) 4686 return -rte_errno; 4687 /* Apply Flows created by application. */ 4688 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4689 flow, next) { 4690 ret = flow_mreg_start_copy_action(dev, flow); 4691 if (ret < 0) 4692 goto error; 4693 ret = flow_drv_apply(dev, flow, &error); 4694 if (ret < 0) 4695 goto error; 4696 flow_rxq_flags_set(dev, flow); 4697 } 4698 return 0; 4699 error: 4700 ret = rte_errno; /* Save rte_errno before cleanup. */ 4701 mlx5_flow_stop(dev, list); 4702 rte_errno = ret; /* Restore rte_errno. */ 4703 return -rte_errno; 4704 } 4705 4706 /** 4707 * Stop all default actions for flows. 4708 * 4709 * @param dev 4710 * Pointer to Ethernet device. 4711 */ 4712 void 4713 mlx5_flow_stop_default(struct rte_eth_dev *dev) 4714 { 4715 flow_mreg_del_default_copy_action(dev); 4716 flow_rxq_flags_clear(dev); 4717 } 4718 4719 /** 4720 * Start all default actions for flows. 4721 * 4722 * @param dev 4723 * Pointer to Ethernet device. 4724 * @return 4725 * 0 on success, a negative errno value otherwise and rte_errno is set. 4726 */ 4727 int 4728 mlx5_flow_start_default(struct rte_eth_dev *dev) 4729 { 4730 struct rte_flow_error error; 4731 4732 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4733 return flow_mreg_add_default_copy_action(dev, &error); 4734 } 4735 4736 /** 4737 * Allocate intermediate resources for flow creation. 4738 * 4739 * @param dev 4740 * Pointer to Ethernet device. 4741 */ 4742 void 4743 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) 4744 { 4745 struct mlx5_priv *priv = dev->data->dev_private; 4746 4747 if (!priv->inter_flows) { 4748 priv->inter_flows = rte_calloc(__func__, 1, 4749 MLX5_NUM_MAX_DEV_FLOWS * 4750 sizeof(struct mlx5_flow) + 4751 (sizeof(struct mlx5_flow_rss_desc) + 4752 sizeof(uint16_t) * UINT16_MAX) * 2, 0); 4753 if (!priv->inter_flows) { 4754 DRV_LOG(ERR, "can't allocate intermediate memory."); 4755 return; 4756 } 4757 } 4758 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows) 4759 [MLX5_NUM_MAX_DEV_FLOWS]; 4760 /* Reset the index. */ 4761 priv->flow_idx = 0; 4762 priv->flow_nested_idx = 0; 4763 } 4764 4765 /** 4766 * Free intermediate resources for flows. 4767 * 4768 * @param dev 4769 * Pointer to Ethernet device. 4770 */ 4771 void 4772 mlx5_flow_free_intermediate(struct rte_eth_dev *dev) 4773 { 4774 struct mlx5_priv *priv = dev->data->dev_private; 4775 4776 rte_free(priv->inter_flows); 4777 priv->inter_flows = NULL; 4778 } 4779 4780 /** 4781 * Verify the flow list is empty 4782 * 4783 * @param dev 4784 * Pointer to Ethernet device. 4785 * 4786 * @return the number of flows not released. 4787 */ 4788 int 4789 mlx5_flow_verify(struct rte_eth_dev *dev) 4790 { 4791 struct mlx5_priv *priv = dev->data->dev_private; 4792 struct rte_flow *flow; 4793 uint32_t idx; 4794 int ret = 0; 4795 4796 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 4797 flow, next) { 4798 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4799 dev->data->port_id, (void *)flow); 4800 ++ret; 4801 } 4802 return ret; 4803 } 4804 4805 /** 4806 * Enable default hairpin egress flow. 4807 * 4808 * @param dev 4809 * Pointer to Ethernet device. 4810 * @param queue 4811 * The queue index. 4812 * 4813 * @return 4814 * 0 on success, a negative errno value otherwise and rte_errno is set. 4815 */ 4816 int 4817 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4818 uint32_t queue) 4819 { 4820 struct mlx5_priv *priv = dev->data->dev_private; 4821 const struct rte_flow_attr attr = { 4822 .egress = 1, 4823 .priority = 0, 4824 }; 4825 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4826 .queue = queue, 4827 }; 4828 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4829 .queue = UINT32_MAX, 4830 }; 4831 struct rte_flow_item items[] = { 4832 { 4833 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4834 .spec = &queue_spec, 4835 .last = NULL, 4836 .mask = &queue_mask, 4837 }, 4838 { 4839 .type = RTE_FLOW_ITEM_TYPE_END, 4840 }, 4841 }; 4842 struct rte_flow_action_jump jump = { 4843 .group = MLX5_HAIRPIN_TX_TABLE, 4844 }; 4845 struct rte_flow_action actions[2]; 4846 uint32_t flow_idx; 4847 struct rte_flow_error error; 4848 4849 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4850 actions[0].conf = &jump; 4851 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4852 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4853 &attr, items, actions, false, &error); 4854 if (!flow_idx) { 4855 DRV_LOG(DEBUG, 4856 "Failed to create ctrl flow: rte_errno(%d)," 4857 " type(%d), message(%s)", 4858 rte_errno, error.type, 4859 error.message ? error.message : " (no stated reason)"); 4860 return -rte_errno; 4861 } 4862 return 0; 4863 } 4864 4865 /** 4866 * Enable a control flow configured from the control plane. 4867 * 4868 * @param dev 4869 * Pointer to Ethernet device. 4870 * @param eth_spec 4871 * An Ethernet flow spec to apply. 4872 * @param eth_mask 4873 * An Ethernet flow mask to apply. 4874 * @param vlan_spec 4875 * A VLAN flow spec to apply. 4876 * @param vlan_mask 4877 * A VLAN flow mask to apply. 4878 * 4879 * @return 4880 * 0 on success, a negative errno value otherwise and rte_errno is set. 4881 */ 4882 int 4883 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4884 struct rte_flow_item_eth *eth_spec, 4885 struct rte_flow_item_eth *eth_mask, 4886 struct rte_flow_item_vlan *vlan_spec, 4887 struct rte_flow_item_vlan *vlan_mask) 4888 { 4889 struct mlx5_priv *priv = dev->data->dev_private; 4890 const struct rte_flow_attr attr = { 4891 .ingress = 1, 4892 .priority = MLX5_FLOW_PRIO_RSVD, 4893 }; 4894 struct rte_flow_item items[] = { 4895 { 4896 .type = RTE_FLOW_ITEM_TYPE_ETH, 4897 .spec = eth_spec, 4898 .last = NULL, 4899 .mask = eth_mask, 4900 }, 4901 { 4902 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4903 RTE_FLOW_ITEM_TYPE_END, 4904 .spec = vlan_spec, 4905 .last = NULL, 4906 .mask = vlan_mask, 4907 }, 4908 { 4909 .type = RTE_FLOW_ITEM_TYPE_END, 4910 }, 4911 }; 4912 uint16_t queue[priv->reta_idx_n]; 4913 struct rte_flow_action_rss action_rss = { 4914 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4915 .level = 0, 4916 .types = priv->rss_conf.rss_hf, 4917 .key_len = priv->rss_conf.rss_key_len, 4918 .queue_num = priv->reta_idx_n, 4919 .key = priv->rss_conf.rss_key, 4920 .queue = queue, 4921 }; 4922 struct rte_flow_action actions[] = { 4923 { 4924 .type = RTE_FLOW_ACTION_TYPE_RSS, 4925 .conf = &action_rss, 4926 }, 4927 { 4928 .type = RTE_FLOW_ACTION_TYPE_END, 4929 }, 4930 }; 4931 uint32_t flow_idx; 4932 struct rte_flow_error error; 4933 unsigned int i; 4934 4935 if (!priv->reta_idx_n || !priv->rxqs_n) { 4936 return 0; 4937 } 4938 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 4939 action_rss.types = 0; 4940 for (i = 0; i != priv->reta_idx_n; ++i) 4941 queue[i] = (*priv->reta_idx)[i]; 4942 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4943 &attr, items, actions, false, &error); 4944 if (!flow_idx) 4945 return -rte_errno; 4946 return 0; 4947 } 4948 4949 /** 4950 * Enable a flow control configured from the control plane. 4951 * 4952 * @param dev 4953 * Pointer to Ethernet device. 4954 * @param eth_spec 4955 * An Ethernet flow spec to apply. 4956 * @param eth_mask 4957 * An Ethernet flow mask to apply. 4958 * 4959 * @return 4960 * 0 on success, a negative errno value otherwise and rte_errno is set. 4961 */ 4962 int 4963 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4964 struct rte_flow_item_eth *eth_spec, 4965 struct rte_flow_item_eth *eth_mask) 4966 { 4967 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4968 } 4969 4970 /** 4971 * Destroy a flow. 4972 * 4973 * @see rte_flow_destroy() 4974 * @see rte_flow_ops 4975 */ 4976 int 4977 mlx5_flow_destroy(struct rte_eth_dev *dev, 4978 struct rte_flow *flow, 4979 struct rte_flow_error *error __rte_unused) 4980 { 4981 struct mlx5_priv *priv = dev->data->dev_private; 4982 4983 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 4984 return 0; 4985 } 4986 4987 /** 4988 * Destroy all flows. 4989 * 4990 * @see rte_flow_flush() 4991 * @see rte_flow_ops 4992 */ 4993 int 4994 mlx5_flow_flush(struct rte_eth_dev *dev, 4995 struct rte_flow_error *error __rte_unused) 4996 { 4997 struct mlx5_priv *priv = dev->data->dev_private; 4998 4999 mlx5_flow_list_flush(dev, &priv->flows, false); 5000 return 0; 5001 } 5002 5003 /** 5004 * Isolated mode. 5005 * 5006 * @see rte_flow_isolate() 5007 * @see rte_flow_ops 5008 */ 5009 int 5010 mlx5_flow_isolate(struct rte_eth_dev *dev, 5011 int enable, 5012 struct rte_flow_error *error) 5013 { 5014 struct mlx5_priv *priv = dev->data->dev_private; 5015 5016 if (dev->data->dev_started) { 5017 rte_flow_error_set(error, EBUSY, 5018 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5019 NULL, 5020 "port must be stopped first"); 5021 return -rte_errno; 5022 } 5023 priv->isolated = !!enable; 5024 if (enable) 5025 dev->dev_ops = &mlx5_dev_ops_isolate; 5026 else 5027 dev->dev_ops = &mlx5_dev_ops; 5028 return 0; 5029 } 5030 5031 /** 5032 * Query a flow. 5033 * 5034 * @see rte_flow_query() 5035 * @see rte_flow_ops 5036 */ 5037 static int 5038 flow_drv_query(struct rte_eth_dev *dev, 5039 uint32_t flow_idx, 5040 const struct rte_flow_action *actions, 5041 void *data, 5042 struct rte_flow_error *error) 5043 { 5044 struct mlx5_priv *priv = dev->data->dev_private; 5045 const struct mlx5_flow_driver_ops *fops; 5046 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5047 [MLX5_IPOOL_RTE_FLOW], 5048 flow_idx); 5049 enum mlx5_flow_drv_type ftype; 5050 5051 if (!flow) { 5052 return rte_flow_error_set(error, ENOENT, 5053 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5054 NULL, 5055 "invalid flow handle"); 5056 } 5057 ftype = flow->drv_type; 5058 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 5059 fops = flow_get_drv_ops(ftype); 5060 5061 return fops->query(dev, flow, actions, data, error); 5062 } 5063 5064 /** 5065 * Query a flow. 5066 * 5067 * @see rte_flow_query() 5068 * @see rte_flow_ops 5069 */ 5070 int 5071 mlx5_flow_query(struct rte_eth_dev *dev, 5072 struct rte_flow *flow, 5073 const struct rte_flow_action *actions, 5074 void *data, 5075 struct rte_flow_error *error) 5076 { 5077 int ret; 5078 5079 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 5080 error); 5081 if (ret < 0) 5082 return ret; 5083 return 0; 5084 } 5085 5086 /** 5087 * Convert a flow director filter to a generic flow. 5088 * 5089 * @param dev 5090 * Pointer to Ethernet device. 5091 * @param fdir_filter 5092 * Flow director filter to add. 5093 * @param attributes 5094 * Generic flow parameters structure. 5095 * 5096 * @return 5097 * 0 on success, a negative errno value otherwise and rte_errno is set. 5098 */ 5099 static int 5100 flow_fdir_filter_convert(struct rte_eth_dev *dev, 5101 const struct rte_eth_fdir_filter *fdir_filter, 5102 struct mlx5_fdir *attributes) 5103 { 5104 struct mlx5_priv *priv = dev->data->dev_private; 5105 const struct rte_eth_fdir_input *input = &fdir_filter->input; 5106 const struct rte_eth_fdir_masks *mask = 5107 &dev->data->dev_conf.fdir_conf.mask; 5108 5109 /* Validate queue number. */ 5110 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 5111 DRV_LOG(ERR, "port %u invalid queue number %d", 5112 dev->data->port_id, fdir_filter->action.rx_queue); 5113 rte_errno = EINVAL; 5114 return -rte_errno; 5115 } 5116 attributes->attr.ingress = 1; 5117 attributes->items[0] = (struct rte_flow_item) { 5118 .type = RTE_FLOW_ITEM_TYPE_ETH, 5119 .spec = &attributes->l2, 5120 .mask = &attributes->l2_mask, 5121 }; 5122 switch (fdir_filter->action.behavior) { 5123 case RTE_ETH_FDIR_ACCEPT: 5124 attributes->actions[0] = (struct rte_flow_action){ 5125 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 5126 .conf = &attributes->queue, 5127 }; 5128 break; 5129 case RTE_ETH_FDIR_REJECT: 5130 attributes->actions[0] = (struct rte_flow_action){ 5131 .type = RTE_FLOW_ACTION_TYPE_DROP, 5132 }; 5133 break; 5134 default: 5135 DRV_LOG(ERR, "port %u invalid behavior %d", 5136 dev->data->port_id, 5137 fdir_filter->action.behavior); 5138 rte_errno = ENOTSUP; 5139 return -rte_errno; 5140 } 5141 attributes->queue.index = fdir_filter->action.rx_queue; 5142 /* Handle L3. */ 5143 switch (fdir_filter->input.flow_type) { 5144 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5145 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5146 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5147 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 5148 .src_addr = input->flow.ip4_flow.src_ip, 5149 .dst_addr = input->flow.ip4_flow.dst_ip, 5150 .time_to_live = input->flow.ip4_flow.ttl, 5151 .type_of_service = input->flow.ip4_flow.tos, 5152 }; 5153 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 5154 .src_addr = mask->ipv4_mask.src_ip, 5155 .dst_addr = mask->ipv4_mask.dst_ip, 5156 .time_to_live = mask->ipv4_mask.ttl, 5157 .type_of_service = mask->ipv4_mask.tos, 5158 .next_proto_id = mask->ipv4_mask.proto, 5159 }; 5160 attributes->items[1] = (struct rte_flow_item){ 5161 .type = RTE_FLOW_ITEM_TYPE_IPV4, 5162 .spec = &attributes->l3, 5163 .mask = &attributes->l3_mask, 5164 }; 5165 break; 5166 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5167 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5168 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5169 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 5170 .hop_limits = input->flow.ipv6_flow.hop_limits, 5171 .proto = input->flow.ipv6_flow.proto, 5172 }; 5173 5174 memcpy(attributes->l3.ipv6.hdr.src_addr, 5175 input->flow.ipv6_flow.src_ip, 5176 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5177 memcpy(attributes->l3.ipv6.hdr.dst_addr, 5178 input->flow.ipv6_flow.dst_ip, 5179 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5180 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 5181 mask->ipv6_mask.src_ip, 5182 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5183 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 5184 mask->ipv6_mask.dst_ip, 5185 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5186 attributes->items[1] = (struct rte_flow_item){ 5187 .type = RTE_FLOW_ITEM_TYPE_IPV6, 5188 .spec = &attributes->l3, 5189 .mask = &attributes->l3_mask, 5190 }; 5191 break; 5192 default: 5193 DRV_LOG(ERR, "port %u invalid flow type%d", 5194 dev->data->port_id, fdir_filter->input.flow_type); 5195 rte_errno = ENOTSUP; 5196 return -rte_errno; 5197 } 5198 /* Handle L4. */ 5199 switch (fdir_filter->input.flow_type) { 5200 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5201 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5202 .src_port = input->flow.udp4_flow.src_port, 5203 .dst_port = input->flow.udp4_flow.dst_port, 5204 }; 5205 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5206 .src_port = mask->src_port_mask, 5207 .dst_port = mask->dst_port_mask, 5208 }; 5209 attributes->items[2] = (struct rte_flow_item){ 5210 .type = RTE_FLOW_ITEM_TYPE_UDP, 5211 .spec = &attributes->l4, 5212 .mask = &attributes->l4_mask, 5213 }; 5214 break; 5215 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5216 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5217 .src_port = input->flow.tcp4_flow.src_port, 5218 .dst_port = input->flow.tcp4_flow.dst_port, 5219 }; 5220 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5221 .src_port = mask->src_port_mask, 5222 .dst_port = mask->dst_port_mask, 5223 }; 5224 attributes->items[2] = (struct rte_flow_item){ 5225 .type = RTE_FLOW_ITEM_TYPE_TCP, 5226 .spec = &attributes->l4, 5227 .mask = &attributes->l4_mask, 5228 }; 5229 break; 5230 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5231 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5232 .src_port = input->flow.udp6_flow.src_port, 5233 .dst_port = input->flow.udp6_flow.dst_port, 5234 }; 5235 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5236 .src_port = mask->src_port_mask, 5237 .dst_port = mask->dst_port_mask, 5238 }; 5239 attributes->items[2] = (struct rte_flow_item){ 5240 .type = RTE_FLOW_ITEM_TYPE_UDP, 5241 .spec = &attributes->l4, 5242 .mask = &attributes->l4_mask, 5243 }; 5244 break; 5245 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5246 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5247 .src_port = input->flow.tcp6_flow.src_port, 5248 .dst_port = input->flow.tcp6_flow.dst_port, 5249 }; 5250 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5251 .src_port = mask->src_port_mask, 5252 .dst_port = mask->dst_port_mask, 5253 }; 5254 attributes->items[2] = (struct rte_flow_item){ 5255 .type = RTE_FLOW_ITEM_TYPE_TCP, 5256 .spec = &attributes->l4, 5257 .mask = &attributes->l4_mask, 5258 }; 5259 break; 5260 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5261 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5262 break; 5263 default: 5264 DRV_LOG(ERR, "port %u invalid flow type%d", 5265 dev->data->port_id, fdir_filter->input.flow_type); 5266 rte_errno = ENOTSUP; 5267 return -rte_errno; 5268 } 5269 return 0; 5270 } 5271 5272 #define FLOW_FDIR_CMP(f1, f2, fld) \ 5273 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 5274 5275 /** 5276 * Compare two FDIR flows. If items and actions are identical, the two flows are 5277 * regarded as same. 5278 * 5279 * @param dev 5280 * Pointer to Ethernet device. 5281 * @param f1 5282 * FDIR flow to compare. 5283 * @param f2 5284 * FDIR flow to compare. 5285 * 5286 * @return 5287 * Zero on match, 1 otherwise. 5288 */ 5289 static int 5290 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 5291 { 5292 if (FLOW_FDIR_CMP(f1, f2, attr) || 5293 FLOW_FDIR_CMP(f1, f2, l2) || 5294 FLOW_FDIR_CMP(f1, f2, l2_mask) || 5295 FLOW_FDIR_CMP(f1, f2, l3) || 5296 FLOW_FDIR_CMP(f1, f2, l3_mask) || 5297 FLOW_FDIR_CMP(f1, f2, l4) || 5298 FLOW_FDIR_CMP(f1, f2, l4_mask) || 5299 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 5300 return 1; 5301 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 5302 FLOW_FDIR_CMP(f1, f2, queue)) 5303 return 1; 5304 return 0; 5305 } 5306 5307 /** 5308 * Search device flow list to find out a matched FDIR flow. 5309 * 5310 * @param dev 5311 * Pointer to Ethernet device. 5312 * @param fdir_flow 5313 * FDIR flow to lookup. 5314 * 5315 * @return 5316 * Index of flow if found, 0 otherwise. 5317 */ 5318 static uint32_t 5319 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5320 { 5321 struct mlx5_priv *priv = dev->data->dev_private; 5322 uint32_t flow_idx = 0; 5323 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5324 5325 MLX5_ASSERT(fdir_flow); 5326 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5327 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { 5328 DRV_LOG(DEBUG, "port %u found FDIR flow %u", 5329 dev->data->port_id, flow_idx); 5330 flow_idx = priv_fdir_flow->rix_flow; 5331 break; 5332 } 5333 } 5334 return flow_idx; 5335 } 5336 5337 /** 5338 * Add new flow director filter and store it in list. 5339 * 5340 * @param dev 5341 * Pointer to Ethernet device. 5342 * @param fdir_filter 5343 * Flow director filter to add. 5344 * 5345 * @return 5346 * 0 on success, a negative errno value otherwise and rte_errno is set. 5347 */ 5348 static int 5349 flow_fdir_filter_add(struct rte_eth_dev *dev, 5350 const struct rte_eth_fdir_filter *fdir_filter) 5351 { 5352 struct mlx5_priv *priv = dev->data->dev_private; 5353 struct mlx5_fdir *fdir_flow; 5354 struct rte_flow *flow; 5355 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5356 uint32_t flow_idx; 5357 int ret; 5358 5359 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5360 if (!fdir_flow) { 5361 rte_errno = ENOMEM; 5362 return -rte_errno; 5363 } 5364 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5365 if (ret) 5366 goto error; 5367 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); 5368 if (flow_idx) { 5369 rte_errno = EEXIST; 5370 goto error; 5371 } 5372 priv_fdir_flow = rte_zmalloc(__func__, sizeof(struct mlx5_fdir_flow), 5373 0); 5374 if (!priv_fdir_flow) { 5375 rte_errno = ENOMEM; 5376 goto error; 5377 } 5378 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5379 fdir_flow->items, fdir_flow->actions, true, 5380 NULL); 5381 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5382 if (!flow) 5383 goto error; 5384 flow->fdir = 1; 5385 priv_fdir_flow->fdir = fdir_flow; 5386 priv_fdir_flow->rix_flow = flow_idx; 5387 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); 5388 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5389 dev->data->port_id, (void *)flow); 5390 return 0; 5391 error: 5392 rte_free(priv_fdir_flow); 5393 rte_free(fdir_flow); 5394 return -rte_errno; 5395 } 5396 5397 /** 5398 * Delete specific filter. 5399 * 5400 * @param dev 5401 * Pointer to Ethernet device. 5402 * @param fdir_filter 5403 * Filter to be deleted. 5404 * 5405 * @return 5406 * 0 on success, a negative errno value otherwise and rte_errno is set. 5407 */ 5408 static int 5409 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5410 const struct rte_eth_fdir_filter *fdir_filter) 5411 { 5412 struct mlx5_priv *priv = dev->data->dev_private; 5413 uint32_t flow_idx; 5414 struct mlx5_fdir fdir_flow = { 5415 .attr.group = 0, 5416 }; 5417 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5418 int ret; 5419 5420 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5421 if (ret) 5422 return -rte_errno; 5423 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5424 /* Find the fdir in priv list */ 5425 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) 5426 break; 5427 } 5428 if (!priv_fdir_flow) 5429 return 0; 5430 LIST_REMOVE(priv_fdir_flow, next); 5431 flow_idx = priv_fdir_flow->rix_flow; 5432 flow_list_destroy(dev, &priv->flows, flow_idx); 5433 rte_free(priv_fdir_flow->fdir); 5434 rte_free(priv_fdir_flow); 5435 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", 5436 dev->data->port_id, flow_idx); 5437 return 0; 5438 } 5439 5440 /** 5441 * Update queue for specific filter. 5442 * 5443 * @param dev 5444 * Pointer to Ethernet device. 5445 * @param fdir_filter 5446 * Filter to be updated. 5447 * 5448 * @return 5449 * 0 on success, a negative errno value otherwise and rte_errno is set. 5450 */ 5451 static int 5452 flow_fdir_filter_update(struct rte_eth_dev *dev, 5453 const struct rte_eth_fdir_filter *fdir_filter) 5454 { 5455 int ret; 5456 5457 ret = flow_fdir_filter_delete(dev, fdir_filter); 5458 if (ret) 5459 return ret; 5460 return flow_fdir_filter_add(dev, fdir_filter); 5461 } 5462 5463 /** 5464 * Flush all filters. 5465 * 5466 * @param dev 5467 * Pointer to Ethernet device. 5468 */ 5469 static void 5470 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5471 { 5472 struct mlx5_priv *priv = dev->data->dev_private; 5473 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5474 5475 while (!LIST_EMPTY(&priv->fdir_flows)) { 5476 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); 5477 LIST_REMOVE(priv_fdir_flow, next); 5478 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); 5479 rte_free(priv_fdir_flow->fdir); 5480 rte_free(priv_fdir_flow); 5481 } 5482 } 5483 5484 /** 5485 * Get flow director information. 5486 * 5487 * @param dev 5488 * Pointer to Ethernet device. 5489 * @param[out] fdir_info 5490 * Resulting flow director information. 5491 */ 5492 static void 5493 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5494 { 5495 struct rte_eth_fdir_masks *mask = 5496 &dev->data->dev_conf.fdir_conf.mask; 5497 5498 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5499 fdir_info->guarant_spc = 0; 5500 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5501 fdir_info->max_flexpayload = 0; 5502 fdir_info->flow_types_mask[0] = 0; 5503 fdir_info->flex_payload_unit = 0; 5504 fdir_info->max_flex_payload_segment_num = 0; 5505 fdir_info->flex_payload_limit = 0; 5506 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5507 } 5508 5509 /** 5510 * Deal with flow director operations. 5511 * 5512 * @param dev 5513 * Pointer to Ethernet device. 5514 * @param filter_op 5515 * Operation to perform. 5516 * @param arg 5517 * Pointer to operation-specific structure. 5518 * 5519 * @return 5520 * 0 on success, a negative errno value otherwise and rte_errno is set. 5521 */ 5522 static int 5523 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5524 void *arg) 5525 { 5526 enum rte_fdir_mode fdir_mode = 5527 dev->data->dev_conf.fdir_conf.mode; 5528 5529 if (filter_op == RTE_ETH_FILTER_NOP) 5530 return 0; 5531 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5532 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5533 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5534 dev->data->port_id, fdir_mode); 5535 rte_errno = EINVAL; 5536 return -rte_errno; 5537 } 5538 switch (filter_op) { 5539 case RTE_ETH_FILTER_ADD: 5540 return flow_fdir_filter_add(dev, arg); 5541 case RTE_ETH_FILTER_UPDATE: 5542 return flow_fdir_filter_update(dev, arg); 5543 case RTE_ETH_FILTER_DELETE: 5544 return flow_fdir_filter_delete(dev, arg); 5545 case RTE_ETH_FILTER_FLUSH: 5546 flow_fdir_filter_flush(dev); 5547 break; 5548 case RTE_ETH_FILTER_INFO: 5549 flow_fdir_info_get(dev, arg); 5550 break; 5551 default: 5552 DRV_LOG(DEBUG, "port %u unknown operation %u", 5553 dev->data->port_id, filter_op); 5554 rte_errno = EINVAL; 5555 return -rte_errno; 5556 } 5557 return 0; 5558 } 5559 5560 /** 5561 * Manage filter operations. 5562 * 5563 * @param dev 5564 * Pointer to Ethernet device structure. 5565 * @param filter_type 5566 * Filter type. 5567 * @param filter_op 5568 * Operation to perform. 5569 * @param arg 5570 * Pointer to operation-specific structure. 5571 * 5572 * @return 5573 * 0 on success, a negative errno value otherwise and rte_errno is set. 5574 */ 5575 int 5576 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5577 enum rte_filter_type filter_type, 5578 enum rte_filter_op filter_op, 5579 void *arg) 5580 { 5581 switch (filter_type) { 5582 case RTE_ETH_FILTER_GENERIC: 5583 if (filter_op != RTE_ETH_FILTER_GET) { 5584 rte_errno = EINVAL; 5585 return -rte_errno; 5586 } 5587 *(const void **)arg = &mlx5_flow_ops; 5588 return 0; 5589 case RTE_ETH_FILTER_FDIR: 5590 return flow_fdir_ctrl_func(dev, filter_op, arg); 5591 default: 5592 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5593 dev->data->port_id, filter_type); 5594 rte_errno = ENOTSUP; 5595 return -rte_errno; 5596 } 5597 return 0; 5598 } 5599 5600 /** 5601 * Create the needed meter and suffix tables. 5602 * 5603 * @param[in] dev 5604 * Pointer to Ethernet device. 5605 * @param[in] fm 5606 * Pointer to the flow meter. 5607 * 5608 * @return 5609 * Pointer to table set on success, NULL otherwise. 5610 */ 5611 struct mlx5_meter_domains_infos * 5612 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5613 const struct mlx5_flow_meter *fm) 5614 { 5615 const struct mlx5_flow_driver_ops *fops; 5616 5617 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5618 return fops->create_mtr_tbls(dev, fm); 5619 } 5620 5621 /** 5622 * Destroy the meter table set. 5623 * 5624 * @param[in] dev 5625 * Pointer to Ethernet device. 5626 * @param[in] tbl 5627 * Pointer to the meter table set. 5628 * 5629 * @return 5630 * 0 on success. 5631 */ 5632 int 5633 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5634 struct mlx5_meter_domains_infos *tbls) 5635 { 5636 const struct mlx5_flow_driver_ops *fops; 5637 5638 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5639 return fops->destroy_mtr_tbls(dev, tbls); 5640 } 5641 5642 /** 5643 * Create policer rules. 5644 * 5645 * @param[in] dev 5646 * Pointer to Ethernet device. 5647 * @param[in] fm 5648 * Pointer to flow meter structure. 5649 * @param[in] attr 5650 * Pointer to flow attributes. 5651 * 5652 * @return 5653 * 0 on success, -1 otherwise. 5654 */ 5655 int 5656 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5657 struct mlx5_flow_meter *fm, 5658 const struct rte_flow_attr *attr) 5659 { 5660 const struct mlx5_flow_driver_ops *fops; 5661 5662 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5663 return fops->create_policer_rules(dev, fm, attr); 5664 } 5665 5666 /** 5667 * Destroy policer rules. 5668 * 5669 * @param[in] fm 5670 * Pointer to flow meter structure. 5671 * @param[in] attr 5672 * Pointer to flow attributes. 5673 * 5674 * @return 5675 * 0 on success, -1 otherwise. 5676 */ 5677 int 5678 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5679 struct mlx5_flow_meter *fm, 5680 const struct rte_flow_attr *attr) 5681 { 5682 const struct mlx5_flow_driver_ops *fops; 5683 5684 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5685 return fops->destroy_policer_rules(dev, fm, attr); 5686 } 5687 5688 /** 5689 * Allocate a counter. 5690 * 5691 * @param[in] dev 5692 * Pointer to Ethernet device structure. 5693 * 5694 * @return 5695 * Index to allocated counter on success, 0 otherwise. 5696 */ 5697 uint32_t 5698 mlx5_counter_alloc(struct rte_eth_dev *dev) 5699 { 5700 const struct mlx5_flow_driver_ops *fops; 5701 struct rte_flow_attr attr = { .transfer = 0 }; 5702 5703 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5704 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5705 return fops->counter_alloc(dev); 5706 } 5707 DRV_LOG(ERR, 5708 "port %u counter allocate is not supported.", 5709 dev->data->port_id); 5710 return 0; 5711 } 5712 5713 /** 5714 * Free a counter. 5715 * 5716 * @param[in] dev 5717 * Pointer to Ethernet device structure. 5718 * @param[in] cnt 5719 * Index to counter to be free. 5720 */ 5721 void 5722 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 5723 { 5724 const struct mlx5_flow_driver_ops *fops; 5725 struct rte_flow_attr attr = { .transfer = 0 }; 5726 5727 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5728 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5729 fops->counter_free(dev, cnt); 5730 return; 5731 } 5732 DRV_LOG(ERR, 5733 "port %u counter free is not supported.", 5734 dev->data->port_id); 5735 } 5736 5737 /** 5738 * Query counter statistics. 5739 * 5740 * @param[in] dev 5741 * Pointer to Ethernet device structure. 5742 * @param[in] cnt 5743 * Index to counter to query. 5744 * @param[in] clear 5745 * Set to clear counter statistics. 5746 * @param[out] pkts 5747 * The counter hits packets number to save. 5748 * @param[out] bytes 5749 * The counter hits bytes number to save. 5750 * 5751 * @return 5752 * 0 on success, a negative errno value otherwise. 5753 */ 5754 int 5755 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 5756 bool clear, uint64_t *pkts, uint64_t *bytes) 5757 { 5758 const struct mlx5_flow_driver_ops *fops; 5759 struct rte_flow_attr attr = { .transfer = 0 }; 5760 5761 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5762 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5763 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5764 } 5765 DRV_LOG(ERR, 5766 "port %u counter query is not supported.", 5767 dev->data->port_id); 5768 return -ENOTSUP; 5769 } 5770 5771 #define MLX5_POOL_QUERY_FREQ_US 1000000 5772 5773 /** 5774 * Get number of all validate pools. 5775 * 5776 * @param[in] sh 5777 * Pointer to mlx5_ibv_shared object. 5778 * 5779 * @return 5780 * The number of all validate pools. 5781 */ 5782 static uint32_t 5783 mlx5_get_all_valid_pool_count(struct mlx5_ibv_shared *sh) 5784 { 5785 uint8_t age, i; 5786 uint32_t pools_n = 0; 5787 struct mlx5_pools_container *cont; 5788 5789 for (age = 0; age < RTE_DIM(sh->cmng.ccont[0]); ++age) { 5790 for (i = 0; i < 2 ; ++i) { 5791 cont = MLX5_CNT_CONTAINER(sh, i, 0, age); 5792 pools_n += rte_atomic16_read(&cont->n_valid); 5793 } 5794 } 5795 return pools_n; 5796 } 5797 5798 /** 5799 * Set the periodic procedure for triggering asynchronous batch queries for all 5800 * the counter pools. 5801 * 5802 * @param[in] sh 5803 * Pointer to mlx5_ibv_shared object. 5804 */ 5805 void 5806 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5807 { 5808 uint32_t pools_n, us; 5809 5810 pools_n = mlx5_get_all_valid_pool_count(sh); 5811 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5812 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5813 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5814 sh->cmng.query_thread_on = 0; 5815 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5816 } else { 5817 sh->cmng.query_thread_on = 1; 5818 } 5819 } 5820 5821 /** 5822 * The periodic procedure for triggering asynchronous batch queries for all the 5823 * counter pools. This function is probably called by the host thread. 5824 * 5825 * @param[in] arg 5826 * The parameter for the alarm process. 5827 */ 5828 void 5829 mlx5_flow_query_alarm(void *arg) 5830 { 5831 struct mlx5_ibv_shared *sh = arg; 5832 struct mlx5_devx_obj *dcs; 5833 uint16_t offset; 5834 int ret; 5835 uint8_t batch = sh->cmng.batch; 5836 uint8_t age = sh->cmng.age; 5837 uint16_t pool_index = sh->cmng.pool_index; 5838 struct mlx5_pools_container *cont; 5839 struct mlx5_pools_container *mcont; 5840 struct mlx5_flow_counter_pool *pool; 5841 5842 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5843 goto set_alarm; 5844 next_container: 5845 cont = MLX5_CNT_CONTAINER(sh, batch, 1, age); 5846 mcont = MLX5_CNT_CONTAINER(sh, batch, 0, age); 5847 /* Check if resize was done and need to flip a container. */ 5848 if (cont != mcont) { 5849 if (cont->pools) { 5850 /* Clean the old container. */ 5851 rte_free(cont->pools); 5852 memset(cont, 0, sizeof(*cont)); 5853 } 5854 rte_cio_wmb(); 5855 /* Flip the host container. */ 5856 sh->cmng.mhi[batch][age] ^= (uint8_t)2; 5857 cont = mcont; 5858 } 5859 if (!cont->pools) { 5860 /* 2 empty containers case is unexpected. */ 5861 if (unlikely(batch != sh->cmng.batch) && 5862 unlikely(age != sh->cmng.age)) { 5863 goto set_alarm; 5864 } 5865 batch ^= 0x1; 5866 pool_index = 0; 5867 if (batch == 0 && pool_index == 0) { 5868 age ^= 0x1; 5869 sh->cmng.batch = batch; 5870 sh->cmng.age = age; 5871 } 5872 goto next_container; 5873 } 5874 pool = cont->pools[pool_index]; 5875 if (pool->raw_hw) 5876 /* There is a pool query in progress. */ 5877 goto set_alarm; 5878 pool->raw_hw = 5879 LIST_FIRST(&sh->cmng.free_stat_raws); 5880 if (!pool->raw_hw) 5881 /* No free counter statistics raw memory. */ 5882 goto set_alarm; 5883 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5884 (&pool->a64_dcs); 5885 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5886 /* 5887 * Identify the counters released between query trigger and query 5888 * handle more effiecntly. The counter released in this gap period 5889 * should wait for a new round of query as the new arrived packets 5890 * will not be taken into account. 5891 */ 5892 rte_atomic64_add(&pool->start_query_gen, 1); 5893 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5894 offset, NULL, NULL, 5895 pool->raw_hw->mem_mng->dm->id, 5896 (void *)(uintptr_t) 5897 (pool->raw_hw->data + offset), 5898 sh->devx_comp, 5899 (uint64_t)(uintptr_t)pool); 5900 if (ret) { 5901 rte_atomic64_sub(&pool->start_query_gen, 1); 5902 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5903 " %d", pool->min_dcs->id); 5904 pool->raw_hw = NULL; 5905 goto set_alarm; 5906 } 5907 pool->raw_hw->min_dcs_id = dcs->id; 5908 LIST_REMOVE(pool->raw_hw, next); 5909 sh->cmng.pending_queries++; 5910 pool_index++; 5911 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5912 batch ^= 0x1; 5913 pool_index = 0; 5914 if (batch == 0 && pool_index == 0) 5915 age ^= 0x1; 5916 } 5917 set_alarm: 5918 sh->cmng.batch = batch; 5919 sh->cmng.pool_index = pool_index; 5920 sh->cmng.age = age; 5921 mlx5_set_query_alarm(sh); 5922 } 5923 5924 /** 5925 * Check and callback event for new aged flow in the counter pool 5926 * 5927 * @param[in] sh 5928 * Pointer to mlx5_ibv_shared object. 5929 * @param[in] pool 5930 * Pointer to Current counter pool. 5931 */ 5932 static void 5933 mlx5_flow_aging_check(struct mlx5_ibv_shared *sh, 5934 struct mlx5_flow_counter_pool *pool) 5935 { 5936 struct mlx5_priv *priv; 5937 struct mlx5_flow_counter *cnt; 5938 struct mlx5_age_info *age_info; 5939 struct mlx5_age_param *age_param; 5940 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 5941 struct mlx5_counter_stats_raw *prev = pool->raw; 5942 uint16_t curr = rte_rdtsc() / (rte_get_tsc_hz() / 10); 5943 uint32_t i; 5944 5945 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 5946 cnt = MLX5_POOL_GET_CNT(pool, i); 5947 age_param = MLX5_CNT_TO_AGE(cnt); 5948 if (rte_atomic16_read(&age_param->state) != AGE_CANDIDATE) 5949 continue; 5950 if (cur->data[i].hits != prev->data[i].hits) { 5951 age_param->expire = curr + age_param->timeout; 5952 continue; 5953 } 5954 if ((uint16_t)(curr - age_param->expire) >= (UINT16_MAX / 2)) 5955 continue; 5956 /** 5957 * Hold the lock first, or if between the 5958 * state AGE_TMOUT and tailq operation the 5959 * release happened, the release procedure 5960 * may delete a non-existent tailq node. 5961 */ 5962 priv = rte_eth_devices[age_param->port_id].data->dev_private; 5963 age_info = GET_PORT_AGE_INFO(priv); 5964 rte_spinlock_lock(&age_info->aged_sl); 5965 /* If the cpmset fails, release happens. */ 5966 if (rte_atomic16_cmpset((volatile uint16_t *) 5967 &age_param->state, 5968 AGE_CANDIDATE, 5969 AGE_TMOUT) == 5970 AGE_CANDIDATE) { 5971 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 5972 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 5973 } 5974 rte_spinlock_unlock(&age_info->aged_sl); 5975 } 5976 for (i = 0; i < sh->max_port; i++) { 5977 age_info = &sh->port[i].age_info; 5978 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 5979 continue; 5980 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) 5981 _rte_eth_dev_callback_process 5982 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 5983 RTE_ETH_EVENT_FLOW_AGED, NULL); 5984 age_info->flags = 0; 5985 } 5986 } 5987 5988 /** 5989 * Handler for the HW respond about ready values from an asynchronous batch 5990 * query. This function is probably called by the host thread. 5991 * 5992 * @param[in] sh 5993 * The pointer to the shared IB device context. 5994 * @param[in] async_id 5995 * The Devx async ID. 5996 * @param[in] status 5997 * The status of the completion. 5998 */ 5999 void 6000 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 6001 uint64_t async_id, int status) 6002 { 6003 struct mlx5_flow_counter_pool *pool = 6004 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 6005 struct mlx5_counter_stats_raw *raw_to_free; 6006 6007 if (unlikely(status)) { 6008 rte_atomic64_sub(&pool->start_query_gen, 1); 6009 raw_to_free = pool->raw_hw; 6010 } else { 6011 raw_to_free = pool->raw; 6012 if (IS_AGE_POOL(pool)) 6013 mlx5_flow_aging_check(sh, pool); 6014 rte_spinlock_lock(&pool->sl); 6015 pool->raw = pool->raw_hw; 6016 rte_spinlock_unlock(&pool->sl); 6017 MLX5_ASSERT(rte_atomic64_read(&pool->end_query_gen) + 1 == 6018 rte_atomic64_read(&pool->start_query_gen)); 6019 rte_atomic64_set(&pool->end_query_gen, 6020 rte_atomic64_read(&pool->start_query_gen)); 6021 /* Be sure the new raw counters data is updated in memory. */ 6022 rte_cio_wmb(); 6023 } 6024 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 6025 pool->raw_hw = NULL; 6026 sh->cmng.pending_queries--; 6027 } 6028 6029 /** 6030 * Translate the rte_flow group index to HW table value. 6031 * 6032 * @param[in] attributes 6033 * Pointer to flow attributes 6034 * @param[in] external 6035 * Value is part of flow rule created by request external to PMD. 6036 * @param[in] group 6037 * rte_flow group index value. 6038 * @param[out] fdb_def_rule 6039 * Whether fdb jump to table 1 is configured. 6040 * @param[out] table 6041 * HW table value. 6042 * @param[out] error 6043 * Pointer to error structure. 6044 * 6045 * @return 6046 * 0 on success, a negative errno value otherwise and rte_errno is set. 6047 */ 6048 int 6049 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 6050 uint32_t group, bool fdb_def_rule, uint32_t *table, 6051 struct rte_flow_error *error) 6052 { 6053 if (attributes->transfer && external && fdb_def_rule) { 6054 if (group == UINT32_MAX) 6055 return rte_flow_error_set 6056 (error, EINVAL, 6057 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6058 NULL, 6059 "group index not supported"); 6060 *table = group + 1; 6061 } else { 6062 *table = group; 6063 } 6064 return 0; 6065 } 6066 6067 /** 6068 * Discover availability of metadata reg_c's. 6069 * 6070 * Iteratively use test flows to check availability. 6071 * 6072 * @param[in] dev 6073 * Pointer to the Ethernet device structure. 6074 * 6075 * @return 6076 * 0 on success, a negative errno value otherwise and rte_errno is set. 6077 */ 6078 int 6079 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 6080 { 6081 struct mlx5_priv *priv = dev->data->dev_private; 6082 struct mlx5_dev_config *config = &priv->config; 6083 enum modify_reg idx; 6084 int n = 0; 6085 6086 /* reg_c[0] and reg_c[1] are reserved. */ 6087 config->flow_mreg_c[n++] = REG_C_0; 6088 config->flow_mreg_c[n++] = REG_C_1; 6089 /* Discover availability of other reg_c's. */ 6090 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 6091 struct rte_flow_attr attr = { 6092 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6093 .priority = MLX5_FLOW_PRIO_RSVD, 6094 .ingress = 1, 6095 }; 6096 struct rte_flow_item items[] = { 6097 [0] = { 6098 .type = RTE_FLOW_ITEM_TYPE_END, 6099 }, 6100 }; 6101 struct rte_flow_action actions[] = { 6102 [0] = { 6103 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6104 .conf = &(struct mlx5_flow_action_copy_mreg){ 6105 .src = REG_C_1, 6106 .dst = idx, 6107 }, 6108 }, 6109 [1] = { 6110 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6111 .conf = &(struct rte_flow_action_jump){ 6112 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6113 }, 6114 }, 6115 [2] = { 6116 .type = RTE_FLOW_ACTION_TYPE_END, 6117 }, 6118 }; 6119 uint32_t flow_idx; 6120 struct rte_flow *flow; 6121 struct rte_flow_error error; 6122 6123 if (!config->dv_flow_en) 6124 break; 6125 /* Create internal flow, validation skips copy action. */ 6126 flow_idx = flow_list_create(dev, NULL, &attr, items, 6127 actions, false, &error); 6128 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 6129 flow_idx); 6130 if (!flow) 6131 continue; 6132 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 6133 config->flow_mreg_c[n++] = idx; 6134 flow_list_destroy(dev, NULL, flow_idx); 6135 } 6136 for (; n < MLX5_MREG_C_NUM; ++n) 6137 config->flow_mreg_c[n] = REG_NONE; 6138 return 0; 6139 } 6140 6141 /** 6142 * Dump flow raw hw data to file 6143 * 6144 * @param[in] dev 6145 * The pointer to Ethernet device. 6146 * @param[in] file 6147 * A pointer to a file for output. 6148 * @param[out] error 6149 * Perform verbose error reporting if not NULL. PMDs initialize this 6150 * structure in case of error only. 6151 * @return 6152 * 0 on success, a nagative value otherwise. 6153 */ 6154 int 6155 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 6156 FILE *file, 6157 struct rte_flow_error *error __rte_unused) 6158 { 6159 struct mlx5_priv *priv = dev->data->dev_private; 6160 struct mlx5_ibv_shared *sh = priv->sh; 6161 6162 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 6163 sh->tx_domain, file); 6164 } 6165 6166 /** 6167 * Get aged-out flows. 6168 * 6169 * @param[in] dev 6170 * Pointer to the Ethernet device structure. 6171 * @param[in] context 6172 * The address of an array of pointers to the aged-out flows contexts. 6173 * @param[in] nb_countexts 6174 * The length of context array pointers. 6175 * @param[out] error 6176 * Perform verbose error reporting if not NULL. Initialized in case of 6177 * error only. 6178 * 6179 * @return 6180 * how many contexts get in success, otherwise negative errno value. 6181 * if nb_contexts is 0, return the amount of all aged contexts. 6182 * if nb_contexts is not 0 , return the amount of aged flows reported 6183 * in the context array. 6184 */ 6185 int 6186 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 6187 uint32_t nb_contexts, struct rte_flow_error *error) 6188 { 6189 const struct mlx5_flow_driver_ops *fops; 6190 struct rte_flow_attr attr = { .transfer = 0 }; 6191 6192 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6193 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6194 return fops->get_aged_flows(dev, contexts, nb_contexts, 6195 error); 6196 } 6197 DRV_LOG(ERR, 6198 "port %u get aged flows is not supported.", 6199 dev->data->port_id); 6200 return -ENOTSUP; 6201 } 6202