1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 /* Verbs header. */ 14 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 15 #ifdef PEDANTIC 16 #pragma GCC diagnostic ignored "-Wpedantic" 17 #endif 18 #include <infiniband/verbs.h> 19 #ifdef PEDANTIC 20 #pragma GCC diagnostic error "-Wpedantic" 21 #endif 22 23 #include <rte_common.h> 24 #include <rte_ether.h> 25 #include <rte_ethdev_driver.h> 26 #include <rte_flow.h> 27 #include <rte_cycles.h> 28 #include <rte_flow_driver.h> 29 #include <rte_malloc.h> 30 #include <rte_ip.h> 31 32 #include <mlx5_glue.h> 33 #include <mlx5_devx_cmds.h> 34 #include <mlx5_prm.h> 35 36 #include "mlx5_defs.h" 37 #include "mlx5.h" 38 #include "mlx5_flow.h" 39 #include "mlx5_rxtx.h" 40 41 /* Dev ops structure defined in mlx5.c */ 42 extern const struct eth_dev_ops mlx5_dev_ops; 43 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 44 45 /** Device flow drivers. */ 46 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 47 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 48 #endif 49 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 50 51 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 52 53 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 54 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 55 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 56 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 57 #endif 58 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 59 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 60 }; 61 62 enum mlx5_expansion { 63 MLX5_EXPANSION_ROOT, 64 MLX5_EXPANSION_ROOT_OUTER, 65 MLX5_EXPANSION_ROOT_ETH_VLAN, 66 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 67 MLX5_EXPANSION_OUTER_ETH, 68 MLX5_EXPANSION_OUTER_ETH_VLAN, 69 MLX5_EXPANSION_OUTER_VLAN, 70 MLX5_EXPANSION_OUTER_IPV4, 71 MLX5_EXPANSION_OUTER_IPV4_UDP, 72 MLX5_EXPANSION_OUTER_IPV4_TCP, 73 MLX5_EXPANSION_OUTER_IPV6, 74 MLX5_EXPANSION_OUTER_IPV6_UDP, 75 MLX5_EXPANSION_OUTER_IPV6_TCP, 76 MLX5_EXPANSION_VXLAN, 77 MLX5_EXPANSION_VXLAN_GPE, 78 MLX5_EXPANSION_GRE, 79 MLX5_EXPANSION_MPLS, 80 MLX5_EXPANSION_ETH, 81 MLX5_EXPANSION_ETH_VLAN, 82 MLX5_EXPANSION_VLAN, 83 MLX5_EXPANSION_IPV4, 84 MLX5_EXPANSION_IPV4_UDP, 85 MLX5_EXPANSION_IPV4_TCP, 86 MLX5_EXPANSION_IPV6, 87 MLX5_EXPANSION_IPV6_UDP, 88 MLX5_EXPANSION_IPV6_TCP, 89 }; 90 91 /** Supported expansion of items. */ 92 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 93 [MLX5_EXPANSION_ROOT] = { 94 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 95 MLX5_EXPANSION_IPV4, 96 MLX5_EXPANSION_IPV6), 97 .type = RTE_FLOW_ITEM_TYPE_END, 98 }, 99 [MLX5_EXPANSION_ROOT_OUTER] = { 100 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 101 MLX5_EXPANSION_OUTER_IPV4, 102 MLX5_EXPANSION_OUTER_IPV6), 103 .type = RTE_FLOW_ITEM_TYPE_END, 104 }, 105 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 107 .type = RTE_FLOW_ITEM_TYPE_END, 108 }, 109 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 110 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 111 .type = RTE_FLOW_ITEM_TYPE_END, 112 }, 113 [MLX5_EXPANSION_OUTER_ETH] = { 114 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 115 MLX5_EXPANSION_OUTER_IPV6, 116 MLX5_EXPANSION_MPLS), 117 .type = RTE_FLOW_ITEM_TYPE_ETH, 118 .rss_types = 0, 119 }, 120 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 121 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 122 .type = RTE_FLOW_ITEM_TYPE_ETH, 123 .rss_types = 0, 124 }, 125 [MLX5_EXPANSION_OUTER_VLAN] = { 126 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 127 MLX5_EXPANSION_OUTER_IPV6), 128 .type = RTE_FLOW_ITEM_TYPE_VLAN, 129 }, 130 [MLX5_EXPANSION_OUTER_IPV4] = { 131 .next = RTE_FLOW_EXPAND_RSS_NEXT 132 (MLX5_EXPANSION_OUTER_IPV4_UDP, 133 MLX5_EXPANSION_OUTER_IPV4_TCP, 134 MLX5_EXPANSION_GRE, 135 MLX5_EXPANSION_IPV4, 136 MLX5_EXPANSION_IPV6), 137 .type = RTE_FLOW_ITEM_TYPE_IPV4, 138 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 139 ETH_RSS_NONFRAG_IPV4_OTHER, 140 }, 141 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 142 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 143 MLX5_EXPANSION_VXLAN_GPE), 144 .type = RTE_FLOW_ITEM_TYPE_UDP, 145 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 146 }, 147 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 148 .type = RTE_FLOW_ITEM_TYPE_TCP, 149 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 150 }, 151 [MLX5_EXPANSION_OUTER_IPV6] = { 152 .next = RTE_FLOW_EXPAND_RSS_NEXT 153 (MLX5_EXPANSION_OUTER_IPV6_UDP, 154 MLX5_EXPANSION_OUTER_IPV6_TCP, 155 MLX5_EXPANSION_IPV4, 156 MLX5_EXPANSION_IPV6), 157 .type = RTE_FLOW_ITEM_TYPE_IPV6, 158 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 159 ETH_RSS_NONFRAG_IPV6_OTHER, 160 }, 161 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 162 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 163 MLX5_EXPANSION_VXLAN_GPE), 164 .type = RTE_FLOW_ITEM_TYPE_UDP, 165 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 166 }, 167 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 168 .type = RTE_FLOW_ITEM_TYPE_TCP, 169 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 170 }, 171 [MLX5_EXPANSION_VXLAN] = { 172 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 173 MLX5_EXPANSION_IPV4, 174 MLX5_EXPANSION_IPV6), 175 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 176 }, 177 [MLX5_EXPANSION_VXLAN_GPE] = { 178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 179 MLX5_EXPANSION_IPV4, 180 MLX5_EXPANSION_IPV6), 181 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 182 }, 183 [MLX5_EXPANSION_GRE] = { 184 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 185 .type = RTE_FLOW_ITEM_TYPE_GRE, 186 }, 187 [MLX5_EXPANSION_MPLS] = { 188 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 189 MLX5_EXPANSION_IPV6), 190 .type = RTE_FLOW_ITEM_TYPE_MPLS, 191 }, 192 [MLX5_EXPANSION_ETH] = { 193 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 194 MLX5_EXPANSION_IPV6), 195 .type = RTE_FLOW_ITEM_TYPE_ETH, 196 }, 197 [MLX5_EXPANSION_ETH_VLAN] = { 198 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 199 .type = RTE_FLOW_ITEM_TYPE_ETH, 200 }, 201 [MLX5_EXPANSION_VLAN] = { 202 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 203 MLX5_EXPANSION_IPV6), 204 .type = RTE_FLOW_ITEM_TYPE_VLAN, 205 }, 206 [MLX5_EXPANSION_IPV4] = { 207 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 208 MLX5_EXPANSION_IPV4_TCP), 209 .type = RTE_FLOW_ITEM_TYPE_IPV4, 210 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 211 ETH_RSS_NONFRAG_IPV4_OTHER, 212 }, 213 [MLX5_EXPANSION_IPV4_UDP] = { 214 .type = RTE_FLOW_ITEM_TYPE_UDP, 215 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 216 }, 217 [MLX5_EXPANSION_IPV4_TCP] = { 218 .type = RTE_FLOW_ITEM_TYPE_TCP, 219 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 220 }, 221 [MLX5_EXPANSION_IPV6] = { 222 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 223 MLX5_EXPANSION_IPV6_TCP), 224 .type = RTE_FLOW_ITEM_TYPE_IPV6, 225 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 226 ETH_RSS_NONFRAG_IPV6_OTHER, 227 }, 228 [MLX5_EXPANSION_IPV6_UDP] = { 229 .type = RTE_FLOW_ITEM_TYPE_UDP, 230 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 231 }, 232 [MLX5_EXPANSION_IPV6_TCP] = { 233 .type = RTE_FLOW_ITEM_TYPE_TCP, 234 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 235 }, 236 }; 237 238 static const struct rte_flow_ops mlx5_flow_ops = { 239 .validate = mlx5_flow_validate, 240 .create = mlx5_flow_create, 241 .destroy = mlx5_flow_destroy, 242 .flush = mlx5_flow_flush, 243 .isolate = mlx5_flow_isolate, 244 .query = mlx5_flow_query, 245 .dev_dump = mlx5_flow_dev_dump, 246 .get_aged_flows = mlx5_flow_get_aged_flows, 247 }; 248 249 /* Convert FDIR request to Generic flow. */ 250 struct mlx5_fdir { 251 struct rte_flow_attr attr; 252 struct rte_flow_item items[4]; 253 struct rte_flow_item_eth l2; 254 struct rte_flow_item_eth l2_mask; 255 union { 256 struct rte_flow_item_ipv4 ipv4; 257 struct rte_flow_item_ipv6 ipv6; 258 } l3; 259 union { 260 struct rte_flow_item_ipv4 ipv4; 261 struct rte_flow_item_ipv6 ipv6; 262 } l3_mask; 263 union { 264 struct rte_flow_item_udp udp; 265 struct rte_flow_item_tcp tcp; 266 } l4; 267 union { 268 struct rte_flow_item_udp udp; 269 struct rte_flow_item_tcp tcp; 270 } l4_mask; 271 struct rte_flow_action actions[2]; 272 struct rte_flow_action_queue queue; 273 }; 274 275 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 276 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 277 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 278 }; 279 280 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 281 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 282 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 283 { 9, 10, 11 }, { 12, 13, 14 }, 284 }; 285 286 /* Tunnel information. */ 287 struct mlx5_flow_tunnel_info { 288 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 289 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 290 }; 291 292 static struct mlx5_flow_tunnel_info tunnels_info[] = { 293 { 294 .tunnel = MLX5_FLOW_LAYER_VXLAN, 295 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 296 }, 297 { 298 .tunnel = MLX5_FLOW_LAYER_GENEVE, 299 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 300 }, 301 { 302 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 303 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 304 }, 305 { 306 .tunnel = MLX5_FLOW_LAYER_GRE, 307 .ptype = RTE_PTYPE_TUNNEL_GRE, 308 }, 309 { 310 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 311 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 312 }, 313 { 314 .tunnel = MLX5_FLOW_LAYER_MPLS, 315 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 316 }, 317 { 318 .tunnel = MLX5_FLOW_LAYER_NVGRE, 319 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 320 }, 321 { 322 .tunnel = MLX5_FLOW_LAYER_IPIP, 323 .ptype = RTE_PTYPE_TUNNEL_IP, 324 }, 325 { 326 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 327 .ptype = RTE_PTYPE_TUNNEL_IP, 328 }, 329 { 330 .tunnel = MLX5_FLOW_LAYER_GTP, 331 .ptype = RTE_PTYPE_TUNNEL_GTPU, 332 }, 333 }; 334 335 /** 336 * Translate tag ID to register. 337 * 338 * @param[in] dev 339 * Pointer to the Ethernet device structure. 340 * @param[in] feature 341 * The feature that request the register. 342 * @param[in] id 343 * The request register ID. 344 * @param[out] error 345 * Error description in case of any. 346 * 347 * @return 348 * The request register on success, a negative errno 349 * value otherwise and rte_errno is set. 350 */ 351 int 352 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 353 enum mlx5_feature_name feature, 354 uint32_t id, 355 struct rte_flow_error *error) 356 { 357 struct mlx5_priv *priv = dev->data->dev_private; 358 struct mlx5_dev_config *config = &priv->config; 359 enum modify_reg start_reg; 360 bool skip_mtr_reg = false; 361 362 switch (feature) { 363 case MLX5_HAIRPIN_RX: 364 return REG_B; 365 case MLX5_HAIRPIN_TX: 366 return REG_A; 367 case MLX5_METADATA_RX: 368 switch (config->dv_xmeta_en) { 369 case MLX5_XMETA_MODE_LEGACY: 370 return REG_B; 371 case MLX5_XMETA_MODE_META16: 372 return REG_C_0; 373 case MLX5_XMETA_MODE_META32: 374 return REG_C_1; 375 } 376 break; 377 case MLX5_METADATA_TX: 378 return REG_A; 379 case MLX5_METADATA_FDB: 380 switch (config->dv_xmeta_en) { 381 case MLX5_XMETA_MODE_LEGACY: 382 return REG_NONE; 383 case MLX5_XMETA_MODE_META16: 384 return REG_C_0; 385 case MLX5_XMETA_MODE_META32: 386 return REG_C_1; 387 } 388 break; 389 case MLX5_FLOW_MARK: 390 switch (config->dv_xmeta_en) { 391 case MLX5_XMETA_MODE_LEGACY: 392 return REG_NONE; 393 case MLX5_XMETA_MODE_META16: 394 return REG_C_1; 395 case MLX5_XMETA_MODE_META32: 396 return REG_C_0; 397 } 398 break; 399 case MLX5_MTR_SFX: 400 /* 401 * If meter color and flow match share one register, flow match 402 * should use the meter color register for match. 403 */ 404 if (priv->mtr_reg_share) 405 return priv->mtr_color_reg; 406 else 407 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 408 REG_C_3; 409 case MLX5_MTR_COLOR: 410 MLX5_ASSERT(priv->mtr_color_reg != REG_NONE); 411 return priv->mtr_color_reg; 412 case MLX5_COPY_MARK: 413 /* 414 * Metadata COPY_MARK register using is in meter suffix sub 415 * flow while with meter. It's safe to share the same register. 416 */ 417 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 418 case MLX5_APP_TAG: 419 /* 420 * If meter is enable, it will engage the register for color 421 * match and flow match. If meter color match is not using the 422 * REG_C_2, need to skip the REG_C_x be used by meter color 423 * match. 424 * If meter is disable, free to use all available registers. 425 */ 426 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 427 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 428 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 429 if (id > (REG_C_7 - start_reg)) 430 return rte_flow_error_set(error, EINVAL, 431 RTE_FLOW_ERROR_TYPE_ITEM, 432 NULL, "invalid tag id"); 433 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 434 return rte_flow_error_set(error, ENOTSUP, 435 RTE_FLOW_ERROR_TYPE_ITEM, 436 NULL, "unsupported tag id"); 437 /* 438 * This case means meter is using the REG_C_x great than 2. 439 * Take care not to conflict with meter color REG_C_x. 440 * If the available index REG_C_y >= REG_C_x, skip the 441 * color register. 442 */ 443 if (skip_mtr_reg && config->flow_mreg_c 444 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 445 if (config->flow_mreg_c 446 [id + 1 + start_reg - REG_C_0] != REG_NONE) 447 return config->flow_mreg_c 448 [id + 1 + start_reg - REG_C_0]; 449 return rte_flow_error_set(error, ENOTSUP, 450 RTE_FLOW_ERROR_TYPE_ITEM, 451 NULL, "unsupported tag id"); 452 } 453 return config->flow_mreg_c[id + start_reg - REG_C_0]; 454 } 455 MLX5_ASSERT(false); 456 return rte_flow_error_set(error, EINVAL, 457 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 458 NULL, "invalid feature name"); 459 } 460 461 /** 462 * Check extensive flow metadata register support. 463 * 464 * @param dev 465 * Pointer to rte_eth_dev structure. 466 * 467 * @return 468 * True if device supports extensive flow metadata register, otherwise false. 469 */ 470 bool 471 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 472 { 473 struct mlx5_priv *priv = dev->data->dev_private; 474 struct mlx5_dev_config *config = &priv->config; 475 476 /* 477 * Having available reg_c can be regarded inclusively as supporting 478 * extensive flow metadata register, which could mean, 479 * - metadata register copy action by modify header. 480 * - 16 modify header actions is supported. 481 * - reg_c's are preserved across different domain (FDB and NIC) on 482 * packet loopback by flow lookup miss. 483 */ 484 return config->flow_mreg_c[2] != REG_NONE; 485 } 486 487 /** 488 * Discover the maximum number of priority available. 489 * 490 * @param[in] dev 491 * Pointer to the Ethernet device structure. 492 * 493 * @return 494 * number of supported flow priority on success, a negative errno 495 * value otherwise and rte_errno is set. 496 */ 497 int 498 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 499 { 500 struct mlx5_priv *priv = dev->data->dev_private; 501 struct { 502 struct ibv_flow_attr attr; 503 struct ibv_flow_spec_eth eth; 504 struct ibv_flow_spec_action_drop drop; 505 } flow_attr = { 506 .attr = { 507 .num_of_specs = 2, 508 .port = (uint8_t)priv->ibv_port, 509 }, 510 .eth = { 511 .type = IBV_FLOW_SPEC_ETH, 512 .size = sizeof(struct ibv_flow_spec_eth), 513 }, 514 .drop = { 515 .size = sizeof(struct ibv_flow_spec_action_drop), 516 .type = IBV_FLOW_SPEC_ACTION_DROP, 517 }, 518 }; 519 struct ibv_flow *flow; 520 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 521 uint16_t vprio[] = { 8, 16 }; 522 int i; 523 int priority = 0; 524 525 if (!drop) { 526 rte_errno = ENOTSUP; 527 return -rte_errno; 528 } 529 for (i = 0; i != RTE_DIM(vprio); i++) { 530 flow_attr.attr.priority = vprio[i] - 1; 531 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 532 if (!flow) 533 break; 534 claim_zero(mlx5_glue->destroy_flow(flow)); 535 priority = vprio[i]; 536 } 537 mlx5_hrxq_drop_release(dev); 538 switch (priority) { 539 case 8: 540 priority = RTE_DIM(priority_map_3); 541 break; 542 case 16: 543 priority = RTE_DIM(priority_map_5); 544 break; 545 default: 546 rte_errno = ENOTSUP; 547 DRV_LOG(ERR, 548 "port %u verbs maximum priority: %d expected 8/16", 549 dev->data->port_id, priority); 550 return -rte_errno; 551 } 552 DRV_LOG(INFO, "port %u flow maximum priority: %d", 553 dev->data->port_id, priority); 554 return priority; 555 } 556 557 /** 558 * Adjust flow priority based on the highest layer and the request priority. 559 * 560 * @param[in] dev 561 * Pointer to the Ethernet device structure. 562 * @param[in] priority 563 * The rule base priority. 564 * @param[in] subpriority 565 * The priority based on the items. 566 * 567 * @return 568 * The new priority. 569 */ 570 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 571 uint32_t subpriority) 572 { 573 uint32_t res = 0; 574 struct mlx5_priv *priv = dev->data->dev_private; 575 576 switch (priv->config.flow_prio) { 577 case RTE_DIM(priority_map_3): 578 res = priority_map_3[priority][subpriority]; 579 break; 580 case RTE_DIM(priority_map_5): 581 res = priority_map_5[priority][subpriority]; 582 break; 583 } 584 return res; 585 } 586 587 /** 588 * Verify the @p item specifications (spec, last, mask) are compatible with the 589 * NIC capabilities. 590 * 591 * @param[in] item 592 * Item specification. 593 * @param[in] mask 594 * @p item->mask or flow default bit-masks. 595 * @param[in] nic_mask 596 * Bit-masks covering supported fields by the NIC to compare with user mask. 597 * @param[in] size 598 * Bit-masks size in bytes. 599 * @param[out] error 600 * Pointer to error structure. 601 * 602 * @return 603 * 0 on success, a negative errno value otherwise and rte_errno is set. 604 */ 605 int 606 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 607 const uint8_t *mask, 608 const uint8_t *nic_mask, 609 unsigned int size, 610 struct rte_flow_error *error) 611 { 612 unsigned int i; 613 614 MLX5_ASSERT(nic_mask); 615 for (i = 0; i < size; ++i) 616 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 617 return rte_flow_error_set(error, ENOTSUP, 618 RTE_FLOW_ERROR_TYPE_ITEM, 619 item, 620 "mask enables non supported" 621 " bits"); 622 if (!item->spec && (item->mask || item->last)) 623 return rte_flow_error_set(error, EINVAL, 624 RTE_FLOW_ERROR_TYPE_ITEM, item, 625 "mask/last without a spec is not" 626 " supported"); 627 if (item->spec && item->last) { 628 uint8_t spec[size]; 629 uint8_t last[size]; 630 unsigned int i; 631 int ret; 632 633 for (i = 0; i < size; ++i) { 634 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 635 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 636 } 637 ret = memcmp(spec, last, size); 638 if (ret != 0) 639 return rte_flow_error_set(error, EINVAL, 640 RTE_FLOW_ERROR_TYPE_ITEM, 641 item, 642 "range is not valid"); 643 } 644 return 0; 645 } 646 647 /** 648 * Adjust the hash fields according to the @p flow information. 649 * 650 * @param[in] dev_flow. 651 * Pointer to the mlx5_flow. 652 * @param[in] tunnel 653 * 1 when the hash field is for a tunnel item. 654 * @param[in] layer_types 655 * ETH_RSS_* types. 656 * @param[in] hash_fields 657 * Item hash fields. 658 * 659 * @return 660 * The hash fields that should be used. 661 */ 662 uint64_t 663 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 664 int tunnel __rte_unused, uint64_t layer_types, 665 uint64_t hash_fields) 666 { 667 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 668 int rss_request_inner = rss_desc->level >= 2; 669 670 /* Check RSS hash level for tunnel. */ 671 if (tunnel && rss_request_inner) 672 hash_fields |= IBV_RX_HASH_INNER; 673 else if (tunnel || rss_request_inner) 674 return 0; 675 #endif 676 /* Check if requested layer matches RSS hash fields. */ 677 if (!(rss_desc->types & layer_types)) 678 return 0; 679 return hash_fields; 680 } 681 682 /** 683 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 684 * if several tunnel rules are used on this queue, the tunnel ptype will be 685 * cleared. 686 * 687 * @param rxq_ctrl 688 * Rx queue to update. 689 */ 690 static void 691 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 692 { 693 unsigned int i; 694 uint32_t tunnel_ptype = 0; 695 696 /* Look up for the ptype to use. */ 697 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 698 if (!rxq_ctrl->flow_tunnels_n[i]) 699 continue; 700 if (!tunnel_ptype) { 701 tunnel_ptype = tunnels_info[i].ptype; 702 } else { 703 tunnel_ptype = 0; 704 break; 705 } 706 } 707 rxq_ctrl->rxq.tunnel = tunnel_ptype; 708 } 709 710 /** 711 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 712 * flow. 713 * 714 * @param[in] dev 715 * Pointer to the Ethernet device structure. 716 * @param[in] dev_handle 717 * Pointer to device flow handle structure. 718 */ 719 static void 720 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 721 struct mlx5_flow_handle *dev_handle) 722 { 723 struct mlx5_priv *priv = dev->data->dev_private; 724 const int mark = dev_handle->mark; 725 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 726 struct mlx5_hrxq *hrxq; 727 unsigned int i; 728 729 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 730 return; 731 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 732 dev_handle->rix_hrxq); 733 if (!hrxq) 734 return; 735 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 736 int idx = hrxq->ind_table->queues[i]; 737 struct mlx5_rxq_ctrl *rxq_ctrl = 738 container_of((*priv->rxqs)[idx], 739 struct mlx5_rxq_ctrl, rxq); 740 741 /* 742 * To support metadata register copy on Tx loopback, 743 * this must be always enabled (metadata may arive 744 * from other port - not from local flows only. 745 */ 746 if (priv->config.dv_flow_en && 747 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 748 mlx5_flow_ext_mreg_supported(dev)) { 749 rxq_ctrl->rxq.mark = 1; 750 rxq_ctrl->flow_mark_n = 1; 751 } else if (mark) { 752 rxq_ctrl->rxq.mark = 1; 753 rxq_ctrl->flow_mark_n++; 754 } 755 if (tunnel) { 756 unsigned int j; 757 758 /* Increase the counter matching the flow. */ 759 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 760 if ((tunnels_info[j].tunnel & 761 dev_handle->layers) == 762 tunnels_info[j].tunnel) { 763 rxq_ctrl->flow_tunnels_n[j]++; 764 break; 765 } 766 } 767 flow_rxq_tunnel_ptype_update(rxq_ctrl); 768 } 769 } 770 } 771 772 /** 773 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 774 * 775 * @param[in] dev 776 * Pointer to the Ethernet device structure. 777 * @param[in] flow 778 * Pointer to flow structure. 779 */ 780 static void 781 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 782 { 783 struct mlx5_priv *priv = dev->data->dev_private; 784 uint32_t handle_idx; 785 struct mlx5_flow_handle *dev_handle; 786 787 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 788 handle_idx, dev_handle, next) 789 flow_drv_rxq_flags_set(dev, dev_handle); 790 } 791 792 /** 793 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 794 * device flow if no other flow uses it with the same kind of request. 795 * 796 * @param dev 797 * Pointer to Ethernet device. 798 * @param[in] dev_handle 799 * Pointer to the device flow handle structure. 800 */ 801 static void 802 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 803 struct mlx5_flow_handle *dev_handle) 804 { 805 struct mlx5_priv *priv = dev->data->dev_private; 806 const int mark = dev_handle->mark; 807 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 808 struct mlx5_hrxq *hrxq; 809 unsigned int i; 810 811 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 812 return; 813 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 814 dev_handle->rix_hrxq); 815 if (!hrxq) 816 return; 817 MLX5_ASSERT(dev->data->dev_started); 818 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 819 int idx = hrxq->ind_table->queues[i]; 820 struct mlx5_rxq_ctrl *rxq_ctrl = 821 container_of((*priv->rxqs)[idx], 822 struct mlx5_rxq_ctrl, rxq); 823 824 if (priv->config.dv_flow_en && 825 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 826 mlx5_flow_ext_mreg_supported(dev)) { 827 rxq_ctrl->rxq.mark = 1; 828 rxq_ctrl->flow_mark_n = 1; 829 } else if (mark) { 830 rxq_ctrl->flow_mark_n--; 831 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 832 } 833 if (tunnel) { 834 unsigned int j; 835 836 /* Decrease the counter matching the flow. */ 837 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 838 if ((tunnels_info[j].tunnel & 839 dev_handle->layers) == 840 tunnels_info[j].tunnel) { 841 rxq_ctrl->flow_tunnels_n[j]--; 842 break; 843 } 844 } 845 flow_rxq_tunnel_ptype_update(rxq_ctrl); 846 } 847 } 848 } 849 850 /** 851 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 852 * @p flow if no other flow uses it with the same kind of request. 853 * 854 * @param dev 855 * Pointer to Ethernet device. 856 * @param[in] flow 857 * Pointer to the flow. 858 */ 859 static void 860 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 861 { 862 struct mlx5_priv *priv = dev->data->dev_private; 863 uint32_t handle_idx; 864 struct mlx5_flow_handle *dev_handle; 865 866 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 867 handle_idx, dev_handle, next) 868 flow_drv_rxq_flags_trim(dev, dev_handle); 869 } 870 871 /** 872 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 873 * 874 * @param dev 875 * Pointer to Ethernet device. 876 */ 877 static void 878 flow_rxq_flags_clear(struct rte_eth_dev *dev) 879 { 880 struct mlx5_priv *priv = dev->data->dev_private; 881 unsigned int i; 882 883 for (i = 0; i != priv->rxqs_n; ++i) { 884 struct mlx5_rxq_ctrl *rxq_ctrl; 885 unsigned int j; 886 887 if (!(*priv->rxqs)[i]) 888 continue; 889 rxq_ctrl = container_of((*priv->rxqs)[i], 890 struct mlx5_rxq_ctrl, rxq); 891 rxq_ctrl->flow_mark_n = 0; 892 rxq_ctrl->rxq.mark = 0; 893 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 894 rxq_ctrl->flow_tunnels_n[j] = 0; 895 rxq_ctrl->rxq.tunnel = 0; 896 } 897 } 898 899 /** 900 * Set the Rx queue dynamic metadata (mask and offset) for a flow 901 * 902 * @param[in] dev 903 * Pointer to the Ethernet device structure. 904 */ 905 void 906 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 907 { 908 struct mlx5_priv *priv = dev->data->dev_private; 909 struct mlx5_rxq_data *data; 910 unsigned int i; 911 912 for (i = 0; i != priv->rxqs_n; ++i) { 913 if (!(*priv->rxqs)[i]) 914 continue; 915 data = (*priv->rxqs)[i]; 916 if (!rte_flow_dynf_metadata_avail()) { 917 data->dynf_meta = 0; 918 data->flow_meta_mask = 0; 919 data->flow_meta_offset = -1; 920 } else { 921 data->dynf_meta = 1; 922 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 923 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 924 } 925 } 926 } 927 928 /* 929 * return a pointer to the desired action in the list of actions. 930 * 931 * @param[in] actions 932 * The list of actions to search the action in. 933 * @param[in] action 934 * The action to find. 935 * 936 * @return 937 * Pointer to the action in the list, if found. NULL otherwise. 938 */ 939 const struct rte_flow_action * 940 mlx5_flow_find_action(const struct rte_flow_action *actions, 941 enum rte_flow_action_type action) 942 { 943 if (actions == NULL) 944 return NULL; 945 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 946 if (actions->type == action) 947 return actions; 948 return NULL; 949 } 950 951 /* 952 * Validate the flag action. 953 * 954 * @param[in] action_flags 955 * Bit-fields that holds the actions detected until now. 956 * @param[in] attr 957 * Attributes of flow that includes this action. 958 * @param[out] error 959 * Pointer to error structure. 960 * 961 * @return 962 * 0 on success, a negative errno value otherwise and rte_errno is set. 963 */ 964 int 965 mlx5_flow_validate_action_flag(uint64_t action_flags, 966 const struct rte_flow_attr *attr, 967 struct rte_flow_error *error) 968 { 969 if (action_flags & MLX5_FLOW_ACTION_MARK) 970 return rte_flow_error_set(error, EINVAL, 971 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 972 "can't mark and flag in same flow"); 973 if (action_flags & MLX5_FLOW_ACTION_FLAG) 974 return rte_flow_error_set(error, EINVAL, 975 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 976 "can't have 2 flag" 977 " actions in same flow"); 978 if (attr->egress) 979 return rte_flow_error_set(error, ENOTSUP, 980 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 981 "flag action not supported for " 982 "egress"); 983 return 0; 984 } 985 986 /* 987 * Validate the mark action. 988 * 989 * @param[in] action 990 * Pointer to the queue action. 991 * @param[in] action_flags 992 * Bit-fields that holds the actions detected until now. 993 * @param[in] attr 994 * Attributes of flow that includes this action. 995 * @param[out] error 996 * Pointer to error structure. 997 * 998 * @return 999 * 0 on success, a negative errno value otherwise and rte_errno is set. 1000 */ 1001 int 1002 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1003 uint64_t action_flags, 1004 const struct rte_flow_attr *attr, 1005 struct rte_flow_error *error) 1006 { 1007 const struct rte_flow_action_mark *mark = action->conf; 1008 1009 if (!mark) 1010 return rte_flow_error_set(error, EINVAL, 1011 RTE_FLOW_ERROR_TYPE_ACTION, 1012 action, 1013 "configuration cannot be null"); 1014 if (mark->id >= MLX5_FLOW_MARK_MAX) 1015 return rte_flow_error_set(error, EINVAL, 1016 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1017 &mark->id, 1018 "mark id must in 0 <= id < " 1019 RTE_STR(MLX5_FLOW_MARK_MAX)); 1020 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1021 return rte_flow_error_set(error, EINVAL, 1022 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1023 "can't flag and mark in same flow"); 1024 if (action_flags & MLX5_FLOW_ACTION_MARK) 1025 return rte_flow_error_set(error, EINVAL, 1026 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1027 "can't have 2 mark actions in same" 1028 " flow"); 1029 if (attr->egress) 1030 return rte_flow_error_set(error, ENOTSUP, 1031 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1032 "mark action not supported for " 1033 "egress"); 1034 return 0; 1035 } 1036 1037 /* 1038 * Validate the drop action. 1039 * 1040 * @param[in] action_flags 1041 * Bit-fields that holds the actions detected until now. 1042 * @param[in] attr 1043 * Attributes of flow that includes this action. 1044 * @param[out] error 1045 * Pointer to error structure. 1046 * 1047 * @return 1048 * 0 on success, a negative errno value otherwise and rte_errno is set. 1049 */ 1050 int 1051 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1052 const struct rte_flow_attr *attr, 1053 struct rte_flow_error *error) 1054 { 1055 if (attr->egress) 1056 return rte_flow_error_set(error, ENOTSUP, 1057 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1058 "drop action not supported for " 1059 "egress"); 1060 return 0; 1061 } 1062 1063 /* 1064 * Validate the queue action. 1065 * 1066 * @param[in] action 1067 * Pointer to the queue action. 1068 * @param[in] action_flags 1069 * Bit-fields that holds the actions detected until now. 1070 * @param[in] dev 1071 * Pointer to the Ethernet device structure. 1072 * @param[in] attr 1073 * Attributes of flow that includes this action. 1074 * @param[out] error 1075 * Pointer to error structure. 1076 * 1077 * @return 1078 * 0 on success, a negative errno value otherwise and rte_errno is set. 1079 */ 1080 int 1081 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1082 uint64_t action_flags, 1083 struct rte_eth_dev *dev, 1084 const struct rte_flow_attr *attr, 1085 struct rte_flow_error *error) 1086 { 1087 struct mlx5_priv *priv = dev->data->dev_private; 1088 const struct rte_flow_action_queue *queue = action->conf; 1089 1090 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1091 return rte_flow_error_set(error, EINVAL, 1092 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1093 "can't have 2 fate actions in" 1094 " same flow"); 1095 if (!priv->rxqs_n) 1096 return rte_flow_error_set(error, EINVAL, 1097 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1098 NULL, "No Rx queues configured"); 1099 if (queue->index >= priv->rxqs_n) 1100 return rte_flow_error_set(error, EINVAL, 1101 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1102 &queue->index, 1103 "queue index out of range"); 1104 if (!(*priv->rxqs)[queue->index]) 1105 return rte_flow_error_set(error, EINVAL, 1106 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1107 &queue->index, 1108 "queue is not configured"); 1109 if (attr->egress) 1110 return rte_flow_error_set(error, ENOTSUP, 1111 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1112 "queue action not supported for " 1113 "egress"); 1114 return 0; 1115 } 1116 1117 /* 1118 * Validate the rss action. 1119 * 1120 * @param[in] action 1121 * Pointer to the queue action. 1122 * @param[in] action_flags 1123 * Bit-fields that holds the actions detected until now. 1124 * @param[in] dev 1125 * Pointer to the Ethernet device structure. 1126 * @param[in] attr 1127 * Attributes of flow that includes this action. 1128 * @param[in] item_flags 1129 * Items that were detected. 1130 * @param[out] error 1131 * Pointer to error structure. 1132 * 1133 * @return 1134 * 0 on success, a negative errno value otherwise and rte_errno is set. 1135 */ 1136 int 1137 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1138 uint64_t action_flags, 1139 struct rte_eth_dev *dev, 1140 const struct rte_flow_attr *attr, 1141 uint64_t item_flags, 1142 struct rte_flow_error *error) 1143 { 1144 struct mlx5_priv *priv = dev->data->dev_private; 1145 const struct rte_flow_action_rss *rss = action->conf; 1146 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1147 unsigned int i; 1148 1149 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1150 return rte_flow_error_set(error, EINVAL, 1151 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1152 "can't have 2 fate actions" 1153 " in same flow"); 1154 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1155 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1156 return rte_flow_error_set(error, ENOTSUP, 1157 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1158 &rss->func, 1159 "RSS hash function not supported"); 1160 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1161 if (rss->level > 2) 1162 #else 1163 if (rss->level > 1) 1164 #endif 1165 return rte_flow_error_set(error, ENOTSUP, 1166 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1167 &rss->level, 1168 "tunnel RSS is not supported"); 1169 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1170 if (rss->key_len == 0 && rss->key != NULL) 1171 return rte_flow_error_set(error, ENOTSUP, 1172 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1173 &rss->key_len, 1174 "RSS hash key length 0"); 1175 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1176 return rte_flow_error_set(error, ENOTSUP, 1177 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1178 &rss->key_len, 1179 "RSS hash key too small"); 1180 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1181 return rte_flow_error_set(error, ENOTSUP, 1182 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1183 &rss->key_len, 1184 "RSS hash key too large"); 1185 if (rss->queue_num > priv->config.ind_table_max_size) 1186 return rte_flow_error_set(error, ENOTSUP, 1187 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1188 &rss->queue_num, 1189 "number of queues too large"); 1190 if (rss->types & MLX5_RSS_HF_MASK) 1191 return rte_flow_error_set(error, ENOTSUP, 1192 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1193 &rss->types, 1194 "some RSS protocols are not" 1195 " supported"); 1196 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1197 !(rss->types & ETH_RSS_IP)) 1198 return rte_flow_error_set(error, EINVAL, 1199 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1200 "L3 partial RSS requested but L3 RSS" 1201 " type not specified"); 1202 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1203 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1204 return rte_flow_error_set(error, EINVAL, 1205 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1206 "L4 partial RSS requested but L4 RSS" 1207 " type not specified"); 1208 if (!priv->rxqs_n) 1209 return rte_flow_error_set(error, EINVAL, 1210 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1211 NULL, "No Rx queues configured"); 1212 if (!rss->queue_num) 1213 return rte_flow_error_set(error, EINVAL, 1214 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1215 NULL, "No queues configured"); 1216 for (i = 0; i != rss->queue_num; ++i) { 1217 if (rss->queue[i] >= priv->rxqs_n) 1218 return rte_flow_error_set 1219 (error, EINVAL, 1220 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1221 &rss->queue[i], "queue index out of range"); 1222 if (!(*priv->rxqs)[rss->queue[i]]) 1223 return rte_flow_error_set 1224 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1225 &rss->queue[i], "queue is not configured"); 1226 } 1227 if (attr->egress) 1228 return rte_flow_error_set(error, ENOTSUP, 1229 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1230 "rss action not supported for " 1231 "egress"); 1232 if (rss->level > 1 && !tunnel) 1233 return rte_flow_error_set(error, EINVAL, 1234 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1235 "inner RSS is not supported for " 1236 "non-tunnel flows"); 1237 return 0; 1238 } 1239 1240 /* 1241 * Validate the count action. 1242 * 1243 * @param[in] dev 1244 * Pointer to the Ethernet device structure. 1245 * @param[in] attr 1246 * Attributes of flow that includes this action. 1247 * @param[out] error 1248 * Pointer to error structure. 1249 * 1250 * @return 1251 * 0 on success, a negative errno value otherwise and rte_errno is set. 1252 */ 1253 int 1254 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1255 const struct rte_flow_attr *attr, 1256 struct rte_flow_error *error) 1257 { 1258 if (attr->egress) 1259 return rte_flow_error_set(error, ENOTSUP, 1260 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1261 "count action not supported for " 1262 "egress"); 1263 return 0; 1264 } 1265 1266 /** 1267 * Verify the @p attributes will be correctly understood by the NIC and store 1268 * them in the @p flow if everything is correct. 1269 * 1270 * @param[in] dev 1271 * Pointer to the Ethernet device structure. 1272 * @param[in] attributes 1273 * Pointer to flow attributes 1274 * @param[out] error 1275 * Pointer to error structure. 1276 * 1277 * @return 1278 * 0 on success, a negative errno value otherwise and rte_errno is set. 1279 */ 1280 int 1281 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1282 const struct rte_flow_attr *attributes, 1283 struct rte_flow_error *error) 1284 { 1285 struct mlx5_priv *priv = dev->data->dev_private; 1286 uint32_t priority_max = priv->config.flow_prio - 1; 1287 1288 if (attributes->group) 1289 return rte_flow_error_set(error, ENOTSUP, 1290 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1291 NULL, "groups is not supported"); 1292 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1293 attributes->priority >= priority_max) 1294 return rte_flow_error_set(error, ENOTSUP, 1295 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1296 NULL, "priority out of range"); 1297 if (attributes->egress) 1298 return rte_flow_error_set(error, ENOTSUP, 1299 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1300 "egress is not supported"); 1301 if (attributes->transfer && !priv->config.dv_esw_en) 1302 return rte_flow_error_set(error, ENOTSUP, 1303 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1304 NULL, "transfer is not supported"); 1305 if (!attributes->ingress) 1306 return rte_flow_error_set(error, EINVAL, 1307 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1308 NULL, 1309 "ingress attribute is mandatory"); 1310 return 0; 1311 } 1312 1313 /** 1314 * Validate ICMP6 item. 1315 * 1316 * @param[in] item 1317 * Item specification. 1318 * @param[in] item_flags 1319 * Bit-fields that holds the items detected until now. 1320 * @param[out] error 1321 * Pointer to error structure. 1322 * 1323 * @return 1324 * 0 on success, a negative errno value otherwise and rte_errno is set. 1325 */ 1326 int 1327 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1328 uint64_t item_flags, 1329 uint8_t target_protocol, 1330 struct rte_flow_error *error) 1331 { 1332 const struct rte_flow_item_icmp6 *mask = item->mask; 1333 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1334 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1335 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1336 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1337 MLX5_FLOW_LAYER_OUTER_L4; 1338 int ret; 1339 1340 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1341 return rte_flow_error_set(error, EINVAL, 1342 RTE_FLOW_ERROR_TYPE_ITEM, item, 1343 "protocol filtering not compatible" 1344 " with ICMP6 layer"); 1345 if (!(item_flags & l3m)) 1346 return rte_flow_error_set(error, EINVAL, 1347 RTE_FLOW_ERROR_TYPE_ITEM, item, 1348 "IPv6 is mandatory to filter on" 1349 " ICMP6"); 1350 if (item_flags & l4m) 1351 return rte_flow_error_set(error, EINVAL, 1352 RTE_FLOW_ERROR_TYPE_ITEM, item, 1353 "multiple L4 layers not supported"); 1354 if (!mask) 1355 mask = &rte_flow_item_icmp6_mask; 1356 ret = mlx5_flow_item_acceptable 1357 (item, (const uint8_t *)mask, 1358 (const uint8_t *)&rte_flow_item_icmp6_mask, 1359 sizeof(struct rte_flow_item_icmp6), error); 1360 if (ret < 0) 1361 return ret; 1362 return 0; 1363 } 1364 1365 /** 1366 * Validate ICMP item. 1367 * 1368 * @param[in] item 1369 * Item specification. 1370 * @param[in] item_flags 1371 * Bit-fields that holds the items detected until now. 1372 * @param[out] error 1373 * Pointer to error structure. 1374 * 1375 * @return 1376 * 0 on success, a negative errno value otherwise and rte_errno is set. 1377 */ 1378 int 1379 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1380 uint64_t item_flags, 1381 uint8_t target_protocol, 1382 struct rte_flow_error *error) 1383 { 1384 const struct rte_flow_item_icmp *mask = item->mask; 1385 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1386 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1387 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1388 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1389 MLX5_FLOW_LAYER_OUTER_L4; 1390 int ret; 1391 1392 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1393 return rte_flow_error_set(error, EINVAL, 1394 RTE_FLOW_ERROR_TYPE_ITEM, item, 1395 "protocol filtering not compatible" 1396 " with ICMP layer"); 1397 if (!(item_flags & l3m)) 1398 return rte_flow_error_set(error, EINVAL, 1399 RTE_FLOW_ERROR_TYPE_ITEM, item, 1400 "IPv4 is mandatory to filter" 1401 " on ICMP"); 1402 if (item_flags & l4m) 1403 return rte_flow_error_set(error, EINVAL, 1404 RTE_FLOW_ERROR_TYPE_ITEM, item, 1405 "multiple L4 layers not supported"); 1406 if (!mask) 1407 mask = &rte_flow_item_icmp_mask; 1408 ret = mlx5_flow_item_acceptable 1409 (item, (const uint8_t *)mask, 1410 (const uint8_t *)&rte_flow_item_icmp_mask, 1411 sizeof(struct rte_flow_item_icmp), error); 1412 if (ret < 0) 1413 return ret; 1414 return 0; 1415 } 1416 1417 /** 1418 * Validate Ethernet item. 1419 * 1420 * @param[in] item 1421 * Item specification. 1422 * @param[in] item_flags 1423 * Bit-fields that holds the items detected until now. 1424 * @param[out] error 1425 * Pointer to error structure. 1426 * 1427 * @return 1428 * 0 on success, a negative errno value otherwise and rte_errno is set. 1429 */ 1430 int 1431 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1432 uint64_t item_flags, 1433 struct rte_flow_error *error) 1434 { 1435 const struct rte_flow_item_eth *mask = item->mask; 1436 const struct rte_flow_item_eth nic_mask = { 1437 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1438 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1439 .type = RTE_BE16(0xffff), 1440 }; 1441 int ret; 1442 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1443 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1444 MLX5_FLOW_LAYER_OUTER_L2; 1445 1446 if (item_flags & ethm) 1447 return rte_flow_error_set(error, ENOTSUP, 1448 RTE_FLOW_ERROR_TYPE_ITEM, item, 1449 "multiple L2 layers not supported"); 1450 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1451 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1452 return rte_flow_error_set(error, EINVAL, 1453 RTE_FLOW_ERROR_TYPE_ITEM, item, 1454 "L2 layer should not follow " 1455 "L3 layers"); 1456 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1457 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1458 return rte_flow_error_set(error, EINVAL, 1459 RTE_FLOW_ERROR_TYPE_ITEM, item, 1460 "L2 layer should not follow VLAN"); 1461 if (!mask) 1462 mask = &rte_flow_item_eth_mask; 1463 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1464 (const uint8_t *)&nic_mask, 1465 sizeof(struct rte_flow_item_eth), 1466 error); 1467 return ret; 1468 } 1469 1470 /** 1471 * Validate VLAN item. 1472 * 1473 * @param[in] item 1474 * Item specification. 1475 * @param[in] item_flags 1476 * Bit-fields that holds the items detected until now. 1477 * @param[in] dev 1478 * Ethernet device flow is being created on. 1479 * @param[out] error 1480 * Pointer to error structure. 1481 * 1482 * @return 1483 * 0 on success, a negative errno value otherwise and rte_errno is set. 1484 */ 1485 int 1486 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1487 uint64_t item_flags, 1488 struct rte_eth_dev *dev, 1489 struct rte_flow_error *error) 1490 { 1491 const struct rte_flow_item_vlan *spec = item->spec; 1492 const struct rte_flow_item_vlan *mask = item->mask; 1493 const struct rte_flow_item_vlan nic_mask = { 1494 .tci = RTE_BE16(UINT16_MAX), 1495 .inner_type = RTE_BE16(UINT16_MAX), 1496 }; 1497 uint16_t vlan_tag = 0; 1498 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1499 int ret; 1500 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1501 MLX5_FLOW_LAYER_INNER_L4) : 1502 (MLX5_FLOW_LAYER_OUTER_L3 | 1503 MLX5_FLOW_LAYER_OUTER_L4); 1504 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1505 MLX5_FLOW_LAYER_OUTER_VLAN; 1506 1507 if (item_flags & vlanm) 1508 return rte_flow_error_set(error, EINVAL, 1509 RTE_FLOW_ERROR_TYPE_ITEM, item, 1510 "multiple VLAN layers not supported"); 1511 else if ((item_flags & l34m) != 0) 1512 return rte_flow_error_set(error, EINVAL, 1513 RTE_FLOW_ERROR_TYPE_ITEM, item, 1514 "VLAN cannot follow L3/L4 layer"); 1515 if (!mask) 1516 mask = &rte_flow_item_vlan_mask; 1517 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1518 (const uint8_t *)&nic_mask, 1519 sizeof(struct rte_flow_item_vlan), 1520 error); 1521 if (ret) 1522 return ret; 1523 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1524 struct mlx5_priv *priv = dev->data->dev_private; 1525 1526 if (priv->vmwa_context) { 1527 /* 1528 * Non-NULL context means we have a virtual machine 1529 * and SR-IOV enabled, we have to create VLAN interface 1530 * to make hypervisor to setup E-Switch vport 1531 * context correctly. We avoid creating the multiple 1532 * VLAN interfaces, so we cannot support VLAN tag mask. 1533 */ 1534 return rte_flow_error_set(error, EINVAL, 1535 RTE_FLOW_ERROR_TYPE_ITEM, 1536 item, 1537 "VLAN tag mask is not" 1538 " supported in virtual" 1539 " environment"); 1540 } 1541 } 1542 if (spec) { 1543 vlan_tag = spec->tci; 1544 vlan_tag &= mask->tci; 1545 } 1546 /* 1547 * From verbs perspective an empty VLAN is equivalent 1548 * to a packet without VLAN layer. 1549 */ 1550 if (!vlan_tag) 1551 return rte_flow_error_set(error, EINVAL, 1552 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1553 item->spec, 1554 "VLAN cannot be empty"); 1555 return 0; 1556 } 1557 1558 /** 1559 * Validate IPV4 item. 1560 * 1561 * @param[in] item 1562 * Item specification. 1563 * @param[in] item_flags 1564 * Bit-fields that holds the items detected until now. 1565 * @param[in] acc_mask 1566 * Acceptable mask, if NULL default internal default mask 1567 * will be used to check whether item fields are supported. 1568 * @param[out] error 1569 * Pointer to error structure. 1570 * 1571 * @return 1572 * 0 on success, a negative errno value otherwise and rte_errno is set. 1573 */ 1574 int 1575 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1576 uint64_t item_flags, 1577 uint64_t last_item, 1578 uint16_t ether_type, 1579 const struct rte_flow_item_ipv4 *acc_mask, 1580 struct rte_flow_error *error) 1581 { 1582 const struct rte_flow_item_ipv4 *mask = item->mask; 1583 const struct rte_flow_item_ipv4 *spec = item->spec; 1584 const struct rte_flow_item_ipv4 nic_mask = { 1585 .hdr = { 1586 .src_addr = RTE_BE32(0xffffffff), 1587 .dst_addr = RTE_BE32(0xffffffff), 1588 .type_of_service = 0xff, 1589 .next_proto_id = 0xff, 1590 }, 1591 }; 1592 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1593 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1594 MLX5_FLOW_LAYER_OUTER_L3; 1595 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1596 MLX5_FLOW_LAYER_OUTER_L4; 1597 int ret; 1598 uint8_t next_proto = 0xFF; 1599 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1600 MLX5_FLOW_LAYER_OUTER_VLAN | 1601 MLX5_FLOW_LAYER_INNER_VLAN); 1602 1603 if ((last_item & l2_vlan) && ether_type && 1604 ether_type != RTE_ETHER_TYPE_IPV4) 1605 return rte_flow_error_set(error, EINVAL, 1606 RTE_FLOW_ERROR_TYPE_ITEM, item, 1607 "IPv4 cannot follow L2/VLAN layer " 1608 "which ether type is not IPv4"); 1609 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1610 if (mask && spec) 1611 next_proto = mask->hdr.next_proto_id & 1612 spec->hdr.next_proto_id; 1613 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1614 return rte_flow_error_set(error, EINVAL, 1615 RTE_FLOW_ERROR_TYPE_ITEM, 1616 item, 1617 "multiple tunnel " 1618 "not supported"); 1619 } 1620 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1621 return rte_flow_error_set(error, EINVAL, 1622 RTE_FLOW_ERROR_TYPE_ITEM, item, 1623 "wrong tunnel type - IPv6 specified " 1624 "but IPv4 item provided"); 1625 if (item_flags & l3m) 1626 return rte_flow_error_set(error, ENOTSUP, 1627 RTE_FLOW_ERROR_TYPE_ITEM, item, 1628 "multiple L3 layers not supported"); 1629 else if (item_flags & l4m) 1630 return rte_flow_error_set(error, EINVAL, 1631 RTE_FLOW_ERROR_TYPE_ITEM, item, 1632 "L3 cannot follow an L4 layer."); 1633 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1634 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1635 return rte_flow_error_set(error, EINVAL, 1636 RTE_FLOW_ERROR_TYPE_ITEM, item, 1637 "L3 cannot follow an NVGRE layer."); 1638 if (!mask) 1639 mask = &rte_flow_item_ipv4_mask; 1640 else if (mask->hdr.next_proto_id != 0 && 1641 mask->hdr.next_proto_id != 0xff) 1642 return rte_flow_error_set(error, EINVAL, 1643 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1644 "partial mask is not supported" 1645 " for protocol"); 1646 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1647 acc_mask ? (const uint8_t *)acc_mask 1648 : (const uint8_t *)&nic_mask, 1649 sizeof(struct rte_flow_item_ipv4), 1650 error); 1651 if (ret < 0) 1652 return ret; 1653 return 0; 1654 } 1655 1656 /** 1657 * Validate IPV6 item. 1658 * 1659 * @param[in] item 1660 * Item specification. 1661 * @param[in] item_flags 1662 * Bit-fields that holds the items detected until now. 1663 * @param[in] acc_mask 1664 * Acceptable mask, if NULL default internal default mask 1665 * will be used to check whether item fields are supported. 1666 * @param[out] error 1667 * Pointer to error structure. 1668 * 1669 * @return 1670 * 0 on success, a negative errno value otherwise and rte_errno is set. 1671 */ 1672 int 1673 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1674 uint64_t item_flags, 1675 uint64_t last_item, 1676 uint16_t ether_type, 1677 const struct rte_flow_item_ipv6 *acc_mask, 1678 struct rte_flow_error *error) 1679 { 1680 const struct rte_flow_item_ipv6 *mask = item->mask; 1681 const struct rte_flow_item_ipv6 *spec = item->spec; 1682 const struct rte_flow_item_ipv6 nic_mask = { 1683 .hdr = { 1684 .src_addr = 1685 "\xff\xff\xff\xff\xff\xff\xff\xff" 1686 "\xff\xff\xff\xff\xff\xff\xff\xff", 1687 .dst_addr = 1688 "\xff\xff\xff\xff\xff\xff\xff\xff" 1689 "\xff\xff\xff\xff\xff\xff\xff\xff", 1690 .vtc_flow = RTE_BE32(0xffffffff), 1691 .proto = 0xff, 1692 }, 1693 }; 1694 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1695 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1696 MLX5_FLOW_LAYER_OUTER_L3; 1697 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1698 MLX5_FLOW_LAYER_OUTER_L4; 1699 int ret; 1700 uint8_t next_proto = 0xFF; 1701 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1702 MLX5_FLOW_LAYER_OUTER_VLAN | 1703 MLX5_FLOW_LAYER_INNER_VLAN); 1704 1705 if ((last_item & l2_vlan) && ether_type && 1706 ether_type != RTE_ETHER_TYPE_IPV6) 1707 return rte_flow_error_set(error, EINVAL, 1708 RTE_FLOW_ERROR_TYPE_ITEM, item, 1709 "IPv6 cannot follow L2/VLAN layer " 1710 "which ether type is not IPv6"); 1711 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1712 if (mask && spec) 1713 next_proto = mask->hdr.proto & spec->hdr.proto; 1714 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1715 return rte_flow_error_set(error, EINVAL, 1716 RTE_FLOW_ERROR_TYPE_ITEM, 1717 item, 1718 "multiple tunnel " 1719 "not supported"); 1720 } 1721 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1722 return rte_flow_error_set(error, EINVAL, 1723 RTE_FLOW_ERROR_TYPE_ITEM, item, 1724 "wrong tunnel type - IPv4 specified " 1725 "but IPv6 item provided"); 1726 if (item_flags & l3m) 1727 return rte_flow_error_set(error, ENOTSUP, 1728 RTE_FLOW_ERROR_TYPE_ITEM, item, 1729 "multiple L3 layers not supported"); 1730 else if (item_flags & l4m) 1731 return rte_flow_error_set(error, EINVAL, 1732 RTE_FLOW_ERROR_TYPE_ITEM, item, 1733 "L3 cannot follow an L4 layer."); 1734 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1735 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1736 return rte_flow_error_set(error, EINVAL, 1737 RTE_FLOW_ERROR_TYPE_ITEM, item, 1738 "L3 cannot follow an NVGRE layer."); 1739 if (!mask) 1740 mask = &rte_flow_item_ipv6_mask; 1741 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1742 acc_mask ? (const uint8_t *)acc_mask 1743 : (const uint8_t *)&nic_mask, 1744 sizeof(struct rte_flow_item_ipv6), 1745 error); 1746 if (ret < 0) 1747 return ret; 1748 return 0; 1749 } 1750 1751 /** 1752 * Validate UDP item. 1753 * 1754 * @param[in] item 1755 * Item specification. 1756 * @param[in] item_flags 1757 * Bit-fields that holds the items detected until now. 1758 * @param[in] target_protocol 1759 * The next protocol in the previous item. 1760 * @param[in] flow_mask 1761 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1762 * @param[out] error 1763 * Pointer to error structure. 1764 * 1765 * @return 1766 * 0 on success, a negative errno value otherwise and rte_errno is set. 1767 */ 1768 int 1769 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1770 uint64_t item_flags, 1771 uint8_t target_protocol, 1772 struct rte_flow_error *error) 1773 { 1774 const struct rte_flow_item_udp *mask = item->mask; 1775 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1776 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1777 MLX5_FLOW_LAYER_OUTER_L3; 1778 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1779 MLX5_FLOW_LAYER_OUTER_L4; 1780 int ret; 1781 1782 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1783 return rte_flow_error_set(error, EINVAL, 1784 RTE_FLOW_ERROR_TYPE_ITEM, item, 1785 "protocol filtering not compatible" 1786 " with UDP layer"); 1787 if (!(item_flags & l3m)) 1788 return rte_flow_error_set(error, EINVAL, 1789 RTE_FLOW_ERROR_TYPE_ITEM, item, 1790 "L3 is mandatory to filter on L4"); 1791 if (item_flags & l4m) 1792 return rte_flow_error_set(error, EINVAL, 1793 RTE_FLOW_ERROR_TYPE_ITEM, item, 1794 "multiple L4 layers not supported"); 1795 if (!mask) 1796 mask = &rte_flow_item_udp_mask; 1797 ret = mlx5_flow_item_acceptable 1798 (item, (const uint8_t *)mask, 1799 (const uint8_t *)&rte_flow_item_udp_mask, 1800 sizeof(struct rte_flow_item_udp), error); 1801 if (ret < 0) 1802 return ret; 1803 return 0; 1804 } 1805 1806 /** 1807 * Validate TCP item. 1808 * 1809 * @param[in] item 1810 * Item specification. 1811 * @param[in] item_flags 1812 * Bit-fields that holds the items detected until now. 1813 * @param[in] target_protocol 1814 * The next protocol in the previous item. 1815 * @param[out] error 1816 * Pointer to error structure. 1817 * 1818 * @return 1819 * 0 on success, a negative errno value otherwise and rte_errno is set. 1820 */ 1821 int 1822 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1823 uint64_t item_flags, 1824 uint8_t target_protocol, 1825 const struct rte_flow_item_tcp *flow_mask, 1826 struct rte_flow_error *error) 1827 { 1828 const struct rte_flow_item_tcp *mask = item->mask; 1829 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1830 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1831 MLX5_FLOW_LAYER_OUTER_L3; 1832 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1833 MLX5_FLOW_LAYER_OUTER_L4; 1834 int ret; 1835 1836 MLX5_ASSERT(flow_mask); 1837 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1838 return rte_flow_error_set(error, EINVAL, 1839 RTE_FLOW_ERROR_TYPE_ITEM, item, 1840 "protocol filtering not compatible" 1841 " with TCP layer"); 1842 if (!(item_flags & l3m)) 1843 return rte_flow_error_set(error, EINVAL, 1844 RTE_FLOW_ERROR_TYPE_ITEM, item, 1845 "L3 is mandatory to filter on L4"); 1846 if (item_flags & l4m) 1847 return rte_flow_error_set(error, EINVAL, 1848 RTE_FLOW_ERROR_TYPE_ITEM, item, 1849 "multiple L4 layers not supported"); 1850 if (!mask) 1851 mask = &rte_flow_item_tcp_mask; 1852 ret = mlx5_flow_item_acceptable 1853 (item, (const uint8_t *)mask, 1854 (const uint8_t *)flow_mask, 1855 sizeof(struct rte_flow_item_tcp), error); 1856 if (ret < 0) 1857 return ret; 1858 return 0; 1859 } 1860 1861 /** 1862 * Validate VXLAN item. 1863 * 1864 * @param[in] item 1865 * Item specification. 1866 * @param[in] item_flags 1867 * Bit-fields that holds the items detected until now. 1868 * @param[in] target_protocol 1869 * The next protocol in the previous item. 1870 * @param[out] error 1871 * Pointer to error structure. 1872 * 1873 * @return 1874 * 0 on success, a negative errno value otherwise and rte_errno is set. 1875 */ 1876 int 1877 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1878 uint64_t item_flags, 1879 struct rte_flow_error *error) 1880 { 1881 const struct rte_flow_item_vxlan *spec = item->spec; 1882 const struct rte_flow_item_vxlan *mask = item->mask; 1883 int ret; 1884 union vni { 1885 uint32_t vlan_id; 1886 uint8_t vni[4]; 1887 } id = { .vlan_id = 0, }; 1888 1889 1890 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1891 return rte_flow_error_set(error, ENOTSUP, 1892 RTE_FLOW_ERROR_TYPE_ITEM, item, 1893 "multiple tunnel layers not" 1894 " supported"); 1895 /* 1896 * Verify only UDPv4 is present as defined in 1897 * https://tools.ietf.org/html/rfc7348 1898 */ 1899 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1900 return rte_flow_error_set(error, EINVAL, 1901 RTE_FLOW_ERROR_TYPE_ITEM, item, 1902 "no outer UDP layer found"); 1903 if (!mask) 1904 mask = &rte_flow_item_vxlan_mask; 1905 ret = mlx5_flow_item_acceptable 1906 (item, (const uint8_t *)mask, 1907 (const uint8_t *)&rte_flow_item_vxlan_mask, 1908 sizeof(struct rte_flow_item_vxlan), 1909 error); 1910 if (ret < 0) 1911 return ret; 1912 if (spec) { 1913 memcpy(&id.vni[1], spec->vni, 3); 1914 memcpy(&id.vni[1], mask->vni, 3); 1915 } 1916 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1917 return rte_flow_error_set(error, ENOTSUP, 1918 RTE_FLOW_ERROR_TYPE_ITEM, item, 1919 "VXLAN tunnel must be fully defined"); 1920 return 0; 1921 } 1922 1923 /** 1924 * Validate VXLAN_GPE item. 1925 * 1926 * @param[in] item 1927 * Item specification. 1928 * @param[in] item_flags 1929 * Bit-fields that holds the items detected until now. 1930 * @param[in] priv 1931 * Pointer to the private data structure. 1932 * @param[in] target_protocol 1933 * The next protocol in the previous item. 1934 * @param[out] error 1935 * Pointer to error structure. 1936 * 1937 * @return 1938 * 0 on success, a negative errno value otherwise and rte_errno is set. 1939 */ 1940 int 1941 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1942 uint64_t item_flags, 1943 struct rte_eth_dev *dev, 1944 struct rte_flow_error *error) 1945 { 1946 struct mlx5_priv *priv = dev->data->dev_private; 1947 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1948 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1949 int ret; 1950 union vni { 1951 uint32_t vlan_id; 1952 uint8_t vni[4]; 1953 } id = { .vlan_id = 0, }; 1954 1955 if (!priv->config.l3_vxlan_en) 1956 return rte_flow_error_set(error, ENOTSUP, 1957 RTE_FLOW_ERROR_TYPE_ITEM, item, 1958 "L3 VXLAN is not enabled by device" 1959 " parameter and/or not configured in" 1960 " firmware"); 1961 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1962 return rte_flow_error_set(error, ENOTSUP, 1963 RTE_FLOW_ERROR_TYPE_ITEM, item, 1964 "multiple tunnel layers not" 1965 " supported"); 1966 /* 1967 * Verify only UDPv4 is present as defined in 1968 * https://tools.ietf.org/html/rfc7348 1969 */ 1970 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1971 return rte_flow_error_set(error, EINVAL, 1972 RTE_FLOW_ERROR_TYPE_ITEM, item, 1973 "no outer UDP layer found"); 1974 if (!mask) 1975 mask = &rte_flow_item_vxlan_gpe_mask; 1976 ret = mlx5_flow_item_acceptable 1977 (item, (const uint8_t *)mask, 1978 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1979 sizeof(struct rte_flow_item_vxlan_gpe), 1980 error); 1981 if (ret < 0) 1982 return ret; 1983 if (spec) { 1984 if (spec->protocol) 1985 return rte_flow_error_set(error, ENOTSUP, 1986 RTE_FLOW_ERROR_TYPE_ITEM, 1987 item, 1988 "VxLAN-GPE protocol" 1989 " not supported"); 1990 memcpy(&id.vni[1], spec->vni, 3); 1991 memcpy(&id.vni[1], mask->vni, 3); 1992 } 1993 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1994 return rte_flow_error_set(error, ENOTSUP, 1995 RTE_FLOW_ERROR_TYPE_ITEM, item, 1996 "VXLAN-GPE tunnel must be fully" 1997 " defined"); 1998 return 0; 1999 } 2000 /** 2001 * Validate GRE Key item. 2002 * 2003 * @param[in] item 2004 * Item specification. 2005 * @param[in] item_flags 2006 * Bit flags to mark detected items. 2007 * @param[in] gre_item 2008 * Pointer to gre_item 2009 * @param[out] error 2010 * Pointer to error structure. 2011 * 2012 * @return 2013 * 0 on success, a negative errno value otherwise and rte_errno is set. 2014 */ 2015 int 2016 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2017 uint64_t item_flags, 2018 const struct rte_flow_item *gre_item, 2019 struct rte_flow_error *error) 2020 { 2021 const rte_be32_t *mask = item->mask; 2022 int ret = 0; 2023 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2024 const struct rte_flow_item_gre *gre_spec; 2025 const struct rte_flow_item_gre *gre_mask; 2026 2027 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2028 return rte_flow_error_set(error, ENOTSUP, 2029 RTE_FLOW_ERROR_TYPE_ITEM, item, 2030 "Multiple GRE key not support"); 2031 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2032 return rte_flow_error_set(error, ENOTSUP, 2033 RTE_FLOW_ERROR_TYPE_ITEM, item, 2034 "No preceding GRE header"); 2035 if (item_flags & MLX5_FLOW_LAYER_INNER) 2036 return rte_flow_error_set(error, ENOTSUP, 2037 RTE_FLOW_ERROR_TYPE_ITEM, item, 2038 "GRE key following a wrong item"); 2039 gre_mask = gre_item->mask; 2040 if (!gre_mask) 2041 gre_mask = &rte_flow_item_gre_mask; 2042 gre_spec = gre_item->spec; 2043 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2044 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2045 return rte_flow_error_set(error, EINVAL, 2046 RTE_FLOW_ERROR_TYPE_ITEM, item, 2047 "Key bit must be on"); 2048 2049 if (!mask) 2050 mask = &gre_key_default_mask; 2051 ret = mlx5_flow_item_acceptable 2052 (item, (const uint8_t *)mask, 2053 (const uint8_t *)&gre_key_default_mask, 2054 sizeof(rte_be32_t), error); 2055 return ret; 2056 } 2057 2058 /** 2059 * Validate GRE item. 2060 * 2061 * @param[in] item 2062 * Item specification. 2063 * @param[in] item_flags 2064 * Bit flags to mark detected items. 2065 * @param[in] target_protocol 2066 * The next protocol in the previous item. 2067 * @param[out] error 2068 * Pointer to error structure. 2069 * 2070 * @return 2071 * 0 on success, a negative errno value otherwise and rte_errno is set. 2072 */ 2073 int 2074 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2075 uint64_t item_flags, 2076 uint8_t target_protocol, 2077 struct rte_flow_error *error) 2078 { 2079 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2080 const struct rte_flow_item_gre *mask = item->mask; 2081 int ret; 2082 const struct rte_flow_item_gre nic_mask = { 2083 .c_rsvd0_ver = RTE_BE16(0xB000), 2084 .protocol = RTE_BE16(UINT16_MAX), 2085 }; 2086 2087 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2088 return rte_flow_error_set(error, EINVAL, 2089 RTE_FLOW_ERROR_TYPE_ITEM, item, 2090 "protocol filtering not compatible" 2091 " with this GRE layer"); 2092 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2093 return rte_flow_error_set(error, ENOTSUP, 2094 RTE_FLOW_ERROR_TYPE_ITEM, item, 2095 "multiple tunnel layers not" 2096 " supported"); 2097 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2098 return rte_flow_error_set(error, ENOTSUP, 2099 RTE_FLOW_ERROR_TYPE_ITEM, item, 2100 "L3 Layer is missing"); 2101 if (!mask) 2102 mask = &rte_flow_item_gre_mask; 2103 ret = mlx5_flow_item_acceptable 2104 (item, (const uint8_t *)mask, 2105 (const uint8_t *)&nic_mask, 2106 sizeof(struct rte_flow_item_gre), error); 2107 if (ret < 0) 2108 return ret; 2109 #ifndef HAVE_MLX5DV_DR 2110 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2111 if (spec && (spec->protocol & mask->protocol)) 2112 return rte_flow_error_set(error, ENOTSUP, 2113 RTE_FLOW_ERROR_TYPE_ITEM, item, 2114 "without MPLS support the" 2115 " specification cannot be used for" 2116 " filtering"); 2117 #endif 2118 #endif 2119 return 0; 2120 } 2121 2122 /** 2123 * Validate Geneve item. 2124 * 2125 * @param[in] item 2126 * Item specification. 2127 * @param[in] itemFlags 2128 * Bit-fields that holds the items detected until now. 2129 * @param[in] enPriv 2130 * Pointer to the private data structure. 2131 * @param[out] error 2132 * Pointer to error structure. 2133 * 2134 * @return 2135 * 0 on success, a negative errno value otherwise and rte_errno is set. 2136 */ 2137 2138 int 2139 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2140 uint64_t item_flags, 2141 struct rte_eth_dev *dev, 2142 struct rte_flow_error *error) 2143 { 2144 struct mlx5_priv *priv = dev->data->dev_private; 2145 const struct rte_flow_item_geneve *spec = item->spec; 2146 const struct rte_flow_item_geneve *mask = item->mask; 2147 int ret; 2148 uint16_t gbhdr; 2149 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2150 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2151 const struct rte_flow_item_geneve nic_mask = { 2152 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2153 .vni = "\xff\xff\xff", 2154 .protocol = RTE_BE16(UINT16_MAX), 2155 }; 2156 2157 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2158 return rte_flow_error_set(error, ENOTSUP, 2159 RTE_FLOW_ERROR_TYPE_ITEM, item, 2160 "L3 Geneve is not enabled by device" 2161 " parameter and/or not configured in" 2162 " firmware"); 2163 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2164 return rte_flow_error_set(error, ENOTSUP, 2165 RTE_FLOW_ERROR_TYPE_ITEM, item, 2166 "multiple tunnel layers not" 2167 " supported"); 2168 /* 2169 * Verify only UDPv4 is present as defined in 2170 * https://tools.ietf.org/html/rfc7348 2171 */ 2172 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2173 return rte_flow_error_set(error, EINVAL, 2174 RTE_FLOW_ERROR_TYPE_ITEM, item, 2175 "no outer UDP layer found"); 2176 if (!mask) 2177 mask = &rte_flow_item_geneve_mask; 2178 ret = mlx5_flow_item_acceptable 2179 (item, (const uint8_t *)mask, 2180 (const uint8_t *)&nic_mask, 2181 sizeof(struct rte_flow_item_geneve), error); 2182 if (ret) 2183 return ret; 2184 if (spec) { 2185 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2186 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2187 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2188 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2189 return rte_flow_error_set(error, ENOTSUP, 2190 RTE_FLOW_ERROR_TYPE_ITEM, 2191 item, 2192 "Geneve protocol unsupported" 2193 " fields are being used"); 2194 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2195 return rte_flow_error_set 2196 (error, ENOTSUP, 2197 RTE_FLOW_ERROR_TYPE_ITEM, 2198 item, 2199 "Unsupported Geneve options length"); 2200 } 2201 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2202 return rte_flow_error_set 2203 (error, ENOTSUP, 2204 RTE_FLOW_ERROR_TYPE_ITEM, item, 2205 "Geneve tunnel must be fully defined"); 2206 return 0; 2207 } 2208 2209 /** 2210 * Validate MPLS item. 2211 * 2212 * @param[in] dev 2213 * Pointer to the rte_eth_dev structure. 2214 * @param[in] item 2215 * Item specification. 2216 * @param[in] item_flags 2217 * Bit-fields that holds the items detected until now. 2218 * @param[in] prev_layer 2219 * The protocol layer indicated in previous item. 2220 * @param[out] error 2221 * Pointer to error structure. 2222 * 2223 * @return 2224 * 0 on success, a negative errno value otherwise and rte_errno is set. 2225 */ 2226 int 2227 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2228 const struct rte_flow_item *item __rte_unused, 2229 uint64_t item_flags __rte_unused, 2230 uint64_t prev_layer __rte_unused, 2231 struct rte_flow_error *error) 2232 { 2233 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2234 const struct rte_flow_item_mpls *mask = item->mask; 2235 struct mlx5_priv *priv = dev->data->dev_private; 2236 int ret; 2237 2238 if (!priv->config.mpls_en) 2239 return rte_flow_error_set(error, ENOTSUP, 2240 RTE_FLOW_ERROR_TYPE_ITEM, item, 2241 "MPLS not supported or" 2242 " disabled in firmware" 2243 " configuration."); 2244 /* MPLS over IP, UDP, GRE is allowed */ 2245 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2246 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2247 MLX5_FLOW_LAYER_GRE))) 2248 return rte_flow_error_set(error, EINVAL, 2249 RTE_FLOW_ERROR_TYPE_ITEM, item, 2250 "protocol filtering not compatible" 2251 " with MPLS layer"); 2252 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2253 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2254 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2255 return rte_flow_error_set(error, ENOTSUP, 2256 RTE_FLOW_ERROR_TYPE_ITEM, item, 2257 "multiple tunnel layers not" 2258 " supported"); 2259 if (!mask) 2260 mask = &rte_flow_item_mpls_mask; 2261 ret = mlx5_flow_item_acceptable 2262 (item, (const uint8_t *)mask, 2263 (const uint8_t *)&rte_flow_item_mpls_mask, 2264 sizeof(struct rte_flow_item_mpls), error); 2265 if (ret < 0) 2266 return ret; 2267 return 0; 2268 #endif 2269 return rte_flow_error_set(error, ENOTSUP, 2270 RTE_FLOW_ERROR_TYPE_ITEM, item, 2271 "MPLS is not supported by Verbs, please" 2272 " update."); 2273 } 2274 2275 /** 2276 * Validate NVGRE item. 2277 * 2278 * @param[in] item 2279 * Item specification. 2280 * @param[in] item_flags 2281 * Bit flags to mark detected items. 2282 * @param[in] target_protocol 2283 * The next protocol in the previous item. 2284 * @param[out] error 2285 * Pointer to error structure. 2286 * 2287 * @return 2288 * 0 on success, a negative errno value otherwise and rte_errno is set. 2289 */ 2290 int 2291 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2292 uint64_t item_flags, 2293 uint8_t target_protocol, 2294 struct rte_flow_error *error) 2295 { 2296 const struct rte_flow_item_nvgre *mask = item->mask; 2297 int ret; 2298 2299 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2300 return rte_flow_error_set(error, EINVAL, 2301 RTE_FLOW_ERROR_TYPE_ITEM, item, 2302 "protocol filtering not compatible" 2303 " with this GRE layer"); 2304 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2305 return rte_flow_error_set(error, ENOTSUP, 2306 RTE_FLOW_ERROR_TYPE_ITEM, item, 2307 "multiple tunnel layers not" 2308 " supported"); 2309 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2310 return rte_flow_error_set(error, ENOTSUP, 2311 RTE_FLOW_ERROR_TYPE_ITEM, item, 2312 "L3 Layer is missing"); 2313 if (!mask) 2314 mask = &rte_flow_item_nvgre_mask; 2315 ret = mlx5_flow_item_acceptable 2316 (item, (const uint8_t *)mask, 2317 (const uint8_t *)&rte_flow_item_nvgre_mask, 2318 sizeof(struct rte_flow_item_nvgre), error); 2319 if (ret < 0) 2320 return ret; 2321 return 0; 2322 } 2323 2324 /* Allocate unique ID for the split Q/RSS subflows. */ 2325 static uint32_t 2326 flow_qrss_get_id(struct rte_eth_dev *dev) 2327 { 2328 struct mlx5_priv *priv = dev->data->dev_private; 2329 uint32_t qrss_id, ret; 2330 2331 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2332 if (ret) 2333 return 0; 2334 MLX5_ASSERT(qrss_id); 2335 return qrss_id; 2336 } 2337 2338 /* Free unique ID for the split Q/RSS subflows. */ 2339 static void 2340 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2341 { 2342 struct mlx5_priv *priv = dev->data->dev_private; 2343 2344 if (qrss_id) 2345 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2346 } 2347 2348 /** 2349 * Release resource related QUEUE/RSS action split. 2350 * 2351 * @param dev 2352 * Pointer to Ethernet device. 2353 * @param flow 2354 * Flow to release id's from. 2355 */ 2356 static void 2357 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2358 struct rte_flow *flow) 2359 { 2360 struct mlx5_priv *priv = dev->data->dev_private; 2361 uint32_t handle_idx; 2362 struct mlx5_flow_handle *dev_handle; 2363 2364 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2365 handle_idx, dev_handle, next) 2366 if (dev_handle->split_flow_id) 2367 flow_qrss_free_id(dev, dev_handle->split_flow_id); 2368 } 2369 2370 static int 2371 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2372 const struct rte_flow_attr *attr __rte_unused, 2373 const struct rte_flow_item items[] __rte_unused, 2374 const struct rte_flow_action actions[] __rte_unused, 2375 bool external __rte_unused, 2376 int hairpin __rte_unused, 2377 struct rte_flow_error *error) 2378 { 2379 return rte_flow_error_set(error, ENOTSUP, 2380 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2381 } 2382 2383 static struct mlx5_flow * 2384 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2385 const struct rte_flow_attr *attr __rte_unused, 2386 const struct rte_flow_item items[] __rte_unused, 2387 const struct rte_flow_action actions[] __rte_unused, 2388 struct rte_flow_error *error) 2389 { 2390 rte_flow_error_set(error, ENOTSUP, 2391 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2392 return NULL; 2393 } 2394 2395 static int 2396 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2397 struct mlx5_flow *dev_flow __rte_unused, 2398 const struct rte_flow_attr *attr __rte_unused, 2399 const struct rte_flow_item items[] __rte_unused, 2400 const struct rte_flow_action actions[] __rte_unused, 2401 struct rte_flow_error *error) 2402 { 2403 return rte_flow_error_set(error, ENOTSUP, 2404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2405 } 2406 2407 static int 2408 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2409 struct rte_flow *flow __rte_unused, 2410 struct rte_flow_error *error) 2411 { 2412 return rte_flow_error_set(error, ENOTSUP, 2413 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2414 } 2415 2416 static void 2417 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2418 struct rte_flow *flow __rte_unused) 2419 { 2420 } 2421 2422 static void 2423 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2424 struct rte_flow *flow __rte_unused) 2425 { 2426 } 2427 2428 static int 2429 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2430 struct rte_flow *flow __rte_unused, 2431 const struct rte_flow_action *actions __rte_unused, 2432 void *data __rte_unused, 2433 struct rte_flow_error *error) 2434 { 2435 return rte_flow_error_set(error, ENOTSUP, 2436 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2437 } 2438 2439 /* Void driver to protect from null pointer reference. */ 2440 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2441 .validate = flow_null_validate, 2442 .prepare = flow_null_prepare, 2443 .translate = flow_null_translate, 2444 .apply = flow_null_apply, 2445 .remove = flow_null_remove, 2446 .destroy = flow_null_destroy, 2447 .query = flow_null_query, 2448 }; 2449 2450 /** 2451 * Select flow driver type according to flow attributes and device 2452 * configuration. 2453 * 2454 * @param[in] dev 2455 * Pointer to the dev structure. 2456 * @param[in] attr 2457 * Pointer to the flow attributes. 2458 * 2459 * @return 2460 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2461 */ 2462 static enum mlx5_flow_drv_type 2463 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2464 { 2465 struct mlx5_priv *priv = dev->data->dev_private; 2466 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2467 2468 if (attr->transfer && priv->config.dv_esw_en) 2469 type = MLX5_FLOW_TYPE_DV; 2470 if (!attr->transfer) 2471 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2472 MLX5_FLOW_TYPE_VERBS; 2473 return type; 2474 } 2475 2476 #define flow_get_drv_ops(type) flow_drv_ops[type] 2477 2478 /** 2479 * Flow driver validation API. This abstracts calling driver specific functions. 2480 * The type of flow driver is determined according to flow attributes. 2481 * 2482 * @param[in] dev 2483 * Pointer to the dev structure. 2484 * @param[in] attr 2485 * Pointer to the flow attributes. 2486 * @param[in] items 2487 * Pointer to the list of items. 2488 * @param[in] actions 2489 * Pointer to the list of actions. 2490 * @param[in] external 2491 * This flow rule is created by request external to PMD. 2492 * @param[in] hairpin 2493 * Number of hairpin TX actions, 0 means classic flow. 2494 * @param[out] error 2495 * Pointer to the error structure. 2496 * 2497 * @return 2498 * 0 on success, a negative errno value otherwise and rte_errno is set. 2499 */ 2500 static inline int 2501 flow_drv_validate(struct rte_eth_dev *dev, 2502 const struct rte_flow_attr *attr, 2503 const struct rte_flow_item items[], 2504 const struct rte_flow_action actions[], 2505 bool external, int hairpin, struct rte_flow_error *error) 2506 { 2507 const struct mlx5_flow_driver_ops *fops; 2508 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2509 2510 fops = flow_get_drv_ops(type); 2511 return fops->validate(dev, attr, items, actions, external, 2512 hairpin, error); 2513 } 2514 2515 /** 2516 * Flow driver preparation API. This abstracts calling driver specific 2517 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2518 * calculates the size of memory required for device flow, allocates the memory, 2519 * initializes the device flow and returns the pointer. 2520 * 2521 * @note 2522 * This function initializes device flow structure such as dv or verbs in 2523 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2524 * rest. For example, adding returning device flow to flow->dev_flow list and 2525 * setting backward reference to the flow should be done out of this function. 2526 * layers field is not filled either. 2527 * 2528 * @param[in] dev 2529 * Pointer to the dev structure. 2530 * @param[in] attr 2531 * Pointer to the flow attributes. 2532 * @param[in] items 2533 * Pointer to the list of items. 2534 * @param[in] actions 2535 * Pointer to the list of actions. 2536 * @param[in] flow_idx 2537 * This memory pool index to the flow. 2538 * @param[out] error 2539 * Pointer to the error structure. 2540 * 2541 * @return 2542 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2543 */ 2544 static inline struct mlx5_flow * 2545 flow_drv_prepare(struct rte_eth_dev *dev, 2546 const struct rte_flow *flow, 2547 const struct rte_flow_attr *attr, 2548 const struct rte_flow_item items[], 2549 const struct rte_flow_action actions[], 2550 uint32_t flow_idx, 2551 struct rte_flow_error *error) 2552 { 2553 const struct mlx5_flow_driver_ops *fops; 2554 enum mlx5_flow_drv_type type = flow->drv_type; 2555 struct mlx5_flow *mlx5_flow = NULL; 2556 2557 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2558 fops = flow_get_drv_ops(type); 2559 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 2560 if (mlx5_flow) 2561 mlx5_flow->flow_idx = flow_idx; 2562 return mlx5_flow; 2563 } 2564 2565 /** 2566 * Flow driver translation API. This abstracts calling driver specific 2567 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2568 * translates a generic flow into a driver flow. flow_drv_prepare() must 2569 * precede. 2570 * 2571 * @note 2572 * dev_flow->layers could be filled as a result of parsing during translation 2573 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2574 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2575 * flow->actions could be overwritten even though all the expanded dev_flows 2576 * have the same actions. 2577 * 2578 * @param[in] dev 2579 * Pointer to the rte dev structure. 2580 * @param[in, out] dev_flow 2581 * Pointer to the mlx5 flow. 2582 * @param[in] attr 2583 * Pointer to the flow attributes. 2584 * @param[in] items 2585 * Pointer to the list of items. 2586 * @param[in] actions 2587 * Pointer to the list of actions. 2588 * @param[out] error 2589 * Pointer to the error structure. 2590 * 2591 * @return 2592 * 0 on success, a negative errno value otherwise and rte_errno is set. 2593 */ 2594 static inline int 2595 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2596 const struct rte_flow_attr *attr, 2597 const struct rte_flow_item items[], 2598 const struct rte_flow_action actions[], 2599 struct rte_flow_error *error) 2600 { 2601 const struct mlx5_flow_driver_ops *fops; 2602 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2603 2604 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2605 fops = flow_get_drv_ops(type); 2606 return fops->translate(dev, dev_flow, attr, items, actions, error); 2607 } 2608 2609 /** 2610 * Flow driver apply API. This abstracts calling driver specific functions. 2611 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2612 * translated driver flows on to device. flow_drv_translate() must precede. 2613 * 2614 * @param[in] dev 2615 * Pointer to Ethernet device structure. 2616 * @param[in, out] flow 2617 * Pointer to flow structure. 2618 * @param[out] error 2619 * Pointer to error structure. 2620 * 2621 * @return 2622 * 0 on success, a negative errno value otherwise and rte_errno is set. 2623 */ 2624 static inline int 2625 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2626 struct rte_flow_error *error) 2627 { 2628 const struct mlx5_flow_driver_ops *fops; 2629 enum mlx5_flow_drv_type type = flow->drv_type; 2630 2631 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2632 fops = flow_get_drv_ops(type); 2633 return fops->apply(dev, flow, error); 2634 } 2635 2636 /** 2637 * Flow driver remove API. This abstracts calling driver specific functions. 2638 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2639 * on device. All the resources of the flow should be freed by calling 2640 * flow_drv_destroy(). 2641 * 2642 * @param[in] dev 2643 * Pointer to Ethernet device. 2644 * @param[in, out] flow 2645 * Pointer to flow structure. 2646 */ 2647 static inline void 2648 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2649 { 2650 const struct mlx5_flow_driver_ops *fops; 2651 enum mlx5_flow_drv_type type = flow->drv_type; 2652 2653 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2654 fops = flow_get_drv_ops(type); 2655 fops->remove(dev, flow); 2656 } 2657 2658 /** 2659 * Flow driver destroy API. This abstracts calling driver specific functions. 2660 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2661 * on device and releases resources of the flow. 2662 * 2663 * @param[in] dev 2664 * Pointer to Ethernet device. 2665 * @param[in, out] flow 2666 * Pointer to flow structure. 2667 */ 2668 static inline void 2669 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2670 { 2671 const struct mlx5_flow_driver_ops *fops; 2672 enum mlx5_flow_drv_type type = flow->drv_type; 2673 2674 flow_mreg_split_qrss_release(dev, flow); 2675 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2676 fops = flow_get_drv_ops(type); 2677 fops->destroy(dev, flow); 2678 } 2679 2680 /** 2681 * Get RSS action from the action list. 2682 * 2683 * @param[in] actions 2684 * Pointer to the list of actions. 2685 * 2686 * @return 2687 * Pointer to the RSS action if exist, else return NULL. 2688 */ 2689 static const struct rte_flow_action_rss* 2690 flow_get_rss_action(const struct rte_flow_action actions[]) 2691 { 2692 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2693 switch (actions->type) { 2694 case RTE_FLOW_ACTION_TYPE_RSS: 2695 return (const struct rte_flow_action_rss *) 2696 actions->conf; 2697 default: 2698 break; 2699 } 2700 } 2701 return NULL; 2702 } 2703 2704 static unsigned int 2705 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2706 { 2707 const struct rte_flow_item *item; 2708 unsigned int has_vlan = 0; 2709 2710 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2711 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2712 has_vlan = 1; 2713 break; 2714 } 2715 } 2716 if (has_vlan) 2717 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2718 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2719 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2720 MLX5_EXPANSION_ROOT_OUTER; 2721 } 2722 2723 /** 2724 * Get layer flags from the prefix flow. 2725 * 2726 * Some flows may be split to several subflows, the prefix subflow gets the 2727 * match items and the suffix sub flow gets the actions. 2728 * Some actions need the user defined match item flags to get the detail for 2729 * the action. 2730 * This function helps the suffix flow to get the item layer flags from prefix 2731 * subflow. 2732 * 2733 * @param[in] dev_flow 2734 * Pointer the created preifx subflow. 2735 * 2736 * @return 2737 * The layers get from prefix subflow. 2738 */ 2739 static inline uint64_t 2740 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 2741 { 2742 uint64_t layers = 0; 2743 2744 /* 2745 * Layers bits could be localization, but usually the compiler will 2746 * help to do the optimization work for source code. 2747 * If no decap actions, use the layers directly. 2748 */ 2749 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 2750 return dev_flow->handle->layers; 2751 /* Convert L3 layers with decap action. */ 2752 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 2753 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2754 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 2755 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2756 /* Convert L4 layers with decap action. */ 2757 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 2758 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 2759 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 2760 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 2761 return layers; 2762 } 2763 2764 /** 2765 * Get metadata split action information. 2766 * 2767 * @param[in] actions 2768 * Pointer to the list of actions. 2769 * @param[out] qrss 2770 * Pointer to the return pointer. 2771 * @param[out] qrss_type 2772 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2773 * if no QUEUE/RSS is found. 2774 * @param[out] encap_idx 2775 * Pointer to the index of the encap action if exists, otherwise the last 2776 * action index. 2777 * 2778 * @return 2779 * Total number of actions. 2780 */ 2781 static int 2782 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 2783 const struct rte_flow_action **qrss, 2784 int *encap_idx) 2785 { 2786 const struct rte_flow_action_raw_encap *raw_encap; 2787 int actions_n = 0; 2788 int raw_decap_idx = -1; 2789 2790 *encap_idx = -1; 2791 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2792 switch (actions->type) { 2793 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2794 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2795 *encap_idx = actions_n; 2796 break; 2797 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 2798 raw_decap_idx = actions_n; 2799 break; 2800 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2801 raw_encap = actions->conf; 2802 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2803 *encap_idx = raw_decap_idx != -1 ? 2804 raw_decap_idx : actions_n; 2805 break; 2806 case RTE_FLOW_ACTION_TYPE_QUEUE: 2807 case RTE_FLOW_ACTION_TYPE_RSS: 2808 *qrss = actions; 2809 break; 2810 default: 2811 break; 2812 } 2813 actions_n++; 2814 } 2815 if (*encap_idx == -1) 2816 *encap_idx = actions_n; 2817 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2818 return actions_n + 1; 2819 } 2820 2821 /** 2822 * Check meter action from the action list. 2823 * 2824 * @param[in] actions 2825 * Pointer to the list of actions. 2826 * @param[out] mtr 2827 * Pointer to the meter exist flag. 2828 * 2829 * @return 2830 * Total number of actions. 2831 */ 2832 static int 2833 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2834 { 2835 int actions_n = 0; 2836 2837 MLX5_ASSERT(mtr); 2838 *mtr = 0; 2839 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2840 switch (actions->type) { 2841 case RTE_FLOW_ACTION_TYPE_METER: 2842 *mtr = 1; 2843 break; 2844 default: 2845 break; 2846 } 2847 actions_n++; 2848 } 2849 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2850 return actions_n + 1; 2851 } 2852 2853 /** 2854 * Check if the flow should be splited due to hairpin. 2855 * The reason for the split is that in current HW we can't 2856 * support encap on Rx, so if a flow have encap we move it 2857 * to Tx. 2858 * 2859 * @param dev 2860 * Pointer to Ethernet device. 2861 * @param[in] attr 2862 * Flow rule attributes. 2863 * @param[in] actions 2864 * Associated actions (list terminated by the END action). 2865 * 2866 * @return 2867 * > 0 the number of actions and the flow should be split, 2868 * 0 when no split required. 2869 */ 2870 static int 2871 flow_check_hairpin_split(struct rte_eth_dev *dev, 2872 const struct rte_flow_attr *attr, 2873 const struct rte_flow_action actions[]) 2874 { 2875 int queue_action = 0; 2876 int action_n = 0; 2877 int encap = 0; 2878 const struct rte_flow_action_queue *queue; 2879 const struct rte_flow_action_rss *rss; 2880 const struct rte_flow_action_raw_encap *raw_encap; 2881 2882 if (!attr->ingress) 2883 return 0; 2884 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2885 switch (actions->type) { 2886 case RTE_FLOW_ACTION_TYPE_QUEUE: 2887 queue = actions->conf; 2888 if (queue == NULL) 2889 return 0; 2890 if (mlx5_rxq_get_type(dev, queue->index) != 2891 MLX5_RXQ_TYPE_HAIRPIN) 2892 return 0; 2893 queue_action = 1; 2894 action_n++; 2895 break; 2896 case RTE_FLOW_ACTION_TYPE_RSS: 2897 rss = actions->conf; 2898 if (rss == NULL || rss->queue_num == 0) 2899 return 0; 2900 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2901 MLX5_RXQ_TYPE_HAIRPIN) 2902 return 0; 2903 queue_action = 1; 2904 action_n++; 2905 break; 2906 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2907 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2908 encap = 1; 2909 action_n++; 2910 break; 2911 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2912 raw_encap = actions->conf; 2913 if (raw_encap->size > 2914 (sizeof(struct rte_flow_item_eth) + 2915 sizeof(struct rte_flow_item_ipv4))) 2916 encap = 1; 2917 action_n++; 2918 break; 2919 default: 2920 action_n++; 2921 break; 2922 } 2923 } 2924 if (encap == 1 && queue_action) 2925 return action_n; 2926 return 0; 2927 } 2928 2929 /* Declare flow create/destroy prototype in advance. */ 2930 static uint32_t 2931 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 2932 const struct rte_flow_attr *attr, 2933 const struct rte_flow_item items[], 2934 const struct rte_flow_action actions[], 2935 bool external, struct rte_flow_error *error); 2936 2937 static void 2938 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 2939 uint32_t flow_idx); 2940 2941 /** 2942 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2943 * 2944 * As mark_id is unique, if there's already a registered flow for the mark_id, 2945 * return by increasing the reference counter of the resource. Otherwise, create 2946 * the resource (mcp_res) and flow. 2947 * 2948 * Flow looks like, 2949 * - If ingress port is ANY and reg_c[1] is mark_id, 2950 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2951 * 2952 * For default flow (zero mark_id), flow is like, 2953 * - If ingress port is ANY, 2954 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2955 * 2956 * @param dev 2957 * Pointer to Ethernet device. 2958 * @param mark_id 2959 * ID of MARK action, zero means default flow for META. 2960 * @param[out] error 2961 * Perform verbose error reporting if not NULL. 2962 * 2963 * @return 2964 * Associated resource on success, NULL otherwise and rte_errno is set. 2965 */ 2966 static struct mlx5_flow_mreg_copy_resource * 2967 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2968 struct rte_flow_error *error) 2969 { 2970 struct mlx5_priv *priv = dev->data->dev_private; 2971 struct rte_flow_attr attr = { 2972 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2973 .ingress = 1, 2974 }; 2975 struct mlx5_rte_flow_item_tag tag_spec = { 2976 .data = mark_id, 2977 }; 2978 struct rte_flow_item items[] = { 2979 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2980 }; 2981 struct rte_flow_action_mark ftag = { 2982 .id = mark_id, 2983 }; 2984 struct mlx5_flow_action_copy_mreg cp_mreg = { 2985 .dst = REG_B, 2986 .src = 0, 2987 }; 2988 struct rte_flow_action_jump jump = { 2989 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2990 }; 2991 struct rte_flow_action actions[] = { 2992 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2993 }; 2994 struct mlx5_flow_mreg_copy_resource *mcp_res; 2995 uint32_t idx = 0; 2996 int ret; 2997 2998 /* Fill the register fileds in the flow. */ 2999 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3000 if (ret < 0) 3001 return NULL; 3002 tag_spec.id = ret; 3003 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3004 if (ret < 0) 3005 return NULL; 3006 cp_mreg.src = ret; 3007 /* Check if already registered. */ 3008 MLX5_ASSERT(priv->mreg_cp_tbl); 3009 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 3010 if (mcp_res) { 3011 /* For non-default rule. */ 3012 if (mark_id != MLX5_DEFAULT_COPY_ID) 3013 mcp_res->refcnt++; 3014 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 3015 mcp_res->refcnt == 1); 3016 return mcp_res; 3017 } 3018 /* Provide the full width of FLAG specific value. */ 3019 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3020 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3021 /* Build a new flow. */ 3022 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3023 items[0] = (struct rte_flow_item){ 3024 .type = (enum rte_flow_item_type) 3025 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3026 .spec = &tag_spec, 3027 }; 3028 items[1] = (struct rte_flow_item){ 3029 .type = RTE_FLOW_ITEM_TYPE_END, 3030 }; 3031 actions[0] = (struct rte_flow_action){ 3032 .type = (enum rte_flow_action_type) 3033 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3034 .conf = &ftag, 3035 }; 3036 actions[1] = (struct rte_flow_action){ 3037 .type = (enum rte_flow_action_type) 3038 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3039 .conf = &cp_mreg, 3040 }; 3041 actions[2] = (struct rte_flow_action){ 3042 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3043 .conf = &jump, 3044 }; 3045 actions[3] = (struct rte_flow_action){ 3046 .type = RTE_FLOW_ACTION_TYPE_END, 3047 }; 3048 } else { 3049 /* Default rule, wildcard match. */ 3050 attr.priority = MLX5_FLOW_PRIO_RSVD; 3051 items[0] = (struct rte_flow_item){ 3052 .type = RTE_FLOW_ITEM_TYPE_END, 3053 }; 3054 actions[0] = (struct rte_flow_action){ 3055 .type = (enum rte_flow_action_type) 3056 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3057 .conf = &cp_mreg, 3058 }; 3059 actions[1] = (struct rte_flow_action){ 3060 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3061 .conf = &jump, 3062 }; 3063 actions[2] = (struct rte_flow_action){ 3064 .type = RTE_FLOW_ACTION_TYPE_END, 3065 }; 3066 } 3067 /* Build a new entry. */ 3068 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3069 if (!mcp_res) { 3070 rte_errno = ENOMEM; 3071 return NULL; 3072 } 3073 mcp_res->idx = idx; 3074 /* 3075 * The copy Flows are not included in any list. There 3076 * ones are referenced from other Flows and can not 3077 * be applied, removed, deleted in ardbitrary order 3078 * by list traversing. 3079 */ 3080 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3081 actions, false, error); 3082 if (!mcp_res->rix_flow) 3083 goto error; 3084 mcp_res->refcnt++; 3085 mcp_res->hlist_ent.key = mark_id; 3086 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3087 &mcp_res->hlist_ent); 3088 MLX5_ASSERT(!ret); 3089 if (ret) 3090 goto error; 3091 return mcp_res; 3092 error: 3093 if (mcp_res->rix_flow) 3094 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3095 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3096 return NULL; 3097 } 3098 3099 /** 3100 * Release flow in RX_CP_TBL. 3101 * 3102 * @param dev 3103 * Pointer to Ethernet device. 3104 * @flow 3105 * Parent flow for wich copying is provided. 3106 */ 3107 static void 3108 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3109 struct rte_flow *flow) 3110 { 3111 struct mlx5_flow_mreg_copy_resource *mcp_res; 3112 struct mlx5_priv *priv = dev->data->dev_private; 3113 3114 if (!flow->rix_mreg_copy) 3115 return; 3116 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3117 flow->rix_mreg_copy); 3118 if (!mcp_res || !priv->mreg_cp_tbl) 3119 return; 3120 if (flow->copy_applied) { 3121 MLX5_ASSERT(mcp_res->appcnt); 3122 flow->copy_applied = 0; 3123 --mcp_res->appcnt; 3124 if (!mcp_res->appcnt) { 3125 struct rte_flow *mcp_flow = mlx5_ipool_get 3126 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3127 mcp_res->rix_flow); 3128 3129 if (mcp_flow) 3130 flow_drv_remove(dev, mcp_flow); 3131 } 3132 } 3133 /* 3134 * We do not check availability of metadata registers here, 3135 * because copy resources are not allocated in this case. 3136 */ 3137 if (--mcp_res->refcnt) 3138 return; 3139 MLX5_ASSERT(mcp_res->rix_flow); 3140 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3141 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3142 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3143 flow->rix_mreg_copy = 0; 3144 } 3145 3146 /** 3147 * Start flow in RX_CP_TBL. 3148 * 3149 * @param dev 3150 * Pointer to Ethernet device. 3151 * @flow 3152 * Parent flow for wich copying is provided. 3153 * 3154 * @return 3155 * 0 on success, a negative errno value otherwise and rte_errno is set. 3156 */ 3157 static int 3158 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3159 struct rte_flow *flow) 3160 { 3161 struct mlx5_flow_mreg_copy_resource *mcp_res; 3162 struct mlx5_priv *priv = dev->data->dev_private; 3163 int ret; 3164 3165 if (!flow->rix_mreg_copy || flow->copy_applied) 3166 return 0; 3167 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3168 flow->rix_mreg_copy); 3169 if (!mcp_res) 3170 return 0; 3171 if (!mcp_res->appcnt) { 3172 struct rte_flow *mcp_flow = mlx5_ipool_get 3173 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3174 mcp_res->rix_flow); 3175 3176 if (mcp_flow) { 3177 ret = flow_drv_apply(dev, mcp_flow, NULL); 3178 if (ret) 3179 return ret; 3180 } 3181 } 3182 ++mcp_res->appcnt; 3183 flow->copy_applied = 1; 3184 return 0; 3185 } 3186 3187 /** 3188 * Stop flow in RX_CP_TBL. 3189 * 3190 * @param dev 3191 * Pointer to Ethernet device. 3192 * @flow 3193 * Parent flow for wich copying is provided. 3194 */ 3195 static void 3196 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3197 struct rte_flow *flow) 3198 { 3199 struct mlx5_flow_mreg_copy_resource *mcp_res; 3200 struct mlx5_priv *priv = dev->data->dev_private; 3201 3202 if (!flow->rix_mreg_copy || !flow->copy_applied) 3203 return; 3204 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3205 flow->rix_mreg_copy); 3206 if (!mcp_res) 3207 return; 3208 MLX5_ASSERT(mcp_res->appcnt); 3209 --mcp_res->appcnt; 3210 flow->copy_applied = 0; 3211 if (!mcp_res->appcnt) { 3212 struct rte_flow *mcp_flow = mlx5_ipool_get 3213 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3214 mcp_res->rix_flow); 3215 3216 if (mcp_flow) 3217 flow_drv_remove(dev, mcp_flow); 3218 } 3219 } 3220 3221 /** 3222 * Remove the default copy action from RX_CP_TBL. 3223 * 3224 * @param dev 3225 * Pointer to Ethernet device. 3226 */ 3227 static void 3228 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3229 { 3230 struct mlx5_flow_mreg_copy_resource *mcp_res; 3231 struct mlx5_priv *priv = dev->data->dev_private; 3232 3233 /* Check if default flow is registered. */ 3234 if (!priv->mreg_cp_tbl) 3235 return; 3236 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3237 MLX5_DEFAULT_COPY_ID); 3238 if (!mcp_res) 3239 return; 3240 MLX5_ASSERT(mcp_res->rix_flow); 3241 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3242 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3243 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3244 } 3245 3246 /** 3247 * Add the default copy action in in RX_CP_TBL. 3248 * 3249 * @param dev 3250 * Pointer to Ethernet device. 3251 * @param[out] error 3252 * Perform verbose error reporting if not NULL. 3253 * 3254 * @return 3255 * 0 for success, negative value otherwise and rte_errno is set. 3256 */ 3257 static int 3258 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3259 struct rte_flow_error *error) 3260 { 3261 struct mlx5_priv *priv = dev->data->dev_private; 3262 struct mlx5_flow_mreg_copy_resource *mcp_res; 3263 3264 /* Check whether extensive metadata feature is engaged. */ 3265 if (!priv->config.dv_flow_en || 3266 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3267 !mlx5_flow_ext_mreg_supported(dev) || 3268 !priv->sh->dv_regc0_mask) 3269 return 0; 3270 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3271 if (!mcp_res) 3272 return -rte_errno; 3273 return 0; 3274 } 3275 3276 /** 3277 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3278 * 3279 * All the flow having Q/RSS action should be split by 3280 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3281 * performs the following, 3282 * - CQE->flow_tag := reg_c[1] (MARK) 3283 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3284 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3285 * but there should be a flow per each MARK ID set by MARK action. 3286 * 3287 * For the aforementioned reason, if there's a MARK action in flow's action 3288 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3289 * the MARK ID to CQE's flow_tag like, 3290 * - If reg_c[1] is mark_id, 3291 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3292 * 3293 * For SET_META action which stores value in reg_c[0], as the destination is 3294 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3295 * MARK ID means the default flow. The default flow looks like, 3296 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3297 * 3298 * @param dev 3299 * Pointer to Ethernet device. 3300 * @param flow 3301 * Pointer to flow structure. 3302 * @param[in] actions 3303 * Pointer to the list of actions. 3304 * @param[out] error 3305 * Perform verbose error reporting if not NULL. 3306 * 3307 * @return 3308 * 0 on success, negative value otherwise and rte_errno is set. 3309 */ 3310 static int 3311 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3312 struct rte_flow *flow, 3313 const struct rte_flow_action *actions, 3314 struct rte_flow_error *error) 3315 { 3316 struct mlx5_priv *priv = dev->data->dev_private; 3317 struct mlx5_dev_config *config = &priv->config; 3318 struct mlx5_flow_mreg_copy_resource *mcp_res; 3319 const struct rte_flow_action_mark *mark; 3320 3321 /* Check whether extensive metadata feature is engaged. */ 3322 if (!config->dv_flow_en || 3323 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3324 !mlx5_flow_ext_mreg_supported(dev) || 3325 !priv->sh->dv_regc0_mask) 3326 return 0; 3327 /* Find MARK action. */ 3328 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3329 switch (actions->type) { 3330 case RTE_FLOW_ACTION_TYPE_FLAG: 3331 mcp_res = flow_mreg_add_copy_action 3332 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3333 if (!mcp_res) 3334 return -rte_errno; 3335 flow->rix_mreg_copy = mcp_res->idx; 3336 if (dev->data->dev_started) { 3337 mcp_res->appcnt++; 3338 flow->copy_applied = 1; 3339 } 3340 return 0; 3341 case RTE_FLOW_ACTION_TYPE_MARK: 3342 mark = (const struct rte_flow_action_mark *) 3343 actions->conf; 3344 mcp_res = 3345 flow_mreg_add_copy_action(dev, mark->id, error); 3346 if (!mcp_res) 3347 return -rte_errno; 3348 flow->rix_mreg_copy = mcp_res->idx; 3349 if (dev->data->dev_started) { 3350 mcp_res->appcnt++; 3351 flow->copy_applied = 1; 3352 } 3353 return 0; 3354 default: 3355 break; 3356 } 3357 } 3358 return 0; 3359 } 3360 3361 #define MLX5_MAX_SPLIT_ACTIONS 24 3362 #define MLX5_MAX_SPLIT_ITEMS 24 3363 3364 /** 3365 * Split the hairpin flow. 3366 * Since HW can't support encap on Rx we move the encap to Tx. 3367 * If the count action is after the encap then we also 3368 * move the count action. in this case the count will also measure 3369 * the outer bytes. 3370 * 3371 * @param dev 3372 * Pointer to Ethernet device. 3373 * @param[in] actions 3374 * Associated actions (list terminated by the END action). 3375 * @param[out] actions_rx 3376 * Rx flow actions. 3377 * @param[out] actions_tx 3378 * Tx flow actions.. 3379 * @param[out] pattern_tx 3380 * The pattern items for the Tx flow. 3381 * @param[out] flow_id 3382 * The flow ID connected to this flow. 3383 * 3384 * @return 3385 * 0 on success. 3386 */ 3387 static int 3388 flow_hairpin_split(struct rte_eth_dev *dev, 3389 const struct rte_flow_action actions[], 3390 struct rte_flow_action actions_rx[], 3391 struct rte_flow_action actions_tx[], 3392 struct rte_flow_item pattern_tx[], 3393 uint32_t *flow_id) 3394 { 3395 struct mlx5_priv *priv = dev->data->dev_private; 3396 const struct rte_flow_action_raw_encap *raw_encap; 3397 const struct rte_flow_action_raw_decap *raw_decap; 3398 struct mlx5_rte_flow_action_set_tag *set_tag; 3399 struct rte_flow_action *tag_action; 3400 struct mlx5_rte_flow_item_tag *tag_item; 3401 struct rte_flow_item *item; 3402 char *addr; 3403 int encap = 0; 3404 3405 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3406 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3407 switch (actions->type) { 3408 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3409 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3410 rte_memcpy(actions_tx, actions, 3411 sizeof(struct rte_flow_action)); 3412 actions_tx++; 3413 break; 3414 case RTE_FLOW_ACTION_TYPE_COUNT: 3415 if (encap) { 3416 rte_memcpy(actions_tx, actions, 3417 sizeof(struct rte_flow_action)); 3418 actions_tx++; 3419 } else { 3420 rte_memcpy(actions_rx, actions, 3421 sizeof(struct rte_flow_action)); 3422 actions_rx++; 3423 } 3424 break; 3425 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3426 raw_encap = actions->conf; 3427 if (raw_encap->size > 3428 (sizeof(struct rte_flow_item_eth) + 3429 sizeof(struct rte_flow_item_ipv4))) { 3430 memcpy(actions_tx, actions, 3431 sizeof(struct rte_flow_action)); 3432 actions_tx++; 3433 encap = 1; 3434 } else { 3435 rte_memcpy(actions_rx, actions, 3436 sizeof(struct rte_flow_action)); 3437 actions_rx++; 3438 } 3439 break; 3440 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3441 raw_decap = actions->conf; 3442 if (raw_decap->size < 3443 (sizeof(struct rte_flow_item_eth) + 3444 sizeof(struct rte_flow_item_ipv4))) { 3445 memcpy(actions_tx, actions, 3446 sizeof(struct rte_flow_action)); 3447 actions_tx++; 3448 } else { 3449 rte_memcpy(actions_rx, actions, 3450 sizeof(struct rte_flow_action)); 3451 actions_rx++; 3452 } 3453 break; 3454 default: 3455 rte_memcpy(actions_rx, actions, 3456 sizeof(struct rte_flow_action)); 3457 actions_rx++; 3458 break; 3459 } 3460 } 3461 /* Add set meta action and end action for the Rx flow. */ 3462 tag_action = actions_rx; 3463 tag_action->type = (enum rte_flow_action_type) 3464 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3465 actions_rx++; 3466 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3467 actions_rx++; 3468 set_tag = (void *)actions_rx; 3469 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3470 MLX5_ASSERT(set_tag->id > REG_NONE); 3471 set_tag->data = *flow_id; 3472 tag_action->conf = set_tag; 3473 /* Create Tx item list. */ 3474 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3475 addr = (void *)&pattern_tx[2]; 3476 item = pattern_tx; 3477 item->type = (enum rte_flow_item_type) 3478 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3479 tag_item = (void *)addr; 3480 tag_item->data = *flow_id; 3481 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3482 MLX5_ASSERT(set_tag->id > REG_NONE); 3483 item->spec = tag_item; 3484 addr += sizeof(struct mlx5_rte_flow_item_tag); 3485 tag_item = (void *)addr; 3486 tag_item->data = UINT32_MAX; 3487 tag_item->id = UINT16_MAX; 3488 item->mask = tag_item; 3489 addr += sizeof(struct mlx5_rte_flow_item_tag); 3490 item->last = NULL; 3491 item++; 3492 item->type = RTE_FLOW_ITEM_TYPE_END; 3493 return 0; 3494 } 3495 3496 /** 3497 * The last stage of splitting chain, just creates the subflow 3498 * without any modification. 3499 * 3500 * @param[in] dev 3501 * Pointer to Ethernet device. 3502 * @param[in] flow 3503 * Parent flow structure pointer. 3504 * @param[in, out] sub_flow 3505 * Pointer to return the created subflow, may be NULL. 3506 * @param[in] prefix_layers 3507 * Prefix subflow layers, may be 0. 3508 * @param[in] attr 3509 * Flow rule attributes. 3510 * @param[in] items 3511 * Pattern specification (list terminated by the END pattern item). 3512 * @param[in] actions 3513 * Associated actions (list terminated by the END action). 3514 * @param[in] external 3515 * This flow rule is created by request external to PMD. 3516 * @param[in] flow_idx 3517 * This memory pool index to the flow. 3518 * @param[out] error 3519 * Perform verbose error reporting if not NULL. 3520 * @return 3521 * 0 on success, negative value otherwise 3522 */ 3523 static int 3524 flow_create_split_inner(struct rte_eth_dev *dev, 3525 struct rte_flow *flow, 3526 struct mlx5_flow **sub_flow, 3527 uint64_t prefix_layers, 3528 const struct rte_flow_attr *attr, 3529 const struct rte_flow_item items[], 3530 const struct rte_flow_action actions[], 3531 bool external, uint32_t flow_idx, 3532 struct rte_flow_error *error) 3533 { 3534 struct mlx5_flow *dev_flow; 3535 3536 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 3537 flow_idx, error); 3538 if (!dev_flow) 3539 return -rte_errno; 3540 dev_flow->flow = flow; 3541 dev_flow->external = external; 3542 /* Subflow object was created, we must include one in the list. */ 3543 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 3544 dev_flow->handle, next); 3545 /* 3546 * If dev_flow is as one of the suffix flow, some actions in suffix 3547 * flow may need some user defined item layer flags. 3548 */ 3549 if (prefix_layers) 3550 dev_flow->handle->layers = prefix_layers; 3551 if (sub_flow) 3552 *sub_flow = dev_flow; 3553 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3554 } 3555 3556 /** 3557 * Split the meter flow. 3558 * 3559 * As meter flow will split to three sub flow, other than meter 3560 * action, the other actions make sense to only meter accepts 3561 * the packet. If it need to be dropped, no other additional 3562 * actions should be take. 3563 * 3564 * One kind of special action which decapsulates the L3 tunnel 3565 * header will be in the prefix sub flow, as not to take the 3566 * L3 tunnel header into account. 3567 * 3568 * @param dev 3569 * Pointer to Ethernet device. 3570 * @param[in] items 3571 * Pattern specification (list terminated by the END pattern item). 3572 * @param[out] sfx_items 3573 * Suffix flow match items (list terminated by the END pattern item). 3574 * @param[in] actions 3575 * Associated actions (list terminated by the END action). 3576 * @param[out] actions_sfx 3577 * Suffix flow actions. 3578 * @param[out] actions_pre 3579 * Prefix flow actions. 3580 * @param[out] pattern_sfx 3581 * The pattern items for the suffix flow. 3582 * @param[out] tag_sfx 3583 * Pointer to suffix flow tag. 3584 * 3585 * @return 3586 * 0 on success. 3587 */ 3588 static int 3589 flow_meter_split_prep(struct rte_eth_dev *dev, 3590 const struct rte_flow_item items[], 3591 struct rte_flow_item sfx_items[], 3592 const struct rte_flow_action actions[], 3593 struct rte_flow_action actions_sfx[], 3594 struct rte_flow_action actions_pre[]) 3595 { 3596 struct rte_flow_action *tag_action = NULL; 3597 struct rte_flow_item *tag_item; 3598 struct mlx5_rte_flow_action_set_tag *set_tag; 3599 struct rte_flow_error error; 3600 const struct rte_flow_action_raw_encap *raw_encap; 3601 const struct rte_flow_action_raw_decap *raw_decap; 3602 struct mlx5_rte_flow_item_tag *tag_spec; 3603 struct mlx5_rte_flow_item_tag *tag_mask; 3604 uint32_t tag_id; 3605 bool copy_vlan = false; 3606 3607 /* Prepare the actions for prefix and suffix flow. */ 3608 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3609 struct rte_flow_action **action_cur = NULL; 3610 3611 switch (actions->type) { 3612 case RTE_FLOW_ACTION_TYPE_METER: 3613 /* Add the extra tag action first. */ 3614 tag_action = actions_pre; 3615 tag_action->type = (enum rte_flow_action_type) 3616 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3617 actions_pre++; 3618 action_cur = &actions_pre; 3619 break; 3620 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3621 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3622 action_cur = &actions_pre; 3623 break; 3624 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3625 raw_encap = actions->conf; 3626 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 3627 action_cur = &actions_pre; 3628 break; 3629 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3630 raw_decap = actions->conf; 3631 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3632 action_cur = &actions_pre; 3633 break; 3634 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3635 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3636 copy_vlan = true; 3637 break; 3638 default: 3639 break; 3640 } 3641 if (!action_cur) 3642 action_cur = &actions_sfx; 3643 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 3644 (*action_cur)++; 3645 } 3646 /* Add end action to the actions. */ 3647 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3648 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3649 actions_pre++; 3650 /* Set the tag. */ 3651 set_tag = (void *)actions_pre; 3652 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3653 /* 3654 * Get the id from the qrss_pool to make qrss share the id with meter. 3655 */ 3656 tag_id = flow_qrss_get_id(dev); 3657 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 3658 assert(tag_action); 3659 tag_action->conf = set_tag; 3660 /* Prepare the suffix subflow items. */ 3661 tag_item = sfx_items++; 3662 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 3663 int item_type = items->type; 3664 3665 switch (item_type) { 3666 case RTE_FLOW_ITEM_TYPE_PORT_ID: 3667 memcpy(sfx_items, items, sizeof(*sfx_items)); 3668 sfx_items++; 3669 break; 3670 case RTE_FLOW_ITEM_TYPE_VLAN: 3671 if (copy_vlan) { 3672 memcpy(sfx_items, items, sizeof(*sfx_items)); 3673 /* 3674 * Convert to internal match item, it is used 3675 * for vlan push and set vid. 3676 */ 3677 sfx_items->type = (enum rte_flow_item_type) 3678 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 3679 sfx_items++; 3680 } 3681 break; 3682 default: 3683 break; 3684 } 3685 } 3686 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 3687 sfx_items++; 3688 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 3689 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 3690 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3691 tag_mask = tag_spec + 1; 3692 tag_mask->data = 0xffffff00; 3693 tag_item->type = (enum rte_flow_item_type) 3694 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3695 tag_item->spec = tag_spec; 3696 tag_item->last = NULL; 3697 tag_item->mask = tag_mask; 3698 return tag_id; 3699 } 3700 3701 /** 3702 * Split action list having QUEUE/RSS for metadata register copy. 3703 * 3704 * Once Q/RSS action is detected in user's action list, the flow action 3705 * should be split in order to copy metadata registers, which will happen in 3706 * RX_CP_TBL like, 3707 * - CQE->flow_tag := reg_c[1] (MARK) 3708 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3709 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3710 * This is because the last action of each flow must be a terminal action 3711 * (QUEUE, RSS or DROP). 3712 * 3713 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3714 * stored and kept in the mlx5_flow structure per each sub_flow. 3715 * 3716 * The Q/RSS action is replaced with, 3717 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3718 * And the following JUMP action is added at the end, 3719 * - JUMP, to RX_CP_TBL. 3720 * 3721 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3722 * flow_create_split_metadata() routine. The flow will look like, 3723 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3724 * 3725 * @param dev 3726 * Pointer to Ethernet device. 3727 * @param[out] split_actions 3728 * Pointer to store split actions to jump to CP_TBL. 3729 * @param[in] actions 3730 * Pointer to the list of original flow actions. 3731 * @param[in] qrss 3732 * Pointer to the Q/RSS action. 3733 * @param[in] actions_n 3734 * Number of original actions. 3735 * @param[out] error 3736 * Perform verbose error reporting if not NULL. 3737 * 3738 * @return 3739 * non-zero unique flow_id on success, otherwise 0 and 3740 * error/rte_error are set. 3741 */ 3742 static uint32_t 3743 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3744 struct rte_flow_action *split_actions, 3745 const struct rte_flow_action *actions, 3746 const struct rte_flow_action *qrss, 3747 int actions_n, struct rte_flow_error *error) 3748 { 3749 struct mlx5_rte_flow_action_set_tag *set_tag; 3750 struct rte_flow_action_jump *jump; 3751 const int qrss_idx = qrss - actions; 3752 uint32_t flow_id = 0; 3753 int ret = 0; 3754 3755 /* 3756 * Given actions will be split 3757 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3758 * - Add jump to mreg CP_TBL. 3759 * As a result, there will be one more action. 3760 */ 3761 ++actions_n; 3762 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3763 set_tag = (void *)(split_actions + actions_n); 3764 /* 3765 * If tag action is not set to void(it means we are not the meter 3766 * suffix flow), add the tag action. Since meter suffix flow already 3767 * has the tag added. 3768 */ 3769 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3770 /* 3771 * Allocate the new subflow ID. This one is unique within 3772 * device and not shared with representors. Otherwise, 3773 * we would have to resolve multi-thread access synch 3774 * issue. Each flow on the shared device is appended 3775 * with source vport identifier, so the resulting 3776 * flows will be unique in the shared (by master and 3777 * representors) domain even if they have coinciding 3778 * IDs. 3779 */ 3780 flow_id = flow_qrss_get_id(dev); 3781 if (!flow_id) 3782 return rte_flow_error_set(error, ENOMEM, 3783 RTE_FLOW_ERROR_TYPE_ACTION, 3784 NULL, "can't allocate id " 3785 "for split Q/RSS subflow"); 3786 /* Internal SET_TAG action to set flow ID. */ 3787 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3788 .data = flow_id, 3789 }; 3790 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3791 if (ret < 0) 3792 return ret; 3793 set_tag->id = ret; 3794 /* Construct new actions array. */ 3795 /* Replace QUEUE/RSS action. */ 3796 split_actions[qrss_idx] = (struct rte_flow_action){ 3797 .type = (enum rte_flow_action_type) 3798 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3799 .conf = set_tag, 3800 }; 3801 } 3802 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3803 jump = (void *)(set_tag + 1); 3804 *jump = (struct rte_flow_action_jump){ 3805 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3806 }; 3807 split_actions[actions_n - 2] = (struct rte_flow_action){ 3808 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3809 .conf = jump, 3810 }; 3811 split_actions[actions_n - 1] = (struct rte_flow_action){ 3812 .type = RTE_FLOW_ACTION_TYPE_END, 3813 }; 3814 return flow_id; 3815 } 3816 3817 /** 3818 * Extend the given action list for Tx metadata copy. 3819 * 3820 * Copy the given action list to the ext_actions and add flow metadata register 3821 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3822 * 3823 * @param[out] ext_actions 3824 * Pointer to the extended action list. 3825 * @param[in] actions 3826 * Pointer to the list of actions. 3827 * @param[in] actions_n 3828 * Number of actions in the list. 3829 * @param[out] error 3830 * Perform verbose error reporting if not NULL. 3831 * @param[in] encap_idx 3832 * The encap action inndex. 3833 * 3834 * @return 3835 * 0 on success, negative value otherwise 3836 */ 3837 static int 3838 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3839 struct rte_flow_action *ext_actions, 3840 const struct rte_flow_action *actions, 3841 int actions_n, struct rte_flow_error *error, 3842 int encap_idx) 3843 { 3844 struct mlx5_flow_action_copy_mreg *cp_mreg = 3845 (struct mlx5_flow_action_copy_mreg *) 3846 (ext_actions + actions_n + 1); 3847 int ret; 3848 3849 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3850 if (ret < 0) 3851 return ret; 3852 cp_mreg->dst = ret; 3853 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3854 if (ret < 0) 3855 return ret; 3856 cp_mreg->src = ret; 3857 if (encap_idx != 0) 3858 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 3859 if (encap_idx == actions_n - 1) { 3860 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3861 .type = (enum rte_flow_action_type) 3862 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3863 .conf = cp_mreg, 3864 }; 3865 ext_actions[actions_n] = (struct rte_flow_action){ 3866 .type = RTE_FLOW_ACTION_TYPE_END, 3867 }; 3868 } else { 3869 ext_actions[encap_idx] = (struct rte_flow_action){ 3870 .type = (enum rte_flow_action_type) 3871 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3872 .conf = cp_mreg, 3873 }; 3874 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 3875 sizeof(*ext_actions) * (actions_n - encap_idx)); 3876 } 3877 return 0; 3878 } 3879 3880 /** 3881 * The splitting for metadata feature. 3882 * 3883 * - Q/RSS action on NIC Rx should be split in order to pass by 3884 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3885 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3886 * 3887 * - All the actions on NIC Tx should have a mreg copy action to 3888 * copy reg_a from WQE to reg_c[0]. 3889 * 3890 * @param dev 3891 * Pointer to Ethernet device. 3892 * @param[in] flow 3893 * Parent flow structure pointer. 3894 * @param[in] prefix_layers 3895 * Prefix flow layer flags. 3896 * @param[in] attr 3897 * Flow rule attributes. 3898 * @param[in] items 3899 * Pattern specification (list terminated by the END pattern item). 3900 * @param[in] actions 3901 * Associated actions (list terminated by the END action). 3902 * @param[in] external 3903 * This flow rule is created by request external to PMD. 3904 * @param[in] flow_idx 3905 * This memory pool index to the flow. 3906 * @param[out] error 3907 * Perform verbose error reporting if not NULL. 3908 * @return 3909 * 0 on success, negative value otherwise 3910 */ 3911 static int 3912 flow_create_split_metadata(struct rte_eth_dev *dev, 3913 struct rte_flow *flow, 3914 uint64_t prefix_layers, 3915 const struct rte_flow_attr *attr, 3916 const struct rte_flow_item items[], 3917 const struct rte_flow_action actions[], 3918 bool external, uint32_t flow_idx, 3919 struct rte_flow_error *error) 3920 { 3921 struct mlx5_priv *priv = dev->data->dev_private; 3922 struct mlx5_dev_config *config = &priv->config; 3923 const struct rte_flow_action *qrss = NULL; 3924 struct rte_flow_action *ext_actions = NULL; 3925 struct mlx5_flow *dev_flow = NULL; 3926 uint32_t qrss_id = 0; 3927 int mtr_sfx = 0; 3928 size_t act_size; 3929 int actions_n; 3930 int encap_idx; 3931 int ret; 3932 3933 /* Check whether extensive metadata feature is engaged. */ 3934 if (!config->dv_flow_en || 3935 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3936 !mlx5_flow_ext_mreg_supported(dev)) 3937 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 3938 attr, items, actions, external, 3939 flow_idx, error); 3940 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 3941 &encap_idx); 3942 if (qrss) { 3943 /* Exclude hairpin flows from splitting. */ 3944 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3945 const struct rte_flow_action_queue *queue; 3946 3947 queue = qrss->conf; 3948 if (mlx5_rxq_get_type(dev, queue->index) == 3949 MLX5_RXQ_TYPE_HAIRPIN) 3950 qrss = NULL; 3951 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3952 const struct rte_flow_action_rss *rss; 3953 3954 rss = qrss->conf; 3955 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3956 MLX5_RXQ_TYPE_HAIRPIN) 3957 qrss = NULL; 3958 } 3959 } 3960 if (qrss) { 3961 /* Check if it is in meter suffix table. */ 3962 mtr_sfx = attr->group == (attr->transfer ? 3963 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3964 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3965 /* 3966 * Q/RSS action on NIC Rx should be split in order to pass by 3967 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3968 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3969 */ 3970 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3971 sizeof(struct rte_flow_action_set_tag) + 3972 sizeof(struct rte_flow_action_jump); 3973 ext_actions = rte_zmalloc(__func__, act_size, 0); 3974 if (!ext_actions) 3975 return rte_flow_error_set(error, ENOMEM, 3976 RTE_FLOW_ERROR_TYPE_ACTION, 3977 NULL, "no memory to split " 3978 "metadata flow"); 3979 /* 3980 * If we are the suffix flow of meter, tag already exist. 3981 * Set the tag action to void. 3982 */ 3983 if (mtr_sfx) 3984 ext_actions[qrss - actions].type = 3985 RTE_FLOW_ACTION_TYPE_VOID; 3986 else 3987 ext_actions[qrss - actions].type = 3988 (enum rte_flow_action_type) 3989 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3990 /* 3991 * Create the new actions list with removed Q/RSS action 3992 * and appended set tag and jump to register copy table 3993 * (RX_CP_TBL). We should preallocate unique tag ID here 3994 * in advance, because it is needed for set tag action. 3995 */ 3996 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3997 qrss, actions_n, error); 3998 if (!mtr_sfx && !qrss_id) { 3999 ret = -rte_errno; 4000 goto exit; 4001 } 4002 } else if (attr->egress && !attr->transfer) { 4003 /* 4004 * All the actions on NIC Tx should have a metadata register 4005 * copy action to copy reg_a from WQE to reg_c[meta] 4006 */ 4007 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4008 sizeof(struct mlx5_flow_action_copy_mreg); 4009 ext_actions = rte_zmalloc(__func__, act_size, 0); 4010 if (!ext_actions) 4011 return rte_flow_error_set(error, ENOMEM, 4012 RTE_FLOW_ERROR_TYPE_ACTION, 4013 NULL, "no memory to split " 4014 "metadata flow"); 4015 /* Create the action list appended with copy register. */ 4016 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 4017 actions_n, error, encap_idx); 4018 if (ret < 0) 4019 goto exit; 4020 } 4021 /* Add the unmodified original or prefix subflow. */ 4022 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr, 4023 items, ext_actions ? ext_actions : 4024 actions, external, flow_idx, error); 4025 if (ret < 0) 4026 goto exit; 4027 MLX5_ASSERT(dev_flow); 4028 if (qrss) { 4029 const struct rte_flow_attr q_attr = { 4030 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4031 .ingress = 1, 4032 }; 4033 /* Internal PMD action to set register. */ 4034 struct mlx5_rte_flow_item_tag q_tag_spec = { 4035 .data = qrss_id, 4036 .id = 0, 4037 }; 4038 struct rte_flow_item q_items[] = { 4039 { 4040 .type = (enum rte_flow_item_type) 4041 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4042 .spec = &q_tag_spec, 4043 .last = NULL, 4044 .mask = NULL, 4045 }, 4046 { 4047 .type = RTE_FLOW_ITEM_TYPE_END, 4048 }, 4049 }; 4050 struct rte_flow_action q_actions[] = { 4051 { 4052 .type = qrss->type, 4053 .conf = qrss->conf, 4054 }, 4055 { 4056 .type = RTE_FLOW_ACTION_TYPE_END, 4057 }, 4058 }; 4059 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4060 4061 /* 4062 * Configure the tag item only if there is no meter subflow. 4063 * Since tag is already marked in the meter suffix subflow 4064 * we can just use the meter suffix items as is. 4065 */ 4066 if (qrss_id) { 4067 /* Not meter subflow. */ 4068 MLX5_ASSERT(!mtr_sfx); 4069 /* 4070 * Put unique id in prefix flow due to it is destroyed 4071 * after suffix flow and id will be freed after there 4072 * is no actual flows with this id and identifier 4073 * reallocation becomes possible (for example, for 4074 * other flows in other threads). 4075 */ 4076 dev_flow->handle->split_flow_id = qrss_id; 4077 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4078 error); 4079 if (ret < 0) 4080 goto exit; 4081 q_tag_spec.id = ret; 4082 } 4083 dev_flow = NULL; 4084 /* Add suffix subflow to execute Q/RSS. */ 4085 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 4086 &q_attr, mtr_sfx ? items : 4087 q_items, q_actions, 4088 external, flow_idx, error); 4089 if (ret < 0) 4090 goto exit; 4091 /* qrss ID should be freed if failed. */ 4092 qrss_id = 0; 4093 MLX5_ASSERT(dev_flow); 4094 } 4095 4096 exit: 4097 /* 4098 * We do not destroy the partially created sub_flows in case of error. 4099 * These ones are included into parent flow list and will be destroyed 4100 * by flow_drv_destroy. 4101 */ 4102 flow_qrss_free_id(dev, qrss_id); 4103 rte_free(ext_actions); 4104 return ret; 4105 } 4106 4107 /** 4108 * The splitting for meter feature. 4109 * 4110 * - The meter flow will be split to two flows as prefix and 4111 * suffix flow. The packets make sense only it pass the prefix 4112 * meter action. 4113 * 4114 * - Reg_C_5 is used for the packet to match betweend prefix and 4115 * suffix flow. 4116 * 4117 * @param dev 4118 * Pointer to Ethernet device. 4119 * @param[in] flow 4120 * Parent flow structure pointer. 4121 * @param[in] attr 4122 * Flow rule attributes. 4123 * @param[in] items 4124 * Pattern specification (list terminated by the END pattern item). 4125 * @param[in] actions 4126 * Associated actions (list terminated by the END action). 4127 * @param[in] external 4128 * This flow rule is created by request external to PMD. 4129 * @param[in] flow_idx 4130 * This memory pool index to the flow. 4131 * @param[out] error 4132 * Perform verbose error reporting if not NULL. 4133 * @return 4134 * 0 on success, negative value otherwise 4135 */ 4136 static int 4137 flow_create_split_meter(struct rte_eth_dev *dev, 4138 struct rte_flow *flow, 4139 const struct rte_flow_attr *attr, 4140 const struct rte_flow_item items[], 4141 const struct rte_flow_action actions[], 4142 bool external, uint32_t flow_idx, 4143 struct rte_flow_error *error) 4144 { 4145 struct mlx5_priv *priv = dev->data->dev_private; 4146 struct rte_flow_action *sfx_actions = NULL; 4147 struct rte_flow_action *pre_actions = NULL; 4148 struct rte_flow_item *sfx_items = NULL; 4149 struct mlx5_flow *dev_flow = NULL; 4150 struct rte_flow_attr sfx_attr = *attr; 4151 uint32_t mtr = 0; 4152 uint32_t mtr_tag_id = 0; 4153 size_t act_size; 4154 size_t item_size; 4155 int actions_n = 0; 4156 int ret; 4157 4158 if (priv->mtr_en) 4159 actions_n = flow_check_meter_action(actions, &mtr); 4160 if (mtr) { 4161 /* The five prefix actions: meter, decap, encap, tag, end. */ 4162 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4163 sizeof(struct mlx5_rte_flow_action_set_tag); 4164 /* tag, vlan, port id, end. */ 4165 #define METER_SUFFIX_ITEM 4 4166 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4167 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4168 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 4169 if (!sfx_actions) 4170 return rte_flow_error_set(error, ENOMEM, 4171 RTE_FLOW_ERROR_TYPE_ACTION, 4172 NULL, "no memory to split " 4173 "meter flow"); 4174 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4175 act_size); 4176 pre_actions = sfx_actions + actions_n; 4177 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4178 actions, sfx_actions, 4179 pre_actions); 4180 if (!mtr_tag_id) { 4181 ret = -rte_errno; 4182 goto exit; 4183 } 4184 /* Add the prefix subflow. */ 4185 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr, 4186 items, pre_actions, external, 4187 flow_idx, error); 4188 if (ret) { 4189 ret = -rte_errno; 4190 goto exit; 4191 } 4192 dev_flow->handle->split_flow_id = mtr_tag_id; 4193 /* Setting the sfx group atrr. */ 4194 sfx_attr.group = sfx_attr.transfer ? 4195 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4196 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4197 } 4198 /* Add the prefix subflow. */ 4199 ret = flow_create_split_metadata(dev, flow, dev_flow ? 4200 flow_get_prefix_layer_flags(dev_flow) : 4201 0, &sfx_attr, 4202 sfx_items ? sfx_items : items, 4203 sfx_actions ? sfx_actions : actions, 4204 external, flow_idx, error); 4205 exit: 4206 if (sfx_actions) 4207 rte_free(sfx_actions); 4208 return ret; 4209 } 4210 4211 /** 4212 * Split the flow to subflow set. The splitters might be linked 4213 * in the chain, like this: 4214 * flow_create_split_outer() calls: 4215 * flow_create_split_meter() calls: 4216 * flow_create_split_metadata(meter_subflow_0) calls: 4217 * flow_create_split_inner(metadata_subflow_0) 4218 * flow_create_split_inner(metadata_subflow_1) 4219 * flow_create_split_inner(metadata_subflow_2) 4220 * flow_create_split_metadata(meter_subflow_1) calls: 4221 * flow_create_split_inner(metadata_subflow_0) 4222 * flow_create_split_inner(metadata_subflow_1) 4223 * flow_create_split_inner(metadata_subflow_2) 4224 * 4225 * This provide flexible way to add new levels of flow splitting. 4226 * The all of successfully created subflows are included to the 4227 * parent flow dev_flow list. 4228 * 4229 * @param dev 4230 * Pointer to Ethernet device. 4231 * @param[in] flow 4232 * Parent flow structure pointer. 4233 * @param[in] attr 4234 * Flow rule attributes. 4235 * @param[in] items 4236 * Pattern specification (list terminated by the END pattern item). 4237 * @param[in] actions 4238 * Associated actions (list terminated by the END action). 4239 * @param[in] external 4240 * This flow rule is created by request external to PMD. 4241 * @param[in] flow_idx 4242 * This memory pool index to the flow. 4243 * @param[out] error 4244 * Perform verbose error reporting if not NULL. 4245 * @return 4246 * 0 on success, negative value otherwise 4247 */ 4248 static int 4249 flow_create_split_outer(struct rte_eth_dev *dev, 4250 struct rte_flow *flow, 4251 const struct rte_flow_attr *attr, 4252 const struct rte_flow_item items[], 4253 const struct rte_flow_action actions[], 4254 bool external, uint32_t flow_idx, 4255 struct rte_flow_error *error) 4256 { 4257 int ret; 4258 4259 ret = flow_create_split_meter(dev, flow, attr, items, 4260 actions, external, flow_idx, error); 4261 MLX5_ASSERT(ret <= 0); 4262 return ret; 4263 } 4264 4265 /** 4266 * Create a flow and add it to @p list. 4267 * 4268 * @param dev 4269 * Pointer to Ethernet device. 4270 * @param list 4271 * Pointer to a TAILQ flow list. If this parameter NULL, 4272 * no list insertion occurred, flow is just created, 4273 * this is caller's responsibility to track the 4274 * created flow. 4275 * @param[in] attr 4276 * Flow rule attributes. 4277 * @param[in] items 4278 * Pattern specification (list terminated by the END pattern item). 4279 * @param[in] actions 4280 * Associated actions (list terminated by the END action). 4281 * @param[in] external 4282 * This flow rule is created by request external to PMD. 4283 * @param[out] error 4284 * Perform verbose error reporting if not NULL. 4285 * 4286 * @return 4287 * A flow index on success, 0 otherwise and rte_errno is set. 4288 */ 4289 static uint32_t 4290 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 4291 const struct rte_flow_attr *attr, 4292 const struct rte_flow_item items[], 4293 const struct rte_flow_action actions[], 4294 bool external, struct rte_flow_error *error) 4295 { 4296 struct mlx5_priv *priv = dev->data->dev_private; 4297 struct rte_flow *flow = NULL; 4298 struct mlx5_flow *dev_flow; 4299 const struct rte_flow_action_rss *rss; 4300 union { 4301 struct rte_flow_expand_rss buf; 4302 uint8_t buffer[2048]; 4303 } expand_buffer; 4304 union { 4305 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4306 uint8_t buffer[2048]; 4307 } actions_rx; 4308 union { 4309 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4310 uint8_t buffer[2048]; 4311 } actions_hairpin_tx; 4312 union { 4313 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4314 uint8_t buffer[2048]; 4315 } items_tx; 4316 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4317 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 4318 priv->rss_desc)[!!priv->flow_idx]; 4319 const struct rte_flow_action *p_actions_rx = actions; 4320 uint32_t i; 4321 uint32_t idx = 0; 4322 int hairpin_flow; 4323 uint32_t hairpin_id = 0; 4324 struct rte_flow_attr attr_tx = { .priority = 0 }; 4325 int ret; 4326 4327 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4328 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 4329 external, hairpin_flow, error); 4330 if (ret < 0) 4331 return 0; 4332 if (hairpin_flow > 0) { 4333 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4334 rte_errno = EINVAL; 4335 return 0; 4336 } 4337 flow_hairpin_split(dev, actions, actions_rx.actions, 4338 actions_hairpin_tx.actions, items_tx.items, 4339 &hairpin_id); 4340 p_actions_rx = actions_rx.actions; 4341 } 4342 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 4343 if (!flow) { 4344 rte_errno = ENOMEM; 4345 goto error_before_flow; 4346 } 4347 flow->drv_type = flow_get_drv_type(dev, attr); 4348 if (hairpin_id != 0) 4349 flow->hairpin_flow_id = hairpin_id; 4350 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4351 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4352 memset(rss_desc, 0, sizeof(*rss_desc)); 4353 rss = flow_get_rss_action(p_actions_rx); 4354 if (rss) { 4355 /* 4356 * The following information is required by 4357 * mlx5_flow_hashfields_adjust() in advance. 4358 */ 4359 rss_desc->level = rss->level; 4360 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4361 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 4362 } 4363 flow->dev_handles = 0; 4364 if (rss && rss->types) { 4365 unsigned int graph_root; 4366 4367 graph_root = find_graph_root(items, rss->level); 4368 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4369 items, rss->types, 4370 mlx5_support_expansion, 4371 graph_root); 4372 MLX5_ASSERT(ret > 0 && 4373 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4374 } else { 4375 buf->entries = 1; 4376 buf->entry[0].pattern = (void *)(uintptr_t)items; 4377 } 4378 /* 4379 * Record the start index when there is a nested call. All sub-flows 4380 * need to be translated before another calling. 4381 * No need to use ping-pong buffer to save memory here. 4382 */ 4383 if (priv->flow_idx) { 4384 MLX5_ASSERT(!priv->flow_nested_idx); 4385 priv->flow_nested_idx = priv->flow_idx; 4386 } 4387 for (i = 0; i < buf->entries; ++i) { 4388 /* 4389 * The splitter may create multiple dev_flows, 4390 * depending on configuration. In the simplest 4391 * case it just creates unmodified original flow. 4392 */ 4393 ret = flow_create_split_outer(dev, flow, attr, 4394 buf->entry[i].pattern, 4395 p_actions_rx, external, idx, 4396 error); 4397 if (ret < 0) 4398 goto error; 4399 } 4400 /* Create the tx flow. */ 4401 if (hairpin_flow) { 4402 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4403 attr_tx.ingress = 0; 4404 attr_tx.egress = 1; 4405 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 4406 actions_hairpin_tx.actions, 4407 idx, error); 4408 if (!dev_flow) 4409 goto error; 4410 dev_flow->flow = flow; 4411 dev_flow->external = 0; 4412 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4413 dev_flow->handle, next); 4414 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4415 items_tx.items, 4416 actions_hairpin_tx.actions, error); 4417 if (ret < 0) 4418 goto error; 4419 } 4420 /* 4421 * Update the metadata register copy table. If extensive 4422 * metadata feature is enabled and registers are supported 4423 * we might create the extra rte_flow for each unique 4424 * MARK/FLAG action ID. 4425 * 4426 * The table is updated for ingress Flows only, because 4427 * the egress Flows belong to the different device and 4428 * copy table should be updated in peer NIC Rx domain. 4429 */ 4430 if (attr->ingress && 4431 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4432 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4433 if (ret) 4434 goto error; 4435 } 4436 /* 4437 * If the flow is external (from application) OR device is started, then 4438 * the flow will be applied immediately. 4439 */ 4440 if (external || dev->data->dev_started) { 4441 ret = flow_drv_apply(dev, flow, error); 4442 if (ret < 0) 4443 goto error; 4444 } 4445 if (list) 4446 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 4447 flow, next); 4448 flow_rxq_flags_set(dev, flow); 4449 /* Nested flow creation index recovery. */ 4450 priv->flow_idx = priv->flow_nested_idx; 4451 if (priv->flow_nested_idx) 4452 priv->flow_nested_idx = 0; 4453 return idx; 4454 error: 4455 MLX5_ASSERT(flow); 4456 ret = rte_errno; /* Save rte_errno before cleanup. */ 4457 flow_mreg_del_copy_action(dev, flow); 4458 flow_drv_destroy(dev, flow); 4459 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 4460 rte_errno = ret; /* Restore rte_errno. */ 4461 error_before_flow: 4462 ret = rte_errno; 4463 if (hairpin_id) 4464 mlx5_flow_id_release(priv->sh->flow_id_pool, 4465 hairpin_id); 4466 rte_errno = ret; 4467 priv->flow_idx = priv->flow_nested_idx; 4468 if (priv->flow_nested_idx) 4469 priv->flow_nested_idx = 0; 4470 return 0; 4471 } 4472 4473 /** 4474 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4475 * incoming packets to table 1. 4476 * 4477 * Other flow rules, requested for group n, will be created in 4478 * e-switch table n+1. 4479 * Jump action to e-switch group n will be created to group n+1. 4480 * 4481 * Used when working in switchdev mode, to utilise advantages of table 1 4482 * and above. 4483 * 4484 * @param dev 4485 * Pointer to Ethernet device. 4486 * 4487 * @return 4488 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4489 */ 4490 struct rte_flow * 4491 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4492 { 4493 const struct rte_flow_attr attr = { 4494 .group = 0, 4495 .priority = 0, 4496 .ingress = 1, 4497 .egress = 0, 4498 .transfer = 1, 4499 }; 4500 const struct rte_flow_item pattern = { 4501 .type = RTE_FLOW_ITEM_TYPE_END, 4502 }; 4503 struct rte_flow_action_jump jump = { 4504 .group = 1, 4505 }; 4506 const struct rte_flow_action actions[] = { 4507 { 4508 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4509 .conf = &jump, 4510 }, 4511 { 4512 .type = RTE_FLOW_ACTION_TYPE_END, 4513 }, 4514 }; 4515 struct mlx5_priv *priv = dev->data->dev_private; 4516 struct rte_flow_error error; 4517 4518 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 4519 &attr, &pattern, 4520 actions, false, &error); 4521 } 4522 4523 /** 4524 * Validate a flow supported by the NIC. 4525 * 4526 * @see rte_flow_validate() 4527 * @see rte_flow_ops 4528 */ 4529 int 4530 mlx5_flow_validate(struct rte_eth_dev *dev, 4531 const struct rte_flow_attr *attr, 4532 const struct rte_flow_item items[], 4533 const struct rte_flow_action actions[], 4534 struct rte_flow_error *error) 4535 { 4536 int hairpin_flow; 4537 4538 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4539 return flow_drv_validate(dev, attr, items, actions, 4540 true, hairpin_flow, error); 4541 } 4542 4543 /** 4544 * Create a flow. 4545 * 4546 * @see rte_flow_create() 4547 * @see rte_flow_ops 4548 */ 4549 struct rte_flow * 4550 mlx5_flow_create(struct rte_eth_dev *dev, 4551 const struct rte_flow_attr *attr, 4552 const struct rte_flow_item items[], 4553 const struct rte_flow_action actions[], 4554 struct rte_flow_error *error) 4555 { 4556 struct mlx5_priv *priv = dev->data->dev_private; 4557 4558 /* 4559 * If the device is not started yet, it is not allowed to created a 4560 * flow from application. PMD default flows and traffic control flows 4561 * are not affected. 4562 */ 4563 if (unlikely(!dev->data->dev_started)) { 4564 DRV_LOG(DEBUG, "port %u is not started when " 4565 "inserting a flow", dev->data->port_id); 4566 rte_flow_error_set(error, ENODEV, 4567 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4568 NULL, 4569 "port not started"); 4570 return NULL; 4571 } 4572 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 4573 attr, items, actions, true, error); 4574 } 4575 4576 /** 4577 * Destroy a flow in a list. 4578 * 4579 * @param dev 4580 * Pointer to Ethernet device. 4581 * @param list 4582 * Pointer to the Indexed flow list. If this parameter NULL, 4583 * there is no flow removal from the list. Be noted that as 4584 * flow is add to the indexed list, memory of the indexed 4585 * list points to maybe changed as flow destroyed. 4586 * @param[in] flow_idx 4587 * Index of flow to destroy. 4588 */ 4589 static void 4590 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 4591 uint32_t flow_idx) 4592 { 4593 struct mlx5_priv *priv = dev->data->dev_private; 4594 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 4595 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 4596 [MLX5_IPOOL_RTE_FLOW], flow_idx); 4597 4598 if (!flow) 4599 return; 4600 /* 4601 * Update RX queue flags only if port is started, otherwise it is 4602 * already clean. 4603 */ 4604 if (dev->data->dev_started) 4605 flow_rxq_flags_trim(dev, flow); 4606 if (flow->hairpin_flow_id) 4607 mlx5_flow_id_release(priv->sh->flow_id_pool, 4608 flow->hairpin_flow_id); 4609 flow_drv_destroy(dev, flow); 4610 if (list) 4611 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 4612 flow_idx, flow, next); 4613 flow_mreg_del_copy_action(dev, flow); 4614 if (flow->fdir) { 4615 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 4616 if (priv_fdir_flow->rix_flow == flow_idx) 4617 break; 4618 } 4619 if (priv_fdir_flow) { 4620 LIST_REMOVE(priv_fdir_flow, next); 4621 rte_free(priv_fdir_flow->fdir); 4622 rte_free(priv_fdir_flow); 4623 } 4624 } 4625 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 4626 } 4627 4628 /** 4629 * Destroy all flows. 4630 * 4631 * @param dev 4632 * Pointer to Ethernet device. 4633 * @param list 4634 * Pointer to the Indexed flow list. 4635 * @param active 4636 * If flushing is called avtively. 4637 */ 4638 void 4639 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 4640 { 4641 uint32_t num_flushed = 0; 4642 4643 while (*list) { 4644 flow_list_destroy(dev, list, *list); 4645 num_flushed++; 4646 } 4647 if (active) { 4648 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 4649 dev->data->port_id, num_flushed); 4650 } 4651 } 4652 4653 /** 4654 * Remove all flows. 4655 * 4656 * @param dev 4657 * Pointer to Ethernet device. 4658 * @param list 4659 * Pointer to the Indexed flow list. 4660 */ 4661 void 4662 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list) 4663 { 4664 struct mlx5_priv *priv = dev->data->dev_private; 4665 struct rte_flow *flow = NULL; 4666 uint32_t idx; 4667 4668 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4669 flow, next) { 4670 flow_drv_remove(dev, flow); 4671 flow_mreg_stop_copy_action(dev, flow); 4672 } 4673 flow_mreg_del_default_copy_action(dev); 4674 flow_rxq_flags_clear(dev); 4675 } 4676 4677 /** 4678 * Add all flows. 4679 * 4680 * @param dev 4681 * Pointer to Ethernet device. 4682 * @param list 4683 * Pointer to the Indexed flow list. 4684 * 4685 * @return 4686 * 0 on success, a negative errno value otherwise and rte_errno is set. 4687 */ 4688 int 4689 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list) 4690 { 4691 struct mlx5_priv *priv = dev->data->dev_private; 4692 struct rte_flow *flow = NULL; 4693 struct rte_flow_error error; 4694 uint32_t idx; 4695 int ret = 0; 4696 4697 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4698 ret = flow_mreg_add_default_copy_action(dev, &error); 4699 if (ret < 0) 4700 return -rte_errno; 4701 /* Apply Flows created by application. */ 4702 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4703 flow, next) { 4704 ret = flow_mreg_start_copy_action(dev, flow); 4705 if (ret < 0) 4706 goto error; 4707 ret = flow_drv_apply(dev, flow, &error); 4708 if (ret < 0) 4709 goto error; 4710 flow_rxq_flags_set(dev, flow); 4711 } 4712 return 0; 4713 error: 4714 ret = rte_errno; /* Save rte_errno before cleanup. */ 4715 mlx5_flow_stop(dev, list); 4716 rte_errno = ret; /* Restore rte_errno. */ 4717 return -rte_errno; 4718 } 4719 4720 /** 4721 * Stop all default actions for flows. 4722 * 4723 * @param dev 4724 * Pointer to Ethernet device. 4725 */ 4726 void 4727 mlx5_flow_stop_default(struct rte_eth_dev *dev) 4728 { 4729 flow_mreg_del_default_copy_action(dev); 4730 flow_rxq_flags_clear(dev); 4731 } 4732 4733 /** 4734 * Start all default actions for flows. 4735 * 4736 * @param dev 4737 * Pointer to Ethernet device. 4738 * @return 4739 * 0 on success, a negative errno value otherwise and rte_errno is set. 4740 */ 4741 int 4742 mlx5_flow_start_default(struct rte_eth_dev *dev) 4743 { 4744 struct rte_flow_error error; 4745 4746 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4747 return flow_mreg_add_default_copy_action(dev, &error); 4748 } 4749 4750 /** 4751 * Allocate intermediate resources for flow creation. 4752 * 4753 * @param dev 4754 * Pointer to Ethernet device. 4755 */ 4756 void 4757 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) 4758 { 4759 struct mlx5_priv *priv = dev->data->dev_private; 4760 4761 if (!priv->inter_flows) { 4762 priv->inter_flows = rte_calloc(__func__, 1, 4763 MLX5_NUM_MAX_DEV_FLOWS * 4764 sizeof(struct mlx5_flow) + 4765 (sizeof(struct mlx5_flow_rss_desc) + 4766 sizeof(uint16_t) * UINT16_MAX) * 2, 0); 4767 if (!priv->inter_flows) { 4768 DRV_LOG(ERR, "can't allocate intermediate memory."); 4769 return; 4770 } 4771 } 4772 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows) 4773 [MLX5_NUM_MAX_DEV_FLOWS]; 4774 /* Reset the index. */ 4775 priv->flow_idx = 0; 4776 priv->flow_nested_idx = 0; 4777 } 4778 4779 /** 4780 * Free intermediate resources for flows. 4781 * 4782 * @param dev 4783 * Pointer to Ethernet device. 4784 */ 4785 void 4786 mlx5_flow_free_intermediate(struct rte_eth_dev *dev) 4787 { 4788 struct mlx5_priv *priv = dev->data->dev_private; 4789 4790 rte_free(priv->inter_flows); 4791 priv->inter_flows = NULL; 4792 } 4793 4794 /** 4795 * Verify the flow list is empty 4796 * 4797 * @param dev 4798 * Pointer to Ethernet device. 4799 * 4800 * @return the number of flows not released. 4801 */ 4802 int 4803 mlx5_flow_verify(struct rte_eth_dev *dev) 4804 { 4805 struct mlx5_priv *priv = dev->data->dev_private; 4806 struct rte_flow *flow; 4807 uint32_t idx; 4808 int ret = 0; 4809 4810 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 4811 flow, next) { 4812 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4813 dev->data->port_id, (void *)flow); 4814 ++ret; 4815 } 4816 return ret; 4817 } 4818 4819 /** 4820 * Enable default hairpin egress flow. 4821 * 4822 * @param dev 4823 * Pointer to Ethernet device. 4824 * @param queue 4825 * The queue index. 4826 * 4827 * @return 4828 * 0 on success, a negative errno value otherwise and rte_errno is set. 4829 */ 4830 int 4831 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4832 uint32_t queue) 4833 { 4834 struct mlx5_priv *priv = dev->data->dev_private; 4835 const struct rte_flow_attr attr = { 4836 .egress = 1, 4837 .priority = 0, 4838 }; 4839 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4840 .queue = queue, 4841 }; 4842 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4843 .queue = UINT32_MAX, 4844 }; 4845 struct rte_flow_item items[] = { 4846 { 4847 .type = (enum rte_flow_item_type) 4848 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4849 .spec = &queue_spec, 4850 .last = NULL, 4851 .mask = &queue_mask, 4852 }, 4853 { 4854 .type = RTE_FLOW_ITEM_TYPE_END, 4855 }, 4856 }; 4857 struct rte_flow_action_jump jump = { 4858 .group = MLX5_HAIRPIN_TX_TABLE, 4859 }; 4860 struct rte_flow_action actions[2]; 4861 uint32_t flow_idx; 4862 struct rte_flow_error error; 4863 4864 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4865 actions[0].conf = &jump; 4866 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4867 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4868 &attr, items, actions, false, &error); 4869 if (!flow_idx) { 4870 DRV_LOG(DEBUG, 4871 "Failed to create ctrl flow: rte_errno(%d)," 4872 " type(%d), message(%s)", 4873 rte_errno, error.type, 4874 error.message ? error.message : " (no stated reason)"); 4875 return -rte_errno; 4876 } 4877 return 0; 4878 } 4879 4880 /** 4881 * Enable a control flow configured from the control plane. 4882 * 4883 * @param dev 4884 * Pointer to Ethernet device. 4885 * @param eth_spec 4886 * An Ethernet flow spec to apply. 4887 * @param eth_mask 4888 * An Ethernet flow mask to apply. 4889 * @param vlan_spec 4890 * A VLAN flow spec to apply. 4891 * @param vlan_mask 4892 * A VLAN flow mask to apply. 4893 * 4894 * @return 4895 * 0 on success, a negative errno value otherwise and rte_errno is set. 4896 */ 4897 int 4898 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4899 struct rte_flow_item_eth *eth_spec, 4900 struct rte_flow_item_eth *eth_mask, 4901 struct rte_flow_item_vlan *vlan_spec, 4902 struct rte_flow_item_vlan *vlan_mask) 4903 { 4904 struct mlx5_priv *priv = dev->data->dev_private; 4905 const struct rte_flow_attr attr = { 4906 .ingress = 1, 4907 .priority = MLX5_FLOW_PRIO_RSVD, 4908 }; 4909 struct rte_flow_item items[] = { 4910 { 4911 .type = RTE_FLOW_ITEM_TYPE_ETH, 4912 .spec = eth_spec, 4913 .last = NULL, 4914 .mask = eth_mask, 4915 }, 4916 { 4917 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4918 RTE_FLOW_ITEM_TYPE_END, 4919 .spec = vlan_spec, 4920 .last = NULL, 4921 .mask = vlan_mask, 4922 }, 4923 { 4924 .type = RTE_FLOW_ITEM_TYPE_END, 4925 }, 4926 }; 4927 uint16_t queue[priv->reta_idx_n]; 4928 struct rte_flow_action_rss action_rss = { 4929 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4930 .level = 0, 4931 .types = priv->rss_conf.rss_hf, 4932 .key_len = priv->rss_conf.rss_key_len, 4933 .queue_num = priv->reta_idx_n, 4934 .key = priv->rss_conf.rss_key, 4935 .queue = queue, 4936 }; 4937 struct rte_flow_action actions[] = { 4938 { 4939 .type = RTE_FLOW_ACTION_TYPE_RSS, 4940 .conf = &action_rss, 4941 }, 4942 { 4943 .type = RTE_FLOW_ACTION_TYPE_END, 4944 }, 4945 }; 4946 uint32_t flow_idx; 4947 struct rte_flow_error error; 4948 unsigned int i; 4949 4950 if (!priv->reta_idx_n || !priv->rxqs_n) { 4951 return 0; 4952 } 4953 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 4954 action_rss.types = 0; 4955 for (i = 0; i != priv->reta_idx_n; ++i) 4956 queue[i] = (*priv->reta_idx)[i]; 4957 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4958 &attr, items, actions, false, &error); 4959 if (!flow_idx) 4960 return -rte_errno; 4961 return 0; 4962 } 4963 4964 /** 4965 * Enable a flow control configured from the control plane. 4966 * 4967 * @param dev 4968 * Pointer to Ethernet device. 4969 * @param eth_spec 4970 * An Ethernet flow spec to apply. 4971 * @param eth_mask 4972 * An Ethernet flow mask to apply. 4973 * 4974 * @return 4975 * 0 on success, a negative errno value otherwise and rte_errno is set. 4976 */ 4977 int 4978 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4979 struct rte_flow_item_eth *eth_spec, 4980 struct rte_flow_item_eth *eth_mask) 4981 { 4982 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4983 } 4984 4985 /** 4986 * Destroy a flow. 4987 * 4988 * @see rte_flow_destroy() 4989 * @see rte_flow_ops 4990 */ 4991 int 4992 mlx5_flow_destroy(struct rte_eth_dev *dev, 4993 struct rte_flow *flow, 4994 struct rte_flow_error *error __rte_unused) 4995 { 4996 struct mlx5_priv *priv = dev->data->dev_private; 4997 4998 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 4999 return 0; 5000 } 5001 5002 /** 5003 * Destroy all flows. 5004 * 5005 * @see rte_flow_flush() 5006 * @see rte_flow_ops 5007 */ 5008 int 5009 mlx5_flow_flush(struct rte_eth_dev *dev, 5010 struct rte_flow_error *error __rte_unused) 5011 { 5012 struct mlx5_priv *priv = dev->data->dev_private; 5013 5014 mlx5_flow_list_flush(dev, &priv->flows, false); 5015 return 0; 5016 } 5017 5018 /** 5019 * Isolated mode. 5020 * 5021 * @see rte_flow_isolate() 5022 * @see rte_flow_ops 5023 */ 5024 int 5025 mlx5_flow_isolate(struct rte_eth_dev *dev, 5026 int enable, 5027 struct rte_flow_error *error) 5028 { 5029 struct mlx5_priv *priv = dev->data->dev_private; 5030 5031 if (dev->data->dev_started) { 5032 rte_flow_error_set(error, EBUSY, 5033 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5034 NULL, 5035 "port must be stopped first"); 5036 return -rte_errno; 5037 } 5038 priv->isolated = !!enable; 5039 if (enable) 5040 dev->dev_ops = &mlx5_dev_ops_isolate; 5041 else 5042 dev->dev_ops = &mlx5_dev_ops; 5043 return 0; 5044 } 5045 5046 /** 5047 * Query a flow. 5048 * 5049 * @see rte_flow_query() 5050 * @see rte_flow_ops 5051 */ 5052 static int 5053 flow_drv_query(struct rte_eth_dev *dev, 5054 uint32_t flow_idx, 5055 const struct rte_flow_action *actions, 5056 void *data, 5057 struct rte_flow_error *error) 5058 { 5059 struct mlx5_priv *priv = dev->data->dev_private; 5060 const struct mlx5_flow_driver_ops *fops; 5061 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5062 [MLX5_IPOOL_RTE_FLOW], 5063 flow_idx); 5064 enum mlx5_flow_drv_type ftype; 5065 5066 if (!flow) { 5067 return rte_flow_error_set(error, ENOENT, 5068 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5069 NULL, 5070 "invalid flow handle"); 5071 } 5072 ftype = flow->drv_type; 5073 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 5074 fops = flow_get_drv_ops(ftype); 5075 5076 return fops->query(dev, flow, actions, data, error); 5077 } 5078 5079 /** 5080 * Query a flow. 5081 * 5082 * @see rte_flow_query() 5083 * @see rte_flow_ops 5084 */ 5085 int 5086 mlx5_flow_query(struct rte_eth_dev *dev, 5087 struct rte_flow *flow, 5088 const struct rte_flow_action *actions, 5089 void *data, 5090 struct rte_flow_error *error) 5091 { 5092 int ret; 5093 5094 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 5095 error); 5096 if (ret < 0) 5097 return ret; 5098 return 0; 5099 } 5100 5101 /** 5102 * Convert a flow director filter to a generic flow. 5103 * 5104 * @param dev 5105 * Pointer to Ethernet device. 5106 * @param fdir_filter 5107 * Flow director filter to add. 5108 * @param attributes 5109 * Generic flow parameters structure. 5110 * 5111 * @return 5112 * 0 on success, a negative errno value otherwise and rte_errno is set. 5113 */ 5114 static int 5115 flow_fdir_filter_convert(struct rte_eth_dev *dev, 5116 const struct rte_eth_fdir_filter *fdir_filter, 5117 struct mlx5_fdir *attributes) 5118 { 5119 struct mlx5_priv *priv = dev->data->dev_private; 5120 const struct rte_eth_fdir_input *input = &fdir_filter->input; 5121 const struct rte_eth_fdir_masks *mask = 5122 &dev->data->dev_conf.fdir_conf.mask; 5123 5124 /* Validate queue number. */ 5125 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 5126 DRV_LOG(ERR, "port %u invalid queue number %d", 5127 dev->data->port_id, fdir_filter->action.rx_queue); 5128 rte_errno = EINVAL; 5129 return -rte_errno; 5130 } 5131 attributes->attr.ingress = 1; 5132 attributes->items[0] = (struct rte_flow_item) { 5133 .type = RTE_FLOW_ITEM_TYPE_ETH, 5134 .spec = &attributes->l2, 5135 .mask = &attributes->l2_mask, 5136 }; 5137 switch (fdir_filter->action.behavior) { 5138 case RTE_ETH_FDIR_ACCEPT: 5139 attributes->actions[0] = (struct rte_flow_action){ 5140 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 5141 .conf = &attributes->queue, 5142 }; 5143 break; 5144 case RTE_ETH_FDIR_REJECT: 5145 attributes->actions[0] = (struct rte_flow_action){ 5146 .type = RTE_FLOW_ACTION_TYPE_DROP, 5147 }; 5148 break; 5149 default: 5150 DRV_LOG(ERR, "port %u invalid behavior %d", 5151 dev->data->port_id, 5152 fdir_filter->action.behavior); 5153 rte_errno = ENOTSUP; 5154 return -rte_errno; 5155 } 5156 attributes->queue.index = fdir_filter->action.rx_queue; 5157 /* Handle L3. */ 5158 switch (fdir_filter->input.flow_type) { 5159 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5160 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5161 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5162 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 5163 .src_addr = input->flow.ip4_flow.src_ip, 5164 .dst_addr = input->flow.ip4_flow.dst_ip, 5165 .time_to_live = input->flow.ip4_flow.ttl, 5166 .type_of_service = input->flow.ip4_flow.tos, 5167 }; 5168 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 5169 .src_addr = mask->ipv4_mask.src_ip, 5170 .dst_addr = mask->ipv4_mask.dst_ip, 5171 .time_to_live = mask->ipv4_mask.ttl, 5172 .type_of_service = mask->ipv4_mask.tos, 5173 .next_proto_id = mask->ipv4_mask.proto, 5174 }; 5175 attributes->items[1] = (struct rte_flow_item){ 5176 .type = RTE_FLOW_ITEM_TYPE_IPV4, 5177 .spec = &attributes->l3, 5178 .mask = &attributes->l3_mask, 5179 }; 5180 break; 5181 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5182 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5183 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5184 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 5185 .hop_limits = input->flow.ipv6_flow.hop_limits, 5186 .proto = input->flow.ipv6_flow.proto, 5187 }; 5188 5189 memcpy(attributes->l3.ipv6.hdr.src_addr, 5190 input->flow.ipv6_flow.src_ip, 5191 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5192 memcpy(attributes->l3.ipv6.hdr.dst_addr, 5193 input->flow.ipv6_flow.dst_ip, 5194 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5195 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 5196 mask->ipv6_mask.src_ip, 5197 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5198 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 5199 mask->ipv6_mask.dst_ip, 5200 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5201 attributes->items[1] = (struct rte_flow_item){ 5202 .type = RTE_FLOW_ITEM_TYPE_IPV6, 5203 .spec = &attributes->l3, 5204 .mask = &attributes->l3_mask, 5205 }; 5206 break; 5207 default: 5208 DRV_LOG(ERR, "port %u invalid flow type%d", 5209 dev->data->port_id, fdir_filter->input.flow_type); 5210 rte_errno = ENOTSUP; 5211 return -rte_errno; 5212 } 5213 /* Handle L4. */ 5214 switch (fdir_filter->input.flow_type) { 5215 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5216 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5217 .src_port = input->flow.udp4_flow.src_port, 5218 .dst_port = input->flow.udp4_flow.dst_port, 5219 }; 5220 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5221 .src_port = mask->src_port_mask, 5222 .dst_port = mask->dst_port_mask, 5223 }; 5224 attributes->items[2] = (struct rte_flow_item){ 5225 .type = RTE_FLOW_ITEM_TYPE_UDP, 5226 .spec = &attributes->l4, 5227 .mask = &attributes->l4_mask, 5228 }; 5229 break; 5230 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5231 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5232 .src_port = input->flow.tcp4_flow.src_port, 5233 .dst_port = input->flow.tcp4_flow.dst_port, 5234 }; 5235 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5236 .src_port = mask->src_port_mask, 5237 .dst_port = mask->dst_port_mask, 5238 }; 5239 attributes->items[2] = (struct rte_flow_item){ 5240 .type = RTE_FLOW_ITEM_TYPE_TCP, 5241 .spec = &attributes->l4, 5242 .mask = &attributes->l4_mask, 5243 }; 5244 break; 5245 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5246 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5247 .src_port = input->flow.udp6_flow.src_port, 5248 .dst_port = input->flow.udp6_flow.dst_port, 5249 }; 5250 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5251 .src_port = mask->src_port_mask, 5252 .dst_port = mask->dst_port_mask, 5253 }; 5254 attributes->items[2] = (struct rte_flow_item){ 5255 .type = RTE_FLOW_ITEM_TYPE_UDP, 5256 .spec = &attributes->l4, 5257 .mask = &attributes->l4_mask, 5258 }; 5259 break; 5260 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5261 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5262 .src_port = input->flow.tcp6_flow.src_port, 5263 .dst_port = input->flow.tcp6_flow.dst_port, 5264 }; 5265 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5266 .src_port = mask->src_port_mask, 5267 .dst_port = mask->dst_port_mask, 5268 }; 5269 attributes->items[2] = (struct rte_flow_item){ 5270 .type = RTE_FLOW_ITEM_TYPE_TCP, 5271 .spec = &attributes->l4, 5272 .mask = &attributes->l4_mask, 5273 }; 5274 break; 5275 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5276 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5277 break; 5278 default: 5279 DRV_LOG(ERR, "port %u invalid flow type%d", 5280 dev->data->port_id, fdir_filter->input.flow_type); 5281 rte_errno = ENOTSUP; 5282 return -rte_errno; 5283 } 5284 return 0; 5285 } 5286 5287 #define FLOW_FDIR_CMP(f1, f2, fld) \ 5288 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 5289 5290 /** 5291 * Compare two FDIR flows. If items and actions are identical, the two flows are 5292 * regarded as same. 5293 * 5294 * @param dev 5295 * Pointer to Ethernet device. 5296 * @param f1 5297 * FDIR flow to compare. 5298 * @param f2 5299 * FDIR flow to compare. 5300 * 5301 * @return 5302 * Zero on match, 1 otherwise. 5303 */ 5304 static int 5305 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 5306 { 5307 if (FLOW_FDIR_CMP(f1, f2, attr) || 5308 FLOW_FDIR_CMP(f1, f2, l2) || 5309 FLOW_FDIR_CMP(f1, f2, l2_mask) || 5310 FLOW_FDIR_CMP(f1, f2, l3) || 5311 FLOW_FDIR_CMP(f1, f2, l3_mask) || 5312 FLOW_FDIR_CMP(f1, f2, l4) || 5313 FLOW_FDIR_CMP(f1, f2, l4_mask) || 5314 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 5315 return 1; 5316 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 5317 FLOW_FDIR_CMP(f1, f2, queue)) 5318 return 1; 5319 return 0; 5320 } 5321 5322 /** 5323 * Search device flow list to find out a matched FDIR flow. 5324 * 5325 * @param dev 5326 * Pointer to Ethernet device. 5327 * @param fdir_flow 5328 * FDIR flow to lookup. 5329 * 5330 * @return 5331 * Index of flow if found, 0 otherwise. 5332 */ 5333 static uint32_t 5334 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5335 { 5336 struct mlx5_priv *priv = dev->data->dev_private; 5337 uint32_t flow_idx = 0; 5338 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5339 5340 MLX5_ASSERT(fdir_flow); 5341 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5342 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { 5343 DRV_LOG(DEBUG, "port %u found FDIR flow %u", 5344 dev->data->port_id, flow_idx); 5345 flow_idx = priv_fdir_flow->rix_flow; 5346 break; 5347 } 5348 } 5349 return flow_idx; 5350 } 5351 5352 /** 5353 * Add new flow director filter and store it in list. 5354 * 5355 * @param dev 5356 * Pointer to Ethernet device. 5357 * @param fdir_filter 5358 * Flow director filter to add. 5359 * 5360 * @return 5361 * 0 on success, a negative errno value otherwise and rte_errno is set. 5362 */ 5363 static int 5364 flow_fdir_filter_add(struct rte_eth_dev *dev, 5365 const struct rte_eth_fdir_filter *fdir_filter) 5366 { 5367 struct mlx5_priv *priv = dev->data->dev_private; 5368 struct mlx5_fdir *fdir_flow; 5369 struct rte_flow *flow; 5370 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5371 uint32_t flow_idx; 5372 int ret; 5373 5374 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5375 if (!fdir_flow) { 5376 rte_errno = ENOMEM; 5377 return -rte_errno; 5378 } 5379 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5380 if (ret) 5381 goto error; 5382 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); 5383 if (flow_idx) { 5384 rte_errno = EEXIST; 5385 goto error; 5386 } 5387 priv_fdir_flow = rte_zmalloc(__func__, sizeof(struct mlx5_fdir_flow), 5388 0); 5389 if (!priv_fdir_flow) { 5390 rte_errno = ENOMEM; 5391 goto error; 5392 } 5393 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5394 fdir_flow->items, fdir_flow->actions, true, 5395 NULL); 5396 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5397 if (!flow) 5398 goto error; 5399 flow->fdir = 1; 5400 priv_fdir_flow->fdir = fdir_flow; 5401 priv_fdir_flow->rix_flow = flow_idx; 5402 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); 5403 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5404 dev->data->port_id, (void *)flow); 5405 return 0; 5406 error: 5407 rte_free(priv_fdir_flow); 5408 rte_free(fdir_flow); 5409 return -rte_errno; 5410 } 5411 5412 /** 5413 * Delete specific filter. 5414 * 5415 * @param dev 5416 * Pointer to Ethernet device. 5417 * @param fdir_filter 5418 * Filter to be deleted. 5419 * 5420 * @return 5421 * 0 on success, a negative errno value otherwise and rte_errno is set. 5422 */ 5423 static int 5424 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5425 const struct rte_eth_fdir_filter *fdir_filter) 5426 { 5427 struct mlx5_priv *priv = dev->data->dev_private; 5428 uint32_t flow_idx; 5429 struct mlx5_fdir fdir_flow = { 5430 .attr.group = 0, 5431 }; 5432 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5433 int ret; 5434 5435 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5436 if (ret) 5437 return -rte_errno; 5438 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5439 /* Find the fdir in priv list */ 5440 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) 5441 break; 5442 } 5443 if (!priv_fdir_flow) 5444 return 0; 5445 LIST_REMOVE(priv_fdir_flow, next); 5446 flow_idx = priv_fdir_flow->rix_flow; 5447 flow_list_destroy(dev, &priv->flows, flow_idx); 5448 rte_free(priv_fdir_flow->fdir); 5449 rte_free(priv_fdir_flow); 5450 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", 5451 dev->data->port_id, flow_idx); 5452 return 0; 5453 } 5454 5455 /** 5456 * Update queue for specific filter. 5457 * 5458 * @param dev 5459 * Pointer to Ethernet device. 5460 * @param fdir_filter 5461 * Filter to be updated. 5462 * 5463 * @return 5464 * 0 on success, a negative errno value otherwise and rte_errno is set. 5465 */ 5466 static int 5467 flow_fdir_filter_update(struct rte_eth_dev *dev, 5468 const struct rte_eth_fdir_filter *fdir_filter) 5469 { 5470 int ret; 5471 5472 ret = flow_fdir_filter_delete(dev, fdir_filter); 5473 if (ret) 5474 return ret; 5475 return flow_fdir_filter_add(dev, fdir_filter); 5476 } 5477 5478 /** 5479 * Flush all filters. 5480 * 5481 * @param dev 5482 * Pointer to Ethernet device. 5483 */ 5484 static void 5485 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5486 { 5487 struct mlx5_priv *priv = dev->data->dev_private; 5488 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5489 5490 while (!LIST_EMPTY(&priv->fdir_flows)) { 5491 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); 5492 LIST_REMOVE(priv_fdir_flow, next); 5493 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); 5494 rte_free(priv_fdir_flow->fdir); 5495 rte_free(priv_fdir_flow); 5496 } 5497 } 5498 5499 /** 5500 * Get flow director information. 5501 * 5502 * @param dev 5503 * Pointer to Ethernet device. 5504 * @param[out] fdir_info 5505 * Resulting flow director information. 5506 */ 5507 static void 5508 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5509 { 5510 struct rte_eth_fdir_masks *mask = 5511 &dev->data->dev_conf.fdir_conf.mask; 5512 5513 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5514 fdir_info->guarant_spc = 0; 5515 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5516 fdir_info->max_flexpayload = 0; 5517 fdir_info->flow_types_mask[0] = 0; 5518 fdir_info->flex_payload_unit = 0; 5519 fdir_info->max_flex_payload_segment_num = 0; 5520 fdir_info->flex_payload_limit = 0; 5521 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5522 } 5523 5524 /** 5525 * Deal with flow director operations. 5526 * 5527 * @param dev 5528 * Pointer to Ethernet device. 5529 * @param filter_op 5530 * Operation to perform. 5531 * @param arg 5532 * Pointer to operation-specific structure. 5533 * 5534 * @return 5535 * 0 on success, a negative errno value otherwise and rte_errno is set. 5536 */ 5537 static int 5538 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5539 void *arg) 5540 { 5541 enum rte_fdir_mode fdir_mode = 5542 dev->data->dev_conf.fdir_conf.mode; 5543 5544 if (filter_op == RTE_ETH_FILTER_NOP) 5545 return 0; 5546 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5547 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5548 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5549 dev->data->port_id, fdir_mode); 5550 rte_errno = EINVAL; 5551 return -rte_errno; 5552 } 5553 switch (filter_op) { 5554 case RTE_ETH_FILTER_ADD: 5555 return flow_fdir_filter_add(dev, arg); 5556 case RTE_ETH_FILTER_UPDATE: 5557 return flow_fdir_filter_update(dev, arg); 5558 case RTE_ETH_FILTER_DELETE: 5559 return flow_fdir_filter_delete(dev, arg); 5560 case RTE_ETH_FILTER_FLUSH: 5561 flow_fdir_filter_flush(dev); 5562 break; 5563 case RTE_ETH_FILTER_INFO: 5564 flow_fdir_info_get(dev, arg); 5565 break; 5566 default: 5567 DRV_LOG(DEBUG, "port %u unknown operation %u", 5568 dev->data->port_id, filter_op); 5569 rte_errno = EINVAL; 5570 return -rte_errno; 5571 } 5572 return 0; 5573 } 5574 5575 /** 5576 * Manage filter operations. 5577 * 5578 * @param dev 5579 * Pointer to Ethernet device structure. 5580 * @param filter_type 5581 * Filter type. 5582 * @param filter_op 5583 * Operation to perform. 5584 * @param arg 5585 * Pointer to operation-specific structure. 5586 * 5587 * @return 5588 * 0 on success, a negative errno value otherwise and rte_errno is set. 5589 */ 5590 int 5591 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5592 enum rte_filter_type filter_type, 5593 enum rte_filter_op filter_op, 5594 void *arg) 5595 { 5596 switch (filter_type) { 5597 case RTE_ETH_FILTER_GENERIC: 5598 if (filter_op != RTE_ETH_FILTER_GET) { 5599 rte_errno = EINVAL; 5600 return -rte_errno; 5601 } 5602 *(const void **)arg = &mlx5_flow_ops; 5603 return 0; 5604 case RTE_ETH_FILTER_FDIR: 5605 return flow_fdir_ctrl_func(dev, filter_op, arg); 5606 default: 5607 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5608 dev->data->port_id, filter_type); 5609 rte_errno = ENOTSUP; 5610 return -rte_errno; 5611 } 5612 return 0; 5613 } 5614 5615 /** 5616 * Create the needed meter and suffix tables. 5617 * 5618 * @param[in] dev 5619 * Pointer to Ethernet device. 5620 * @param[in] fm 5621 * Pointer to the flow meter. 5622 * 5623 * @return 5624 * Pointer to table set on success, NULL otherwise. 5625 */ 5626 struct mlx5_meter_domains_infos * 5627 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5628 const struct mlx5_flow_meter *fm) 5629 { 5630 const struct mlx5_flow_driver_ops *fops; 5631 5632 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5633 return fops->create_mtr_tbls(dev, fm); 5634 } 5635 5636 /** 5637 * Destroy the meter table set. 5638 * 5639 * @param[in] dev 5640 * Pointer to Ethernet device. 5641 * @param[in] tbl 5642 * Pointer to the meter table set. 5643 * 5644 * @return 5645 * 0 on success. 5646 */ 5647 int 5648 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5649 struct mlx5_meter_domains_infos *tbls) 5650 { 5651 const struct mlx5_flow_driver_ops *fops; 5652 5653 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5654 return fops->destroy_mtr_tbls(dev, tbls); 5655 } 5656 5657 /** 5658 * Create policer rules. 5659 * 5660 * @param[in] dev 5661 * Pointer to Ethernet device. 5662 * @param[in] fm 5663 * Pointer to flow meter structure. 5664 * @param[in] attr 5665 * Pointer to flow attributes. 5666 * 5667 * @return 5668 * 0 on success, -1 otherwise. 5669 */ 5670 int 5671 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5672 struct mlx5_flow_meter *fm, 5673 const struct rte_flow_attr *attr) 5674 { 5675 const struct mlx5_flow_driver_ops *fops; 5676 5677 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5678 return fops->create_policer_rules(dev, fm, attr); 5679 } 5680 5681 /** 5682 * Destroy policer rules. 5683 * 5684 * @param[in] fm 5685 * Pointer to flow meter structure. 5686 * @param[in] attr 5687 * Pointer to flow attributes. 5688 * 5689 * @return 5690 * 0 on success, -1 otherwise. 5691 */ 5692 int 5693 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5694 struct mlx5_flow_meter *fm, 5695 const struct rte_flow_attr *attr) 5696 { 5697 const struct mlx5_flow_driver_ops *fops; 5698 5699 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5700 return fops->destroy_policer_rules(dev, fm, attr); 5701 } 5702 5703 /** 5704 * Allocate a counter. 5705 * 5706 * @param[in] dev 5707 * Pointer to Ethernet device structure. 5708 * 5709 * @return 5710 * Index to allocated counter on success, 0 otherwise. 5711 */ 5712 uint32_t 5713 mlx5_counter_alloc(struct rte_eth_dev *dev) 5714 { 5715 const struct mlx5_flow_driver_ops *fops; 5716 struct rte_flow_attr attr = { .transfer = 0 }; 5717 5718 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5719 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5720 return fops->counter_alloc(dev); 5721 } 5722 DRV_LOG(ERR, 5723 "port %u counter allocate is not supported.", 5724 dev->data->port_id); 5725 return 0; 5726 } 5727 5728 /** 5729 * Free a counter. 5730 * 5731 * @param[in] dev 5732 * Pointer to Ethernet device structure. 5733 * @param[in] cnt 5734 * Index to counter to be free. 5735 */ 5736 void 5737 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 5738 { 5739 const struct mlx5_flow_driver_ops *fops; 5740 struct rte_flow_attr attr = { .transfer = 0 }; 5741 5742 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5743 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5744 fops->counter_free(dev, cnt); 5745 return; 5746 } 5747 DRV_LOG(ERR, 5748 "port %u counter free is not supported.", 5749 dev->data->port_id); 5750 } 5751 5752 /** 5753 * Query counter statistics. 5754 * 5755 * @param[in] dev 5756 * Pointer to Ethernet device structure. 5757 * @param[in] cnt 5758 * Index to counter to query. 5759 * @param[in] clear 5760 * Set to clear counter statistics. 5761 * @param[out] pkts 5762 * The counter hits packets number to save. 5763 * @param[out] bytes 5764 * The counter hits bytes number to save. 5765 * 5766 * @return 5767 * 0 on success, a negative errno value otherwise. 5768 */ 5769 int 5770 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 5771 bool clear, uint64_t *pkts, uint64_t *bytes) 5772 { 5773 const struct mlx5_flow_driver_ops *fops; 5774 struct rte_flow_attr attr = { .transfer = 0 }; 5775 5776 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5777 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5778 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5779 } 5780 DRV_LOG(ERR, 5781 "port %u counter query is not supported.", 5782 dev->data->port_id); 5783 return -ENOTSUP; 5784 } 5785 5786 #define MLX5_POOL_QUERY_FREQ_US 1000000 5787 5788 /** 5789 * Get number of all validate pools. 5790 * 5791 * @param[in] sh 5792 * Pointer to mlx5_ibv_shared object. 5793 * 5794 * @return 5795 * The number of all validate pools. 5796 */ 5797 static uint32_t 5798 mlx5_get_all_valid_pool_count(struct mlx5_ibv_shared *sh) 5799 { 5800 uint8_t age, i; 5801 uint32_t pools_n = 0; 5802 struct mlx5_pools_container *cont; 5803 5804 for (age = 0; age < RTE_DIM(sh->cmng.ccont[0]); ++age) { 5805 for (i = 0; i < 2 ; ++i) { 5806 cont = MLX5_CNT_CONTAINER(sh, i, 0, age); 5807 pools_n += rte_atomic16_read(&cont->n_valid); 5808 } 5809 } 5810 return pools_n; 5811 } 5812 5813 /** 5814 * Set the periodic procedure for triggering asynchronous batch queries for all 5815 * the counter pools. 5816 * 5817 * @param[in] sh 5818 * Pointer to mlx5_ibv_shared object. 5819 */ 5820 void 5821 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5822 { 5823 uint32_t pools_n, us; 5824 5825 pools_n = mlx5_get_all_valid_pool_count(sh); 5826 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5827 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5828 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5829 sh->cmng.query_thread_on = 0; 5830 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5831 } else { 5832 sh->cmng.query_thread_on = 1; 5833 } 5834 } 5835 5836 /** 5837 * The periodic procedure for triggering asynchronous batch queries for all the 5838 * counter pools. This function is probably called by the host thread. 5839 * 5840 * @param[in] arg 5841 * The parameter for the alarm process. 5842 */ 5843 void 5844 mlx5_flow_query_alarm(void *arg) 5845 { 5846 struct mlx5_ibv_shared *sh = arg; 5847 struct mlx5_devx_obj *dcs; 5848 uint16_t offset; 5849 int ret; 5850 uint8_t batch = sh->cmng.batch; 5851 uint8_t age = sh->cmng.age; 5852 uint16_t pool_index = sh->cmng.pool_index; 5853 struct mlx5_pools_container *cont; 5854 struct mlx5_pools_container *mcont; 5855 struct mlx5_flow_counter_pool *pool; 5856 5857 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5858 goto set_alarm; 5859 next_container: 5860 cont = MLX5_CNT_CONTAINER(sh, batch, 1, age); 5861 mcont = MLX5_CNT_CONTAINER(sh, batch, 0, age); 5862 /* Check if resize was done and need to flip a container. */ 5863 if (cont != mcont) { 5864 if (cont->pools) { 5865 /* Clean the old container. */ 5866 rte_free(cont->pools); 5867 memset(cont, 0, sizeof(*cont)); 5868 } 5869 rte_cio_wmb(); 5870 /* Flip the host container. */ 5871 sh->cmng.mhi[batch][age] ^= (uint8_t)2; 5872 cont = mcont; 5873 } 5874 if (!cont->pools) { 5875 /* 2 empty containers case is unexpected. */ 5876 if (unlikely(batch != sh->cmng.batch) && 5877 unlikely(age != sh->cmng.age)) { 5878 goto set_alarm; 5879 } 5880 batch ^= 0x1; 5881 pool_index = 0; 5882 if (batch == 0 && pool_index == 0) { 5883 age ^= 0x1; 5884 sh->cmng.batch = batch; 5885 sh->cmng.age = age; 5886 } 5887 goto next_container; 5888 } 5889 pool = cont->pools[pool_index]; 5890 if (pool->raw_hw) 5891 /* There is a pool query in progress. */ 5892 goto set_alarm; 5893 pool->raw_hw = 5894 LIST_FIRST(&sh->cmng.free_stat_raws); 5895 if (!pool->raw_hw) 5896 /* No free counter statistics raw memory. */ 5897 goto set_alarm; 5898 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5899 (&pool->a64_dcs); 5900 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5901 /* 5902 * Identify the counters released between query trigger and query 5903 * handle more effiecntly. The counter released in this gap period 5904 * should wait for a new round of query as the new arrived packets 5905 * will not be taken into account. 5906 */ 5907 rte_atomic64_add(&pool->start_query_gen, 1); 5908 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5909 offset, NULL, NULL, 5910 pool->raw_hw->mem_mng->dm->id, 5911 (void *)(uintptr_t) 5912 (pool->raw_hw->data + offset), 5913 sh->devx_comp, 5914 (uint64_t)(uintptr_t)pool); 5915 if (ret) { 5916 rte_atomic64_sub(&pool->start_query_gen, 1); 5917 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5918 " %d", pool->min_dcs->id); 5919 pool->raw_hw = NULL; 5920 goto set_alarm; 5921 } 5922 pool->raw_hw->min_dcs_id = dcs->id; 5923 LIST_REMOVE(pool->raw_hw, next); 5924 sh->cmng.pending_queries++; 5925 pool_index++; 5926 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5927 batch ^= 0x1; 5928 pool_index = 0; 5929 if (batch == 0 && pool_index == 0) 5930 age ^= 0x1; 5931 } 5932 set_alarm: 5933 sh->cmng.batch = batch; 5934 sh->cmng.pool_index = pool_index; 5935 sh->cmng.age = age; 5936 mlx5_set_query_alarm(sh); 5937 } 5938 5939 /** 5940 * Check and callback event for new aged flow in the counter pool 5941 * 5942 * @param[in] sh 5943 * Pointer to mlx5_ibv_shared object. 5944 * @param[in] pool 5945 * Pointer to Current counter pool. 5946 */ 5947 static void 5948 mlx5_flow_aging_check(struct mlx5_ibv_shared *sh, 5949 struct mlx5_flow_counter_pool *pool) 5950 { 5951 struct mlx5_priv *priv; 5952 struct mlx5_flow_counter *cnt; 5953 struct mlx5_age_info *age_info; 5954 struct mlx5_age_param *age_param; 5955 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 5956 struct mlx5_counter_stats_raw *prev = pool->raw; 5957 uint16_t curr = rte_rdtsc() / (rte_get_tsc_hz() / 10); 5958 uint32_t i; 5959 5960 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 5961 cnt = MLX5_POOL_GET_CNT(pool, i); 5962 age_param = MLX5_CNT_TO_AGE(cnt); 5963 if (rte_atomic16_read(&age_param->state) != AGE_CANDIDATE) 5964 continue; 5965 if (cur->data[i].hits != prev->data[i].hits) { 5966 age_param->expire = curr + age_param->timeout; 5967 continue; 5968 } 5969 if ((uint16_t)(curr - age_param->expire) >= (UINT16_MAX / 2)) 5970 continue; 5971 /** 5972 * Hold the lock first, or if between the 5973 * state AGE_TMOUT and tailq operation the 5974 * release happened, the release procedure 5975 * may delete a non-existent tailq node. 5976 */ 5977 priv = rte_eth_devices[age_param->port_id].data->dev_private; 5978 age_info = GET_PORT_AGE_INFO(priv); 5979 rte_spinlock_lock(&age_info->aged_sl); 5980 /* If the cpmset fails, release happens. */ 5981 if (rte_atomic16_cmpset((volatile uint16_t *) 5982 &age_param->state, 5983 AGE_CANDIDATE, 5984 AGE_TMOUT) == 5985 AGE_CANDIDATE) { 5986 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 5987 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 5988 } 5989 rte_spinlock_unlock(&age_info->aged_sl); 5990 } 5991 for (i = 0; i < sh->max_port; i++) { 5992 age_info = &sh->port[i].age_info; 5993 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 5994 continue; 5995 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) 5996 _rte_eth_dev_callback_process 5997 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 5998 RTE_ETH_EVENT_FLOW_AGED, NULL); 5999 age_info->flags = 0; 6000 } 6001 } 6002 6003 /** 6004 * Handler for the HW respond about ready values from an asynchronous batch 6005 * query. This function is probably called by the host thread. 6006 * 6007 * @param[in] sh 6008 * The pointer to the shared IB device context. 6009 * @param[in] async_id 6010 * The Devx async ID. 6011 * @param[in] status 6012 * The status of the completion. 6013 */ 6014 void 6015 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 6016 uint64_t async_id, int status) 6017 { 6018 struct mlx5_flow_counter_pool *pool = 6019 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 6020 struct mlx5_counter_stats_raw *raw_to_free; 6021 6022 if (unlikely(status)) { 6023 rte_atomic64_sub(&pool->start_query_gen, 1); 6024 raw_to_free = pool->raw_hw; 6025 } else { 6026 raw_to_free = pool->raw; 6027 if (IS_AGE_POOL(pool)) 6028 mlx5_flow_aging_check(sh, pool); 6029 rte_spinlock_lock(&pool->sl); 6030 pool->raw = pool->raw_hw; 6031 rte_spinlock_unlock(&pool->sl); 6032 MLX5_ASSERT(rte_atomic64_read(&pool->end_query_gen) + 1 == 6033 rte_atomic64_read(&pool->start_query_gen)); 6034 rte_atomic64_set(&pool->end_query_gen, 6035 rte_atomic64_read(&pool->start_query_gen)); 6036 /* Be sure the new raw counters data is updated in memory. */ 6037 rte_cio_wmb(); 6038 } 6039 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 6040 pool->raw_hw = NULL; 6041 sh->cmng.pending_queries--; 6042 } 6043 6044 /** 6045 * Translate the rte_flow group index to HW table value. 6046 * 6047 * @param[in] attributes 6048 * Pointer to flow attributes 6049 * @param[in] external 6050 * Value is part of flow rule created by request external to PMD. 6051 * @param[in] group 6052 * rte_flow group index value. 6053 * @param[out] fdb_def_rule 6054 * Whether fdb jump to table 1 is configured. 6055 * @param[out] table 6056 * HW table value. 6057 * @param[out] error 6058 * Pointer to error structure. 6059 * 6060 * @return 6061 * 0 on success, a negative errno value otherwise and rte_errno is set. 6062 */ 6063 int 6064 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 6065 uint32_t group, bool fdb_def_rule, uint32_t *table, 6066 struct rte_flow_error *error) 6067 { 6068 if (attributes->transfer && external && fdb_def_rule) { 6069 if (group == UINT32_MAX) 6070 return rte_flow_error_set 6071 (error, EINVAL, 6072 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6073 NULL, 6074 "group index not supported"); 6075 *table = group + 1; 6076 } else { 6077 *table = group; 6078 } 6079 return 0; 6080 } 6081 6082 /** 6083 * Discover availability of metadata reg_c's. 6084 * 6085 * Iteratively use test flows to check availability. 6086 * 6087 * @param[in] dev 6088 * Pointer to the Ethernet device structure. 6089 * 6090 * @return 6091 * 0 on success, a negative errno value otherwise and rte_errno is set. 6092 */ 6093 int 6094 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 6095 { 6096 struct mlx5_priv *priv = dev->data->dev_private; 6097 struct mlx5_dev_config *config = &priv->config; 6098 enum modify_reg idx; 6099 int n = 0; 6100 6101 /* reg_c[0] and reg_c[1] are reserved. */ 6102 config->flow_mreg_c[n++] = REG_C_0; 6103 config->flow_mreg_c[n++] = REG_C_1; 6104 /* Discover availability of other reg_c's. */ 6105 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 6106 struct rte_flow_attr attr = { 6107 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6108 .priority = MLX5_FLOW_PRIO_RSVD, 6109 .ingress = 1, 6110 }; 6111 struct rte_flow_item items[] = { 6112 [0] = { 6113 .type = RTE_FLOW_ITEM_TYPE_END, 6114 }, 6115 }; 6116 struct rte_flow_action actions[] = { 6117 [0] = { 6118 .type = (enum rte_flow_action_type) 6119 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6120 .conf = &(struct mlx5_flow_action_copy_mreg){ 6121 .src = REG_C_1, 6122 .dst = idx, 6123 }, 6124 }, 6125 [1] = { 6126 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6127 .conf = &(struct rte_flow_action_jump){ 6128 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6129 }, 6130 }, 6131 [2] = { 6132 .type = RTE_FLOW_ACTION_TYPE_END, 6133 }, 6134 }; 6135 uint32_t flow_idx; 6136 struct rte_flow *flow; 6137 struct rte_flow_error error; 6138 6139 if (!config->dv_flow_en) 6140 break; 6141 /* Create internal flow, validation skips copy action. */ 6142 flow_idx = flow_list_create(dev, NULL, &attr, items, 6143 actions, false, &error); 6144 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 6145 flow_idx); 6146 if (!flow) 6147 continue; 6148 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 6149 config->flow_mreg_c[n++] = idx; 6150 flow_list_destroy(dev, NULL, flow_idx); 6151 } 6152 for (; n < MLX5_MREG_C_NUM; ++n) 6153 config->flow_mreg_c[n] = REG_NONE; 6154 return 0; 6155 } 6156 6157 /** 6158 * Dump flow raw hw data to file 6159 * 6160 * @param[in] dev 6161 * The pointer to Ethernet device. 6162 * @param[in] file 6163 * A pointer to a file for output. 6164 * @param[out] error 6165 * Perform verbose error reporting if not NULL. PMDs initialize this 6166 * structure in case of error only. 6167 * @return 6168 * 0 on success, a nagative value otherwise. 6169 */ 6170 int 6171 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 6172 FILE *file, 6173 struct rte_flow_error *error __rte_unused) 6174 { 6175 struct mlx5_priv *priv = dev->data->dev_private; 6176 struct mlx5_ibv_shared *sh = priv->sh; 6177 6178 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 6179 sh->tx_domain, file); 6180 } 6181 6182 /** 6183 * Get aged-out flows. 6184 * 6185 * @param[in] dev 6186 * Pointer to the Ethernet device structure. 6187 * @param[in] context 6188 * The address of an array of pointers to the aged-out flows contexts. 6189 * @param[in] nb_countexts 6190 * The length of context array pointers. 6191 * @param[out] error 6192 * Perform verbose error reporting if not NULL. Initialized in case of 6193 * error only. 6194 * 6195 * @return 6196 * how many contexts get in success, otherwise negative errno value. 6197 * if nb_contexts is 0, return the amount of all aged contexts. 6198 * if nb_contexts is not 0 , return the amount of aged flows reported 6199 * in the context array. 6200 */ 6201 int 6202 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 6203 uint32_t nb_contexts, struct rte_flow_error *error) 6204 { 6205 const struct mlx5_flow_driver_ops *fops; 6206 struct rte_flow_attr attr = { .transfer = 0 }; 6207 6208 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6209 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6210 return fops->get_aged_flows(dev, contexts, nb_contexts, 6211 error); 6212 } 6213 DRV_LOG(ERR, 6214 "port %u get aged flows is not supported.", 6215 dev->data->port_id); 6216 return -ENOTSUP; 6217 } 6218