1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 /* Verbs header. */ 14 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 15 #ifdef PEDANTIC 16 #pragma GCC diagnostic ignored "-Wpedantic" 17 #endif 18 #include <infiniband/verbs.h> 19 #ifdef PEDANTIC 20 #pragma GCC diagnostic error "-Wpedantic" 21 #endif 22 23 #include <rte_common.h> 24 #include <rte_ether.h> 25 #include <rte_ethdev_driver.h> 26 #include <rte_flow.h> 27 #include <rte_cycles.h> 28 #include <rte_flow_driver.h> 29 #include <rte_malloc.h> 30 #include <rte_ip.h> 31 32 #include <mlx5_glue.h> 33 #include <mlx5_devx_cmds.h> 34 #include <mlx5_prm.h> 35 36 #include "mlx5_defs.h" 37 #include "mlx5.h" 38 #include "mlx5_flow.h" 39 #include "mlx5_rxtx.h" 40 41 /** Device flow drivers. */ 42 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 43 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 44 #endif 45 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 46 47 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 48 49 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 50 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 51 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 52 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 53 #endif 54 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 55 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 56 }; 57 58 enum mlx5_expansion { 59 MLX5_EXPANSION_ROOT, 60 MLX5_EXPANSION_ROOT_OUTER, 61 MLX5_EXPANSION_ROOT_ETH_VLAN, 62 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 63 MLX5_EXPANSION_OUTER_ETH, 64 MLX5_EXPANSION_OUTER_ETH_VLAN, 65 MLX5_EXPANSION_OUTER_VLAN, 66 MLX5_EXPANSION_OUTER_IPV4, 67 MLX5_EXPANSION_OUTER_IPV4_UDP, 68 MLX5_EXPANSION_OUTER_IPV4_TCP, 69 MLX5_EXPANSION_OUTER_IPV6, 70 MLX5_EXPANSION_OUTER_IPV6_UDP, 71 MLX5_EXPANSION_OUTER_IPV6_TCP, 72 MLX5_EXPANSION_VXLAN, 73 MLX5_EXPANSION_VXLAN_GPE, 74 MLX5_EXPANSION_GRE, 75 MLX5_EXPANSION_MPLS, 76 MLX5_EXPANSION_ETH, 77 MLX5_EXPANSION_ETH_VLAN, 78 MLX5_EXPANSION_VLAN, 79 MLX5_EXPANSION_IPV4, 80 MLX5_EXPANSION_IPV4_UDP, 81 MLX5_EXPANSION_IPV4_TCP, 82 MLX5_EXPANSION_IPV6, 83 MLX5_EXPANSION_IPV6_UDP, 84 MLX5_EXPANSION_IPV6_TCP, 85 }; 86 87 /** Supported expansion of items. */ 88 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 89 [MLX5_EXPANSION_ROOT] = { 90 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 91 MLX5_EXPANSION_IPV4, 92 MLX5_EXPANSION_IPV6), 93 .type = RTE_FLOW_ITEM_TYPE_END, 94 }, 95 [MLX5_EXPANSION_ROOT_OUTER] = { 96 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 97 MLX5_EXPANSION_OUTER_IPV4, 98 MLX5_EXPANSION_OUTER_IPV6), 99 .type = RTE_FLOW_ITEM_TYPE_END, 100 }, 101 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 102 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 103 .type = RTE_FLOW_ITEM_TYPE_END, 104 }, 105 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 107 .type = RTE_FLOW_ITEM_TYPE_END, 108 }, 109 [MLX5_EXPANSION_OUTER_ETH] = { 110 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 111 MLX5_EXPANSION_OUTER_IPV6, 112 MLX5_EXPANSION_MPLS), 113 .type = RTE_FLOW_ITEM_TYPE_ETH, 114 .rss_types = 0, 115 }, 116 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 117 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 118 .type = RTE_FLOW_ITEM_TYPE_ETH, 119 .rss_types = 0, 120 }, 121 [MLX5_EXPANSION_OUTER_VLAN] = { 122 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 123 MLX5_EXPANSION_OUTER_IPV6), 124 .type = RTE_FLOW_ITEM_TYPE_VLAN, 125 }, 126 [MLX5_EXPANSION_OUTER_IPV4] = { 127 .next = RTE_FLOW_EXPAND_RSS_NEXT 128 (MLX5_EXPANSION_OUTER_IPV4_UDP, 129 MLX5_EXPANSION_OUTER_IPV4_TCP, 130 MLX5_EXPANSION_GRE, 131 MLX5_EXPANSION_IPV4, 132 MLX5_EXPANSION_IPV6), 133 .type = RTE_FLOW_ITEM_TYPE_IPV4, 134 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 135 ETH_RSS_NONFRAG_IPV4_OTHER, 136 }, 137 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 138 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 139 MLX5_EXPANSION_VXLAN_GPE), 140 .type = RTE_FLOW_ITEM_TYPE_UDP, 141 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 142 }, 143 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 144 .type = RTE_FLOW_ITEM_TYPE_TCP, 145 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 146 }, 147 [MLX5_EXPANSION_OUTER_IPV6] = { 148 .next = RTE_FLOW_EXPAND_RSS_NEXT 149 (MLX5_EXPANSION_OUTER_IPV6_UDP, 150 MLX5_EXPANSION_OUTER_IPV6_TCP, 151 MLX5_EXPANSION_IPV4, 152 MLX5_EXPANSION_IPV6), 153 .type = RTE_FLOW_ITEM_TYPE_IPV6, 154 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 155 ETH_RSS_NONFRAG_IPV6_OTHER, 156 }, 157 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 158 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 159 MLX5_EXPANSION_VXLAN_GPE), 160 .type = RTE_FLOW_ITEM_TYPE_UDP, 161 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 162 }, 163 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 164 .type = RTE_FLOW_ITEM_TYPE_TCP, 165 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 166 }, 167 [MLX5_EXPANSION_VXLAN] = { 168 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 169 MLX5_EXPANSION_IPV4, 170 MLX5_EXPANSION_IPV6), 171 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 172 }, 173 [MLX5_EXPANSION_VXLAN_GPE] = { 174 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 175 MLX5_EXPANSION_IPV4, 176 MLX5_EXPANSION_IPV6), 177 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 178 }, 179 [MLX5_EXPANSION_GRE] = { 180 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 181 .type = RTE_FLOW_ITEM_TYPE_GRE, 182 }, 183 [MLX5_EXPANSION_MPLS] = { 184 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 185 MLX5_EXPANSION_IPV6), 186 .type = RTE_FLOW_ITEM_TYPE_MPLS, 187 }, 188 [MLX5_EXPANSION_ETH] = { 189 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 190 MLX5_EXPANSION_IPV6), 191 .type = RTE_FLOW_ITEM_TYPE_ETH, 192 }, 193 [MLX5_EXPANSION_ETH_VLAN] = { 194 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 195 .type = RTE_FLOW_ITEM_TYPE_ETH, 196 }, 197 [MLX5_EXPANSION_VLAN] = { 198 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 199 MLX5_EXPANSION_IPV6), 200 .type = RTE_FLOW_ITEM_TYPE_VLAN, 201 }, 202 [MLX5_EXPANSION_IPV4] = { 203 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 204 MLX5_EXPANSION_IPV4_TCP), 205 .type = RTE_FLOW_ITEM_TYPE_IPV4, 206 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 207 ETH_RSS_NONFRAG_IPV4_OTHER, 208 }, 209 [MLX5_EXPANSION_IPV4_UDP] = { 210 .type = RTE_FLOW_ITEM_TYPE_UDP, 211 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 212 }, 213 [MLX5_EXPANSION_IPV4_TCP] = { 214 .type = RTE_FLOW_ITEM_TYPE_TCP, 215 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 216 }, 217 [MLX5_EXPANSION_IPV6] = { 218 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 219 MLX5_EXPANSION_IPV6_TCP), 220 .type = RTE_FLOW_ITEM_TYPE_IPV6, 221 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 222 ETH_RSS_NONFRAG_IPV6_OTHER, 223 }, 224 [MLX5_EXPANSION_IPV6_UDP] = { 225 .type = RTE_FLOW_ITEM_TYPE_UDP, 226 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 227 }, 228 [MLX5_EXPANSION_IPV6_TCP] = { 229 .type = RTE_FLOW_ITEM_TYPE_TCP, 230 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 231 }, 232 }; 233 234 static const struct rte_flow_ops mlx5_flow_ops = { 235 .validate = mlx5_flow_validate, 236 .create = mlx5_flow_create, 237 .destroy = mlx5_flow_destroy, 238 .flush = mlx5_flow_flush, 239 .isolate = mlx5_flow_isolate, 240 .query = mlx5_flow_query, 241 .dev_dump = mlx5_flow_dev_dump, 242 .get_aged_flows = mlx5_flow_get_aged_flows, 243 }; 244 245 /* Convert FDIR request to Generic flow. */ 246 struct mlx5_fdir { 247 struct rte_flow_attr attr; 248 struct rte_flow_item items[4]; 249 struct rte_flow_item_eth l2; 250 struct rte_flow_item_eth l2_mask; 251 union { 252 struct rte_flow_item_ipv4 ipv4; 253 struct rte_flow_item_ipv6 ipv6; 254 } l3; 255 union { 256 struct rte_flow_item_ipv4 ipv4; 257 struct rte_flow_item_ipv6 ipv6; 258 } l3_mask; 259 union { 260 struct rte_flow_item_udp udp; 261 struct rte_flow_item_tcp tcp; 262 } l4; 263 union { 264 struct rte_flow_item_udp udp; 265 struct rte_flow_item_tcp tcp; 266 } l4_mask; 267 struct rte_flow_action actions[2]; 268 struct rte_flow_action_queue queue; 269 }; 270 271 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 272 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 273 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 274 }; 275 276 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 277 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 278 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 279 { 9, 10, 11 }, { 12, 13, 14 }, 280 }; 281 282 /* Tunnel information. */ 283 struct mlx5_flow_tunnel_info { 284 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 285 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 286 }; 287 288 static struct mlx5_flow_tunnel_info tunnels_info[] = { 289 { 290 .tunnel = MLX5_FLOW_LAYER_VXLAN, 291 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 292 }, 293 { 294 .tunnel = MLX5_FLOW_LAYER_GENEVE, 295 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 296 }, 297 { 298 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 299 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 300 }, 301 { 302 .tunnel = MLX5_FLOW_LAYER_GRE, 303 .ptype = RTE_PTYPE_TUNNEL_GRE, 304 }, 305 { 306 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 307 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 308 }, 309 { 310 .tunnel = MLX5_FLOW_LAYER_MPLS, 311 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 312 }, 313 { 314 .tunnel = MLX5_FLOW_LAYER_NVGRE, 315 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 316 }, 317 { 318 .tunnel = MLX5_FLOW_LAYER_IPIP, 319 .ptype = RTE_PTYPE_TUNNEL_IP, 320 }, 321 { 322 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 323 .ptype = RTE_PTYPE_TUNNEL_IP, 324 }, 325 { 326 .tunnel = MLX5_FLOW_LAYER_GTP, 327 .ptype = RTE_PTYPE_TUNNEL_GTPU, 328 }, 329 }; 330 331 /** 332 * Translate tag ID to register. 333 * 334 * @param[in] dev 335 * Pointer to the Ethernet device structure. 336 * @param[in] feature 337 * The feature that request the register. 338 * @param[in] id 339 * The request register ID. 340 * @param[out] error 341 * Error description in case of any. 342 * 343 * @return 344 * The request register on success, a negative errno 345 * value otherwise and rte_errno is set. 346 */ 347 int 348 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 349 enum mlx5_feature_name feature, 350 uint32_t id, 351 struct rte_flow_error *error) 352 { 353 struct mlx5_priv *priv = dev->data->dev_private; 354 struct mlx5_dev_config *config = &priv->config; 355 enum modify_reg start_reg; 356 bool skip_mtr_reg = false; 357 358 switch (feature) { 359 case MLX5_HAIRPIN_RX: 360 return REG_B; 361 case MLX5_HAIRPIN_TX: 362 return REG_A; 363 case MLX5_METADATA_RX: 364 switch (config->dv_xmeta_en) { 365 case MLX5_XMETA_MODE_LEGACY: 366 return REG_B; 367 case MLX5_XMETA_MODE_META16: 368 return REG_C_0; 369 case MLX5_XMETA_MODE_META32: 370 return REG_C_1; 371 } 372 break; 373 case MLX5_METADATA_TX: 374 return REG_A; 375 case MLX5_METADATA_FDB: 376 switch (config->dv_xmeta_en) { 377 case MLX5_XMETA_MODE_LEGACY: 378 return REG_NONE; 379 case MLX5_XMETA_MODE_META16: 380 return REG_C_0; 381 case MLX5_XMETA_MODE_META32: 382 return REG_C_1; 383 } 384 break; 385 case MLX5_FLOW_MARK: 386 switch (config->dv_xmeta_en) { 387 case MLX5_XMETA_MODE_LEGACY: 388 return REG_NONE; 389 case MLX5_XMETA_MODE_META16: 390 return REG_C_1; 391 case MLX5_XMETA_MODE_META32: 392 return REG_C_0; 393 } 394 break; 395 case MLX5_MTR_SFX: 396 /* 397 * If meter color and flow match share one register, flow match 398 * should use the meter color register for match. 399 */ 400 if (priv->mtr_reg_share) 401 return priv->mtr_color_reg; 402 else 403 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 404 REG_C_3; 405 case MLX5_MTR_COLOR: 406 MLX5_ASSERT(priv->mtr_color_reg != REG_NONE); 407 return priv->mtr_color_reg; 408 case MLX5_COPY_MARK: 409 /* 410 * Metadata COPY_MARK register using is in meter suffix sub 411 * flow while with meter. It's safe to share the same register. 412 */ 413 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 414 case MLX5_APP_TAG: 415 /* 416 * If meter is enable, it will engage the register for color 417 * match and flow match. If meter color match is not using the 418 * REG_C_2, need to skip the REG_C_x be used by meter color 419 * match. 420 * If meter is disable, free to use all available registers. 421 */ 422 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 423 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 424 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 425 if (id > (REG_C_7 - start_reg)) 426 return rte_flow_error_set(error, EINVAL, 427 RTE_FLOW_ERROR_TYPE_ITEM, 428 NULL, "invalid tag id"); 429 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 430 return rte_flow_error_set(error, ENOTSUP, 431 RTE_FLOW_ERROR_TYPE_ITEM, 432 NULL, "unsupported tag id"); 433 /* 434 * This case means meter is using the REG_C_x great than 2. 435 * Take care not to conflict with meter color REG_C_x. 436 * If the available index REG_C_y >= REG_C_x, skip the 437 * color register. 438 */ 439 if (skip_mtr_reg && config->flow_mreg_c 440 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 441 if (id >= (REG_C_7 - start_reg)) 442 return rte_flow_error_set(error, EINVAL, 443 RTE_FLOW_ERROR_TYPE_ITEM, 444 NULL, "invalid tag id"); 445 if (config->flow_mreg_c 446 [id + 1 + start_reg - REG_C_0] != REG_NONE) 447 return config->flow_mreg_c 448 [id + 1 + start_reg - REG_C_0]; 449 return rte_flow_error_set(error, ENOTSUP, 450 RTE_FLOW_ERROR_TYPE_ITEM, 451 NULL, "unsupported tag id"); 452 } 453 return config->flow_mreg_c[id + start_reg - REG_C_0]; 454 } 455 MLX5_ASSERT(false); 456 return rte_flow_error_set(error, EINVAL, 457 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 458 NULL, "invalid feature name"); 459 } 460 461 /** 462 * Check extensive flow metadata register support. 463 * 464 * @param dev 465 * Pointer to rte_eth_dev structure. 466 * 467 * @return 468 * True if device supports extensive flow metadata register, otherwise false. 469 */ 470 bool 471 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 472 { 473 struct mlx5_priv *priv = dev->data->dev_private; 474 struct mlx5_dev_config *config = &priv->config; 475 476 /* 477 * Having available reg_c can be regarded inclusively as supporting 478 * extensive flow metadata register, which could mean, 479 * - metadata register copy action by modify header. 480 * - 16 modify header actions is supported. 481 * - reg_c's are preserved across different domain (FDB and NIC) on 482 * packet loopback by flow lookup miss. 483 */ 484 return config->flow_mreg_c[2] != REG_NONE; 485 } 486 487 /** 488 * Discover the maximum number of priority available. 489 * 490 * @param[in] dev 491 * Pointer to the Ethernet device structure. 492 * 493 * @return 494 * number of supported flow priority on success, a negative errno 495 * value otherwise and rte_errno is set. 496 */ 497 int 498 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 499 { 500 struct mlx5_priv *priv = dev->data->dev_private; 501 struct { 502 struct ibv_flow_attr attr; 503 struct ibv_flow_spec_eth eth; 504 struct ibv_flow_spec_action_drop drop; 505 } flow_attr = { 506 .attr = { 507 .num_of_specs = 2, 508 .port = (uint8_t)priv->dev_port, 509 }, 510 .eth = { 511 .type = IBV_FLOW_SPEC_ETH, 512 .size = sizeof(struct ibv_flow_spec_eth), 513 }, 514 .drop = { 515 .size = sizeof(struct ibv_flow_spec_action_drop), 516 .type = IBV_FLOW_SPEC_ACTION_DROP, 517 }, 518 }; 519 struct ibv_flow *flow; 520 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 521 uint16_t vprio[] = { 8, 16 }; 522 int i; 523 int priority = 0; 524 525 if (!drop) { 526 rte_errno = ENOTSUP; 527 return -rte_errno; 528 } 529 for (i = 0; i != RTE_DIM(vprio); i++) { 530 flow_attr.attr.priority = vprio[i] - 1; 531 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 532 if (!flow) 533 break; 534 claim_zero(mlx5_glue->destroy_flow(flow)); 535 priority = vprio[i]; 536 } 537 mlx5_hrxq_drop_release(dev); 538 switch (priority) { 539 case 8: 540 priority = RTE_DIM(priority_map_3); 541 break; 542 case 16: 543 priority = RTE_DIM(priority_map_5); 544 break; 545 default: 546 rte_errno = ENOTSUP; 547 DRV_LOG(ERR, 548 "port %u verbs maximum priority: %d expected 8/16", 549 dev->data->port_id, priority); 550 return -rte_errno; 551 } 552 DRV_LOG(INFO, "port %u flow maximum priority: %d", 553 dev->data->port_id, priority); 554 return priority; 555 } 556 557 /** 558 * Adjust flow priority based on the highest layer and the request priority. 559 * 560 * @param[in] dev 561 * Pointer to the Ethernet device structure. 562 * @param[in] priority 563 * The rule base priority. 564 * @param[in] subpriority 565 * The priority based on the items. 566 * 567 * @return 568 * The new priority. 569 */ 570 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 571 uint32_t subpriority) 572 { 573 uint32_t res = 0; 574 struct mlx5_priv *priv = dev->data->dev_private; 575 576 switch (priv->config.flow_prio) { 577 case RTE_DIM(priority_map_3): 578 res = priority_map_3[priority][subpriority]; 579 break; 580 case RTE_DIM(priority_map_5): 581 res = priority_map_5[priority][subpriority]; 582 break; 583 } 584 return res; 585 } 586 587 /** 588 * Verify the @p item specifications (spec, last, mask) are compatible with the 589 * NIC capabilities. 590 * 591 * @param[in] item 592 * Item specification. 593 * @param[in] mask 594 * @p item->mask or flow default bit-masks. 595 * @param[in] nic_mask 596 * Bit-masks covering supported fields by the NIC to compare with user mask. 597 * @param[in] size 598 * Bit-masks size in bytes. 599 * @param[out] error 600 * Pointer to error structure. 601 * 602 * @return 603 * 0 on success, a negative errno value otherwise and rte_errno is set. 604 */ 605 int 606 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 607 const uint8_t *mask, 608 const uint8_t *nic_mask, 609 unsigned int size, 610 struct rte_flow_error *error) 611 { 612 unsigned int i; 613 614 MLX5_ASSERT(nic_mask); 615 for (i = 0; i < size; ++i) 616 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 617 return rte_flow_error_set(error, ENOTSUP, 618 RTE_FLOW_ERROR_TYPE_ITEM, 619 item, 620 "mask enables non supported" 621 " bits"); 622 if (!item->spec && (item->mask || item->last)) 623 return rte_flow_error_set(error, EINVAL, 624 RTE_FLOW_ERROR_TYPE_ITEM, item, 625 "mask/last without a spec is not" 626 " supported"); 627 if (item->spec && item->last) { 628 uint8_t spec[size]; 629 uint8_t last[size]; 630 unsigned int i; 631 int ret; 632 633 for (i = 0; i < size; ++i) { 634 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 635 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 636 } 637 ret = memcmp(spec, last, size); 638 if (ret != 0) 639 return rte_flow_error_set(error, EINVAL, 640 RTE_FLOW_ERROR_TYPE_ITEM, 641 item, 642 "range is not valid"); 643 } 644 return 0; 645 } 646 647 /** 648 * Adjust the hash fields according to the @p flow information. 649 * 650 * @param[in] dev_flow. 651 * Pointer to the mlx5_flow. 652 * @param[in] tunnel 653 * 1 when the hash field is for a tunnel item. 654 * @param[in] layer_types 655 * ETH_RSS_* types. 656 * @param[in] hash_fields 657 * Item hash fields. 658 * 659 * @return 660 * The hash fields that should be used. 661 */ 662 uint64_t 663 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 664 int tunnel __rte_unused, uint64_t layer_types, 665 uint64_t hash_fields) 666 { 667 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 668 int rss_request_inner = rss_desc->level >= 2; 669 670 /* Check RSS hash level for tunnel. */ 671 if (tunnel && rss_request_inner) 672 hash_fields |= IBV_RX_HASH_INNER; 673 else if (tunnel || rss_request_inner) 674 return 0; 675 #endif 676 /* Check if requested layer matches RSS hash fields. */ 677 if (!(rss_desc->types & layer_types)) 678 return 0; 679 return hash_fields; 680 } 681 682 /** 683 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 684 * if several tunnel rules are used on this queue, the tunnel ptype will be 685 * cleared. 686 * 687 * @param rxq_ctrl 688 * Rx queue to update. 689 */ 690 static void 691 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 692 { 693 unsigned int i; 694 uint32_t tunnel_ptype = 0; 695 696 /* Look up for the ptype to use. */ 697 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 698 if (!rxq_ctrl->flow_tunnels_n[i]) 699 continue; 700 if (!tunnel_ptype) { 701 tunnel_ptype = tunnels_info[i].ptype; 702 } else { 703 tunnel_ptype = 0; 704 break; 705 } 706 } 707 rxq_ctrl->rxq.tunnel = tunnel_ptype; 708 } 709 710 /** 711 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 712 * flow. 713 * 714 * @param[in] dev 715 * Pointer to the Ethernet device structure. 716 * @param[in] dev_handle 717 * Pointer to device flow handle structure. 718 */ 719 static void 720 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 721 struct mlx5_flow_handle *dev_handle) 722 { 723 struct mlx5_priv *priv = dev->data->dev_private; 724 const int mark = dev_handle->mark; 725 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 726 struct mlx5_hrxq *hrxq; 727 unsigned int i; 728 729 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 730 return; 731 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 732 dev_handle->rix_hrxq); 733 if (!hrxq) 734 return; 735 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 736 int idx = hrxq->ind_table->queues[i]; 737 struct mlx5_rxq_ctrl *rxq_ctrl = 738 container_of((*priv->rxqs)[idx], 739 struct mlx5_rxq_ctrl, rxq); 740 741 /* 742 * To support metadata register copy on Tx loopback, 743 * this must be always enabled (metadata may arive 744 * from other port - not from local flows only. 745 */ 746 if (priv->config.dv_flow_en && 747 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 748 mlx5_flow_ext_mreg_supported(dev)) { 749 rxq_ctrl->rxq.mark = 1; 750 rxq_ctrl->flow_mark_n = 1; 751 } else if (mark) { 752 rxq_ctrl->rxq.mark = 1; 753 rxq_ctrl->flow_mark_n++; 754 } 755 if (tunnel) { 756 unsigned int j; 757 758 /* Increase the counter matching the flow. */ 759 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 760 if ((tunnels_info[j].tunnel & 761 dev_handle->layers) == 762 tunnels_info[j].tunnel) { 763 rxq_ctrl->flow_tunnels_n[j]++; 764 break; 765 } 766 } 767 flow_rxq_tunnel_ptype_update(rxq_ctrl); 768 } 769 } 770 } 771 772 /** 773 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 774 * 775 * @param[in] dev 776 * Pointer to the Ethernet device structure. 777 * @param[in] flow 778 * Pointer to flow structure. 779 */ 780 static void 781 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 782 { 783 struct mlx5_priv *priv = dev->data->dev_private; 784 uint32_t handle_idx; 785 struct mlx5_flow_handle *dev_handle; 786 787 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 788 handle_idx, dev_handle, next) 789 flow_drv_rxq_flags_set(dev, dev_handle); 790 } 791 792 /** 793 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 794 * device flow if no other flow uses it with the same kind of request. 795 * 796 * @param dev 797 * Pointer to Ethernet device. 798 * @param[in] dev_handle 799 * Pointer to the device flow handle structure. 800 */ 801 static void 802 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 803 struct mlx5_flow_handle *dev_handle) 804 { 805 struct mlx5_priv *priv = dev->data->dev_private; 806 const int mark = dev_handle->mark; 807 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 808 struct mlx5_hrxq *hrxq; 809 unsigned int i; 810 811 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 812 return; 813 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 814 dev_handle->rix_hrxq); 815 if (!hrxq) 816 return; 817 MLX5_ASSERT(dev->data->dev_started); 818 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 819 int idx = hrxq->ind_table->queues[i]; 820 struct mlx5_rxq_ctrl *rxq_ctrl = 821 container_of((*priv->rxqs)[idx], 822 struct mlx5_rxq_ctrl, rxq); 823 824 if (priv->config.dv_flow_en && 825 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 826 mlx5_flow_ext_mreg_supported(dev)) { 827 rxq_ctrl->rxq.mark = 1; 828 rxq_ctrl->flow_mark_n = 1; 829 } else if (mark) { 830 rxq_ctrl->flow_mark_n--; 831 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 832 } 833 if (tunnel) { 834 unsigned int j; 835 836 /* Decrease the counter matching the flow. */ 837 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 838 if ((tunnels_info[j].tunnel & 839 dev_handle->layers) == 840 tunnels_info[j].tunnel) { 841 rxq_ctrl->flow_tunnels_n[j]--; 842 break; 843 } 844 } 845 flow_rxq_tunnel_ptype_update(rxq_ctrl); 846 } 847 } 848 } 849 850 /** 851 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 852 * @p flow if no other flow uses it with the same kind of request. 853 * 854 * @param dev 855 * Pointer to Ethernet device. 856 * @param[in] flow 857 * Pointer to the flow. 858 */ 859 static void 860 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 861 { 862 struct mlx5_priv *priv = dev->data->dev_private; 863 uint32_t handle_idx; 864 struct mlx5_flow_handle *dev_handle; 865 866 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 867 handle_idx, dev_handle, next) 868 flow_drv_rxq_flags_trim(dev, dev_handle); 869 } 870 871 /** 872 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 873 * 874 * @param dev 875 * Pointer to Ethernet device. 876 */ 877 static void 878 flow_rxq_flags_clear(struct rte_eth_dev *dev) 879 { 880 struct mlx5_priv *priv = dev->data->dev_private; 881 unsigned int i; 882 883 for (i = 0; i != priv->rxqs_n; ++i) { 884 struct mlx5_rxq_ctrl *rxq_ctrl; 885 unsigned int j; 886 887 if (!(*priv->rxqs)[i]) 888 continue; 889 rxq_ctrl = container_of((*priv->rxqs)[i], 890 struct mlx5_rxq_ctrl, rxq); 891 rxq_ctrl->flow_mark_n = 0; 892 rxq_ctrl->rxq.mark = 0; 893 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 894 rxq_ctrl->flow_tunnels_n[j] = 0; 895 rxq_ctrl->rxq.tunnel = 0; 896 } 897 } 898 899 /** 900 * Set the Rx queue dynamic metadata (mask and offset) for a flow 901 * 902 * @param[in] dev 903 * Pointer to the Ethernet device structure. 904 */ 905 void 906 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 907 { 908 struct mlx5_priv *priv = dev->data->dev_private; 909 struct mlx5_rxq_data *data; 910 unsigned int i; 911 912 for (i = 0; i != priv->rxqs_n; ++i) { 913 if (!(*priv->rxqs)[i]) 914 continue; 915 data = (*priv->rxqs)[i]; 916 if (!rte_flow_dynf_metadata_avail()) { 917 data->dynf_meta = 0; 918 data->flow_meta_mask = 0; 919 data->flow_meta_offset = -1; 920 } else { 921 data->dynf_meta = 1; 922 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 923 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 924 } 925 } 926 } 927 928 /* 929 * return a pointer to the desired action in the list of actions. 930 * 931 * @param[in] actions 932 * The list of actions to search the action in. 933 * @param[in] action 934 * The action to find. 935 * 936 * @return 937 * Pointer to the action in the list, if found. NULL otherwise. 938 */ 939 const struct rte_flow_action * 940 mlx5_flow_find_action(const struct rte_flow_action *actions, 941 enum rte_flow_action_type action) 942 { 943 if (actions == NULL) 944 return NULL; 945 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 946 if (actions->type == action) 947 return actions; 948 return NULL; 949 } 950 951 /* 952 * Validate the flag action. 953 * 954 * @param[in] action_flags 955 * Bit-fields that holds the actions detected until now. 956 * @param[in] attr 957 * Attributes of flow that includes this action. 958 * @param[out] error 959 * Pointer to error structure. 960 * 961 * @return 962 * 0 on success, a negative errno value otherwise and rte_errno is set. 963 */ 964 int 965 mlx5_flow_validate_action_flag(uint64_t action_flags, 966 const struct rte_flow_attr *attr, 967 struct rte_flow_error *error) 968 { 969 if (action_flags & MLX5_FLOW_ACTION_MARK) 970 return rte_flow_error_set(error, EINVAL, 971 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 972 "can't mark and flag in same flow"); 973 if (action_flags & MLX5_FLOW_ACTION_FLAG) 974 return rte_flow_error_set(error, EINVAL, 975 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 976 "can't have 2 flag" 977 " actions in same flow"); 978 if (attr->egress) 979 return rte_flow_error_set(error, ENOTSUP, 980 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 981 "flag action not supported for " 982 "egress"); 983 return 0; 984 } 985 986 /* 987 * Validate the mark action. 988 * 989 * @param[in] action 990 * Pointer to the queue action. 991 * @param[in] action_flags 992 * Bit-fields that holds the actions detected until now. 993 * @param[in] attr 994 * Attributes of flow that includes this action. 995 * @param[out] error 996 * Pointer to error structure. 997 * 998 * @return 999 * 0 on success, a negative errno value otherwise and rte_errno is set. 1000 */ 1001 int 1002 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1003 uint64_t action_flags, 1004 const struct rte_flow_attr *attr, 1005 struct rte_flow_error *error) 1006 { 1007 const struct rte_flow_action_mark *mark = action->conf; 1008 1009 if (!mark) 1010 return rte_flow_error_set(error, EINVAL, 1011 RTE_FLOW_ERROR_TYPE_ACTION, 1012 action, 1013 "configuration cannot be null"); 1014 if (mark->id >= MLX5_FLOW_MARK_MAX) 1015 return rte_flow_error_set(error, EINVAL, 1016 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1017 &mark->id, 1018 "mark id must in 0 <= id < " 1019 RTE_STR(MLX5_FLOW_MARK_MAX)); 1020 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1021 return rte_flow_error_set(error, EINVAL, 1022 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1023 "can't flag and mark in same flow"); 1024 if (action_flags & MLX5_FLOW_ACTION_MARK) 1025 return rte_flow_error_set(error, EINVAL, 1026 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1027 "can't have 2 mark actions in same" 1028 " flow"); 1029 if (attr->egress) 1030 return rte_flow_error_set(error, ENOTSUP, 1031 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1032 "mark action not supported for " 1033 "egress"); 1034 return 0; 1035 } 1036 1037 /* 1038 * Validate the drop action. 1039 * 1040 * @param[in] action_flags 1041 * Bit-fields that holds the actions detected until now. 1042 * @param[in] attr 1043 * Attributes of flow that includes this action. 1044 * @param[out] error 1045 * Pointer to error structure. 1046 * 1047 * @return 1048 * 0 on success, a negative errno value otherwise and rte_errno is set. 1049 */ 1050 int 1051 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1052 const struct rte_flow_attr *attr, 1053 struct rte_flow_error *error) 1054 { 1055 if (attr->egress) 1056 return rte_flow_error_set(error, ENOTSUP, 1057 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1058 "drop action not supported for " 1059 "egress"); 1060 return 0; 1061 } 1062 1063 /* 1064 * Validate the queue action. 1065 * 1066 * @param[in] action 1067 * Pointer to the queue action. 1068 * @param[in] action_flags 1069 * Bit-fields that holds the actions detected until now. 1070 * @param[in] dev 1071 * Pointer to the Ethernet device structure. 1072 * @param[in] attr 1073 * Attributes of flow that includes this action. 1074 * @param[out] error 1075 * Pointer to error structure. 1076 * 1077 * @return 1078 * 0 on success, a negative errno value otherwise and rte_errno is set. 1079 */ 1080 int 1081 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1082 uint64_t action_flags, 1083 struct rte_eth_dev *dev, 1084 const struct rte_flow_attr *attr, 1085 struct rte_flow_error *error) 1086 { 1087 struct mlx5_priv *priv = dev->data->dev_private; 1088 const struct rte_flow_action_queue *queue = action->conf; 1089 1090 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1091 return rte_flow_error_set(error, EINVAL, 1092 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1093 "can't have 2 fate actions in" 1094 " same flow"); 1095 if (!priv->rxqs_n) 1096 return rte_flow_error_set(error, EINVAL, 1097 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1098 NULL, "No Rx queues configured"); 1099 if (queue->index >= priv->rxqs_n) 1100 return rte_flow_error_set(error, EINVAL, 1101 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1102 &queue->index, 1103 "queue index out of range"); 1104 if (!(*priv->rxqs)[queue->index]) 1105 return rte_flow_error_set(error, EINVAL, 1106 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1107 &queue->index, 1108 "queue is not configured"); 1109 if (attr->egress) 1110 return rte_flow_error_set(error, ENOTSUP, 1111 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1112 "queue action not supported for " 1113 "egress"); 1114 return 0; 1115 } 1116 1117 /* 1118 * Validate the rss action. 1119 * 1120 * @param[in] action 1121 * Pointer to the queue action. 1122 * @param[in] action_flags 1123 * Bit-fields that holds the actions detected until now. 1124 * @param[in] dev 1125 * Pointer to the Ethernet device structure. 1126 * @param[in] attr 1127 * Attributes of flow that includes this action. 1128 * @param[in] item_flags 1129 * Items that were detected. 1130 * @param[out] error 1131 * Pointer to error structure. 1132 * 1133 * @return 1134 * 0 on success, a negative errno value otherwise and rte_errno is set. 1135 */ 1136 int 1137 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1138 uint64_t action_flags, 1139 struct rte_eth_dev *dev, 1140 const struct rte_flow_attr *attr, 1141 uint64_t item_flags, 1142 struct rte_flow_error *error) 1143 { 1144 struct mlx5_priv *priv = dev->data->dev_private; 1145 const struct rte_flow_action_rss *rss = action->conf; 1146 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1147 unsigned int i; 1148 1149 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1150 return rte_flow_error_set(error, EINVAL, 1151 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1152 "can't have 2 fate actions" 1153 " in same flow"); 1154 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1155 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1156 return rte_flow_error_set(error, ENOTSUP, 1157 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1158 &rss->func, 1159 "RSS hash function not supported"); 1160 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1161 if (rss->level > 2) 1162 #else 1163 if (rss->level > 1) 1164 #endif 1165 return rte_flow_error_set(error, ENOTSUP, 1166 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1167 &rss->level, 1168 "tunnel RSS is not supported"); 1169 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1170 if (rss->key_len == 0 && rss->key != NULL) 1171 return rte_flow_error_set(error, ENOTSUP, 1172 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1173 &rss->key_len, 1174 "RSS hash key length 0"); 1175 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1176 return rte_flow_error_set(error, ENOTSUP, 1177 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1178 &rss->key_len, 1179 "RSS hash key too small"); 1180 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1181 return rte_flow_error_set(error, ENOTSUP, 1182 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1183 &rss->key_len, 1184 "RSS hash key too large"); 1185 if (rss->queue_num > priv->config.ind_table_max_size) 1186 return rte_flow_error_set(error, ENOTSUP, 1187 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1188 &rss->queue_num, 1189 "number of queues too large"); 1190 if (rss->types & MLX5_RSS_HF_MASK) 1191 return rte_flow_error_set(error, ENOTSUP, 1192 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1193 &rss->types, 1194 "some RSS protocols are not" 1195 " supported"); 1196 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1197 !(rss->types & ETH_RSS_IP)) 1198 return rte_flow_error_set(error, EINVAL, 1199 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1200 "L3 partial RSS requested but L3 RSS" 1201 " type not specified"); 1202 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1203 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1204 return rte_flow_error_set(error, EINVAL, 1205 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1206 "L4 partial RSS requested but L4 RSS" 1207 " type not specified"); 1208 if (!priv->rxqs_n) 1209 return rte_flow_error_set(error, EINVAL, 1210 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1211 NULL, "No Rx queues configured"); 1212 if (!rss->queue_num) 1213 return rte_flow_error_set(error, EINVAL, 1214 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1215 NULL, "No queues configured"); 1216 for (i = 0; i != rss->queue_num; ++i) { 1217 if (rss->queue[i] >= priv->rxqs_n) 1218 return rte_flow_error_set 1219 (error, EINVAL, 1220 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1221 &rss->queue[i], "queue index out of range"); 1222 if (!(*priv->rxqs)[rss->queue[i]]) 1223 return rte_flow_error_set 1224 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1225 &rss->queue[i], "queue is not configured"); 1226 } 1227 if (attr->egress) 1228 return rte_flow_error_set(error, ENOTSUP, 1229 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1230 "rss action not supported for " 1231 "egress"); 1232 if (rss->level > 1 && !tunnel) 1233 return rte_flow_error_set(error, EINVAL, 1234 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1235 "inner RSS is not supported for " 1236 "non-tunnel flows"); 1237 return 0; 1238 } 1239 1240 /* 1241 * Validate the count action. 1242 * 1243 * @param[in] dev 1244 * Pointer to the Ethernet device structure. 1245 * @param[in] attr 1246 * Attributes of flow that includes this action. 1247 * @param[out] error 1248 * Pointer to error structure. 1249 * 1250 * @return 1251 * 0 on success, a negative errno value otherwise and rte_errno is set. 1252 */ 1253 int 1254 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1255 const struct rte_flow_attr *attr, 1256 struct rte_flow_error *error) 1257 { 1258 if (attr->egress) 1259 return rte_flow_error_set(error, ENOTSUP, 1260 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1261 "count action not supported for " 1262 "egress"); 1263 return 0; 1264 } 1265 1266 /** 1267 * Verify the @p attributes will be correctly understood by the NIC and store 1268 * them in the @p flow if everything is correct. 1269 * 1270 * @param[in] dev 1271 * Pointer to the Ethernet device structure. 1272 * @param[in] attributes 1273 * Pointer to flow attributes 1274 * @param[out] error 1275 * Pointer to error structure. 1276 * 1277 * @return 1278 * 0 on success, a negative errno value otherwise and rte_errno is set. 1279 */ 1280 int 1281 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1282 const struct rte_flow_attr *attributes, 1283 struct rte_flow_error *error) 1284 { 1285 struct mlx5_priv *priv = dev->data->dev_private; 1286 uint32_t priority_max = priv->config.flow_prio - 1; 1287 1288 if (attributes->group) 1289 return rte_flow_error_set(error, ENOTSUP, 1290 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1291 NULL, "groups is not supported"); 1292 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1293 attributes->priority >= priority_max) 1294 return rte_flow_error_set(error, ENOTSUP, 1295 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1296 NULL, "priority out of range"); 1297 if (attributes->egress) 1298 return rte_flow_error_set(error, ENOTSUP, 1299 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1300 "egress is not supported"); 1301 if (attributes->transfer && !priv->config.dv_esw_en) 1302 return rte_flow_error_set(error, ENOTSUP, 1303 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1304 NULL, "transfer is not supported"); 1305 if (!attributes->ingress) 1306 return rte_flow_error_set(error, EINVAL, 1307 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1308 NULL, 1309 "ingress attribute is mandatory"); 1310 return 0; 1311 } 1312 1313 /** 1314 * Validate ICMP6 item. 1315 * 1316 * @param[in] item 1317 * Item specification. 1318 * @param[in] item_flags 1319 * Bit-fields that holds the items detected until now. 1320 * @param[out] error 1321 * Pointer to error structure. 1322 * 1323 * @return 1324 * 0 on success, a negative errno value otherwise and rte_errno is set. 1325 */ 1326 int 1327 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1328 uint64_t item_flags, 1329 uint8_t target_protocol, 1330 struct rte_flow_error *error) 1331 { 1332 const struct rte_flow_item_icmp6 *mask = item->mask; 1333 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1334 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1335 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1336 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1337 MLX5_FLOW_LAYER_OUTER_L4; 1338 int ret; 1339 1340 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1341 return rte_flow_error_set(error, EINVAL, 1342 RTE_FLOW_ERROR_TYPE_ITEM, item, 1343 "protocol filtering not compatible" 1344 " with ICMP6 layer"); 1345 if (!(item_flags & l3m)) 1346 return rte_flow_error_set(error, EINVAL, 1347 RTE_FLOW_ERROR_TYPE_ITEM, item, 1348 "IPv6 is mandatory to filter on" 1349 " ICMP6"); 1350 if (item_flags & l4m) 1351 return rte_flow_error_set(error, EINVAL, 1352 RTE_FLOW_ERROR_TYPE_ITEM, item, 1353 "multiple L4 layers not supported"); 1354 if (!mask) 1355 mask = &rte_flow_item_icmp6_mask; 1356 ret = mlx5_flow_item_acceptable 1357 (item, (const uint8_t *)mask, 1358 (const uint8_t *)&rte_flow_item_icmp6_mask, 1359 sizeof(struct rte_flow_item_icmp6), error); 1360 if (ret < 0) 1361 return ret; 1362 return 0; 1363 } 1364 1365 /** 1366 * Validate ICMP item. 1367 * 1368 * @param[in] item 1369 * Item specification. 1370 * @param[in] item_flags 1371 * Bit-fields that holds the items detected until now. 1372 * @param[out] error 1373 * Pointer to error structure. 1374 * 1375 * @return 1376 * 0 on success, a negative errno value otherwise and rte_errno is set. 1377 */ 1378 int 1379 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1380 uint64_t item_flags, 1381 uint8_t target_protocol, 1382 struct rte_flow_error *error) 1383 { 1384 const struct rte_flow_item_icmp *mask = item->mask; 1385 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1386 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1387 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1388 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1389 MLX5_FLOW_LAYER_OUTER_L4; 1390 int ret; 1391 1392 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1393 return rte_flow_error_set(error, EINVAL, 1394 RTE_FLOW_ERROR_TYPE_ITEM, item, 1395 "protocol filtering not compatible" 1396 " with ICMP layer"); 1397 if (!(item_flags & l3m)) 1398 return rte_flow_error_set(error, EINVAL, 1399 RTE_FLOW_ERROR_TYPE_ITEM, item, 1400 "IPv4 is mandatory to filter" 1401 " on ICMP"); 1402 if (item_flags & l4m) 1403 return rte_flow_error_set(error, EINVAL, 1404 RTE_FLOW_ERROR_TYPE_ITEM, item, 1405 "multiple L4 layers not supported"); 1406 if (!mask) 1407 mask = &rte_flow_item_icmp_mask; 1408 ret = mlx5_flow_item_acceptable 1409 (item, (const uint8_t *)mask, 1410 (const uint8_t *)&rte_flow_item_icmp_mask, 1411 sizeof(struct rte_flow_item_icmp), error); 1412 if (ret < 0) 1413 return ret; 1414 return 0; 1415 } 1416 1417 /** 1418 * Validate Ethernet item. 1419 * 1420 * @param[in] item 1421 * Item specification. 1422 * @param[in] item_flags 1423 * Bit-fields that holds the items detected until now. 1424 * @param[out] error 1425 * Pointer to error structure. 1426 * 1427 * @return 1428 * 0 on success, a negative errno value otherwise and rte_errno is set. 1429 */ 1430 int 1431 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1432 uint64_t item_flags, 1433 struct rte_flow_error *error) 1434 { 1435 const struct rte_flow_item_eth *mask = item->mask; 1436 const struct rte_flow_item_eth nic_mask = { 1437 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1438 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1439 .type = RTE_BE16(0xffff), 1440 }; 1441 int ret; 1442 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1443 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1444 MLX5_FLOW_LAYER_OUTER_L2; 1445 1446 if (item_flags & ethm) 1447 return rte_flow_error_set(error, ENOTSUP, 1448 RTE_FLOW_ERROR_TYPE_ITEM, item, 1449 "multiple L2 layers not supported"); 1450 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1451 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1452 return rte_flow_error_set(error, EINVAL, 1453 RTE_FLOW_ERROR_TYPE_ITEM, item, 1454 "L2 layer should not follow " 1455 "L3 layers"); 1456 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1457 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1458 return rte_flow_error_set(error, EINVAL, 1459 RTE_FLOW_ERROR_TYPE_ITEM, item, 1460 "L2 layer should not follow VLAN"); 1461 if (!mask) 1462 mask = &rte_flow_item_eth_mask; 1463 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1464 (const uint8_t *)&nic_mask, 1465 sizeof(struct rte_flow_item_eth), 1466 error); 1467 return ret; 1468 } 1469 1470 /** 1471 * Validate VLAN item. 1472 * 1473 * @param[in] item 1474 * Item specification. 1475 * @param[in] item_flags 1476 * Bit-fields that holds the items detected until now. 1477 * @param[in] dev 1478 * Ethernet device flow is being created on. 1479 * @param[out] error 1480 * Pointer to error structure. 1481 * 1482 * @return 1483 * 0 on success, a negative errno value otherwise and rte_errno is set. 1484 */ 1485 int 1486 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1487 uint64_t item_flags, 1488 struct rte_eth_dev *dev, 1489 struct rte_flow_error *error) 1490 { 1491 const struct rte_flow_item_vlan *spec = item->spec; 1492 const struct rte_flow_item_vlan *mask = item->mask; 1493 const struct rte_flow_item_vlan nic_mask = { 1494 .tci = RTE_BE16(UINT16_MAX), 1495 .inner_type = RTE_BE16(UINT16_MAX), 1496 }; 1497 uint16_t vlan_tag = 0; 1498 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1499 int ret; 1500 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1501 MLX5_FLOW_LAYER_INNER_L4) : 1502 (MLX5_FLOW_LAYER_OUTER_L3 | 1503 MLX5_FLOW_LAYER_OUTER_L4); 1504 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1505 MLX5_FLOW_LAYER_OUTER_VLAN; 1506 1507 if (item_flags & vlanm) 1508 return rte_flow_error_set(error, EINVAL, 1509 RTE_FLOW_ERROR_TYPE_ITEM, item, 1510 "multiple VLAN layers not supported"); 1511 else if ((item_flags & l34m) != 0) 1512 return rte_flow_error_set(error, EINVAL, 1513 RTE_FLOW_ERROR_TYPE_ITEM, item, 1514 "VLAN cannot follow L3/L4 layer"); 1515 if (!mask) 1516 mask = &rte_flow_item_vlan_mask; 1517 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1518 (const uint8_t *)&nic_mask, 1519 sizeof(struct rte_flow_item_vlan), 1520 error); 1521 if (ret) 1522 return ret; 1523 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1524 struct mlx5_priv *priv = dev->data->dev_private; 1525 1526 if (priv->vmwa_context) { 1527 /* 1528 * Non-NULL context means we have a virtual machine 1529 * and SR-IOV enabled, we have to create VLAN interface 1530 * to make hypervisor to setup E-Switch vport 1531 * context correctly. We avoid creating the multiple 1532 * VLAN interfaces, so we cannot support VLAN tag mask. 1533 */ 1534 return rte_flow_error_set(error, EINVAL, 1535 RTE_FLOW_ERROR_TYPE_ITEM, 1536 item, 1537 "VLAN tag mask is not" 1538 " supported in virtual" 1539 " environment"); 1540 } 1541 } 1542 if (spec) { 1543 vlan_tag = spec->tci; 1544 vlan_tag &= mask->tci; 1545 } 1546 /* 1547 * From verbs perspective an empty VLAN is equivalent 1548 * to a packet without VLAN layer. 1549 */ 1550 if (!vlan_tag) 1551 return rte_flow_error_set(error, EINVAL, 1552 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1553 item->spec, 1554 "VLAN cannot be empty"); 1555 return 0; 1556 } 1557 1558 /** 1559 * Validate IPV4 item. 1560 * 1561 * @param[in] item 1562 * Item specification. 1563 * @param[in] item_flags 1564 * Bit-fields that holds the items detected until now. 1565 * @param[in] acc_mask 1566 * Acceptable mask, if NULL default internal default mask 1567 * will be used to check whether item fields are supported. 1568 * @param[out] error 1569 * Pointer to error structure. 1570 * 1571 * @return 1572 * 0 on success, a negative errno value otherwise and rte_errno is set. 1573 */ 1574 int 1575 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1576 uint64_t item_flags, 1577 uint64_t last_item, 1578 uint16_t ether_type, 1579 const struct rte_flow_item_ipv4 *acc_mask, 1580 struct rte_flow_error *error) 1581 { 1582 const struct rte_flow_item_ipv4 *mask = item->mask; 1583 const struct rte_flow_item_ipv4 *spec = item->spec; 1584 const struct rte_flow_item_ipv4 nic_mask = { 1585 .hdr = { 1586 .src_addr = RTE_BE32(0xffffffff), 1587 .dst_addr = RTE_BE32(0xffffffff), 1588 .type_of_service = 0xff, 1589 .next_proto_id = 0xff, 1590 }, 1591 }; 1592 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1593 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1594 MLX5_FLOW_LAYER_OUTER_L3; 1595 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1596 MLX5_FLOW_LAYER_OUTER_L4; 1597 int ret; 1598 uint8_t next_proto = 0xFF; 1599 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1600 MLX5_FLOW_LAYER_OUTER_VLAN | 1601 MLX5_FLOW_LAYER_INNER_VLAN); 1602 1603 if ((last_item & l2_vlan) && ether_type && 1604 ether_type != RTE_ETHER_TYPE_IPV4) 1605 return rte_flow_error_set(error, EINVAL, 1606 RTE_FLOW_ERROR_TYPE_ITEM, item, 1607 "IPv4 cannot follow L2/VLAN layer " 1608 "which ether type is not IPv4"); 1609 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1610 if (mask && spec) 1611 next_proto = mask->hdr.next_proto_id & 1612 spec->hdr.next_proto_id; 1613 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1614 return rte_flow_error_set(error, EINVAL, 1615 RTE_FLOW_ERROR_TYPE_ITEM, 1616 item, 1617 "multiple tunnel " 1618 "not supported"); 1619 } 1620 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1621 return rte_flow_error_set(error, EINVAL, 1622 RTE_FLOW_ERROR_TYPE_ITEM, item, 1623 "wrong tunnel type - IPv6 specified " 1624 "but IPv4 item provided"); 1625 if (item_flags & l3m) 1626 return rte_flow_error_set(error, ENOTSUP, 1627 RTE_FLOW_ERROR_TYPE_ITEM, item, 1628 "multiple L3 layers not supported"); 1629 else if (item_flags & l4m) 1630 return rte_flow_error_set(error, EINVAL, 1631 RTE_FLOW_ERROR_TYPE_ITEM, item, 1632 "L3 cannot follow an L4 layer."); 1633 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1634 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1635 return rte_flow_error_set(error, EINVAL, 1636 RTE_FLOW_ERROR_TYPE_ITEM, item, 1637 "L3 cannot follow an NVGRE layer."); 1638 if (!mask) 1639 mask = &rte_flow_item_ipv4_mask; 1640 else if (mask->hdr.next_proto_id != 0 && 1641 mask->hdr.next_proto_id != 0xff) 1642 return rte_flow_error_set(error, EINVAL, 1643 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1644 "partial mask is not supported" 1645 " for protocol"); 1646 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1647 acc_mask ? (const uint8_t *)acc_mask 1648 : (const uint8_t *)&nic_mask, 1649 sizeof(struct rte_flow_item_ipv4), 1650 error); 1651 if (ret < 0) 1652 return ret; 1653 return 0; 1654 } 1655 1656 /** 1657 * Validate IPV6 item. 1658 * 1659 * @param[in] item 1660 * Item specification. 1661 * @param[in] item_flags 1662 * Bit-fields that holds the items detected until now. 1663 * @param[in] acc_mask 1664 * Acceptable mask, if NULL default internal default mask 1665 * will be used to check whether item fields are supported. 1666 * @param[out] error 1667 * Pointer to error structure. 1668 * 1669 * @return 1670 * 0 on success, a negative errno value otherwise and rte_errno is set. 1671 */ 1672 int 1673 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1674 uint64_t item_flags, 1675 uint64_t last_item, 1676 uint16_t ether_type, 1677 const struct rte_flow_item_ipv6 *acc_mask, 1678 struct rte_flow_error *error) 1679 { 1680 const struct rte_flow_item_ipv6 *mask = item->mask; 1681 const struct rte_flow_item_ipv6 *spec = item->spec; 1682 const struct rte_flow_item_ipv6 nic_mask = { 1683 .hdr = { 1684 .src_addr = 1685 "\xff\xff\xff\xff\xff\xff\xff\xff" 1686 "\xff\xff\xff\xff\xff\xff\xff\xff", 1687 .dst_addr = 1688 "\xff\xff\xff\xff\xff\xff\xff\xff" 1689 "\xff\xff\xff\xff\xff\xff\xff\xff", 1690 .vtc_flow = RTE_BE32(0xffffffff), 1691 .proto = 0xff, 1692 }, 1693 }; 1694 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1695 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1696 MLX5_FLOW_LAYER_OUTER_L3; 1697 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1698 MLX5_FLOW_LAYER_OUTER_L4; 1699 int ret; 1700 uint8_t next_proto = 0xFF; 1701 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1702 MLX5_FLOW_LAYER_OUTER_VLAN | 1703 MLX5_FLOW_LAYER_INNER_VLAN); 1704 1705 if ((last_item & l2_vlan) && ether_type && 1706 ether_type != RTE_ETHER_TYPE_IPV6) 1707 return rte_flow_error_set(error, EINVAL, 1708 RTE_FLOW_ERROR_TYPE_ITEM, item, 1709 "IPv6 cannot follow L2/VLAN layer " 1710 "which ether type is not IPv6"); 1711 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1712 if (mask && spec) 1713 next_proto = mask->hdr.proto & spec->hdr.proto; 1714 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1715 return rte_flow_error_set(error, EINVAL, 1716 RTE_FLOW_ERROR_TYPE_ITEM, 1717 item, 1718 "multiple tunnel " 1719 "not supported"); 1720 } 1721 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1722 return rte_flow_error_set(error, EINVAL, 1723 RTE_FLOW_ERROR_TYPE_ITEM, item, 1724 "wrong tunnel type - IPv4 specified " 1725 "but IPv6 item provided"); 1726 if (item_flags & l3m) 1727 return rte_flow_error_set(error, ENOTSUP, 1728 RTE_FLOW_ERROR_TYPE_ITEM, item, 1729 "multiple L3 layers not supported"); 1730 else if (item_flags & l4m) 1731 return rte_flow_error_set(error, EINVAL, 1732 RTE_FLOW_ERROR_TYPE_ITEM, item, 1733 "L3 cannot follow an L4 layer."); 1734 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1735 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1736 return rte_flow_error_set(error, EINVAL, 1737 RTE_FLOW_ERROR_TYPE_ITEM, item, 1738 "L3 cannot follow an NVGRE layer."); 1739 if (!mask) 1740 mask = &rte_flow_item_ipv6_mask; 1741 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1742 acc_mask ? (const uint8_t *)acc_mask 1743 : (const uint8_t *)&nic_mask, 1744 sizeof(struct rte_flow_item_ipv6), 1745 error); 1746 if (ret < 0) 1747 return ret; 1748 return 0; 1749 } 1750 1751 /** 1752 * Validate UDP item. 1753 * 1754 * @param[in] item 1755 * Item specification. 1756 * @param[in] item_flags 1757 * Bit-fields that holds the items detected until now. 1758 * @param[in] target_protocol 1759 * The next protocol in the previous item. 1760 * @param[in] flow_mask 1761 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1762 * @param[out] error 1763 * Pointer to error structure. 1764 * 1765 * @return 1766 * 0 on success, a negative errno value otherwise and rte_errno is set. 1767 */ 1768 int 1769 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1770 uint64_t item_flags, 1771 uint8_t target_protocol, 1772 struct rte_flow_error *error) 1773 { 1774 const struct rte_flow_item_udp *mask = item->mask; 1775 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1776 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1777 MLX5_FLOW_LAYER_OUTER_L3; 1778 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1779 MLX5_FLOW_LAYER_OUTER_L4; 1780 int ret; 1781 1782 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1783 return rte_flow_error_set(error, EINVAL, 1784 RTE_FLOW_ERROR_TYPE_ITEM, item, 1785 "protocol filtering not compatible" 1786 " with UDP layer"); 1787 if (!(item_flags & l3m)) 1788 return rte_flow_error_set(error, EINVAL, 1789 RTE_FLOW_ERROR_TYPE_ITEM, item, 1790 "L3 is mandatory to filter on L4"); 1791 if (item_flags & l4m) 1792 return rte_flow_error_set(error, EINVAL, 1793 RTE_FLOW_ERROR_TYPE_ITEM, item, 1794 "multiple L4 layers not supported"); 1795 if (!mask) 1796 mask = &rte_flow_item_udp_mask; 1797 ret = mlx5_flow_item_acceptable 1798 (item, (const uint8_t *)mask, 1799 (const uint8_t *)&rte_flow_item_udp_mask, 1800 sizeof(struct rte_flow_item_udp), error); 1801 if (ret < 0) 1802 return ret; 1803 return 0; 1804 } 1805 1806 /** 1807 * Validate TCP item. 1808 * 1809 * @param[in] item 1810 * Item specification. 1811 * @param[in] item_flags 1812 * Bit-fields that holds the items detected until now. 1813 * @param[in] target_protocol 1814 * The next protocol in the previous item. 1815 * @param[out] error 1816 * Pointer to error structure. 1817 * 1818 * @return 1819 * 0 on success, a negative errno value otherwise and rte_errno is set. 1820 */ 1821 int 1822 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1823 uint64_t item_flags, 1824 uint8_t target_protocol, 1825 const struct rte_flow_item_tcp *flow_mask, 1826 struct rte_flow_error *error) 1827 { 1828 const struct rte_flow_item_tcp *mask = item->mask; 1829 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1830 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1831 MLX5_FLOW_LAYER_OUTER_L3; 1832 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1833 MLX5_FLOW_LAYER_OUTER_L4; 1834 int ret; 1835 1836 MLX5_ASSERT(flow_mask); 1837 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1838 return rte_flow_error_set(error, EINVAL, 1839 RTE_FLOW_ERROR_TYPE_ITEM, item, 1840 "protocol filtering not compatible" 1841 " with TCP layer"); 1842 if (!(item_flags & l3m)) 1843 return rte_flow_error_set(error, EINVAL, 1844 RTE_FLOW_ERROR_TYPE_ITEM, item, 1845 "L3 is mandatory to filter on L4"); 1846 if (item_flags & l4m) 1847 return rte_flow_error_set(error, EINVAL, 1848 RTE_FLOW_ERROR_TYPE_ITEM, item, 1849 "multiple L4 layers not supported"); 1850 if (!mask) 1851 mask = &rte_flow_item_tcp_mask; 1852 ret = mlx5_flow_item_acceptable 1853 (item, (const uint8_t *)mask, 1854 (const uint8_t *)flow_mask, 1855 sizeof(struct rte_flow_item_tcp), error); 1856 if (ret < 0) 1857 return ret; 1858 return 0; 1859 } 1860 1861 /** 1862 * Validate VXLAN item. 1863 * 1864 * @param[in] item 1865 * Item specification. 1866 * @param[in] item_flags 1867 * Bit-fields that holds the items detected until now. 1868 * @param[in] target_protocol 1869 * The next protocol in the previous item. 1870 * @param[out] error 1871 * Pointer to error structure. 1872 * 1873 * @return 1874 * 0 on success, a negative errno value otherwise and rte_errno is set. 1875 */ 1876 int 1877 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1878 uint64_t item_flags, 1879 struct rte_flow_error *error) 1880 { 1881 const struct rte_flow_item_vxlan *spec = item->spec; 1882 const struct rte_flow_item_vxlan *mask = item->mask; 1883 int ret; 1884 union vni { 1885 uint32_t vlan_id; 1886 uint8_t vni[4]; 1887 } id = { .vlan_id = 0, }; 1888 1889 1890 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1891 return rte_flow_error_set(error, ENOTSUP, 1892 RTE_FLOW_ERROR_TYPE_ITEM, item, 1893 "multiple tunnel layers not" 1894 " supported"); 1895 /* 1896 * Verify only UDPv4 is present as defined in 1897 * https://tools.ietf.org/html/rfc7348 1898 */ 1899 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1900 return rte_flow_error_set(error, EINVAL, 1901 RTE_FLOW_ERROR_TYPE_ITEM, item, 1902 "no outer UDP layer found"); 1903 if (!mask) 1904 mask = &rte_flow_item_vxlan_mask; 1905 ret = mlx5_flow_item_acceptable 1906 (item, (const uint8_t *)mask, 1907 (const uint8_t *)&rte_flow_item_vxlan_mask, 1908 sizeof(struct rte_flow_item_vxlan), 1909 error); 1910 if (ret < 0) 1911 return ret; 1912 if (spec) { 1913 memcpy(&id.vni[1], spec->vni, 3); 1914 memcpy(&id.vni[1], mask->vni, 3); 1915 } 1916 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1917 return rte_flow_error_set(error, ENOTSUP, 1918 RTE_FLOW_ERROR_TYPE_ITEM, item, 1919 "VXLAN tunnel must be fully defined"); 1920 return 0; 1921 } 1922 1923 /** 1924 * Validate VXLAN_GPE item. 1925 * 1926 * @param[in] item 1927 * Item specification. 1928 * @param[in] item_flags 1929 * Bit-fields that holds the items detected until now. 1930 * @param[in] priv 1931 * Pointer to the private data structure. 1932 * @param[in] target_protocol 1933 * The next protocol in the previous item. 1934 * @param[out] error 1935 * Pointer to error structure. 1936 * 1937 * @return 1938 * 0 on success, a negative errno value otherwise and rte_errno is set. 1939 */ 1940 int 1941 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1942 uint64_t item_flags, 1943 struct rte_eth_dev *dev, 1944 struct rte_flow_error *error) 1945 { 1946 struct mlx5_priv *priv = dev->data->dev_private; 1947 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1948 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1949 int ret; 1950 union vni { 1951 uint32_t vlan_id; 1952 uint8_t vni[4]; 1953 } id = { .vlan_id = 0, }; 1954 1955 if (!priv->config.l3_vxlan_en) 1956 return rte_flow_error_set(error, ENOTSUP, 1957 RTE_FLOW_ERROR_TYPE_ITEM, item, 1958 "L3 VXLAN is not enabled by device" 1959 " parameter and/or not configured in" 1960 " firmware"); 1961 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1962 return rte_flow_error_set(error, ENOTSUP, 1963 RTE_FLOW_ERROR_TYPE_ITEM, item, 1964 "multiple tunnel layers not" 1965 " supported"); 1966 /* 1967 * Verify only UDPv4 is present as defined in 1968 * https://tools.ietf.org/html/rfc7348 1969 */ 1970 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1971 return rte_flow_error_set(error, EINVAL, 1972 RTE_FLOW_ERROR_TYPE_ITEM, item, 1973 "no outer UDP layer found"); 1974 if (!mask) 1975 mask = &rte_flow_item_vxlan_gpe_mask; 1976 ret = mlx5_flow_item_acceptable 1977 (item, (const uint8_t *)mask, 1978 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1979 sizeof(struct rte_flow_item_vxlan_gpe), 1980 error); 1981 if (ret < 0) 1982 return ret; 1983 if (spec) { 1984 if (spec->protocol) 1985 return rte_flow_error_set(error, ENOTSUP, 1986 RTE_FLOW_ERROR_TYPE_ITEM, 1987 item, 1988 "VxLAN-GPE protocol" 1989 " not supported"); 1990 memcpy(&id.vni[1], spec->vni, 3); 1991 memcpy(&id.vni[1], mask->vni, 3); 1992 } 1993 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1994 return rte_flow_error_set(error, ENOTSUP, 1995 RTE_FLOW_ERROR_TYPE_ITEM, item, 1996 "VXLAN-GPE tunnel must be fully" 1997 " defined"); 1998 return 0; 1999 } 2000 /** 2001 * Validate GRE Key item. 2002 * 2003 * @param[in] item 2004 * Item specification. 2005 * @param[in] item_flags 2006 * Bit flags to mark detected items. 2007 * @param[in] gre_item 2008 * Pointer to gre_item 2009 * @param[out] error 2010 * Pointer to error structure. 2011 * 2012 * @return 2013 * 0 on success, a negative errno value otherwise and rte_errno is set. 2014 */ 2015 int 2016 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2017 uint64_t item_flags, 2018 const struct rte_flow_item *gre_item, 2019 struct rte_flow_error *error) 2020 { 2021 const rte_be32_t *mask = item->mask; 2022 int ret = 0; 2023 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2024 const struct rte_flow_item_gre *gre_spec; 2025 const struct rte_flow_item_gre *gre_mask; 2026 2027 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2028 return rte_flow_error_set(error, ENOTSUP, 2029 RTE_FLOW_ERROR_TYPE_ITEM, item, 2030 "Multiple GRE key not support"); 2031 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2032 return rte_flow_error_set(error, ENOTSUP, 2033 RTE_FLOW_ERROR_TYPE_ITEM, item, 2034 "No preceding GRE header"); 2035 if (item_flags & MLX5_FLOW_LAYER_INNER) 2036 return rte_flow_error_set(error, ENOTSUP, 2037 RTE_FLOW_ERROR_TYPE_ITEM, item, 2038 "GRE key following a wrong item"); 2039 gre_mask = gre_item->mask; 2040 if (!gre_mask) 2041 gre_mask = &rte_flow_item_gre_mask; 2042 gre_spec = gre_item->spec; 2043 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2044 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2045 return rte_flow_error_set(error, EINVAL, 2046 RTE_FLOW_ERROR_TYPE_ITEM, item, 2047 "Key bit must be on"); 2048 2049 if (!mask) 2050 mask = &gre_key_default_mask; 2051 ret = mlx5_flow_item_acceptable 2052 (item, (const uint8_t *)mask, 2053 (const uint8_t *)&gre_key_default_mask, 2054 sizeof(rte_be32_t), error); 2055 return ret; 2056 } 2057 2058 /** 2059 * Validate GRE item. 2060 * 2061 * @param[in] item 2062 * Item specification. 2063 * @param[in] item_flags 2064 * Bit flags to mark detected items. 2065 * @param[in] target_protocol 2066 * The next protocol in the previous item. 2067 * @param[out] error 2068 * Pointer to error structure. 2069 * 2070 * @return 2071 * 0 on success, a negative errno value otherwise and rte_errno is set. 2072 */ 2073 int 2074 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2075 uint64_t item_flags, 2076 uint8_t target_protocol, 2077 struct rte_flow_error *error) 2078 { 2079 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2080 const struct rte_flow_item_gre *mask = item->mask; 2081 int ret; 2082 const struct rte_flow_item_gre nic_mask = { 2083 .c_rsvd0_ver = RTE_BE16(0xB000), 2084 .protocol = RTE_BE16(UINT16_MAX), 2085 }; 2086 2087 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2088 return rte_flow_error_set(error, EINVAL, 2089 RTE_FLOW_ERROR_TYPE_ITEM, item, 2090 "protocol filtering not compatible" 2091 " with this GRE layer"); 2092 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2093 return rte_flow_error_set(error, ENOTSUP, 2094 RTE_FLOW_ERROR_TYPE_ITEM, item, 2095 "multiple tunnel layers not" 2096 " supported"); 2097 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2098 return rte_flow_error_set(error, ENOTSUP, 2099 RTE_FLOW_ERROR_TYPE_ITEM, item, 2100 "L3 Layer is missing"); 2101 if (!mask) 2102 mask = &rte_flow_item_gre_mask; 2103 ret = mlx5_flow_item_acceptable 2104 (item, (const uint8_t *)mask, 2105 (const uint8_t *)&nic_mask, 2106 sizeof(struct rte_flow_item_gre), error); 2107 if (ret < 0) 2108 return ret; 2109 #ifndef HAVE_MLX5DV_DR 2110 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2111 if (spec && (spec->protocol & mask->protocol)) 2112 return rte_flow_error_set(error, ENOTSUP, 2113 RTE_FLOW_ERROR_TYPE_ITEM, item, 2114 "without MPLS support the" 2115 " specification cannot be used for" 2116 " filtering"); 2117 #endif 2118 #endif 2119 return 0; 2120 } 2121 2122 /** 2123 * Validate Geneve item. 2124 * 2125 * @param[in] item 2126 * Item specification. 2127 * @param[in] itemFlags 2128 * Bit-fields that holds the items detected until now. 2129 * @param[in] enPriv 2130 * Pointer to the private data structure. 2131 * @param[out] error 2132 * Pointer to error structure. 2133 * 2134 * @return 2135 * 0 on success, a negative errno value otherwise and rte_errno is set. 2136 */ 2137 2138 int 2139 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2140 uint64_t item_flags, 2141 struct rte_eth_dev *dev, 2142 struct rte_flow_error *error) 2143 { 2144 struct mlx5_priv *priv = dev->data->dev_private; 2145 const struct rte_flow_item_geneve *spec = item->spec; 2146 const struct rte_flow_item_geneve *mask = item->mask; 2147 int ret; 2148 uint16_t gbhdr; 2149 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2150 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2151 const struct rte_flow_item_geneve nic_mask = { 2152 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2153 .vni = "\xff\xff\xff", 2154 .protocol = RTE_BE16(UINT16_MAX), 2155 }; 2156 2157 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2158 return rte_flow_error_set(error, ENOTSUP, 2159 RTE_FLOW_ERROR_TYPE_ITEM, item, 2160 "L3 Geneve is not enabled by device" 2161 " parameter and/or not configured in" 2162 " firmware"); 2163 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2164 return rte_flow_error_set(error, ENOTSUP, 2165 RTE_FLOW_ERROR_TYPE_ITEM, item, 2166 "multiple tunnel layers not" 2167 " supported"); 2168 /* 2169 * Verify only UDPv4 is present as defined in 2170 * https://tools.ietf.org/html/rfc7348 2171 */ 2172 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2173 return rte_flow_error_set(error, EINVAL, 2174 RTE_FLOW_ERROR_TYPE_ITEM, item, 2175 "no outer UDP layer found"); 2176 if (!mask) 2177 mask = &rte_flow_item_geneve_mask; 2178 ret = mlx5_flow_item_acceptable 2179 (item, (const uint8_t *)mask, 2180 (const uint8_t *)&nic_mask, 2181 sizeof(struct rte_flow_item_geneve), error); 2182 if (ret) 2183 return ret; 2184 if (spec) { 2185 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2186 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2187 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2188 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2189 return rte_flow_error_set(error, ENOTSUP, 2190 RTE_FLOW_ERROR_TYPE_ITEM, 2191 item, 2192 "Geneve protocol unsupported" 2193 " fields are being used"); 2194 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2195 return rte_flow_error_set 2196 (error, ENOTSUP, 2197 RTE_FLOW_ERROR_TYPE_ITEM, 2198 item, 2199 "Unsupported Geneve options length"); 2200 } 2201 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2202 return rte_flow_error_set 2203 (error, ENOTSUP, 2204 RTE_FLOW_ERROR_TYPE_ITEM, item, 2205 "Geneve tunnel must be fully defined"); 2206 return 0; 2207 } 2208 2209 /** 2210 * Validate MPLS item. 2211 * 2212 * @param[in] dev 2213 * Pointer to the rte_eth_dev structure. 2214 * @param[in] item 2215 * Item specification. 2216 * @param[in] item_flags 2217 * Bit-fields that holds the items detected until now. 2218 * @param[in] prev_layer 2219 * The protocol layer indicated in previous item. 2220 * @param[out] error 2221 * Pointer to error structure. 2222 * 2223 * @return 2224 * 0 on success, a negative errno value otherwise and rte_errno is set. 2225 */ 2226 int 2227 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2228 const struct rte_flow_item *item __rte_unused, 2229 uint64_t item_flags __rte_unused, 2230 uint64_t prev_layer __rte_unused, 2231 struct rte_flow_error *error) 2232 { 2233 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2234 const struct rte_flow_item_mpls *mask = item->mask; 2235 struct mlx5_priv *priv = dev->data->dev_private; 2236 int ret; 2237 2238 if (!priv->config.mpls_en) 2239 return rte_flow_error_set(error, ENOTSUP, 2240 RTE_FLOW_ERROR_TYPE_ITEM, item, 2241 "MPLS not supported or" 2242 " disabled in firmware" 2243 " configuration."); 2244 /* MPLS over IP, UDP, GRE is allowed */ 2245 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2246 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2247 MLX5_FLOW_LAYER_GRE))) 2248 return rte_flow_error_set(error, EINVAL, 2249 RTE_FLOW_ERROR_TYPE_ITEM, item, 2250 "protocol filtering not compatible" 2251 " with MPLS layer"); 2252 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2253 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2254 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2255 return rte_flow_error_set(error, ENOTSUP, 2256 RTE_FLOW_ERROR_TYPE_ITEM, item, 2257 "multiple tunnel layers not" 2258 " supported"); 2259 if (!mask) 2260 mask = &rte_flow_item_mpls_mask; 2261 ret = mlx5_flow_item_acceptable 2262 (item, (const uint8_t *)mask, 2263 (const uint8_t *)&rte_flow_item_mpls_mask, 2264 sizeof(struct rte_flow_item_mpls), error); 2265 if (ret < 0) 2266 return ret; 2267 return 0; 2268 #else 2269 return rte_flow_error_set(error, ENOTSUP, 2270 RTE_FLOW_ERROR_TYPE_ITEM, item, 2271 "MPLS is not supported by Verbs, please" 2272 " update."); 2273 #endif 2274 } 2275 2276 /** 2277 * Validate NVGRE item. 2278 * 2279 * @param[in] item 2280 * Item specification. 2281 * @param[in] item_flags 2282 * Bit flags to mark detected items. 2283 * @param[in] target_protocol 2284 * The next protocol in the previous item. 2285 * @param[out] error 2286 * Pointer to error structure. 2287 * 2288 * @return 2289 * 0 on success, a negative errno value otherwise and rte_errno is set. 2290 */ 2291 int 2292 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2293 uint64_t item_flags, 2294 uint8_t target_protocol, 2295 struct rte_flow_error *error) 2296 { 2297 const struct rte_flow_item_nvgre *mask = item->mask; 2298 int ret; 2299 2300 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2301 return rte_flow_error_set(error, EINVAL, 2302 RTE_FLOW_ERROR_TYPE_ITEM, item, 2303 "protocol filtering not compatible" 2304 " with this GRE layer"); 2305 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2306 return rte_flow_error_set(error, ENOTSUP, 2307 RTE_FLOW_ERROR_TYPE_ITEM, item, 2308 "multiple tunnel layers not" 2309 " supported"); 2310 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2311 return rte_flow_error_set(error, ENOTSUP, 2312 RTE_FLOW_ERROR_TYPE_ITEM, item, 2313 "L3 Layer is missing"); 2314 if (!mask) 2315 mask = &rte_flow_item_nvgre_mask; 2316 ret = mlx5_flow_item_acceptable 2317 (item, (const uint8_t *)mask, 2318 (const uint8_t *)&rte_flow_item_nvgre_mask, 2319 sizeof(struct rte_flow_item_nvgre), error); 2320 if (ret < 0) 2321 return ret; 2322 return 0; 2323 } 2324 2325 /* Allocate unique ID for the split Q/RSS subflows. */ 2326 static uint32_t 2327 flow_qrss_get_id(struct rte_eth_dev *dev) 2328 { 2329 struct mlx5_priv *priv = dev->data->dev_private; 2330 uint32_t qrss_id, ret; 2331 2332 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2333 if (ret) 2334 return 0; 2335 MLX5_ASSERT(qrss_id); 2336 return qrss_id; 2337 } 2338 2339 /* Free unique ID for the split Q/RSS subflows. */ 2340 static void 2341 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2342 { 2343 struct mlx5_priv *priv = dev->data->dev_private; 2344 2345 if (qrss_id) 2346 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2347 } 2348 2349 /** 2350 * Release resource related QUEUE/RSS action split. 2351 * 2352 * @param dev 2353 * Pointer to Ethernet device. 2354 * @param flow 2355 * Flow to release id's from. 2356 */ 2357 static void 2358 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2359 struct rte_flow *flow) 2360 { 2361 struct mlx5_priv *priv = dev->data->dev_private; 2362 uint32_t handle_idx; 2363 struct mlx5_flow_handle *dev_handle; 2364 2365 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2366 handle_idx, dev_handle, next) 2367 if (dev_handle->split_flow_id) 2368 flow_qrss_free_id(dev, dev_handle->split_flow_id); 2369 } 2370 2371 static int 2372 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2373 const struct rte_flow_attr *attr __rte_unused, 2374 const struct rte_flow_item items[] __rte_unused, 2375 const struct rte_flow_action actions[] __rte_unused, 2376 bool external __rte_unused, 2377 int hairpin __rte_unused, 2378 struct rte_flow_error *error) 2379 { 2380 return rte_flow_error_set(error, ENOTSUP, 2381 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2382 } 2383 2384 static struct mlx5_flow * 2385 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2386 const struct rte_flow_attr *attr __rte_unused, 2387 const struct rte_flow_item items[] __rte_unused, 2388 const struct rte_flow_action actions[] __rte_unused, 2389 struct rte_flow_error *error) 2390 { 2391 rte_flow_error_set(error, ENOTSUP, 2392 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2393 return NULL; 2394 } 2395 2396 static int 2397 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2398 struct mlx5_flow *dev_flow __rte_unused, 2399 const struct rte_flow_attr *attr __rte_unused, 2400 const struct rte_flow_item items[] __rte_unused, 2401 const struct rte_flow_action actions[] __rte_unused, 2402 struct rte_flow_error *error) 2403 { 2404 return rte_flow_error_set(error, ENOTSUP, 2405 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2406 } 2407 2408 static int 2409 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2410 struct rte_flow *flow __rte_unused, 2411 struct rte_flow_error *error) 2412 { 2413 return rte_flow_error_set(error, ENOTSUP, 2414 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2415 } 2416 2417 static void 2418 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2419 struct rte_flow *flow __rte_unused) 2420 { 2421 } 2422 2423 static void 2424 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2425 struct rte_flow *flow __rte_unused) 2426 { 2427 } 2428 2429 static int 2430 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2431 struct rte_flow *flow __rte_unused, 2432 const struct rte_flow_action *actions __rte_unused, 2433 void *data __rte_unused, 2434 struct rte_flow_error *error) 2435 { 2436 return rte_flow_error_set(error, ENOTSUP, 2437 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2438 } 2439 2440 /* Void driver to protect from null pointer reference. */ 2441 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2442 .validate = flow_null_validate, 2443 .prepare = flow_null_prepare, 2444 .translate = flow_null_translate, 2445 .apply = flow_null_apply, 2446 .remove = flow_null_remove, 2447 .destroy = flow_null_destroy, 2448 .query = flow_null_query, 2449 }; 2450 2451 /** 2452 * Select flow driver type according to flow attributes and device 2453 * configuration. 2454 * 2455 * @param[in] dev 2456 * Pointer to the dev structure. 2457 * @param[in] attr 2458 * Pointer to the flow attributes. 2459 * 2460 * @return 2461 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2462 */ 2463 static enum mlx5_flow_drv_type 2464 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2465 { 2466 struct mlx5_priv *priv = dev->data->dev_private; 2467 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2468 2469 if (attr->transfer && priv->config.dv_esw_en) 2470 type = MLX5_FLOW_TYPE_DV; 2471 if (!attr->transfer) 2472 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2473 MLX5_FLOW_TYPE_VERBS; 2474 return type; 2475 } 2476 2477 #define flow_get_drv_ops(type) flow_drv_ops[type] 2478 2479 /** 2480 * Flow driver validation API. This abstracts calling driver specific functions. 2481 * The type of flow driver is determined according to flow attributes. 2482 * 2483 * @param[in] dev 2484 * Pointer to the dev structure. 2485 * @param[in] attr 2486 * Pointer to the flow attributes. 2487 * @param[in] items 2488 * Pointer to the list of items. 2489 * @param[in] actions 2490 * Pointer to the list of actions. 2491 * @param[in] external 2492 * This flow rule is created by request external to PMD. 2493 * @param[in] hairpin 2494 * Number of hairpin TX actions, 0 means classic flow. 2495 * @param[out] error 2496 * Pointer to the error structure. 2497 * 2498 * @return 2499 * 0 on success, a negative errno value otherwise and rte_errno is set. 2500 */ 2501 static inline int 2502 flow_drv_validate(struct rte_eth_dev *dev, 2503 const struct rte_flow_attr *attr, 2504 const struct rte_flow_item items[], 2505 const struct rte_flow_action actions[], 2506 bool external, int hairpin, struct rte_flow_error *error) 2507 { 2508 const struct mlx5_flow_driver_ops *fops; 2509 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2510 2511 fops = flow_get_drv_ops(type); 2512 return fops->validate(dev, attr, items, actions, external, 2513 hairpin, error); 2514 } 2515 2516 /** 2517 * Flow driver preparation API. This abstracts calling driver specific 2518 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2519 * calculates the size of memory required for device flow, allocates the memory, 2520 * initializes the device flow and returns the pointer. 2521 * 2522 * @note 2523 * This function initializes device flow structure such as dv or verbs in 2524 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2525 * rest. For example, adding returning device flow to flow->dev_flow list and 2526 * setting backward reference to the flow should be done out of this function. 2527 * layers field is not filled either. 2528 * 2529 * @param[in] dev 2530 * Pointer to the dev structure. 2531 * @param[in] attr 2532 * Pointer to the flow attributes. 2533 * @param[in] items 2534 * Pointer to the list of items. 2535 * @param[in] actions 2536 * Pointer to the list of actions. 2537 * @param[in] flow_idx 2538 * This memory pool index to the flow. 2539 * @param[out] error 2540 * Pointer to the error structure. 2541 * 2542 * @return 2543 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2544 */ 2545 static inline struct mlx5_flow * 2546 flow_drv_prepare(struct rte_eth_dev *dev, 2547 const struct rte_flow *flow, 2548 const struct rte_flow_attr *attr, 2549 const struct rte_flow_item items[], 2550 const struct rte_flow_action actions[], 2551 uint32_t flow_idx, 2552 struct rte_flow_error *error) 2553 { 2554 const struct mlx5_flow_driver_ops *fops; 2555 enum mlx5_flow_drv_type type = flow->drv_type; 2556 struct mlx5_flow *mlx5_flow = NULL; 2557 2558 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2559 fops = flow_get_drv_ops(type); 2560 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 2561 if (mlx5_flow) 2562 mlx5_flow->flow_idx = flow_idx; 2563 return mlx5_flow; 2564 } 2565 2566 /** 2567 * Flow driver translation API. This abstracts calling driver specific 2568 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2569 * translates a generic flow into a driver flow. flow_drv_prepare() must 2570 * precede. 2571 * 2572 * @note 2573 * dev_flow->layers could be filled as a result of parsing during translation 2574 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2575 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2576 * flow->actions could be overwritten even though all the expanded dev_flows 2577 * have the same actions. 2578 * 2579 * @param[in] dev 2580 * Pointer to the rte dev structure. 2581 * @param[in, out] dev_flow 2582 * Pointer to the mlx5 flow. 2583 * @param[in] attr 2584 * Pointer to the flow attributes. 2585 * @param[in] items 2586 * Pointer to the list of items. 2587 * @param[in] actions 2588 * Pointer to the list of actions. 2589 * @param[out] error 2590 * Pointer to the error structure. 2591 * 2592 * @return 2593 * 0 on success, a negative errno value otherwise and rte_errno is set. 2594 */ 2595 static inline int 2596 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2597 const struct rte_flow_attr *attr, 2598 const struct rte_flow_item items[], 2599 const struct rte_flow_action actions[], 2600 struct rte_flow_error *error) 2601 { 2602 const struct mlx5_flow_driver_ops *fops; 2603 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2604 2605 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2606 fops = flow_get_drv_ops(type); 2607 return fops->translate(dev, dev_flow, attr, items, actions, error); 2608 } 2609 2610 /** 2611 * Flow driver apply API. This abstracts calling driver specific functions. 2612 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2613 * translated driver flows on to device. flow_drv_translate() must precede. 2614 * 2615 * @param[in] dev 2616 * Pointer to Ethernet device structure. 2617 * @param[in, out] flow 2618 * Pointer to flow structure. 2619 * @param[out] error 2620 * Pointer to error structure. 2621 * 2622 * @return 2623 * 0 on success, a negative errno value otherwise and rte_errno is set. 2624 */ 2625 static inline int 2626 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2627 struct rte_flow_error *error) 2628 { 2629 const struct mlx5_flow_driver_ops *fops; 2630 enum mlx5_flow_drv_type type = flow->drv_type; 2631 2632 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2633 fops = flow_get_drv_ops(type); 2634 return fops->apply(dev, flow, error); 2635 } 2636 2637 /** 2638 * Flow driver remove API. This abstracts calling driver specific functions. 2639 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2640 * on device. All the resources of the flow should be freed by calling 2641 * flow_drv_destroy(). 2642 * 2643 * @param[in] dev 2644 * Pointer to Ethernet device. 2645 * @param[in, out] flow 2646 * Pointer to flow structure. 2647 */ 2648 static inline void 2649 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2650 { 2651 const struct mlx5_flow_driver_ops *fops; 2652 enum mlx5_flow_drv_type type = flow->drv_type; 2653 2654 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2655 fops = flow_get_drv_ops(type); 2656 fops->remove(dev, flow); 2657 } 2658 2659 /** 2660 * Flow driver destroy API. This abstracts calling driver specific functions. 2661 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2662 * on device and releases resources of the flow. 2663 * 2664 * @param[in] dev 2665 * Pointer to Ethernet device. 2666 * @param[in, out] flow 2667 * Pointer to flow structure. 2668 */ 2669 static inline void 2670 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2671 { 2672 const struct mlx5_flow_driver_ops *fops; 2673 enum mlx5_flow_drv_type type = flow->drv_type; 2674 2675 flow_mreg_split_qrss_release(dev, flow); 2676 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2677 fops = flow_get_drv_ops(type); 2678 fops->destroy(dev, flow); 2679 } 2680 2681 /** 2682 * Get RSS action from the action list. 2683 * 2684 * @param[in] actions 2685 * Pointer to the list of actions. 2686 * 2687 * @return 2688 * Pointer to the RSS action if exist, else return NULL. 2689 */ 2690 static const struct rte_flow_action_rss* 2691 flow_get_rss_action(const struct rte_flow_action actions[]) 2692 { 2693 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2694 switch (actions->type) { 2695 case RTE_FLOW_ACTION_TYPE_RSS: 2696 return (const struct rte_flow_action_rss *) 2697 actions->conf; 2698 default: 2699 break; 2700 } 2701 } 2702 return NULL; 2703 } 2704 2705 static unsigned int 2706 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2707 { 2708 const struct rte_flow_item *item; 2709 unsigned int has_vlan = 0; 2710 2711 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2712 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2713 has_vlan = 1; 2714 break; 2715 } 2716 } 2717 if (has_vlan) 2718 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2719 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2720 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2721 MLX5_EXPANSION_ROOT_OUTER; 2722 } 2723 2724 /** 2725 * Get layer flags from the prefix flow. 2726 * 2727 * Some flows may be split to several subflows, the prefix subflow gets the 2728 * match items and the suffix sub flow gets the actions. 2729 * Some actions need the user defined match item flags to get the detail for 2730 * the action. 2731 * This function helps the suffix flow to get the item layer flags from prefix 2732 * subflow. 2733 * 2734 * @param[in] dev_flow 2735 * Pointer the created preifx subflow. 2736 * 2737 * @return 2738 * The layers get from prefix subflow. 2739 */ 2740 static inline uint64_t 2741 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 2742 { 2743 uint64_t layers = 0; 2744 2745 /* 2746 * Layers bits could be localization, but usually the compiler will 2747 * help to do the optimization work for source code. 2748 * If no decap actions, use the layers directly. 2749 */ 2750 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 2751 return dev_flow->handle->layers; 2752 /* Convert L3 layers with decap action. */ 2753 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 2754 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2755 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 2756 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2757 /* Convert L4 layers with decap action. */ 2758 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 2759 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 2760 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 2761 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 2762 return layers; 2763 } 2764 2765 /** 2766 * Get metadata split action information. 2767 * 2768 * @param[in] actions 2769 * Pointer to the list of actions. 2770 * @param[out] qrss 2771 * Pointer to the return pointer. 2772 * @param[out] qrss_type 2773 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2774 * if no QUEUE/RSS is found. 2775 * @param[out] encap_idx 2776 * Pointer to the index of the encap action if exists, otherwise the last 2777 * action index. 2778 * 2779 * @return 2780 * Total number of actions. 2781 */ 2782 static int 2783 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 2784 const struct rte_flow_action **qrss, 2785 int *encap_idx) 2786 { 2787 const struct rte_flow_action_raw_encap *raw_encap; 2788 int actions_n = 0; 2789 int raw_decap_idx = -1; 2790 2791 *encap_idx = -1; 2792 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2793 switch (actions->type) { 2794 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2795 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2796 *encap_idx = actions_n; 2797 break; 2798 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 2799 raw_decap_idx = actions_n; 2800 break; 2801 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2802 raw_encap = actions->conf; 2803 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2804 *encap_idx = raw_decap_idx != -1 ? 2805 raw_decap_idx : actions_n; 2806 break; 2807 case RTE_FLOW_ACTION_TYPE_QUEUE: 2808 case RTE_FLOW_ACTION_TYPE_RSS: 2809 *qrss = actions; 2810 break; 2811 default: 2812 break; 2813 } 2814 actions_n++; 2815 } 2816 if (*encap_idx == -1) 2817 *encap_idx = actions_n; 2818 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2819 return actions_n + 1; 2820 } 2821 2822 /** 2823 * Check meter action from the action list. 2824 * 2825 * @param[in] actions 2826 * Pointer to the list of actions. 2827 * @param[out] mtr 2828 * Pointer to the meter exist flag. 2829 * 2830 * @return 2831 * Total number of actions. 2832 */ 2833 static int 2834 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2835 { 2836 int actions_n = 0; 2837 2838 MLX5_ASSERT(mtr); 2839 *mtr = 0; 2840 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2841 switch (actions->type) { 2842 case RTE_FLOW_ACTION_TYPE_METER: 2843 *mtr = 1; 2844 break; 2845 default: 2846 break; 2847 } 2848 actions_n++; 2849 } 2850 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2851 return actions_n + 1; 2852 } 2853 2854 /** 2855 * Check if the flow should be splited due to hairpin. 2856 * The reason for the split is that in current HW we can't 2857 * support encap on Rx, so if a flow have encap we move it 2858 * to Tx. 2859 * 2860 * @param dev 2861 * Pointer to Ethernet device. 2862 * @param[in] attr 2863 * Flow rule attributes. 2864 * @param[in] actions 2865 * Associated actions (list terminated by the END action). 2866 * 2867 * @return 2868 * > 0 the number of actions and the flow should be split, 2869 * 0 when no split required. 2870 */ 2871 static int 2872 flow_check_hairpin_split(struct rte_eth_dev *dev, 2873 const struct rte_flow_attr *attr, 2874 const struct rte_flow_action actions[]) 2875 { 2876 int queue_action = 0; 2877 int action_n = 0; 2878 int encap = 0; 2879 const struct rte_flow_action_queue *queue; 2880 const struct rte_flow_action_rss *rss; 2881 const struct rte_flow_action_raw_encap *raw_encap; 2882 2883 if (!attr->ingress) 2884 return 0; 2885 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2886 switch (actions->type) { 2887 case RTE_FLOW_ACTION_TYPE_QUEUE: 2888 queue = actions->conf; 2889 if (queue == NULL) 2890 return 0; 2891 if (mlx5_rxq_get_type(dev, queue->index) != 2892 MLX5_RXQ_TYPE_HAIRPIN) 2893 return 0; 2894 queue_action = 1; 2895 action_n++; 2896 break; 2897 case RTE_FLOW_ACTION_TYPE_RSS: 2898 rss = actions->conf; 2899 if (rss == NULL || rss->queue_num == 0) 2900 return 0; 2901 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2902 MLX5_RXQ_TYPE_HAIRPIN) 2903 return 0; 2904 queue_action = 1; 2905 action_n++; 2906 break; 2907 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2908 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2909 encap = 1; 2910 action_n++; 2911 break; 2912 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2913 raw_encap = actions->conf; 2914 if (raw_encap->size > 2915 (sizeof(struct rte_flow_item_eth) + 2916 sizeof(struct rte_flow_item_ipv4))) 2917 encap = 1; 2918 action_n++; 2919 break; 2920 default: 2921 action_n++; 2922 break; 2923 } 2924 } 2925 if (encap == 1 && queue_action) 2926 return action_n; 2927 return 0; 2928 } 2929 2930 /* Declare flow create/destroy prototype in advance. */ 2931 static uint32_t 2932 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 2933 const struct rte_flow_attr *attr, 2934 const struct rte_flow_item items[], 2935 const struct rte_flow_action actions[], 2936 bool external, struct rte_flow_error *error); 2937 2938 static void 2939 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 2940 uint32_t flow_idx); 2941 2942 /** 2943 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2944 * 2945 * As mark_id is unique, if there's already a registered flow for the mark_id, 2946 * return by increasing the reference counter of the resource. Otherwise, create 2947 * the resource (mcp_res) and flow. 2948 * 2949 * Flow looks like, 2950 * - If ingress port is ANY and reg_c[1] is mark_id, 2951 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2952 * 2953 * For default flow (zero mark_id), flow is like, 2954 * - If ingress port is ANY, 2955 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2956 * 2957 * @param dev 2958 * Pointer to Ethernet device. 2959 * @param mark_id 2960 * ID of MARK action, zero means default flow for META. 2961 * @param[out] error 2962 * Perform verbose error reporting if not NULL. 2963 * 2964 * @return 2965 * Associated resource on success, NULL otherwise and rte_errno is set. 2966 */ 2967 static struct mlx5_flow_mreg_copy_resource * 2968 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2969 struct rte_flow_error *error) 2970 { 2971 struct mlx5_priv *priv = dev->data->dev_private; 2972 struct rte_flow_attr attr = { 2973 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2974 .ingress = 1, 2975 }; 2976 struct mlx5_rte_flow_item_tag tag_spec = { 2977 .data = mark_id, 2978 }; 2979 struct rte_flow_item items[] = { 2980 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2981 }; 2982 struct rte_flow_action_mark ftag = { 2983 .id = mark_id, 2984 }; 2985 struct mlx5_flow_action_copy_mreg cp_mreg = { 2986 .dst = REG_B, 2987 .src = 0, 2988 }; 2989 struct rte_flow_action_jump jump = { 2990 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2991 }; 2992 struct rte_flow_action actions[] = { 2993 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2994 }; 2995 struct mlx5_flow_mreg_copy_resource *mcp_res; 2996 uint32_t idx = 0; 2997 int ret; 2998 2999 /* Fill the register fileds in the flow. */ 3000 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 3001 if (ret < 0) 3002 return NULL; 3003 tag_spec.id = ret; 3004 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3005 if (ret < 0) 3006 return NULL; 3007 cp_mreg.src = ret; 3008 /* Check if already registered. */ 3009 MLX5_ASSERT(priv->mreg_cp_tbl); 3010 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 3011 if (mcp_res) { 3012 /* For non-default rule. */ 3013 if (mark_id != MLX5_DEFAULT_COPY_ID) 3014 mcp_res->refcnt++; 3015 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 3016 mcp_res->refcnt == 1); 3017 return mcp_res; 3018 } 3019 /* Provide the full width of FLAG specific value. */ 3020 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3021 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3022 /* Build a new flow. */ 3023 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3024 items[0] = (struct rte_flow_item){ 3025 .type = (enum rte_flow_item_type) 3026 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3027 .spec = &tag_spec, 3028 }; 3029 items[1] = (struct rte_flow_item){ 3030 .type = RTE_FLOW_ITEM_TYPE_END, 3031 }; 3032 actions[0] = (struct rte_flow_action){ 3033 .type = (enum rte_flow_action_type) 3034 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3035 .conf = &ftag, 3036 }; 3037 actions[1] = (struct rte_flow_action){ 3038 .type = (enum rte_flow_action_type) 3039 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3040 .conf = &cp_mreg, 3041 }; 3042 actions[2] = (struct rte_flow_action){ 3043 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3044 .conf = &jump, 3045 }; 3046 actions[3] = (struct rte_flow_action){ 3047 .type = RTE_FLOW_ACTION_TYPE_END, 3048 }; 3049 } else { 3050 /* Default rule, wildcard match. */ 3051 attr.priority = MLX5_FLOW_PRIO_RSVD; 3052 items[0] = (struct rte_flow_item){ 3053 .type = RTE_FLOW_ITEM_TYPE_END, 3054 }; 3055 actions[0] = (struct rte_flow_action){ 3056 .type = (enum rte_flow_action_type) 3057 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3058 .conf = &cp_mreg, 3059 }; 3060 actions[1] = (struct rte_flow_action){ 3061 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3062 .conf = &jump, 3063 }; 3064 actions[2] = (struct rte_flow_action){ 3065 .type = RTE_FLOW_ACTION_TYPE_END, 3066 }; 3067 } 3068 /* Build a new entry. */ 3069 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3070 if (!mcp_res) { 3071 rte_errno = ENOMEM; 3072 return NULL; 3073 } 3074 mcp_res->idx = idx; 3075 /* 3076 * The copy Flows are not included in any list. There 3077 * ones are referenced from other Flows and can not 3078 * be applied, removed, deleted in ardbitrary order 3079 * by list traversing. 3080 */ 3081 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3082 actions, false, error); 3083 if (!mcp_res->rix_flow) 3084 goto error; 3085 mcp_res->refcnt++; 3086 mcp_res->hlist_ent.key = mark_id; 3087 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3088 &mcp_res->hlist_ent); 3089 MLX5_ASSERT(!ret); 3090 if (ret) 3091 goto error; 3092 return mcp_res; 3093 error: 3094 if (mcp_res->rix_flow) 3095 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3096 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3097 return NULL; 3098 } 3099 3100 /** 3101 * Release flow in RX_CP_TBL. 3102 * 3103 * @param dev 3104 * Pointer to Ethernet device. 3105 * @flow 3106 * Parent flow for wich copying is provided. 3107 */ 3108 static void 3109 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3110 struct rte_flow *flow) 3111 { 3112 struct mlx5_flow_mreg_copy_resource *mcp_res; 3113 struct mlx5_priv *priv = dev->data->dev_private; 3114 3115 if (!flow->rix_mreg_copy) 3116 return; 3117 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3118 flow->rix_mreg_copy); 3119 if (!mcp_res || !priv->mreg_cp_tbl) 3120 return; 3121 if (flow->copy_applied) { 3122 MLX5_ASSERT(mcp_res->appcnt); 3123 flow->copy_applied = 0; 3124 --mcp_res->appcnt; 3125 if (!mcp_res->appcnt) { 3126 struct rte_flow *mcp_flow = mlx5_ipool_get 3127 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3128 mcp_res->rix_flow); 3129 3130 if (mcp_flow) 3131 flow_drv_remove(dev, mcp_flow); 3132 } 3133 } 3134 /* 3135 * We do not check availability of metadata registers here, 3136 * because copy resources are not allocated in this case. 3137 */ 3138 if (--mcp_res->refcnt) 3139 return; 3140 MLX5_ASSERT(mcp_res->rix_flow); 3141 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3142 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3143 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3144 flow->rix_mreg_copy = 0; 3145 } 3146 3147 /** 3148 * Start flow in RX_CP_TBL. 3149 * 3150 * @param dev 3151 * Pointer to Ethernet device. 3152 * @flow 3153 * Parent flow for wich copying is provided. 3154 * 3155 * @return 3156 * 0 on success, a negative errno value otherwise and rte_errno is set. 3157 */ 3158 static int 3159 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3160 struct rte_flow *flow) 3161 { 3162 struct mlx5_flow_mreg_copy_resource *mcp_res; 3163 struct mlx5_priv *priv = dev->data->dev_private; 3164 int ret; 3165 3166 if (!flow->rix_mreg_copy || flow->copy_applied) 3167 return 0; 3168 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3169 flow->rix_mreg_copy); 3170 if (!mcp_res) 3171 return 0; 3172 if (!mcp_res->appcnt) { 3173 struct rte_flow *mcp_flow = mlx5_ipool_get 3174 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3175 mcp_res->rix_flow); 3176 3177 if (mcp_flow) { 3178 ret = flow_drv_apply(dev, mcp_flow, NULL); 3179 if (ret) 3180 return ret; 3181 } 3182 } 3183 ++mcp_res->appcnt; 3184 flow->copy_applied = 1; 3185 return 0; 3186 } 3187 3188 /** 3189 * Stop flow in RX_CP_TBL. 3190 * 3191 * @param dev 3192 * Pointer to Ethernet device. 3193 * @flow 3194 * Parent flow for wich copying is provided. 3195 */ 3196 static void 3197 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3198 struct rte_flow *flow) 3199 { 3200 struct mlx5_flow_mreg_copy_resource *mcp_res; 3201 struct mlx5_priv *priv = dev->data->dev_private; 3202 3203 if (!flow->rix_mreg_copy || !flow->copy_applied) 3204 return; 3205 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3206 flow->rix_mreg_copy); 3207 if (!mcp_res) 3208 return; 3209 MLX5_ASSERT(mcp_res->appcnt); 3210 --mcp_res->appcnt; 3211 flow->copy_applied = 0; 3212 if (!mcp_res->appcnt) { 3213 struct rte_flow *mcp_flow = mlx5_ipool_get 3214 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3215 mcp_res->rix_flow); 3216 3217 if (mcp_flow) 3218 flow_drv_remove(dev, mcp_flow); 3219 } 3220 } 3221 3222 /** 3223 * Remove the default copy action from RX_CP_TBL. 3224 * 3225 * @param dev 3226 * Pointer to Ethernet device. 3227 */ 3228 static void 3229 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3230 { 3231 struct mlx5_flow_mreg_copy_resource *mcp_res; 3232 struct mlx5_priv *priv = dev->data->dev_private; 3233 3234 /* Check if default flow is registered. */ 3235 if (!priv->mreg_cp_tbl) 3236 return; 3237 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3238 MLX5_DEFAULT_COPY_ID); 3239 if (!mcp_res) 3240 return; 3241 MLX5_ASSERT(mcp_res->rix_flow); 3242 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3243 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3244 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3245 } 3246 3247 /** 3248 * Add the default copy action in in RX_CP_TBL. 3249 * 3250 * @param dev 3251 * Pointer to Ethernet device. 3252 * @param[out] error 3253 * Perform verbose error reporting if not NULL. 3254 * 3255 * @return 3256 * 0 for success, negative value otherwise and rte_errno is set. 3257 */ 3258 static int 3259 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3260 struct rte_flow_error *error) 3261 { 3262 struct mlx5_priv *priv = dev->data->dev_private; 3263 struct mlx5_flow_mreg_copy_resource *mcp_res; 3264 3265 /* Check whether extensive metadata feature is engaged. */ 3266 if (!priv->config.dv_flow_en || 3267 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3268 !mlx5_flow_ext_mreg_supported(dev) || 3269 !priv->sh->dv_regc0_mask) 3270 return 0; 3271 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3272 if (!mcp_res) 3273 return -rte_errno; 3274 return 0; 3275 } 3276 3277 /** 3278 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3279 * 3280 * All the flow having Q/RSS action should be split by 3281 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3282 * performs the following, 3283 * - CQE->flow_tag := reg_c[1] (MARK) 3284 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3285 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3286 * but there should be a flow per each MARK ID set by MARK action. 3287 * 3288 * For the aforementioned reason, if there's a MARK action in flow's action 3289 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3290 * the MARK ID to CQE's flow_tag like, 3291 * - If reg_c[1] is mark_id, 3292 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3293 * 3294 * For SET_META action which stores value in reg_c[0], as the destination is 3295 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3296 * MARK ID means the default flow. The default flow looks like, 3297 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3298 * 3299 * @param dev 3300 * Pointer to Ethernet device. 3301 * @param flow 3302 * Pointer to flow structure. 3303 * @param[in] actions 3304 * Pointer to the list of actions. 3305 * @param[out] error 3306 * Perform verbose error reporting if not NULL. 3307 * 3308 * @return 3309 * 0 on success, negative value otherwise and rte_errno is set. 3310 */ 3311 static int 3312 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3313 struct rte_flow *flow, 3314 const struct rte_flow_action *actions, 3315 struct rte_flow_error *error) 3316 { 3317 struct mlx5_priv *priv = dev->data->dev_private; 3318 struct mlx5_dev_config *config = &priv->config; 3319 struct mlx5_flow_mreg_copy_resource *mcp_res; 3320 const struct rte_flow_action_mark *mark; 3321 3322 /* Check whether extensive metadata feature is engaged. */ 3323 if (!config->dv_flow_en || 3324 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3325 !mlx5_flow_ext_mreg_supported(dev) || 3326 !priv->sh->dv_regc0_mask) 3327 return 0; 3328 /* Find MARK action. */ 3329 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3330 switch (actions->type) { 3331 case RTE_FLOW_ACTION_TYPE_FLAG: 3332 mcp_res = flow_mreg_add_copy_action 3333 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3334 if (!mcp_res) 3335 return -rte_errno; 3336 flow->rix_mreg_copy = mcp_res->idx; 3337 if (dev->data->dev_started) { 3338 mcp_res->appcnt++; 3339 flow->copy_applied = 1; 3340 } 3341 return 0; 3342 case RTE_FLOW_ACTION_TYPE_MARK: 3343 mark = (const struct rte_flow_action_mark *) 3344 actions->conf; 3345 mcp_res = 3346 flow_mreg_add_copy_action(dev, mark->id, error); 3347 if (!mcp_res) 3348 return -rte_errno; 3349 flow->rix_mreg_copy = mcp_res->idx; 3350 if (dev->data->dev_started) { 3351 mcp_res->appcnt++; 3352 flow->copy_applied = 1; 3353 } 3354 return 0; 3355 default: 3356 break; 3357 } 3358 } 3359 return 0; 3360 } 3361 3362 #define MLX5_MAX_SPLIT_ACTIONS 24 3363 #define MLX5_MAX_SPLIT_ITEMS 24 3364 3365 /** 3366 * Split the hairpin flow. 3367 * Since HW can't support encap on Rx we move the encap to Tx. 3368 * If the count action is after the encap then we also 3369 * move the count action. in this case the count will also measure 3370 * the outer bytes. 3371 * 3372 * @param dev 3373 * Pointer to Ethernet device. 3374 * @param[in] actions 3375 * Associated actions (list terminated by the END action). 3376 * @param[out] actions_rx 3377 * Rx flow actions. 3378 * @param[out] actions_tx 3379 * Tx flow actions.. 3380 * @param[out] pattern_tx 3381 * The pattern items for the Tx flow. 3382 * @param[out] flow_id 3383 * The flow ID connected to this flow. 3384 * 3385 * @return 3386 * 0 on success. 3387 */ 3388 static int 3389 flow_hairpin_split(struct rte_eth_dev *dev, 3390 const struct rte_flow_action actions[], 3391 struct rte_flow_action actions_rx[], 3392 struct rte_flow_action actions_tx[], 3393 struct rte_flow_item pattern_tx[], 3394 uint32_t *flow_id) 3395 { 3396 struct mlx5_priv *priv = dev->data->dev_private; 3397 const struct rte_flow_action_raw_encap *raw_encap; 3398 const struct rte_flow_action_raw_decap *raw_decap; 3399 struct mlx5_rte_flow_action_set_tag *set_tag; 3400 struct rte_flow_action *tag_action; 3401 struct mlx5_rte_flow_item_tag *tag_item; 3402 struct rte_flow_item *item; 3403 char *addr; 3404 int encap = 0; 3405 3406 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3407 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3408 switch (actions->type) { 3409 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3410 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3411 rte_memcpy(actions_tx, actions, 3412 sizeof(struct rte_flow_action)); 3413 actions_tx++; 3414 break; 3415 case RTE_FLOW_ACTION_TYPE_COUNT: 3416 if (encap) { 3417 rte_memcpy(actions_tx, actions, 3418 sizeof(struct rte_flow_action)); 3419 actions_tx++; 3420 } else { 3421 rte_memcpy(actions_rx, actions, 3422 sizeof(struct rte_flow_action)); 3423 actions_rx++; 3424 } 3425 break; 3426 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3427 raw_encap = actions->conf; 3428 if (raw_encap->size > 3429 (sizeof(struct rte_flow_item_eth) + 3430 sizeof(struct rte_flow_item_ipv4))) { 3431 memcpy(actions_tx, actions, 3432 sizeof(struct rte_flow_action)); 3433 actions_tx++; 3434 encap = 1; 3435 } else { 3436 rte_memcpy(actions_rx, actions, 3437 sizeof(struct rte_flow_action)); 3438 actions_rx++; 3439 } 3440 break; 3441 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3442 raw_decap = actions->conf; 3443 if (raw_decap->size < 3444 (sizeof(struct rte_flow_item_eth) + 3445 sizeof(struct rte_flow_item_ipv4))) { 3446 memcpy(actions_tx, actions, 3447 sizeof(struct rte_flow_action)); 3448 actions_tx++; 3449 } else { 3450 rte_memcpy(actions_rx, actions, 3451 sizeof(struct rte_flow_action)); 3452 actions_rx++; 3453 } 3454 break; 3455 default: 3456 rte_memcpy(actions_rx, actions, 3457 sizeof(struct rte_flow_action)); 3458 actions_rx++; 3459 break; 3460 } 3461 } 3462 /* Add set meta action and end action for the Rx flow. */ 3463 tag_action = actions_rx; 3464 tag_action->type = (enum rte_flow_action_type) 3465 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3466 actions_rx++; 3467 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3468 actions_rx++; 3469 set_tag = (void *)actions_rx; 3470 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3471 MLX5_ASSERT(set_tag->id > REG_NONE); 3472 set_tag->data = *flow_id; 3473 tag_action->conf = set_tag; 3474 /* Create Tx item list. */ 3475 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3476 addr = (void *)&pattern_tx[2]; 3477 item = pattern_tx; 3478 item->type = (enum rte_flow_item_type) 3479 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3480 tag_item = (void *)addr; 3481 tag_item->data = *flow_id; 3482 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3483 MLX5_ASSERT(set_tag->id > REG_NONE); 3484 item->spec = tag_item; 3485 addr += sizeof(struct mlx5_rte_flow_item_tag); 3486 tag_item = (void *)addr; 3487 tag_item->data = UINT32_MAX; 3488 tag_item->id = UINT16_MAX; 3489 item->mask = tag_item; 3490 addr += sizeof(struct mlx5_rte_flow_item_tag); 3491 item->last = NULL; 3492 item++; 3493 item->type = RTE_FLOW_ITEM_TYPE_END; 3494 return 0; 3495 } 3496 3497 /** 3498 * The last stage of splitting chain, just creates the subflow 3499 * without any modification. 3500 * 3501 * @param[in] dev 3502 * Pointer to Ethernet device. 3503 * @param[in] flow 3504 * Parent flow structure pointer. 3505 * @param[in, out] sub_flow 3506 * Pointer to return the created subflow, may be NULL. 3507 * @param[in] prefix_layers 3508 * Prefix subflow layers, may be 0. 3509 * @param[in] attr 3510 * Flow rule attributes. 3511 * @param[in] items 3512 * Pattern specification (list terminated by the END pattern item). 3513 * @param[in] actions 3514 * Associated actions (list terminated by the END action). 3515 * @param[in] external 3516 * This flow rule is created by request external to PMD. 3517 * @param[in] flow_idx 3518 * This memory pool index to the flow. 3519 * @param[out] error 3520 * Perform verbose error reporting if not NULL. 3521 * @return 3522 * 0 on success, negative value otherwise 3523 */ 3524 static int 3525 flow_create_split_inner(struct rte_eth_dev *dev, 3526 struct rte_flow *flow, 3527 struct mlx5_flow **sub_flow, 3528 uint64_t prefix_layers, 3529 const struct rte_flow_attr *attr, 3530 const struct rte_flow_item items[], 3531 const struct rte_flow_action actions[], 3532 bool external, uint32_t flow_idx, 3533 struct rte_flow_error *error) 3534 { 3535 struct mlx5_flow *dev_flow; 3536 3537 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 3538 flow_idx, error); 3539 if (!dev_flow) 3540 return -rte_errno; 3541 dev_flow->flow = flow; 3542 dev_flow->external = external; 3543 /* Subflow object was created, we must include one in the list. */ 3544 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 3545 dev_flow->handle, next); 3546 /* 3547 * If dev_flow is as one of the suffix flow, some actions in suffix 3548 * flow may need some user defined item layer flags. 3549 */ 3550 if (prefix_layers) 3551 dev_flow->handle->layers = prefix_layers; 3552 if (sub_flow) 3553 *sub_flow = dev_flow; 3554 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3555 } 3556 3557 /** 3558 * Split the meter flow. 3559 * 3560 * As meter flow will split to three sub flow, other than meter 3561 * action, the other actions make sense to only meter accepts 3562 * the packet. If it need to be dropped, no other additional 3563 * actions should be take. 3564 * 3565 * One kind of special action which decapsulates the L3 tunnel 3566 * header will be in the prefix sub flow, as not to take the 3567 * L3 tunnel header into account. 3568 * 3569 * @param dev 3570 * Pointer to Ethernet device. 3571 * @param[in] items 3572 * Pattern specification (list terminated by the END pattern item). 3573 * @param[out] sfx_items 3574 * Suffix flow match items (list terminated by the END pattern item). 3575 * @param[in] actions 3576 * Associated actions (list terminated by the END action). 3577 * @param[out] actions_sfx 3578 * Suffix flow actions. 3579 * @param[out] actions_pre 3580 * Prefix flow actions. 3581 * @param[out] pattern_sfx 3582 * The pattern items for the suffix flow. 3583 * @param[out] tag_sfx 3584 * Pointer to suffix flow tag. 3585 * 3586 * @return 3587 * 0 on success. 3588 */ 3589 static int 3590 flow_meter_split_prep(struct rte_eth_dev *dev, 3591 const struct rte_flow_item items[], 3592 struct rte_flow_item sfx_items[], 3593 const struct rte_flow_action actions[], 3594 struct rte_flow_action actions_sfx[], 3595 struct rte_flow_action actions_pre[]) 3596 { 3597 struct rte_flow_action *tag_action = NULL; 3598 struct rte_flow_item *tag_item; 3599 struct mlx5_rte_flow_action_set_tag *set_tag; 3600 struct rte_flow_error error; 3601 const struct rte_flow_action_raw_encap *raw_encap; 3602 const struct rte_flow_action_raw_decap *raw_decap; 3603 struct mlx5_rte_flow_item_tag *tag_spec; 3604 struct mlx5_rte_flow_item_tag *tag_mask; 3605 uint32_t tag_id; 3606 bool copy_vlan = false; 3607 3608 /* Prepare the actions for prefix and suffix flow. */ 3609 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3610 struct rte_flow_action **action_cur = NULL; 3611 3612 switch (actions->type) { 3613 case RTE_FLOW_ACTION_TYPE_METER: 3614 /* Add the extra tag action first. */ 3615 tag_action = actions_pre; 3616 tag_action->type = (enum rte_flow_action_type) 3617 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3618 actions_pre++; 3619 action_cur = &actions_pre; 3620 break; 3621 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3622 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3623 action_cur = &actions_pre; 3624 break; 3625 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3626 raw_encap = actions->conf; 3627 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 3628 action_cur = &actions_pre; 3629 break; 3630 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3631 raw_decap = actions->conf; 3632 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3633 action_cur = &actions_pre; 3634 break; 3635 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3636 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3637 copy_vlan = true; 3638 break; 3639 default: 3640 break; 3641 } 3642 if (!action_cur) 3643 action_cur = &actions_sfx; 3644 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 3645 (*action_cur)++; 3646 } 3647 /* Add end action to the actions. */ 3648 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3649 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3650 actions_pre++; 3651 /* Set the tag. */ 3652 set_tag = (void *)actions_pre; 3653 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3654 /* 3655 * Get the id from the qrss_pool to make qrss share the id with meter. 3656 */ 3657 tag_id = flow_qrss_get_id(dev); 3658 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 3659 assert(tag_action); 3660 tag_action->conf = set_tag; 3661 /* Prepare the suffix subflow items. */ 3662 tag_item = sfx_items++; 3663 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 3664 int item_type = items->type; 3665 3666 switch (item_type) { 3667 case RTE_FLOW_ITEM_TYPE_PORT_ID: 3668 memcpy(sfx_items, items, sizeof(*sfx_items)); 3669 sfx_items++; 3670 break; 3671 case RTE_FLOW_ITEM_TYPE_VLAN: 3672 if (copy_vlan) { 3673 memcpy(sfx_items, items, sizeof(*sfx_items)); 3674 /* 3675 * Convert to internal match item, it is used 3676 * for vlan push and set vid. 3677 */ 3678 sfx_items->type = (enum rte_flow_item_type) 3679 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 3680 sfx_items++; 3681 } 3682 break; 3683 default: 3684 break; 3685 } 3686 } 3687 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 3688 sfx_items++; 3689 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 3690 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 3691 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3692 tag_mask = tag_spec + 1; 3693 tag_mask->data = 0xffffff00; 3694 tag_item->type = (enum rte_flow_item_type) 3695 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3696 tag_item->spec = tag_spec; 3697 tag_item->last = NULL; 3698 tag_item->mask = tag_mask; 3699 return tag_id; 3700 } 3701 3702 /** 3703 * Split action list having QUEUE/RSS for metadata register copy. 3704 * 3705 * Once Q/RSS action is detected in user's action list, the flow action 3706 * should be split in order to copy metadata registers, which will happen in 3707 * RX_CP_TBL like, 3708 * - CQE->flow_tag := reg_c[1] (MARK) 3709 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3710 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3711 * This is because the last action of each flow must be a terminal action 3712 * (QUEUE, RSS or DROP). 3713 * 3714 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3715 * stored and kept in the mlx5_flow structure per each sub_flow. 3716 * 3717 * The Q/RSS action is replaced with, 3718 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3719 * And the following JUMP action is added at the end, 3720 * - JUMP, to RX_CP_TBL. 3721 * 3722 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3723 * flow_create_split_metadata() routine. The flow will look like, 3724 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3725 * 3726 * @param dev 3727 * Pointer to Ethernet device. 3728 * @param[out] split_actions 3729 * Pointer to store split actions to jump to CP_TBL. 3730 * @param[in] actions 3731 * Pointer to the list of original flow actions. 3732 * @param[in] qrss 3733 * Pointer to the Q/RSS action. 3734 * @param[in] actions_n 3735 * Number of original actions. 3736 * @param[out] error 3737 * Perform verbose error reporting if not NULL. 3738 * 3739 * @return 3740 * non-zero unique flow_id on success, otherwise 0 and 3741 * error/rte_error are set. 3742 */ 3743 static uint32_t 3744 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3745 struct rte_flow_action *split_actions, 3746 const struct rte_flow_action *actions, 3747 const struct rte_flow_action *qrss, 3748 int actions_n, struct rte_flow_error *error) 3749 { 3750 struct mlx5_rte_flow_action_set_tag *set_tag; 3751 struct rte_flow_action_jump *jump; 3752 const int qrss_idx = qrss - actions; 3753 uint32_t flow_id = 0; 3754 int ret = 0; 3755 3756 /* 3757 * Given actions will be split 3758 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3759 * - Add jump to mreg CP_TBL. 3760 * As a result, there will be one more action. 3761 */ 3762 ++actions_n; 3763 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3764 set_tag = (void *)(split_actions + actions_n); 3765 /* 3766 * If tag action is not set to void(it means we are not the meter 3767 * suffix flow), add the tag action. Since meter suffix flow already 3768 * has the tag added. 3769 */ 3770 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3771 /* 3772 * Allocate the new subflow ID. This one is unique within 3773 * device and not shared with representors. Otherwise, 3774 * we would have to resolve multi-thread access synch 3775 * issue. Each flow on the shared device is appended 3776 * with source vport identifier, so the resulting 3777 * flows will be unique in the shared (by master and 3778 * representors) domain even if they have coinciding 3779 * IDs. 3780 */ 3781 flow_id = flow_qrss_get_id(dev); 3782 if (!flow_id) 3783 return rte_flow_error_set(error, ENOMEM, 3784 RTE_FLOW_ERROR_TYPE_ACTION, 3785 NULL, "can't allocate id " 3786 "for split Q/RSS subflow"); 3787 /* Internal SET_TAG action to set flow ID. */ 3788 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3789 .data = flow_id, 3790 }; 3791 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3792 if (ret < 0) 3793 return ret; 3794 set_tag->id = ret; 3795 /* Construct new actions array. */ 3796 /* Replace QUEUE/RSS action. */ 3797 split_actions[qrss_idx] = (struct rte_flow_action){ 3798 .type = (enum rte_flow_action_type) 3799 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3800 .conf = set_tag, 3801 }; 3802 } 3803 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3804 jump = (void *)(set_tag + 1); 3805 *jump = (struct rte_flow_action_jump){ 3806 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3807 }; 3808 split_actions[actions_n - 2] = (struct rte_flow_action){ 3809 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3810 .conf = jump, 3811 }; 3812 split_actions[actions_n - 1] = (struct rte_flow_action){ 3813 .type = RTE_FLOW_ACTION_TYPE_END, 3814 }; 3815 return flow_id; 3816 } 3817 3818 /** 3819 * Extend the given action list for Tx metadata copy. 3820 * 3821 * Copy the given action list to the ext_actions and add flow metadata register 3822 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3823 * 3824 * @param[out] ext_actions 3825 * Pointer to the extended action list. 3826 * @param[in] actions 3827 * Pointer to the list of actions. 3828 * @param[in] actions_n 3829 * Number of actions in the list. 3830 * @param[out] error 3831 * Perform verbose error reporting if not NULL. 3832 * @param[in] encap_idx 3833 * The encap action inndex. 3834 * 3835 * @return 3836 * 0 on success, negative value otherwise 3837 */ 3838 static int 3839 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3840 struct rte_flow_action *ext_actions, 3841 const struct rte_flow_action *actions, 3842 int actions_n, struct rte_flow_error *error, 3843 int encap_idx) 3844 { 3845 struct mlx5_flow_action_copy_mreg *cp_mreg = 3846 (struct mlx5_flow_action_copy_mreg *) 3847 (ext_actions + actions_n + 1); 3848 int ret; 3849 3850 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3851 if (ret < 0) 3852 return ret; 3853 cp_mreg->dst = ret; 3854 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3855 if (ret < 0) 3856 return ret; 3857 cp_mreg->src = ret; 3858 if (encap_idx != 0) 3859 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 3860 if (encap_idx == actions_n - 1) { 3861 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3862 .type = (enum rte_flow_action_type) 3863 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3864 .conf = cp_mreg, 3865 }; 3866 ext_actions[actions_n] = (struct rte_flow_action){ 3867 .type = RTE_FLOW_ACTION_TYPE_END, 3868 }; 3869 } else { 3870 ext_actions[encap_idx] = (struct rte_flow_action){ 3871 .type = (enum rte_flow_action_type) 3872 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3873 .conf = cp_mreg, 3874 }; 3875 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 3876 sizeof(*ext_actions) * (actions_n - encap_idx)); 3877 } 3878 return 0; 3879 } 3880 3881 /** 3882 * The splitting for metadata feature. 3883 * 3884 * - Q/RSS action on NIC Rx should be split in order to pass by 3885 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3886 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3887 * 3888 * - All the actions on NIC Tx should have a mreg copy action to 3889 * copy reg_a from WQE to reg_c[0]. 3890 * 3891 * @param dev 3892 * Pointer to Ethernet device. 3893 * @param[in] flow 3894 * Parent flow structure pointer. 3895 * @param[in] prefix_layers 3896 * Prefix flow layer flags. 3897 * @param[in] attr 3898 * Flow rule attributes. 3899 * @param[in] items 3900 * Pattern specification (list terminated by the END pattern item). 3901 * @param[in] actions 3902 * Associated actions (list terminated by the END action). 3903 * @param[in] external 3904 * This flow rule is created by request external to PMD. 3905 * @param[in] flow_idx 3906 * This memory pool index to the flow. 3907 * @param[out] error 3908 * Perform verbose error reporting if not NULL. 3909 * @return 3910 * 0 on success, negative value otherwise 3911 */ 3912 static int 3913 flow_create_split_metadata(struct rte_eth_dev *dev, 3914 struct rte_flow *flow, 3915 uint64_t prefix_layers, 3916 const struct rte_flow_attr *attr, 3917 const struct rte_flow_item items[], 3918 const struct rte_flow_action actions[], 3919 bool external, uint32_t flow_idx, 3920 struct rte_flow_error *error) 3921 { 3922 struct mlx5_priv *priv = dev->data->dev_private; 3923 struct mlx5_dev_config *config = &priv->config; 3924 const struct rte_flow_action *qrss = NULL; 3925 struct rte_flow_action *ext_actions = NULL; 3926 struct mlx5_flow *dev_flow = NULL; 3927 uint32_t qrss_id = 0; 3928 int mtr_sfx = 0; 3929 size_t act_size; 3930 int actions_n; 3931 int encap_idx; 3932 int ret; 3933 3934 /* Check whether extensive metadata feature is engaged. */ 3935 if (!config->dv_flow_en || 3936 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3937 !mlx5_flow_ext_mreg_supported(dev)) 3938 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 3939 attr, items, actions, external, 3940 flow_idx, error); 3941 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 3942 &encap_idx); 3943 if (qrss) { 3944 /* Exclude hairpin flows from splitting. */ 3945 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3946 const struct rte_flow_action_queue *queue; 3947 3948 queue = qrss->conf; 3949 if (mlx5_rxq_get_type(dev, queue->index) == 3950 MLX5_RXQ_TYPE_HAIRPIN) 3951 qrss = NULL; 3952 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3953 const struct rte_flow_action_rss *rss; 3954 3955 rss = qrss->conf; 3956 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3957 MLX5_RXQ_TYPE_HAIRPIN) 3958 qrss = NULL; 3959 } 3960 } 3961 if (qrss) { 3962 /* Check if it is in meter suffix table. */ 3963 mtr_sfx = attr->group == (attr->transfer ? 3964 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3965 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3966 /* 3967 * Q/RSS action on NIC Rx should be split in order to pass by 3968 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3969 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3970 */ 3971 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3972 sizeof(struct rte_flow_action_set_tag) + 3973 sizeof(struct rte_flow_action_jump); 3974 ext_actions = rte_zmalloc(__func__, act_size, 0); 3975 if (!ext_actions) 3976 return rte_flow_error_set(error, ENOMEM, 3977 RTE_FLOW_ERROR_TYPE_ACTION, 3978 NULL, "no memory to split " 3979 "metadata flow"); 3980 /* 3981 * If we are the suffix flow of meter, tag already exist. 3982 * Set the tag action to void. 3983 */ 3984 if (mtr_sfx) 3985 ext_actions[qrss - actions].type = 3986 RTE_FLOW_ACTION_TYPE_VOID; 3987 else 3988 ext_actions[qrss - actions].type = 3989 (enum rte_flow_action_type) 3990 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3991 /* 3992 * Create the new actions list with removed Q/RSS action 3993 * and appended set tag and jump to register copy table 3994 * (RX_CP_TBL). We should preallocate unique tag ID here 3995 * in advance, because it is needed for set tag action. 3996 */ 3997 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3998 qrss, actions_n, error); 3999 if (!mtr_sfx && !qrss_id) { 4000 ret = -rte_errno; 4001 goto exit; 4002 } 4003 } else if (attr->egress && !attr->transfer) { 4004 /* 4005 * All the actions on NIC Tx should have a metadata register 4006 * copy action to copy reg_a from WQE to reg_c[meta] 4007 */ 4008 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 4009 sizeof(struct mlx5_flow_action_copy_mreg); 4010 ext_actions = rte_zmalloc(__func__, act_size, 0); 4011 if (!ext_actions) 4012 return rte_flow_error_set(error, ENOMEM, 4013 RTE_FLOW_ERROR_TYPE_ACTION, 4014 NULL, "no memory to split " 4015 "metadata flow"); 4016 /* Create the action list appended with copy register. */ 4017 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 4018 actions_n, error, encap_idx); 4019 if (ret < 0) 4020 goto exit; 4021 } 4022 /* Add the unmodified original or prefix subflow. */ 4023 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr, 4024 items, ext_actions ? ext_actions : 4025 actions, external, flow_idx, error); 4026 if (ret < 0) 4027 goto exit; 4028 MLX5_ASSERT(dev_flow); 4029 if (qrss) { 4030 const struct rte_flow_attr q_attr = { 4031 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4032 .ingress = 1, 4033 }; 4034 /* Internal PMD action to set register. */ 4035 struct mlx5_rte_flow_item_tag q_tag_spec = { 4036 .data = qrss_id, 4037 .id = 0, 4038 }; 4039 struct rte_flow_item q_items[] = { 4040 { 4041 .type = (enum rte_flow_item_type) 4042 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4043 .spec = &q_tag_spec, 4044 .last = NULL, 4045 .mask = NULL, 4046 }, 4047 { 4048 .type = RTE_FLOW_ITEM_TYPE_END, 4049 }, 4050 }; 4051 struct rte_flow_action q_actions[] = { 4052 { 4053 .type = qrss->type, 4054 .conf = qrss->conf, 4055 }, 4056 { 4057 .type = RTE_FLOW_ACTION_TYPE_END, 4058 }, 4059 }; 4060 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4061 4062 /* 4063 * Configure the tag item only if there is no meter subflow. 4064 * Since tag is already marked in the meter suffix subflow 4065 * we can just use the meter suffix items as is. 4066 */ 4067 if (qrss_id) { 4068 /* Not meter subflow. */ 4069 MLX5_ASSERT(!mtr_sfx); 4070 /* 4071 * Put unique id in prefix flow due to it is destroyed 4072 * after suffix flow and id will be freed after there 4073 * is no actual flows with this id and identifier 4074 * reallocation becomes possible (for example, for 4075 * other flows in other threads). 4076 */ 4077 dev_flow->handle->split_flow_id = qrss_id; 4078 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4079 error); 4080 if (ret < 0) 4081 goto exit; 4082 q_tag_spec.id = ret; 4083 } 4084 dev_flow = NULL; 4085 /* Add suffix subflow to execute Q/RSS. */ 4086 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 4087 &q_attr, mtr_sfx ? items : 4088 q_items, q_actions, 4089 external, flow_idx, error); 4090 if (ret < 0) 4091 goto exit; 4092 /* qrss ID should be freed if failed. */ 4093 qrss_id = 0; 4094 MLX5_ASSERT(dev_flow); 4095 } 4096 4097 exit: 4098 /* 4099 * We do not destroy the partially created sub_flows in case of error. 4100 * These ones are included into parent flow list and will be destroyed 4101 * by flow_drv_destroy. 4102 */ 4103 flow_qrss_free_id(dev, qrss_id); 4104 rte_free(ext_actions); 4105 return ret; 4106 } 4107 4108 /** 4109 * The splitting for meter feature. 4110 * 4111 * - The meter flow will be split to two flows as prefix and 4112 * suffix flow. The packets make sense only it pass the prefix 4113 * meter action. 4114 * 4115 * - Reg_C_5 is used for the packet to match betweend prefix and 4116 * suffix flow. 4117 * 4118 * @param dev 4119 * Pointer to Ethernet device. 4120 * @param[in] flow 4121 * Parent flow structure pointer. 4122 * @param[in] attr 4123 * Flow rule attributes. 4124 * @param[in] items 4125 * Pattern specification (list terminated by the END pattern item). 4126 * @param[in] actions 4127 * Associated actions (list terminated by the END action). 4128 * @param[in] external 4129 * This flow rule is created by request external to PMD. 4130 * @param[in] flow_idx 4131 * This memory pool index to the flow. 4132 * @param[out] error 4133 * Perform verbose error reporting if not NULL. 4134 * @return 4135 * 0 on success, negative value otherwise 4136 */ 4137 static int 4138 flow_create_split_meter(struct rte_eth_dev *dev, 4139 struct rte_flow *flow, 4140 const struct rte_flow_attr *attr, 4141 const struct rte_flow_item items[], 4142 const struct rte_flow_action actions[], 4143 bool external, uint32_t flow_idx, 4144 struct rte_flow_error *error) 4145 { 4146 struct mlx5_priv *priv = dev->data->dev_private; 4147 struct rte_flow_action *sfx_actions = NULL; 4148 struct rte_flow_action *pre_actions = NULL; 4149 struct rte_flow_item *sfx_items = NULL; 4150 struct mlx5_flow *dev_flow = NULL; 4151 struct rte_flow_attr sfx_attr = *attr; 4152 uint32_t mtr = 0; 4153 uint32_t mtr_tag_id = 0; 4154 size_t act_size; 4155 size_t item_size; 4156 int actions_n = 0; 4157 int ret; 4158 4159 if (priv->mtr_en) 4160 actions_n = flow_check_meter_action(actions, &mtr); 4161 if (mtr) { 4162 /* The five prefix actions: meter, decap, encap, tag, end. */ 4163 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4164 sizeof(struct mlx5_rte_flow_action_set_tag); 4165 /* tag, vlan, port id, end. */ 4166 #define METER_SUFFIX_ITEM 4 4167 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4168 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4169 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 4170 if (!sfx_actions) 4171 return rte_flow_error_set(error, ENOMEM, 4172 RTE_FLOW_ERROR_TYPE_ACTION, 4173 NULL, "no memory to split " 4174 "meter flow"); 4175 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4176 act_size); 4177 pre_actions = sfx_actions + actions_n; 4178 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4179 actions, sfx_actions, 4180 pre_actions); 4181 if (!mtr_tag_id) { 4182 ret = -rte_errno; 4183 goto exit; 4184 } 4185 /* Add the prefix subflow. */ 4186 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr, 4187 items, pre_actions, external, 4188 flow_idx, error); 4189 if (ret) { 4190 ret = -rte_errno; 4191 goto exit; 4192 } 4193 dev_flow->handle->split_flow_id = mtr_tag_id; 4194 /* Setting the sfx group atrr. */ 4195 sfx_attr.group = sfx_attr.transfer ? 4196 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4197 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4198 } 4199 /* Add the prefix subflow. */ 4200 ret = flow_create_split_metadata(dev, flow, dev_flow ? 4201 flow_get_prefix_layer_flags(dev_flow) : 4202 0, &sfx_attr, 4203 sfx_items ? sfx_items : items, 4204 sfx_actions ? sfx_actions : actions, 4205 external, flow_idx, error); 4206 exit: 4207 if (sfx_actions) 4208 rte_free(sfx_actions); 4209 return ret; 4210 } 4211 4212 /** 4213 * Split the flow to subflow set. The splitters might be linked 4214 * in the chain, like this: 4215 * flow_create_split_outer() calls: 4216 * flow_create_split_meter() calls: 4217 * flow_create_split_metadata(meter_subflow_0) calls: 4218 * flow_create_split_inner(metadata_subflow_0) 4219 * flow_create_split_inner(metadata_subflow_1) 4220 * flow_create_split_inner(metadata_subflow_2) 4221 * flow_create_split_metadata(meter_subflow_1) calls: 4222 * flow_create_split_inner(metadata_subflow_0) 4223 * flow_create_split_inner(metadata_subflow_1) 4224 * flow_create_split_inner(metadata_subflow_2) 4225 * 4226 * This provide flexible way to add new levels of flow splitting. 4227 * The all of successfully created subflows are included to the 4228 * parent flow dev_flow list. 4229 * 4230 * @param dev 4231 * Pointer to Ethernet device. 4232 * @param[in] flow 4233 * Parent flow structure pointer. 4234 * @param[in] attr 4235 * Flow rule attributes. 4236 * @param[in] items 4237 * Pattern specification (list terminated by the END pattern item). 4238 * @param[in] actions 4239 * Associated actions (list terminated by the END action). 4240 * @param[in] external 4241 * This flow rule is created by request external to PMD. 4242 * @param[in] flow_idx 4243 * This memory pool index to the flow. 4244 * @param[out] error 4245 * Perform verbose error reporting if not NULL. 4246 * @return 4247 * 0 on success, negative value otherwise 4248 */ 4249 static int 4250 flow_create_split_outer(struct rte_eth_dev *dev, 4251 struct rte_flow *flow, 4252 const struct rte_flow_attr *attr, 4253 const struct rte_flow_item items[], 4254 const struct rte_flow_action actions[], 4255 bool external, uint32_t flow_idx, 4256 struct rte_flow_error *error) 4257 { 4258 int ret; 4259 4260 ret = flow_create_split_meter(dev, flow, attr, items, 4261 actions, external, flow_idx, error); 4262 MLX5_ASSERT(ret <= 0); 4263 return ret; 4264 } 4265 4266 /** 4267 * Create a flow and add it to @p list. 4268 * 4269 * @param dev 4270 * Pointer to Ethernet device. 4271 * @param list 4272 * Pointer to a TAILQ flow list. If this parameter NULL, 4273 * no list insertion occurred, flow is just created, 4274 * this is caller's responsibility to track the 4275 * created flow. 4276 * @param[in] attr 4277 * Flow rule attributes. 4278 * @param[in] items 4279 * Pattern specification (list terminated by the END pattern item). 4280 * @param[in] actions 4281 * Associated actions (list terminated by the END action). 4282 * @param[in] external 4283 * This flow rule is created by request external to PMD. 4284 * @param[out] error 4285 * Perform verbose error reporting if not NULL. 4286 * 4287 * @return 4288 * A flow index on success, 0 otherwise and rte_errno is set. 4289 */ 4290 static uint32_t 4291 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 4292 const struct rte_flow_attr *attr, 4293 const struct rte_flow_item items[], 4294 const struct rte_flow_action actions[], 4295 bool external, struct rte_flow_error *error) 4296 { 4297 struct mlx5_priv *priv = dev->data->dev_private; 4298 struct rte_flow *flow = NULL; 4299 struct mlx5_flow *dev_flow; 4300 const struct rte_flow_action_rss *rss; 4301 union { 4302 struct rte_flow_expand_rss buf; 4303 uint8_t buffer[2048]; 4304 } expand_buffer; 4305 union { 4306 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4307 uint8_t buffer[2048]; 4308 } actions_rx; 4309 union { 4310 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4311 uint8_t buffer[2048]; 4312 } actions_hairpin_tx; 4313 union { 4314 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4315 uint8_t buffer[2048]; 4316 } items_tx; 4317 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4318 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 4319 priv->rss_desc)[!!priv->flow_idx]; 4320 const struct rte_flow_action *p_actions_rx = actions; 4321 uint32_t i; 4322 uint32_t idx = 0; 4323 int hairpin_flow; 4324 uint32_t hairpin_id = 0; 4325 struct rte_flow_attr attr_tx = { .priority = 0 }; 4326 int ret; 4327 4328 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4329 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 4330 external, hairpin_flow, error); 4331 if (ret < 0) 4332 return 0; 4333 if (hairpin_flow > 0) { 4334 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4335 rte_errno = EINVAL; 4336 return 0; 4337 } 4338 flow_hairpin_split(dev, actions, actions_rx.actions, 4339 actions_hairpin_tx.actions, items_tx.items, 4340 &hairpin_id); 4341 p_actions_rx = actions_rx.actions; 4342 } 4343 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 4344 if (!flow) { 4345 rte_errno = ENOMEM; 4346 goto error_before_flow; 4347 } 4348 flow->drv_type = flow_get_drv_type(dev, attr); 4349 if (hairpin_id != 0) 4350 flow->hairpin_flow_id = hairpin_id; 4351 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4352 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4353 memset(rss_desc, 0, sizeof(*rss_desc)); 4354 rss = flow_get_rss_action(p_actions_rx); 4355 if (rss) { 4356 /* 4357 * The following information is required by 4358 * mlx5_flow_hashfields_adjust() in advance. 4359 */ 4360 rss_desc->level = rss->level; 4361 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4362 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 4363 } 4364 flow->dev_handles = 0; 4365 if (rss && rss->types) { 4366 unsigned int graph_root; 4367 4368 graph_root = find_graph_root(items, rss->level); 4369 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4370 items, rss->types, 4371 mlx5_support_expansion, 4372 graph_root); 4373 MLX5_ASSERT(ret > 0 && 4374 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4375 } else { 4376 buf->entries = 1; 4377 buf->entry[0].pattern = (void *)(uintptr_t)items; 4378 } 4379 /* 4380 * Record the start index when there is a nested call. All sub-flows 4381 * need to be translated before another calling. 4382 * No need to use ping-pong buffer to save memory here. 4383 */ 4384 if (priv->flow_idx) { 4385 MLX5_ASSERT(!priv->flow_nested_idx); 4386 priv->flow_nested_idx = priv->flow_idx; 4387 } 4388 for (i = 0; i < buf->entries; ++i) { 4389 /* 4390 * The splitter may create multiple dev_flows, 4391 * depending on configuration. In the simplest 4392 * case it just creates unmodified original flow. 4393 */ 4394 ret = flow_create_split_outer(dev, flow, attr, 4395 buf->entry[i].pattern, 4396 p_actions_rx, external, idx, 4397 error); 4398 if (ret < 0) 4399 goto error; 4400 } 4401 /* Create the tx flow. */ 4402 if (hairpin_flow) { 4403 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4404 attr_tx.ingress = 0; 4405 attr_tx.egress = 1; 4406 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 4407 actions_hairpin_tx.actions, 4408 idx, error); 4409 if (!dev_flow) 4410 goto error; 4411 dev_flow->flow = flow; 4412 dev_flow->external = 0; 4413 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4414 dev_flow->handle, next); 4415 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4416 items_tx.items, 4417 actions_hairpin_tx.actions, error); 4418 if (ret < 0) 4419 goto error; 4420 } 4421 /* 4422 * Update the metadata register copy table. If extensive 4423 * metadata feature is enabled and registers are supported 4424 * we might create the extra rte_flow for each unique 4425 * MARK/FLAG action ID. 4426 * 4427 * The table is updated for ingress Flows only, because 4428 * the egress Flows belong to the different device and 4429 * copy table should be updated in peer NIC Rx domain. 4430 */ 4431 if (attr->ingress && 4432 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4433 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4434 if (ret) 4435 goto error; 4436 } 4437 /* 4438 * If the flow is external (from application) OR device is started, then 4439 * the flow will be applied immediately. 4440 */ 4441 if (external || dev->data->dev_started) { 4442 ret = flow_drv_apply(dev, flow, error); 4443 if (ret < 0) 4444 goto error; 4445 } 4446 if (list) 4447 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 4448 flow, next); 4449 flow_rxq_flags_set(dev, flow); 4450 /* Nested flow creation index recovery. */ 4451 priv->flow_idx = priv->flow_nested_idx; 4452 if (priv->flow_nested_idx) 4453 priv->flow_nested_idx = 0; 4454 return idx; 4455 error: 4456 MLX5_ASSERT(flow); 4457 ret = rte_errno; /* Save rte_errno before cleanup. */ 4458 flow_mreg_del_copy_action(dev, flow); 4459 flow_drv_destroy(dev, flow); 4460 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 4461 rte_errno = ret; /* Restore rte_errno. */ 4462 error_before_flow: 4463 ret = rte_errno; 4464 if (hairpin_id) 4465 mlx5_flow_id_release(priv->sh->flow_id_pool, 4466 hairpin_id); 4467 rte_errno = ret; 4468 priv->flow_idx = priv->flow_nested_idx; 4469 if (priv->flow_nested_idx) 4470 priv->flow_nested_idx = 0; 4471 return 0; 4472 } 4473 4474 /** 4475 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4476 * incoming packets to table 1. 4477 * 4478 * Other flow rules, requested for group n, will be created in 4479 * e-switch table n+1. 4480 * Jump action to e-switch group n will be created to group n+1. 4481 * 4482 * Used when working in switchdev mode, to utilise advantages of table 1 4483 * and above. 4484 * 4485 * @param dev 4486 * Pointer to Ethernet device. 4487 * 4488 * @return 4489 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4490 */ 4491 struct rte_flow * 4492 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4493 { 4494 const struct rte_flow_attr attr = { 4495 .group = 0, 4496 .priority = 0, 4497 .ingress = 1, 4498 .egress = 0, 4499 .transfer = 1, 4500 }; 4501 const struct rte_flow_item pattern = { 4502 .type = RTE_FLOW_ITEM_TYPE_END, 4503 }; 4504 struct rte_flow_action_jump jump = { 4505 .group = 1, 4506 }; 4507 const struct rte_flow_action actions[] = { 4508 { 4509 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4510 .conf = &jump, 4511 }, 4512 { 4513 .type = RTE_FLOW_ACTION_TYPE_END, 4514 }, 4515 }; 4516 struct mlx5_priv *priv = dev->data->dev_private; 4517 struct rte_flow_error error; 4518 4519 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 4520 &attr, &pattern, 4521 actions, false, &error); 4522 } 4523 4524 /** 4525 * Validate a flow supported by the NIC. 4526 * 4527 * @see rte_flow_validate() 4528 * @see rte_flow_ops 4529 */ 4530 int 4531 mlx5_flow_validate(struct rte_eth_dev *dev, 4532 const struct rte_flow_attr *attr, 4533 const struct rte_flow_item items[], 4534 const struct rte_flow_action actions[], 4535 struct rte_flow_error *error) 4536 { 4537 int hairpin_flow; 4538 4539 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4540 return flow_drv_validate(dev, attr, items, actions, 4541 true, hairpin_flow, error); 4542 } 4543 4544 /** 4545 * Create a flow. 4546 * 4547 * @see rte_flow_create() 4548 * @see rte_flow_ops 4549 */ 4550 struct rte_flow * 4551 mlx5_flow_create(struct rte_eth_dev *dev, 4552 const struct rte_flow_attr *attr, 4553 const struct rte_flow_item items[], 4554 const struct rte_flow_action actions[], 4555 struct rte_flow_error *error) 4556 { 4557 struct mlx5_priv *priv = dev->data->dev_private; 4558 4559 /* 4560 * If the device is not started yet, it is not allowed to created a 4561 * flow from application. PMD default flows and traffic control flows 4562 * are not affected. 4563 */ 4564 if (unlikely(!dev->data->dev_started)) { 4565 DRV_LOG(DEBUG, "port %u is not started when " 4566 "inserting a flow", dev->data->port_id); 4567 rte_flow_error_set(error, ENODEV, 4568 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4569 NULL, 4570 "port not started"); 4571 return NULL; 4572 } 4573 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 4574 attr, items, actions, true, error); 4575 } 4576 4577 /** 4578 * Destroy a flow in a list. 4579 * 4580 * @param dev 4581 * Pointer to Ethernet device. 4582 * @param list 4583 * Pointer to the Indexed flow list. If this parameter NULL, 4584 * there is no flow removal from the list. Be noted that as 4585 * flow is add to the indexed list, memory of the indexed 4586 * list points to maybe changed as flow destroyed. 4587 * @param[in] flow_idx 4588 * Index of flow to destroy. 4589 */ 4590 static void 4591 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 4592 uint32_t flow_idx) 4593 { 4594 struct mlx5_priv *priv = dev->data->dev_private; 4595 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 4596 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 4597 [MLX5_IPOOL_RTE_FLOW], flow_idx); 4598 4599 if (!flow) 4600 return; 4601 /* 4602 * Update RX queue flags only if port is started, otherwise it is 4603 * already clean. 4604 */ 4605 if (dev->data->dev_started) 4606 flow_rxq_flags_trim(dev, flow); 4607 if (flow->hairpin_flow_id) 4608 mlx5_flow_id_release(priv->sh->flow_id_pool, 4609 flow->hairpin_flow_id); 4610 flow_drv_destroy(dev, flow); 4611 if (list) 4612 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 4613 flow_idx, flow, next); 4614 flow_mreg_del_copy_action(dev, flow); 4615 if (flow->fdir) { 4616 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 4617 if (priv_fdir_flow->rix_flow == flow_idx) 4618 break; 4619 } 4620 if (priv_fdir_flow) { 4621 LIST_REMOVE(priv_fdir_flow, next); 4622 rte_free(priv_fdir_flow->fdir); 4623 rte_free(priv_fdir_flow); 4624 } 4625 } 4626 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 4627 } 4628 4629 /** 4630 * Destroy all flows. 4631 * 4632 * @param dev 4633 * Pointer to Ethernet device. 4634 * @param list 4635 * Pointer to the Indexed flow list. 4636 * @param active 4637 * If flushing is called avtively. 4638 */ 4639 void 4640 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 4641 { 4642 uint32_t num_flushed = 0; 4643 4644 while (*list) { 4645 flow_list_destroy(dev, list, *list); 4646 num_flushed++; 4647 } 4648 if (active) { 4649 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 4650 dev->data->port_id, num_flushed); 4651 } 4652 } 4653 4654 /** 4655 * Remove all flows. 4656 * 4657 * @param dev 4658 * Pointer to Ethernet device. 4659 * @param list 4660 * Pointer to the Indexed flow list. 4661 */ 4662 void 4663 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list) 4664 { 4665 struct mlx5_priv *priv = dev->data->dev_private; 4666 struct rte_flow *flow = NULL; 4667 uint32_t idx; 4668 4669 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4670 flow, next) { 4671 flow_drv_remove(dev, flow); 4672 flow_mreg_stop_copy_action(dev, flow); 4673 } 4674 flow_mreg_del_default_copy_action(dev); 4675 flow_rxq_flags_clear(dev); 4676 } 4677 4678 /** 4679 * Add all flows. 4680 * 4681 * @param dev 4682 * Pointer to Ethernet device. 4683 * @param list 4684 * Pointer to the Indexed flow list. 4685 * 4686 * @return 4687 * 0 on success, a negative errno value otherwise and rte_errno is set. 4688 */ 4689 int 4690 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list) 4691 { 4692 struct mlx5_priv *priv = dev->data->dev_private; 4693 struct rte_flow *flow = NULL; 4694 struct rte_flow_error error; 4695 uint32_t idx; 4696 int ret = 0; 4697 4698 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4699 ret = flow_mreg_add_default_copy_action(dev, &error); 4700 if (ret < 0) 4701 return -rte_errno; 4702 /* Apply Flows created by application. */ 4703 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4704 flow, next) { 4705 ret = flow_mreg_start_copy_action(dev, flow); 4706 if (ret < 0) 4707 goto error; 4708 ret = flow_drv_apply(dev, flow, &error); 4709 if (ret < 0) 4710 goto error; 4711 flow_rxq_flags_set(dev, flow); 4712 } 4713 return 0; 4714 error: 4715 ret = rte_errno; /* Save rte_errno before cleanup. */ 4716 mlx5_flow_stop(dev, list); 4717 rte_errno = ret; /* Restore rte_errno. */ 4718 return -rte_errno; 4719 } 4720 4721 /** 4722 * Stop all default actions for flows. 4723 * 4724 * @param dev 4725 * Pointer to Ethernet device. 4726 */ 4727 void 4728 mlx5_flow_stop_default(struct rte_eth_dev *dev) 4729 { 4730 flow_mreg_del_default_copy_action(dev); 4731 flow_rxq_flags_clear(dev); 4732 } 4733 4734 /** 4735 * Start all default actions for flows. 4736 * 4737 * @param dev 4738 * Pointer to Ethernet device. 4739 * @return 4740 * 0 on success, a negative errno value otherwise and rte_errno is set. 4741 */ 4742 int 4743 mlx5_flow_start_default(struct rte_eth_dev *dev) 4744 { 4745 struct rte_flow_error error; 4746 4747 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4748 return flow_mreg_add_default_copy_action(dev, &error); 4749 } 4750 4751 /** 4752 * Allocate intermediate resources for flow creation. 4753 * 4754 * @param dev 4755 * Pointer to Ethernet device. 4756 */ 4757 void 4758 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) 4759 { 4760 struct mlx5_priv *priv = dev->data->dev_private; 4761 4762 if (!priv->inter_flows) { 4763 priv->inter_flows = rte_calloc(__func__, 1, 4764 MLX5_NUM_MAX_DEV_FLOWS * 4765 sizeof(struct mlx5_flow) + 4766 (sizeof(struct mlx5_flow_rss_desc) + 4767 sizeof(uint16_t) * UINT16_MAX) * 2, 0); 4768 if (!priv->inter_flows) { 4769 DRV_LOG(ERR, "can't allocate intermediate memory."); 4770 return; 4771 } 4772 } 4773 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows) 4774 [MLX5_NUM_MAX_DEV_FLOWS]; 4775 /* Reset the index. */ 4776 priv->flow_idx = 0; 4777 priv->flow_nested_idx = 0; 4778 } 4779 4780 /** 4781 * Free intermediate resources for flows. 4782 * 4783 * @param dev 4784 * Pointer to Ethernet device. 4785 */ 4786 void 4787 mlx5_flow_free_intermediate(struct rte_eth_dev *dev) 4788 { 4789 struct mlx5_priv *priv = dev->data->dev_private; 4790 4791 rte_free(priv->inter_flows); 4792 priv->inter_flows = NULL; 4793 } 4794 4795 /** 4796 * Verify the flow list is empty 4797 * 4798 * @param dev 4799 * Pointer to Ethernet device. 4800 * 4801 * @return the number of flows not released. 4802 */ 4803 int 4804 mlx5_flow_verify(struct rte_eth_dev *dev) 4805 { 4806 struct mlx5_priv *priv = dev->data->dev_private; 4807 struct rte_flow *flow; 4808 uint32_t idx; 4809 int ret = 0; 4810 4811 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 4812 flow, next) { 4813 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4814 dev->data->port_id, (void *)flow); 4815 ++ret; 4816 } 4817 return ret; 4818 } 4819 4820 /** 4821 * Enable default hairpin egress flow. 4822 * 4823 * @param dev 4824 * Pointer to Ethernet device. 4825 * @param queue 4826 * The queue index. 4827 * 4828 * @return 4829 * 0 on success, a negative errno value otherwise and rte_errno is set. 4830 */ 4831 int 4832 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4833 uint32_t queue) 4834 { 4835 struct mlx5_priv *priv = dev->data->dev_private; 4836 const struct rte_flow_attr attr = { 4837 .egress = 1, 4838 .priority = 0, 4839 }; 4840 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4841 .queue = queue, 4842 }; 4843 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4844 .queue = UINT32_MAX, 4845 }; 4846 struct rte_flow_item items[] = { 4847 { 4848 .type = (enum rte_flow_item_type) 4849 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4850 .spec = &queue_spec, 4851 .last = NULL, 4852 .mask = &queue_mask, 4853 }, 4854 { 4855 .type = RTE_FLOW_ITEM_TYPE_END, 4856 }, 4857 }; 4858 struct rte_flow_action_jump jump = { 4859 .group = MLX5_HAIRPIN_TX_TABLE, 4860 }; 4861 struct rte_flow_action actions[2]; 4862 uint32_t flow_idx; 4863 struct rte_flow_error error; 4864 4865 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4866 actions[0].conf = &jump; 4867 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4868 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4869 &attr, items, actions, false, &error); 4870 if (!flow_idx) { 4871 DRV_LOG(DEBUG, 4872 "Failed to create ctrl flow: rte_errno(%d)," 4873 " type(%d), message(%s)", 4874 rte_errno, error.type, 4875 error.message ? error.message : " (no stated reason)"); 4876 return -rte_errno; 4877 } 4878 return 0; 4879 } 4880 4881 /** 4882 * Enable a control flow configured from the control plane. 4883 * 4884 * @param dev 4885 * Pointer to Ethernet device. 4886 * @param eth_spec 4887 * An Ethernet flow spec to apply. 4888 * @param eth_mask 4889 * An Ethernet flow mask to apply. 4890 * @param vlan_spec 4891 * A VLAN flow spec to apply. 4892 * @param vlan_mask 4893 * A VLAN flow mask to apply. 4894 * 4895 * @return 4896 * 0 on success, a negative errno value otherwise and rte_errno is set. 4897 */ 4898 int 4899 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4900 struct rte_flow_item_eth *eth_spec, 4901 struct rte_flow_item_eth *eth_mask, 4902 struct rte_flow_item_vlan *vlan_spec, 4903 struct rte_flow_item_vlan *vlan_mask) 4904 { 4905 struct mlx5_priv *priv = dev->data->dev_private; 4906 const struct rte_flow_attr attr = { 4907 .ingress = 1, 4908 .priority = MLX5_FLOW_PRIO_RSVD, 4909 }; 4910 struct rte_flow_item items[] = { 4911 { 4912 .type = RTE_FLOW_ITEM_TYPE_ETH, 4913 .spec = eth_spec, 4914 .last = NULL, 4915 .mask = eth_mask, 4916 }, 4917 { 4918 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4919 RTE_FLOW_ITEM_TYPE_END, 4920 .spec = vlan_spec, 4921 .last = NULL, 4922 .mask = vlan_mask, 4923 }, 4924 { 4925 .type = RTE_FLOW_ITEM_TYPE_END, 4926 }, 4927 }; 4928 uint16_t queue[priv->reta_idx_n]; 4929 struct rte_flow_action_rss action_rss = { 4930 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4931 .level = 0, 4932 .types = priv->rss_conf.rss_hf, 4933 .key_len = priv->rss_conf.rss_key_len, 4934 .queue_num = priv->reta_idx_n, 4935 .key = priv->rss_conf.rss_key, 4936 .queue = queue, 4937 }; 4938 struct rte_flow_action actions[] = { 4939 { 4940 .type = RTE_FLOW_ACTION_TYPE_RSS, 4941 .conf = &action_rss, 4942 }, 4943 { 4944 .type = RTE_FLOW_ACTION_TYPE_END, 4945 }, 4946 }; 4947 uint32_t flow_idx; 4948 struct rte_flow_error error; 4949 unsigned int i; 4950 4951 if (!priv->reta_idx_n || !priv->rxqs_n) { 4952 return 0; 4953 } 4954 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) 4955 action_rss.types = 0; 4956 for (i = 0; i != priv->reta_idx_n; ++i) 4957 queue[i] = (*priv->reta_idx)[i]; 4958 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4959 &attr, items, actions, false, &error); 4960 if (!flow_idx) 4961 return -rte_errno; 4962 return 0; 4963 } 4964 4965 /** 4966 * Enable a flow control configured from the control plane. 4967 * 4968 * @param dev 4969 * Pointer to Ethernet device. 4970 * @param eth_spec 4971 * An Ethernet flow spec to apply. 4972 * @param eth_mask 4973 * An Ethernet flow mask to apply. 4974 * 4975 * @return 4976 * 0 on success, a negative errno value otherwise and rte_errno is set. 4977 */ 4978 int 4979 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4980 struct rte_flow_item_eth *eth_spec, 4981 struct rte_flow_item_eth *eth_mask) 4982 { 4983 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4984 } 4985 4986 /** 4987 * Destroy a flow. 4988 * 4989 * @see rte_flow_destroy() 4990 * @see rte_flow_ops 4991 */ 4992 int 4993 mlx5_flow_destroy(struct rte_eth_dev *dev, 4994 struct rte_flow *flow, 4995 struct rte_flow_error *error __rte_unused) 4996 { 4997 struct mlx5_priv *priv = dev->data->dev_private; 4998 4999 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 5000 return 0; 5001 } 5002 5003 /** 5004 * Destroy all flows. 5005 * 5006 * @see rte_flow_flush() 5007 * @see rte_flow_ops 5008 */ 5009 int 5010 mlx5_flow_flush(struct rte_eth_dev *dev, 5011 struct rte_flow_error *error __rte_unused) 5012 { 5013 struct mlx5_priv *priv = dev->data->dev_private; 5014 5015 mlx5_flow_list_flush(dev, &priv->flows, false); 5016 return 0; 5017 } 5018 5019 /** 5020 * Isolated mode. 5021 * 5022 * @see rte_flow_isolate() 5023 * @see rte_flow_ops 5024 */ 5025 int 5026 mlx5_flow_isolate(struct rte_eth_dev *dev, 5027 int enable, 5028 struct rte_flow_error *error) 5029 { 5030 struct mlx5_priv *priv = dev->data->dev_private; 5031 5032 if (dev->data->dev_started) { 5033 rte_flow_error_set(error, EBUSY, 5034 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5035 NULL, 5036 "port must be stopped first"); 5037 return -rte_errno; 5038 } 5039 priv->isolated = !!enable; 5040 if (enable) 5041 dev->dev_ops = &mlx5_os_dev_ops_isolate; 5042 else 5043 dev->dev_ops = &mlx5_os_dev_ops; 5044 return 0; 5045 } 5046 5047 /** 5048 * Query a flow. 5049 * 5050 * @see rte_flow_query() 5051 * @see rte_flow_ops 5052 */ 5053 static int 5054 flow_drv_query(struct rte_eth_dev *dev, 5055 uint32_t flow_idx, 5056 const struct rte_flow_action *actions, 5057 void *data, 5058 struct rte_flow_error *error) 5059 { 5060 struct mlx5_priv *priv = dev->data->dev_private; 5061 const struct mlx5_flow_driver_ops *fops; 5062 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5063 [MLX5_IPOOL_RTE_FLOW], 5064 flow_idx); 5065 enum mlx5_flow_drv_type ftype; 5066 5067 if (!flow) { 5068 return rte_flow_error_set(error, ENOENT, 5069 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5070 NULL, 5071 "invalid flow handle"); 5072 } 5073 ftype = flow->drv_type; 5074 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 5075 fops = flow_get_drv_ops(ftype); 5076 5077 return fops->query(dev, flow, actions, data, error); 5078 } 5079 5080 /** 5081 * Query a flow. 5082 * 5083 * @see rte_flow_query() 5084 * @see rte_flow_ops 5085 */ 5086 int 5087 mlx5_flow_query(struct rte_eth_dev *dev, 5088 struct rte_flow *flow, 5089 const struct rte_flow_action *actions, 5090 void *data, 5091 struct rte_flow_error *error) 5092 { 5093 int ret; 5094 5095 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 5096 error); 5097 if (ret < 0) 5098 return ret; 5099 return 0; 5100 } 5101 5102 /** 5103 * Convert a flow director filter to a generic flow. 5104 * 5105 * @param dev 5106 * Pointer to Ethernet device. 5107 * @param fdir_filter 5108 * Flow director filter to add. 5109 * @param attributes 5110 * Generic flow parameters structure. 5111 * 5112 * @return 5113 * 0 on success, a negative errno value otherwise and rte_errno is set. 5114 */ 5115 static int 5116 flow_fdir_filter_convert(struct rte_eth_dev *dev, 5117 const struct rte_eth_fdir_filter *fdir_filter, 5118 struct mlx5_fdir *attributes) 5119 { 5120 struct mlx5_priv *priv = dev->data->dev_private; 5121 const struct rte_eth_fdir_input *input = &fdir_filter->input; 5122 const struct rte_eth_fdir_masks *mask = 5123 &dev->data->dev_conf.fdir_conf.mask; 5124 5125 /* Validate queue number. */ 5126 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 5127 DRV_LOG(ERR, "port %u invalid queue number %d", 5128 dev->data->port_id, fdir_filter->action.rx_queue); 5129 rte_errno = EINVAL; 5130 return -rte_errno; 5131 } 5132 attributes->attr.ingress = 1; 5133 attributes->items[0] = (struct rte_flow_item) { 5134 .type = RTE_FLOW_ITEM_TYPE_ETH, 5135 .spec = &attributes->l2, 5136 .mask = &attributes->l2_mask, 5137 }; 5138 switch (fdir_filter->action.behavior) { 5139 case RTE_ETH_FDIR_ACCEPT: 5140 attributes->actions[0] = (struct rte_flow_action){ 5141 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 5142 .conf = &attributes->queue, 5143 }; 5144 break; 5145 case RTE_ETH_FDIR_REJECT: 5146 attributes->actions[0] = (struct rte_flow_action){ 5147 .type = RTE_FLOW_ACTION_TYPE_DROP, 5148 }; 5149 break; 5150 default: 5151 DRV_LOG(ERR, "port %u invalid behavior %d", 5152 dev->data->port_id, 5153 fdir_filter->action.behavior); 5154 rte_errno = ENOTSUP; 5155 return -rte_errno; 5156 } 5157 attributes->queue.index = fdir_filter->action.rx_queue; 5158 /* Handle L3. */ 5159 switch (fdir_filter->input.flow_type) { 5160 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5161 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5162 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5163 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 5164 .src_addr = input->flow.ip4_flow.src_ip, 5165 .dst_addr = input->flow.ip4_flow.dst_ip, 5166 .time_to_live = input->flow.ip4_flow.ttl, 5167 .type_of_service = input->flow.ip4_flow.tos, 5168 }; 5169 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 5170 .src_addr = mask->ipv4_mask.src_ip, 5171 .dst_addr = mask->ipv4_mask.dst_ip, 5172 .time_to_live = mask->ipv4_mask.ttl, 5173 .type_of_service = mask->ipv4_mask.tos, 5174 .next_proto_id = mask->ipv4_mask.proto, 5175 }; 5176 attributes->items[1] = (struct rte_flow_item){ 5177 .type = RTE_FLOW_ITEM_TYPE_IPV4, 5178 .spec = &attributes->l3, 5179 .mask = &attributes->l3_mask, 5180 }; 5181 break; 5182 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5183 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5184 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5185 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 5186 .hop_limits = input->flow.ipv6_flow.hop_limits, 5187 .proto = input->flow.ipv6_flow.proto, 5188 }; 5189 5190 memcpy(attributes->l3.ipv6.hdr.src_addr, 5191 input->flow.ipv6_flow.src_ip, 5192 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5193 memcpy(attributes->l3.ipv6.hdr.dst_addr, 5194 input->flow.ipv6_flow.dst_ip, 5195 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5196 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 5197 mask->ipv6_mask.src_ip, 5198 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5199 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 5200 mask->ipv6_mask.dst_ip, 5201 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5202 attributes->items[1] = (struct rte_flow_item){ 5203 .type = RTE_FLOW_ITEM_TYPE_IPV6, 5204 .spec = &attributes->l3, 5205 .mask = &attributes->l3_mask, 5206 }; 5207 break; 5208 default: 5209 DRV_LOG(ERR, "port %u invalid flow type%d", 5210 dev->data->port_id, fdir_filter->input.flow_type); 5211 rte_errno = ENOTSUP; 5212 return -rte_errno; 5213 } 5214 /* Handle L4. */ 5215 switch (fdir_filter->input.flow_type) { 5216 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5217 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5218 .src_port = input->flow.udp4_flow.src_port, 5219 .dst_port = input->flow.udp4_flow.dst_port, 5220 }; 5221 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5222 .src_port = mask->src_port_mask, 5223 .dst_port = mask->dst_port_mask, 5224 }; 5225 attributes->items[2] = (struct rte_flow_item){ 5226 .type = RTE_FLOW_ITEM_TYPE_UDP, 5227 .spec = &attributes->l4, 5228 .mask = &attributes->l4_mask, 5229 }; 5230 break; 5231 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5232 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5233 .src_port = input->flow.tcp4_flow.src_port, 5234 .dst_port = input->flow.tcp4_flow.dst_port, 5235 }; 5236 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5237 .src_port = mask->src_port_mask, 5238 .dst_port = mask->dst_port_mask, 5239 }; 5240 attributes->items[2] = (struct rte_flow_item){ 5241 .type = RTE_FLOW_ITEM_TYPE_TCP, 5242 .spec = &attributes->l4, 5243 .mask = &attributes->l4_mask, 5244 }; 5245 break; 5246 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5247 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5248 .src_port = input->flow.udp6_flow.src_port, 5249 .dst_port = input->flow.udp6_flow.dst_port, 5250 }; 5251 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5252 .src_port = mask->src_port_mask, 5253 .dst_port = mask->dst_port_mask, 5254 }; 5255 attributes->items[2] = (struct rte_flow_item){ 5256 .type = RTE_FLOW_ITEM_TYPE_UDP, 5257 .spec = &attributes->l4, 5258 .mask = &attributes->l4_mask, 5259 }; 5260 break; 5261 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5262 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5263 .src_port = input->flow.tcp6_flow.src_port, 5264 .dst_port = input->flow.tcp6_flow.dst_port, 5265 }; 5266 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5267 .src_port = mask->src_port_mask, 5268 .dst_port = mask->dst_port_mask, 5269 }; 5270 attributes->items[2] = (struct rte_flow_item){ 5271 .type = RTE_FLOW_ITEM_TYPE_TCP, 5272 .spec = &attributes->l4, 5273 .mask = &attributes->l4_mask, 5274 }; 5275 break; 5276 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5277 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5278 break; 5279 default: 5280 DRV_LOG(ERR, "port %u invalid flow type%d", 5281 dev->data->port_id, fdir_filter->input.flow_type); 5282 rte_errno = ENOTSUP; 5283 return -rte_errno; 5284 } 5285 return 0; 5286 } 5287 5288 #define FLOW_FDIR_CMP(f1, f2, fld) \ 5289 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 5290 5291 /** 5292 * Compare two FDIR flows. If items and actions are identical, the two flows are 5293 * regarded as same. 5294 * 5295 * @param dev 5296 * Pointer to Ethernet device. 5297 * @param f1 5298 * FDIR flow to compare. 5299 * @param f2 5300 * FDIR flow to compare. 5301 * 5302 * @return 5303 * Zero on match, 1 otherwise. 5304 */ 5305 static int 5306 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 5307 { 5308 if (FLOW_FDIR_CMP(f1, f2, attr) || 5309 FLOW_FDIR_CMP(f1, f2, l2) || 5310 FLOW_FDIR_CMP(f1, f2, l2_mask) || 5311 FLOW_FDIR_CMP(f1, f2, l3) || 5312 FLOW_FDIR_CMP(f1, f2, l3_mask) || 5313 FLOW_FDIR_CMP(f1, f2, l4) || 5314 FLOW_FDIR_CMP(f1, f2, l4_mask) || 5315 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 5316 return 1; 5317 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 5318 FLOW_FDIR_CMP(f1, f2, queue)) 5319 return 1; 5320 return 0; 5321 } 5322 5323 /** 5324 * Search device flow list to find out a matched FDIR flow. 5325 * 5326 * @param dev 5327 * Pointer to Ethernet device. 5328 * @param fdir_flow 5329 * FDIR flow to lookup. 5330 * 5331 * @return 5332 * Index of flow if found, 0 otherwise. 5333 */ 5334 static uint32_t 5335 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5336 { 5337 struct mlx5_priv *priv = dev->data->dev_private; 5338 uint32_t flow_idx = 0; 5339 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5340 5341 MLX5_ASSERT(fdir_flow); 5342 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5343 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { 5344 DRV_LOG(DEBUG, "port %u found FDIR flow %u", 5345 dev->data->port_id, flow_idx); 5346 flow_idx = priv_fdir_flow->rix_flow; 5347 break; 5348 } 5349 } 5350 return flow_idx; 5351 } 5352 5353 /** 5354 * Add new flow director filter and store it in list. 5355 * 5356 * @param dev 5357 * Pointer to Ethernet device. 5358 * @param fdir_filter 5359 * Flow director filter to add. 5360 * 5361 * @return 5362 * 0 on success, a negative errno value otherwise and rte_errno is set. 5363 */ 5364 static int 5365 flow_fdir_filter_add(struct rte_eth_dev *dev, 5366 const struct rte_eth_fdir_filter *fdir_filter) 5367 { 5368 struct mlx5_priv *priv = dev->data->dev_private; 5369 struct mlx5_fdir *fdir_flow; 5370 struct rte_flow *flow; 5371 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5372 uint32_t flow_idx; 5373 int ret; 5374 5375 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5376 if (!fdir_flow) { 5377 rte_errno = ENOMEM; 5378 return -rte_errno; 5379 } 5380 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5381 if (ret) 5382 goto error; 5383 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); 5384 if (flow_idx) { 5385 rte_errno = EEXIST; 5386 goto error; 5387 } 5388 priv_fdir_flow = rte_zmalloc(__func__, sizeof(struct mlx5_fdir_flow), 5389 0); 5390 if (!priv_fdir_flow) { 5391 rte_errno = ENOMEM; 5392 goto error; 5393 } 5394 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5395 fdir_flow->items, fdir_flow->actions, true, 5396 NULL); 5397 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5398 if (!flow) 5399 goto error; 5400 flow->fdir = 1; 5401 priv_fdir_flow->fdir = fdir_flow; 5402 priv_fdir_flow->rix_flow = flow_idx; 5403 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); 5404 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5405 dev->data->port_id, (void *)flow); 5406 return 0; 5407 error: 5408 rte_free(priv_fdir_flow); 5409 rte_free(fdir_flow); 5410 return -rte_errno; 5411 } 5412 5413 /** 5414 * Delete specific filter. 5415 * 5416 * @param dev 5417 * Pointer to Ethernet device. 5418 * @param fdir_filter 5419 * Filter to be deleted. 5420 * 5421 * @return 5422 * 0 on success, a negative errno value otherwise and rte_errno is set. 5423 */ 5424 static int 5425 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5426 const struct rte_eth_fdir_filter *fdir_filter) 5427 { 5428 struct mlx5_priv *priv = dev->data->dev_private; 5429 uint32_t flow_idx; 5430 struct mlx5_fdir fdir_flow = { 5431 .attr.group = 0, 5432 }; 5433 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5434 int ret; 5435 5436 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5437 if (ret) 5438 return -rte_errno; 5439 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5440 /* Find the fdir in priv list */ 5441 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) 5442 break; 5443 } 5444 if (!priv_fdir_flow) 5445 return 0; 5446 LIST_REMOVE(priv_fdir_flow, next); 5447 flow_idx = priv_fdir_flow->rix_flow; 5448 flow_list_destroy(dev, &priv->flows, flow_idx); 5449 rte_free(priv_fdir_flow->fdir); 5450 rte_free(priv_fdir_flow); 5451 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", 5452 dev->data->port_id, flow_idx); 5453 return 0; 5454 } 5455 5456 /** 5457 * Update queue for specific filter. 5458 * 5459 * @param dev 5460 * Pointer to Ethernet device. 5461 * @param fdir_filter 5462 * Filter to be updated. 5463 * 5464 * @return 5465 * 0 on success, a negative errno value otherwise and rte_errno is set. 5466 */ 5467 static int 5468 flow_fdir_filter_update(struct rte_eth_dev *dev, 5469 const struct rte_eth_fdir_filter *fdir_filter) 5470 { 5471 int ret; 5472 5473 ret = flow_fdir_filter_delete(dev, fdir_filter); 5474 if (ret) 5475 return ret; 5476 return flow_fdir_filter_add(dev, fdir_filter); 5477 } 5478 5479 /** 5480 * Flush all filters. 5481 * 5482 * @param dev 5483 * Pointer to Ethernet device. 5484 */ 5485 static void 5486 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5487 { 5488 struct mlx5_priv *priv = dev->data->dev_private; 5489 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5490 5491 while (!LIST_EMPTY(&priv->fdir_flows)) { 5492 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); 5493 LIST_REMOVE(priv_fdir_flow, next); 5494 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); 5495 rte_free(priv_fdir_flow->fdir); 5496 rte_free(priv_fdir_flow); 5497 } 5498 } 5499 5500 /** 5501 * Get flow director information. 5502 * 5503 * @param dev 5504 * Pointer to Ethernet device. 5505 * @param[out] fdir_info 5506 * Resulting flow director information. 5507 */ 5508 static void 5509 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5510 { 5511 struct rte_eth_fdir_masks *mask = 5512 &dev->data->dev_conf.fdir_conf.mask; 5513 5514 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5515 fdir_info->guarant_spc = 0; 5516 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5517 fdir_info->max_flexpayload = 0; 5518 fdir_info->flow_types_mask[0] = 0; 5519 fdir_info->flex_payload_unit = 0; 5520 fdir_info->max_flex_payload_segment_num = 0; 5521 fdir_info->flex_payload_limit = 0; 5522 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5523 } 5524 5525 /** 5526 * Deal with flow director operations. 5527 * 5528 * @param dev 5529 * Pointer to Ethernet device. 5530 * @param filter_op 5531 * Operation to perform. 5532 * @param arg 5533 * Pointer to operation-specific structure. 5534 * 5535 * @return 5536 * 0 on success, a negative errno value otherwise and rte_errno is set. 5537 */ 5538 static int 5539 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5540 void *arg) 5541 { 5542 enum rte_fdir_mode fdir_mode = 5543 dev->data->dev_conf.fdir_conf.mode; 5544 5545 if (filter_op == RTE_ETH_FILTER_NOP) 5546 return 0; 5547 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5548 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5549 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5550 dev->data->port_id, fdir_mode); 5551 rte_errno = EINVAL; 5552 return -rte_errno; 5553 } 5554 switch (filter_op) { 5555 case RTE_ETH_FILTER_ADD: 5556 return flow_fdir_filter_add(dev, arg); 5557 case RTE_ETH_FILTER_UPDATE: 5558 return flow_fdir_filter_update(dev, arg); 5559 case RTE_ETH_FILTER_DELETE: 5560 return flow_fdir_filter_delete(dev, arg); 5561 case RTE_ETH_FILTER_FLUSH: 5562 flow_fdir_filter_flush(dev); 5563 break; 5564 case RTE_ETH_FILTER_INFO: 5565 flow_fdir_info_get(dev, arg); 5566 break; 5567 default: 5568 DRV_LOG(DEBUG, "port %u unknown operation %u", 5569 dev->data->port_id, filter_op); 5570 rte_errno = EINVAL; 5571 return -rte_errno; 5572 } 5573 return 0; 5574 } 5575 5576 /** 5577 * Manage filter operations. 5578 * 5579 * @param dev 5580 * Pointer to Ethernet device structure. 5581 * @param filter_type 5582 * Filter type. 5583 * @param filter_op 5584 * Operation to perform. 5585 * @param arg 5586 * Pointer to operation-specific structure. 5587 * 5588 * @return 5589 * 0 on success, a negative errno value otherwise and rte_errno is set. 5590 */ 5591 int 5592 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5593 enum rte_filter_type filter_type, 5594 enum rte_filter_op filter_op, 5595 void *arg) 5596 { 5597 switch (filter_type) { 5598 case RTE_ETH_FILTER_GENERIC: 5599 if (filter_op != RTE_ETH_FILTER_GET) { 5600 rte_errno = EINVAL; 5601 return -rte_errno; 5602 } 5603 *(const void **)arg = &mlx5_flow_ops; 5604 return 0; 5605 case RTE_ETH_FILTER_FDIR: 5606 return flow_fdir_ctrl_func(dev, filter_op, arg); 5607 default: 5608 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5609 dev->data->port_id, filter_type); 5610 rte_errno = ENOTSUP; 5611 return -rte_errno; 5612 } 5613 return 0; 5614 } 5615 5616 /** 5617 * Create the needed meter and suffix tables. 5618 * 5619 * @param[in] dev 5620 * Pointer to Ethernet device. 5621 * @param[in] fm 5622 * Pointer to the flow meter. 5623 * 5624 * @return 5625 * Pointer to table set on success, NULL otherwise. 5626 */ 5627 struct mlx5_meter_domains_infos * 5628 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5629 const struct mlx5_flow_meter *fm) 5630 { 5631 const struct mlx5_flow_driver_ops *fops; 5632 5633 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5634 return fops->create_mtr_tbls(dev, fm); 5635 } 5636 5637 /** 5638 * Destroy the meter table set. 5639 * 5640 * @param[in] dev 5641 * Pointer to Ethernet device. 5642 * @param[in] tbl 5643 * Pointer to the meter table set. 5644 * 5645 * @return 5646 * 0 on success. 5647 */ 5648 int 5649 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5650 struct mlx5_meter_domains_infos *tbls) 5651 { 5652 const struct mlx5_flow_driver_ops *fops; 5653 5654 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5655 return fops->destroy_mtr_tbls(dev, tbls); 5656 } 5657 5658 /** 5659 * Create policer rules. 5660 * 5661 * @param[in] dev 5662 * Pointer to Ethernet device. 5663 * @param[in] fm 5664 * Pointer to flow meter structure. 5665 * @param[in] attr 5666 * Pointer to flow attributes. 5667 * 5668 * @return 5669 * 0 on success, -1 otherwise. 5670 */ 5671 int 5672 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5673 struct mlx5_flow_meter *fm, 5674 const struct rte_flow_attr *attr) 5675 { 5676 const struct mlx5_flow_driver_ops *fops; 5677 5678 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5679 return fops->create_policer_rules(dev, fm, attr); 5680 } 5681 5682 /** 5683 * Destroy policer rules. 5684 * 5685 * @param[in] fm 5686 * Pointer to flow meter structure. 5687 * @param[in] attr 5688 * Pointer to flow attributes. 5689 * 5690 * @return 5691 * 0 on success, -1 otherwise. 5692 */ 5693 int 5694 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5695 struct mlx5_flow_meter *fm, 5696 const struct rte_flow_attr *attr) 5697 { 5698 const struct mlx5_flow_driver_ops *fops; 5699 5700 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5701 return fops->destroy_policer_rules(dev, fm, attr); 5702 } 5703 5704 /** 5705 * Allocate a counter. 5706 * 5707 * @param[in] dev 5708 * Pointer to Ethernet device structure. 5709 * 5710 * @return 5711 * Index to allocated counter on success, 0 otherwise. 5712 */ 5713 uint32_t 5714 mlx5_counter_alloc(struct rte_eth_dev *dev) 5715 { 5716 const struct mlx5_flow_driver_ops *fops; 5717 struct rte_flow_attr attr = { .transfer = 0 }; 5718 5719 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5720 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5721 return fops->counter_alloc(dev); 5722 } 5723 DRV_LOG(ERR, 5724 "port %u counter allocate is not supported.", 5725 dev->data->port_id); 5726 return 0; 5727 } 5728 5729 /** 5730 * Free a counter. 5731 * 5732 * @param[in] dev 5733 * Pointer to Ethernet device structure. 5734 * @param[in] cnt 5735 * Index to counter to be free. 5736 */ 5737 void 5738 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 5739 { 5740 const struct mlx5_flow_driver_ops *fops; 5741 struct rte_flow_attr attr = { .transfer = 0 }; 5742 5743 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5744 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5745 fops->counter_free(dev, cnt); 5746 return; 5747 } 5748 DRV_LOG(ERR, 5749 "port %u counter free is not supported.", 5750 dev->data->port_id); 5751 } 5752 5753 /** 5754 * Query counter statistics. 5755 * 5756 * @param[in] dev 5757 * Pointer to Ethernet device structure. 5758 * @param[in] cnt 5759 * Index to counter to query. 5760 * @param[in] clear 5761 * Set to clear counter statistics. 5762 * @param[out] pkts 5763 * The counter hits packets number to save. 5764 * @param[out] bytes 5765 * The counter hits bytes number to save. 5766 * 5767 * @return 5768 * 0 on success, a negative errno value otherwise. 5769 */ 5770 int 5771 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 5772 bool clear, uint64_t *pkts, uint64_t *bytes) 5773 { 5774 const struct mlx5_flow_driver_ops *fops; 5775 struct rte_flow_attr attr = { .transfer = 0 }; 5776 5777 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5778 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5779 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5780 } 5781 DRV_LOG(ERR, 5782 "port %u counter query is not supported.", 5783 dev->data->port_id); 5784 return -ENOTSUP; 5785 } 5786 5787 #define MLX5_POOL_QUERY_FREQ_US 1000000 5788 5789 /** 5790 * Get number of all validate pools. 5791 * 5792 * @param[in] sh 5793 * Pointer to mlx5_dev_ctx_shared object. 5794 * 5795 * @return 5796 * The number of all validate pools. 5797 */ 5798 static uint32_t 5799 mlx5_get_all_valid_pool_count(struct mlx5_dev_ctx_shared *sh) 5800 { 5801 int i; 5802 uint32_t pools_n = 0; 5803 5804 for (i = 0; i < MLX5_CCONT_TYPE_MAX; ++i) 5805 pools_n += rte_atomic16_read(&sh->cmng.ccont[i].n_valid); 5806 return pools_n; 5807 } 5808 5809 /** 5810 * Set the periodic procedure for triggering asynchronous batch queries for all 5811 * the counter pools. 5812 * 5813 * @param[in] sh 5814 * Pointer to mlx5_dev_ctx_shared object. 5815 */ 5816 void 5817 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 5818 { 5819 uint32_t pools_n, us; 5820 5821 pools_n = mlx5_get_all_valid_pool_count(sh); 5822 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5823 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5824 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5825 sh->cmng.query_thread_on = 0; 5826 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5827 } else { 5828 sh->cmng.query_thread_on = 1; 5829 } 5830 } 5831 5832 /** 5833 * The periodic procedure for triggering asynchronous batch queries for all the 5834 * counter pools. This function is probably called by the host thread. 5835 * 5836 * @param[in] arg 5837 * The parameter for the alarm process. 5838 */ 5839 void 5840 mlx5_flow_query_alarm(void *arg) 5841 { 5842 struct mlx5_dev_ctx_shared *sh = arg; 5843 struct mlx5_devx_obj *dcs; 5844 uint16_t offset; 5845 int ret; 5846 uint8_t batch = sh->cmng.batch; 5847 uint8_t age = sh->cmng.age; 5848 uint16_t pool_index = sh->cmng.pool_index; 5849 struct mlx5_pools_container *cont; 5850 struct mlx5_flow_counter_pool *pool; 5851 int cont_loop = MLX5_CCONT_TYPE_MAX; 5852 5853 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5854 goto set_alarm; 5855 next_container: 5856 cont = MLX5_CNT_CONTAINER(sh, batch, age); 5857 rte_spinlock_lock(&cont->resize_sl); 5858 if (!cont->pools) { 5859 rte_spinlock_unlock(&cont->resize_sl); 5860 /* Check if all the containers are empty. */ 5861 if (unlikely(--cont_loop == 0)) 5862 goto set_alarm; 5863 batch ^= 0x1; 5864 pool_index = 0; 5865 if (batch == 0 && pool_index == 0) { 5866 age ^= 0x1; 5867 sh->cmng.batch = batch; 5868 sh->cmng.age = age; 5869 } 5870 goto next_container; 5871 } 5872 pool = cont->pools[pool_index]; 5873 rte_spinlock_unlock(&cont->resize_sl); 5874 if (pool->raw_hw) 5875 /* There is a pool query in progress. */ 5876 goto set_alarm; 5877 pool->raw_hw = 5878 LIST_FIRST(&sh->cmng.free_stat_raws); 5879 if (!pool->raw_hw) 5880 /* No free counter statistics raw memory. */ 5881 goto set_alarm; 5882 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5883 (&pool->a64_dcs); 5884 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5885 /* 5886 * Identify the counters released between query trigger and query 5887 * handle more effiecntly. The counter released in this gap period 5888 * should wait for a new round of query as the new arrived packets 5889 * will not be taken into account. 5890 */ 5891 rte_atomic64_add(&pool->start_query_gen, 1); 5892 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5893 offset, NULL, NULL, 5894 pool->raw_hw->mem_mng->dm->id, 5895 (void *)(uintptr_t) 5896 (pool->raw_hw->data + offset), 5897 sh->devx_comp, 5898 (uint64_t)(uintptr_t)pool); 5899 if (ret) { 5900 rte_atomic64_sub(&pool->start_query_gen, 1); 5901 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5902 " %d", pool->min_dcs->id); 5903 pool->raw_hw = NULL; 5904 goto set_alarm; 5905 } 5906 pool->raw_hw->min_dcs_id = dcs->id; 5907 LIST_REMOVE(pool->raw_hw, next); 5908 sh->cmng.pending_queries++; 5909 pool_index++; 5910 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5911 batch ^= 0x1; 5912 pool_index = 0; 5913 if (batch == 0 && pool_index == 0) 5914 age ^= 0x1; 5915 } 5916 set_alarm: 5917 sh->cmng.batch = batch; 5918 sh->cmng.pool_index = pool_index; 5919 sh->cmng.age = age; 5920 mlx5_set_query_alarm(sh); 5921 } 5922 5923 /** 5924 * Check and callback event for new aged flow in the counter pool 5925 * 5926 * @param[in] sh 5927 * Pointer to mlx5_dev_ctx_shared object. 5928 * @param[in] pool 5929 * Pointer to Current counter pool. 5930 */ 5931 static void 5932 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 5933 struct mlx5_flow_counter_pool *pool) 5934 { 5935 struct mlx5_priv *priv; 5936 struct mlx5_flow_counter *cnt; 5937 struct mlx5_age_info *age_info; 5938 struct mlx5_age_param *age_param; 5939 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 5940 struct mlx5_counter_stats_raw *prev = pool->raw; 5941 uint16_t curr = rte_rdtsc() / (rte_get_tsc_hz() / 10); 5942 uint32_t i; 5943 5944 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 5945 cnt = MLX5_POOL_GET_CNT(pool, i); 5946 age_param = MLX5_CNT_TO_AGE(cnt); 5947 if (rte_atomic16_read(&age_param->state) != AGE_CANDIDATE) 5948 continue; 5949 if (cur->data[i].hits != prev->data[i].hits) { 5950 age_param->expire = curr + age_param->timeout; 5951 continue; 5952 } 5953 if ((uint16_t)(curr - age_param->expire) >= (UINT16_MAX / 2)) 5954 continue; 5955 /** 5956 * Hold the lock first, or if between the 5957 * state AGE_TMOUT and tailq operation the 5958 * release happened, the release procedure 5959 * may delete a non-existent tailq node. 5960 */ 5961 priv = rte_eth_devices[age_param->port_id].data->dev_private; 5962 age_info = GET_PORT_AGE_INFO(priv); 5963 rte_spinlock_lock(&age_info->aged_sl); 5964 /* If the cpmset fails, release happens. */ 5965 if (rte_atomic16_cmpset((volatile uint16_t *) 5966 &age_param->state, 5967 AGE_CANDIDATE, 5968 AGE_TMOUT) == 5969 AGE_CANDIDATE) { 5970 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 5971 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 5972 } 5973 rte_spinlock_unlock(&age_info->aged_sl); 5974 } 5975 for (i = 0; i < sh->max_port; i++) { 5976 age_info = &sh->port[i].age_info; 5977 if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) 5978 continue; 5979 if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) 5980 _rte_eth_dev_callback_process 5981 (&rte_eth_devices[sh->port[i].devx_ih_port_id], 5982 RTE_ETH_EVENT_FLOW_AGED, NULL); 5983 age_info->flags = 0; 5984 } 5985 } 5986 5987 /** 5988 * Handler for the HW respond about ready values from an asynchronous batch 5989 * query. This function is probably called by the host thread. 5990 * 5991 * @param[in] sh 5992 * The pointer to the shared device context. 5993 * @param[in] async_id 5994 * The Devx async ID. 5995 * @param[in] status 5996 * The status of the completion. 5997 */ 5998 void 5999 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 6000 uint64_t async_id, int status) 6001 { 6002 struct mlx5_flow_counter_pool *pool = 6003 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 6004 struct mlx5_counter_stats_raw *raw_to_free; 6005 6006 if (unlikely(status)) { 6007 rte_atomic64_sub(&pool->start_query_gen, 1); 6008 raw_to_free = pool->raw_hw; 6009 } else { 6010 raw_to_free = pool->raw; 6011 if (IS_AGE_POOL(pool)) 6012 mlx5_flow_aging_check(sh, pool); 6013 rte_spinlock_lock(&pool->sl); 6014 pool->raw = pool->raw_hw; 6015 rte_spinlock_unlock(&pool->sl); 6016 MLX5_ASSERT(rte_atomic64_read(&pool->end_query_gen) + 1 == 6017 rte_atomic64_read(&pool->start_query_gen)); 6018 rte_atomic64_set(&pool->end_query_gen, 6019 rte_atomic64_read(&pool->start_query_gen)); 6020 /* Be sure the new raw counters data is updated in memory. */ 6021 rte_cio_wmb(); 6022 } 6023 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 6024 pool->raw_hw = NULL; 6025 sh->cmng.pending_queries--; 6026 } 6027 6028 /** 6029 * Translate the rte_flow group index to HW table value. 6030 * 6031 * @param[in] attributes 6032 * Pointer to flow attributes 6033 * @param[in] external 6034 * Value is part of flow rule created by request external to PMD. 6035 * @param[in] group 6036 * rte_flow group index value. 6037 * @param[out] fdb_def_rule 6038 * Whether fdb jump to table 1 is configured. 6039 * @param[out] table 6040 * HW table value. 6041 * @param[out] error 6042 * Pointer to error structure. 6043 * 6044 * @return 6045 * 0 on success, a negative errno value otherwise and rte_errno is set. 6046 */ 6047 int 6048 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 6049 uint32_t group, bool fdb_def_rule, uint32_t *table, 6050 struct rte_flow_error *error) 6051 { 6052 if (attributes->transfer && external && fdb_def_rule) { 6053 if (group == UINT32_MAX) 6054 return rte_flow_error_set 6055 (error, EINVAL, 6056 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 6057 NULL, 6058 "group index not supported"); 6059 *table = group + 1; 6060 } else { 6061 *table = group; 6062 } 6063 return 0; 6064 } 6065 6066 /** 6067 * Discover availability of metadata reg_c's. 6068 * 6069 * Iteratively use test flows to check availability. 6070 * 6071 * @param[in] dev 6072 * Pointer to the Ethernet device structure. 6073 * 6074 * @return 6075 * 0 on success, a negative errno value otherwise and rte_errno is set. 6076 */ 6077 int 6078 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 6079 { 6080 struct mlx5_priv *priv = dev->data->dev_private; 6081 struct mlx5_dev_config *config = &priv->config; 6082 enum modify_reg idx; 6083 int n = 0; 6084 6085 /* reg_c[0] and reg_c[1] are reserved. */ 6086 config->flow_mreg_c[n++] = REG_C_0; 6087 config->flow_mreg_c[n++] = REG_C_1; 6088 /* Discover availability of other reg_c's. */ 6089 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 6090 struct rte_flow_attr attr = { 6091 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6092 .priority = MLX5_FLOW_PRIO_RSVD, 6093 .ingress = 1, 6094 }; 6095 struct rte_flow_item items[] = { 6096 [0] = { 6097 .type = RTE_FLOW_ITEM_TYPE_END, 6098 }, 6099 }; 6100 struct rte_flow_action actions[] = { 6101 [0] = { 6102 .type = (enum rte_flow_action_type) 6103 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6104 .conf = &(struct mlx5_flow_action_copy_mreg){ 6105 .src = REG_C_1, 6106 .dst = idx, 6107 }, 6108 }, 6109 [1] = { 6110 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6111 .conf = &(struct rte_flow_action_jump){ 6112 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6113 }, 6114 }, 6115 [2] = { 6116 .type = RTE_FLOW_ACTION_TYPE_END, 6117 }, 6118 }; 6119 uint32_t flow_idx; 6120 struct rte_flow *flow; 6121 struct rte_flow_error error; 6122 6123 if (!config->dv_flow_en) 6124 break; 6125 /* Create internal flow, validation skips copy action. */ 6126 flow_idx = flow_list_create(dev, NULL, &attr, items, 6127 actions, false, &error); 6128 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 6129 flow_idx); 6130 if (!flow) 6131 continue; 6132 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 6133 config->flow_mreg_c[n++] = idx; 6134 flow_list_destroy(dev, NULL, flow_idx); 6135 } 6136 for (; n < MLX5_MREG_C_NUM; ++n) 6137 config->flow_mreg_c[n] = REG_NONE; 6138 return 0; 6139 } 6140 6141 /** 6142 * Dump flow raw hw data to file 6143 * 6144 * @param[in] dev 6145 * The pointer to Ethernet device. 6146 * @param[in] file 6147 * A pointer to a file for output. 6148 * @param[out] error 6149 * Perform verbose error reporting if not NULL. PMDs initialize this 6150 * structure in case of error only. 6151 * @return 6152 * 0 on success, a nagative value otherwise. 6153 */ 6154 int 6155 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 6156 FILE *file, 6157 struct rte_flow_error *error __rte_unused) 6158 { 6159 struct mlx5_priv *priv = dev->data->dev_private; 6160 struct mlx5_dev_ctx_shared *sh = priv->sh; 6161 6162 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 6163 sh->tx_domain, file); 6164 } 6165 6166 /** 6167 * Get aged-out flows. 6168 * 6169 * @param[in] dev 6170 * Pointer to the Ethernet device structure. 6171 * @param[in] context 6172 * The address of an array of pointers to the aged-out flows contexts. 6173 * @param[in] nb_countexts 6174 * The length of context array pointers. 6175 * @param[out] error 6176 * Perform verbose error reporting if not NULL. Initialized in case of 6177 * error only. 6178 * 6179 * @return 6180 * how many contexts get in success, otherwise negative errno value. 6181 * if nb_contexts is 0, return the amount of all aged contexts. 6182 * if nb_contexts is not 0 , return the amount of aged flows reported 6183 * in the context array. 6184 */ 6185 int 6186 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 6187 uint32_t nb_contexts, struct rte_flow_error *error) 6188 { 6189 const struct mlx5_flow_driver_ops *fops; 6190 struct rte_flow_attr attr = { .transfer = 0 }; 6191 6192 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 6193 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 6194 return fops->get_aged_flows(dev, contexts, nb_contexts, 6195 error); 6196 } 6197 DRV_LOG(ERR, 6198 "port %u get aged flows is not supported.", 6199 dev->data->port_id); 6200 return -ENOTSUP; 6201 } 6202