1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 /* Verbs header. */ 14 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 15 #ifdef PEDANTIC 16 #pragma GCC diagnostic ignored "-Wpedantic" 17 #endif 18 #include <infiniband/verbs.h> 19 #ifdef PEDANTIC 20 #pragma GCC diagnostic error "-Wpedantic" 21 #endif 22 23 #include <rte_common.h> 24 #include <rte_ether.h> 25 #include <rte_ethdev_driver.h> 26 #include <rte_flow.h> 27 #include <rte_flow_driver.h> 28 #include <rte_malloc.h> 29 #include <rte_ip.h> 30 31 #include <mlx5_glue.h> 32 #include <mlx5_devx_cmds.h> 33 #include <mlx5_prm.h> 34 35 #include "mlx5_defs.h" 36 #include "mlx5.h" 37 #include "mlx5_flow.h" 38 #include "mlx5_rxtx.h" 39 40 /* Dev ops structure defined in mlx5.c */ 41 extern const struct eth_dev_ops mlx5_dev_ops; 42 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 43 44 /** Device flow drivers. */ 45 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 46 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 47 #endif 48 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 49 50 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 51 52 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 53 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 54 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 55 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 56 #endif 57 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 58 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 59 }; 60 61 enum mlx5_expansion { 62 MLX5_EXPANSION_ROOT, 63 MLX5_EXPANSION_ROOT_OUTER, 64 MLX5_EXPANSION_ROOT_ETH_VLAN, 65 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 66 MLX5_EXPANSION_OUTER_ETH, 67 MLX5_EXPANSION_OUTER_ETH_VLAN, 68 MLX5_EXPANSION_OUTER_VLAN, 69 MLX5_EXPANSION_OUTER_IPV4, 70 MLX5_EXPANSION_OUTER_IPV4_UDP, 71 MLX5_EXPANSION_OUTER_IPV4_TCP, 72 MLX5_EXPANSION_OUTER_IPV6, 73 MLX5_EXPANSION_OUTER_IPV6_UDP, 74 MLX5_EXPANSION_OUTER_IPV6_TCP, 75 MLX5_EXPANSION_VXLAN, 76 MLX5_EXPANSION_VXLAN_GPE, 77 MLX5_EXPANSION_GRE, 78 MLX5_EXPANSION_MPLS, 79 MLX5_EXPANSION_ETH, 80 MLX5_EXPANSION_ETH_VLAN, 81 MLX5_EXPANSION_VLAN, 82 MLX5_EXPANSION_IPV4, 83 MLX5_EXPANSION_IPV4_UDP, 84 MLX5_EXPANSION_IPV4_TCP, 85 MLX5_EXPANSION_IPV6, 86 MLX5_EXPANSION_IPV6_UDP, 87 MLX5_EXPANSION_IPV6_TCP, 88 }; 89 90 /** Supported expansion of items. */ 91 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 92 [MLX5_EXPANSION_ROOT] = { 93 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 94 MLX5_EXPANSION_IPV4, 95 MLX5_EXPANSION_IPV6), 96 .type = RTE_FLOW_ITEM_TYPE_END, 97 }, 98 [MLX5_EXPANSION_ROOT_OUTER] = { 99 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 100 MLX5_EXPANSION_OUTER_IPV4, 101 MLX5_EXPANSION_OUTER_IPV6), 102 .type = RTE_FLOW_ITEM_TYPE_END, 103 }, 104 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 105 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 106 .type = RTE_FLOW_ITEM_TYPE_END, 107 }, 108 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 109 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 110 .type = RTE_FLOW_ITEM_TYPE_END, 111 }, 112 [MLX5_EXPANSION_OUTER_ETH] = { 113 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 114 MLX5_EXPANSION_OUTER_IPV6, 115 MLX5_EXPANSION_MPLS), 116 .type = RTE_FLOW_ITEM_TYPE_ETH, 117 .rss_types = 0, 118 }, 119 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 120 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 121 .type = RTE_FLOW_ITEM_TYPE_ETH, 122 .rss_types = 0, 123 }, 124 [MLX5_EXPANSION_OUTER_VLAN] = { 125 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 126 MLX5_EXPANSION_OUTER_IPV6), 127 .type = RTE_FLOW_ITEM_TYPE_VLAN, 128 }, 129 [MLX5_EXPANSION_OUTER_IPV4] = { 130 .next = RTE_FLOW_EXPAND_RSS_NEXT 131 (MLX5_EXPANSION_OUTER_IPV4_UDP, 132 MLX5_EXPANSION_OUTER_IPV4_TCP, 133 MLX5_EXPANSION_GRE, 134 MLX5_EXPANSION_IPV4, 135 MLX5_EXPANSION_IPV6), 136 .type = RTE_FLOW_ITEM_TYPE_IPV4, 137 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 138 ETH_RSS_NONFRAG_IPV4_OTHER, 139 }, 140 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 141 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 142 MLX5_EXPANSION_VXLAN_GPE), 143 .type = RTE_FLOW_ITEM_TYPE_UDP, 144 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 145 }, 146 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 147 .type = RTE_FLOW_ITEM_TYPE_TCP, 148 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 149 }, 150 [MLX5_EXPANSION_OUTER_IPV6] = { 151 .next = RTE_FLOW_EXPAND_RSS_NEXT 152 (MLX5_EXPANSION_OUTER_IPV6_UDP, 153 MLX5_EXPANSION_OUTER_IPV6_TCP, 154 MLX5_EXPANSION_IPV4, 155 MLX5_EXPANSION_IPV6), 156 .type = RTE_FLOW_ITEM_TYPE_IPV6, 157 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 158 ETH_RSS_NONFRAG_IPV6_OTHER, 159 }, 160 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 161 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 162 MLX5_EXPANSION_VXLAN_GPE), 163 .type = RTE_FLOW_ITEM_TYPE_UDP, 164 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 165 }, 166 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 167 .type = RTE_FLOW_ITEM_TYPE_TCP, 168 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 169 }, 170 [MLX5_EXPANSION_VXLAN] = { 171 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 172 MLX5_EXPANSION_IPV4, 173 MLX5_EXPANSION_IPV6), 174 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 175 }, 176 [MLX5_EXPANSION_VXLAN_GPE] = { 177 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 178 MLX5_EXPANSION_IPV4, 179 MLX5_EXPANSION_IPV6), 180 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 181 }, 182 [MLX5_EXPANSION_GRE] = { 183 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 184 .type = RTE_FLOW_ITEM_TYPE_GRE, 185 }, 186 [MLX5_EXPANSION_MPLS] = { 187 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 188 MLX5_EXPANSION_IPV6), 189 .type = RTE_FLOW_ITEM_TYPE_MPLS, 190 }, 191 [MLX5_EXPANSION_ETH] = { 192 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 193 MLX5_EXPANSION_IPV6), 194 .type = RTE_FLOW_ITEM_TYPE_ETH, 195 }, 196 [MLX5_EXPANSION_ETH_VLAN] = { 197 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 198 .type = RTE_FLOW_ITEM_TYPE_ETH, 199 }, 200 [MLX5_EXPANSION_VLAN] = { 201 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 202 MLX5_EXPANSION_IPV6), 203 .type = RTE_FLOW_ITEM_TYPE_VLAN, 204 }, 205 [MLX5_EXPANSION_IPV4] = { 206 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 207 MLX5_EXPANSION_IPV4_TCP), 208 .type = RTE_FLOW_ITEM_TYPE_IPV4, 209 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 210 ETH_RSS_NONFRAG_IPV4_OTHER, 211 }, 212 [MLX5_EXPANSION_IPV4_UDP] = { 213 .type = RTE_FLOW_ITEM_TYPE_UDP, 214 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 215 }, 216 [MLX5_EXPANSION_IPV4_TCP] = { 217 .type = RTE_FLOW_ITEM_TYPE_TCP, 218 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 219 }, 220 [MLX5_EXPANSION_IPV6] = { 221 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 222 MLX5_EXPANSION_IPV6_TCP), 223 .type = RTE_FLOW_ITEM_TYPE_IPV6, 224 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 225 ETH_RSS_NONFRAG_IPV6_OTHER, 226 }, 227 [MLX5_EXPANSION_IPV6_UDP] = { 228 .type = RTE_FLOW_ITEM_TYPE_UDP, 229 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 230 }, 231 [MLX5_EXPANSION_IPV6_TCP] = { 232 .type = RTE_FLOW_ITEM_TYPE_TCP, 233 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 234 }, 235 }; 236 237 static const struct rte_flow_ops mlx5_flow_ops = { 238 .validate = mlx5_flow_validate, 239 .create = mlx5_flow_create, 240 .destroy = mlx5_flow_destroy, 241 .flush = mlx5_flow_flush, 242 .isolate = mlx5_flow_isolate, 243 .query = mlx5_flow_query, 244 .dev_dump = mlx5_flow_dev_dump, 245 }; 246 247 /* Convert FDIR request to Generic flow. */ 248 struct mlx5_fdir { 249 struct rte_flow_attr attr; 250 struct rte_flow_item items[4]; 251 struct rte_flow_item_eth l2; 252 struct rte_flow_item_eth l2_mask; 253 union { 254 struct rte_flow_item_ipv4 ipv4; 255 struct rte_flow_item_ipv6 ipv6; 256 } l3; 257 union { 258 struct rte_flow_item_ipv4 ipv4; 259 struct rte_flow_item_ipv6 ipv6; 260 } l3_mask; 261 union { 262 struct rte_flow_item_udp udp; 263 struct rte_flow_item_tcp tcp; 264 } l4; 265 union { 266 struct rte_flow_item_udp udp; 267 struct rte_flow_item_tcp tcp; 268 } l4_mask; 269 struct rte_flow_action actions[2]; 270 struct rte_flow_action_queue queue; 271 }; 272 273 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 274 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 275 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 276 }; 277 278 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 279 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 280 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 281 { 9, 10, 11 }, { 12, 13, 14 }, 282 }; 283 284 /* Tunnel information. */ 285 struct mlx5_flow_tunnel_info { 286 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 287 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 288 }; 289 290 static struct mlx5_flow_tunnel_info tunnels_info[] = { 291 { 292 .tunnel = MLX5_FLOW_LAYER_VXLAN, 293 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 294 }, 295 { 296 .tunnel = MLX5_FLOW_LAYER_GENEVE, 297 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 298 }, 299 { 300 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 301 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 302 }, 303 { 304 .tunnel = MLX5_FLOW_LAYER_GRE, 305 .ptype = RTE_PTYPE_TUNNEL_GRE, 306 }, 307 { 308 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 309 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 310 }, 311 { 312 .tunnel = MLX5_FLOW_LAYER_MPLS, 313 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 314 }, 315 { 316 .tunnel = MLX5_FLOW_LAYER_NVGRE, 317 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 318 }, 319 { 320 .tunnel = MLX5_FLOW_LAYER_IPIP, 321 .ptype = RTE_PTYPE_TUNNEL_IP, 322 }, 323 { 324 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 325 .ptype = RTE_PTYPE_TUNNEL_IP, 326 }, 327 { 328 .tunnel = MLX5_FLOW_LAYER_GTP, 329 .ptype = RTE_PTYPE_TUNNEL_GTPU, 330 }, 331 }; 332 333 /** 334 * Translate tag ID to register. 335 * 336 * @param[in] dev 337 * Pointer to the Ethernet device structure. 338 * @param[in] feature 339 * The feature that request the register. 340 * @param[in] id 341 * The request register ID. 342 * @param[out] error 343 * Error description in case of any. 344 * 345 * @return 346 * The request register on success, a negative errno 347 * value otherwise and rte_errno is set. 348 */ 349 int 350 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 351 enum mlx5_feature_name feature, 352 uint32_t id, 353 struct rte_flow_error *error) 354 { 355 struct mlx5_priv *priv = dev->data->dev_private; 356 struct mlx5_dev_config *config = &priv->config; 357 enum modify_reg start_reg; 358 bool skip_mtr_reg = false; 359 360 switch (feature) { 361 case MLX5_HAIRPIN_RX: 362 return REG_B; 363 case MLX5_HAIRPIN_TX: 364 return REG_A; 365 case MLX5_METADATA_RX: 366 switch (config->dv_xmeta_en) { 367 case MLX5_XMETA_MODE_LEGACY: 368 return REG_B; 369 case MLX5_XMETA_MODE_META16: 370 return REG_C_0; 371 case MLX5_XMETA_MODE_META32: 372 return REG_C_1; 373 } 374 break; 375 case MLX5_METADATA_TX: 376 return REG_A; 377 case MLX5_METADATA_FDB: 378 switch (config->dv_xmeta_en) { 379 case MLX5_XMETA_MODE_LEGACY: 380 return REG_NONE; 381 case MLX5_XMETA_MODE_META16: 382 return REG_C_0; 383 case MLX5_XMETA_MODE_META32: 384 return REG_C_1; 385 } 386 break; 387 case MLX5_FLOW_MARK: 388 switch (config->dv_xmeta_en) { 389 case MLX5_XMETA_MODE_LEGACY: 390 return REG_NONE; 391 case MLX5_XMETA_MODE_META16: 392 return REG_C_1; 393 case MLX5_XMETA_MODE_META32: 394 return REG_C_0; 395 } 396 break; 397 case MLX5_MTR_SFX: 398 /* 399 * If meter color and flow match share one register, flow match 400 * should use the meter color register for match. 401 */ 402 if (priv->mtr_reg_share) 403 return priv->mtr_color_reg; 404 else 405 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 406 REG_C_3; 407 case MLX5_MTR_COLOR: 408 MLX5_ASSERT(priv->mtr_color_reg != REG_NONE); 409 return priv->mtr_color_reg; 410 case MLX5_COPY_MARK: 411 /* 412 * Metadata COPY_MARK register using is in meter suffix sub 413 * flow while with meter. It's safe to share the same register. 414 */ 415 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 416 case MLX5_APP_TAG: 417 /* 418 * If meter is enable, it will engage the register for color 419 * match and flow match. If meter color match is not using the 420 * REG_C_2, need to skip the REG_C_x be used by meter color 421 * match. 422 * If meter is disable, free to use all available registers. 423 */ 424 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 425 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 426 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 427 if (id > (REG_C_7 - start_reg)) 428 return rte_flow_error_set(error, EINVAL, 429 RTE_FLOW_ERROR_TYPE_ITEM, 430 NULL, "invalid tag id"); 431 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 432 return rte_flow_error_set(error, ENOTSUP, 433 RTE_FLOW_ERROR_TYPE_ITEM, 434 NULL, "unsupported tag id"); 435 /* 436 * This case means meter is using the REG_C_x great than 2. 437 * Take care not to conflict with meter color REG_C_x. 438 * If the available index REG_C_y >= REG_C_x, skip the 439 * color register. 440 */ 441 if (skip_mtr_reg && config->flow_mreg_c 442 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 443 if (config->flow_mreg_c 444 [id + 1 + start_reg - REG_C_0] != REG_NONE) 445 return config->flow_mreg_c 446 [id + 1 + start_reg - REG_C_0]; 447 return rte_flow_error_set(error, ENOTSUP, 448 RTE_FLOW_ERROR_TYPE_ITEM, 449 NULL, "unsupported tag id"); 450 } 451 return config->flow_mreg_c[id + start_reg - REG_C_0]; 452 } 453 MLX5_ASSERT(false); 454 return rte_flow_error_set(error, EINVAL, 455 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 456 NULL, "invalid feature name"); 457 } 458 459 /** 460 * Check extensive flow metadata register support. 461 * 462 * @param dev 463 * Pointer to rte_eth_dev structure. 464 * 465 * @return 466 * True if device supports extensive flow metadata register, otherwise false. 467 */ 468 bool 469 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 470 { 471 struct mlx5_priv *priv = dev->data->dev_private; 472 struct mlx5_dev_config *config = &priv->config; 473 474 /* 475 * Having available reg_c can be regarded inclusively as supporting 476 * extensive flow metadata register, which could mean, 477 * - metadata register copy action by modify header. 478 * - 16 modify header actions is supported. 479 * - reg_c's are preserved across different domain (FDB and NIC) on 480 * packet loopback by flow lookup miss. 481 */ 482 return config->flow_mreg_c[2] != REG_NONE; 483 } 484 485 /** 486 * Discover the maximum number of priority available. 487 * 488 * @param[in] dev 489 * Pointer to the Ethernet device structure. 490 * 491 * @return 492 * number of supported flow priority on success, a negative errno 493 * value otherwise and rte_errno is set. 494 */ 495 int 496 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 497 { 498 struct mlx5_priv *priv = dev->data->dev_private; 499 struct { 500 struct ibv_flow_attr attr; 501 struct ibv_flow_spec_eth eth; 502 struct ibv_flow_spec_action_drop drop; 503 } flow_attr = { 504 .attr = { 505 .num_of_specs = 2, 506 .port = (uint8_t)priv->ibv_port, 507 }, 508 .eth = { 509 .type = IBV_FLOW_SPEC_ETH, 510 .size = sizeof(struct ibv_flow_spec_eth), 511 }, 512 .drop = { 513 .size = sizeof(struct ibv_flow_spec_action_drop), 514 .type = IBV_FLOW_SPEC_ACTION_DROP, 515 }, 516 }; 517 struct ibv_flow *flow; 518 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 519 uint16_t vprio[] = { 8, 16 }; 520 int i; 521 int priority = 0; 522 523 if (!drop) { 524 rte_errno = ENOTSUP; 525 return -rte_errno; 526 } 527 for (i = 0; i != RTE_DIM(vprio); i++) { 528 flow_attr.attr.priority = vprio[i] - 1; 529 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 530 if (!flow) 531 break; 532 claim_zero(mlx5_glue->destroy_flow(flow)); 533 priority = vprio[i]; 534 } 535 mlx5_hrxq_drop_release(dev); 536 switch (priority) { 537 case 8: 538 priority = RTE_DIM(priority_map_3); 539 break; 540 case 16: 541 priority = RTE_DIM(priority_map_5); 542 break; 543 default: 544 rte_errno = ENOTSUP; 545 DRV_LOG(ERR, 546 "port %u verbs maximum priority: %d expected 8/16", 547 dev->data->port_id, priority); 548 return -rte_errno; 549 } 550 DRV_LOG(INFO, "port %u flow maximum priority: %d", 551 dev->data->port_id, priority); 552 return priority; 553 } 554 555 /** 556 * Adjust flow priority based on the highest layer and the request priority. 557 * 558 * @param[in] dev 559 * Pointer to the Ethernet device structure. 560 * @param[in] priority 561 * The rule base priority. 562 * @param[in] subpriority 563 * The priority based on the items. 564 * 565 * @return 566 * The new priority. 567 */ 568 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 569 uint32_t subpriority) 570 { 571 uint32_t res = 0; 572 struct mlx5_priv *priv = dev->data->dev_private; 573 574 switch (priv->config.flow_prio) { 575 case RTE_DIM(priority_map_3): 576 res = priority_map_3[priority][subpriority]; 577 break; 578 case RTE_DIM(priority_map_5): 579 res = priority_map_5[priority][subpriority]; 580 break; 581 } 582 return res; 583 } 584 585 /** 586 * Verify the @p item specifications (spec, last, mask) are compatible with the 587 * NIC capabilities. 588 * 589 * @param[in] item 590 * Item specification. 591 * @param[in] mask 592 * @p item->mask or flow default bit-masks. 593 * @param[in] nic_mask 594 * Bit-masks covering supported fields by the NIC to compare with user mask. 595 * @param[in] size 596 * Bit-masks size in bytes. 597 * @param[out] error 598 * Pointer to error structure. 599 * 600 * @return 601 * 0 on success, a negative errno value otherwise and rte_errno is set. 602 */ 603 int 604 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 605 const uint8_t *mask, 606 const uint8_t *nic_mask, 607 unsigned int size, 608 struct rte_flow_error *error) 609 { 610 unsigned int i; 611 612 MLX5_ASSERT(nic_mask); 613 for (i = 0; i < size; ++i) 614 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 615 return rte_flow_error_set(error, ENOTSUP, 616 RTE_FLOW_ERROR_TYPE_ITEM, 617 item, 618 "mask enables non supported" 619 " bits"); 620 if (!item->spec && (item->mask || item->last)) 621 return rte_flow_error_set(error, EINVAL, 622 RTE_FLOW_ERROR_TYPE_ITEM, item, 623 "mask/last without a spec is not" 624 " supported"); 625 if (item->spec && item->last) { 626 uint8_t spec[size]; 627 uint8_t last[size]; 628 unsigned int i; 629 int ret; 630 631 for (i = 0; i < size; ++i) { 632 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 633 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 634 } 635 ret = memcmp(spec, last, size); 636 if (ret != 0) 637 return rte_flow_error_set(error, EINVAL, 638 RTE_FLOW_ERROR_TYPE_ITEM, 639 item, 640 "range is not valid"); 641 } 642 return 0; 643 } 644 645 /** 646 * Adjust the hash fields according to the @p flow information. 647 * 648 * @param[in] dev_flow. 649 * Pointer to the mlx5_flow. 650 * @param[in] tunnel 651 * 1 when the hash field is for a tunnel item. 652 * @param[in] layer_types 653 * ETH_RSS_* types. 654 * @param[in] hash_fields 655 * Item hash fields. 656 * 657 * @return 658 * The hash fields that should be used. 659 */ 660 uint64_t 661 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, 662 int tunnel __rte_unused, uint64_t layer_types, 663 uint64_t hash_fields) 664 { 665 struct rte_flow *flow = dev_flow->flow; 666 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 667 int rss_request_inner = flow->rss.level >= 2; 668 669 /* Check RSS hash level for tunnel. */ 670 if (tunnel && rss_request_inner) 671 hash_fields |= IBV_RX_HASH_INNER; 672 else if (tunnel || rss_request_inner) 673 return 0; 674 #endif 675 /* Check if requested layer matches RSS hash fields. */ 676 if (!(flow->rss.types & layer_types)) 677 return 0; 678 return hash_fields; 679 } 680 681 /** 682 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 683 * if several tunnel rules are used on this queue, the tunnel ptype will be 684 * cleared. 685 * 686 * @param rxq_ctrl 687 * Rx queue to update. 688 */ 689 static void 690 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 691 { 692 unsigned int i; 693 uint32_t tunnel_ptype = 0; 694 695 /* Look up for the ptype to use. */ 696 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 697 if (!rxq_ctrl->flow_tunnels_n[i]) 698 continue; 699 if (!tunnel_ptype) { 700 tunnel_ptype = tunnels_info[i].ptype; 701 } else { 702 tunnel_ptype = 0; 703 break; 704 } 705 } 706 rxq_ctrl->rxq.tunnel = tunnel_ptype; 707 } 708 709 /** 710 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 711 * flow. 712 * 713 * @param[in] dev 714 * Pointer to the Ethernet device structure. 715 * @param[in] flow 716 * Pointer to flow structure. 717 * @param[in] dev_handle 718 * Pointer to device flow handle structure. 719 */ 720 static void 721 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow, 722 struct mlx5_flow_handle *dev_handle) 723 { 724 struct mlx5_priv *priv = dev->data->dev_private; 725 const int mark = !!(dev_handle->act_flags & 726 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 727 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 728 unsigned int i; 729 730 for (i = 0; i != flow->rss.queue_num; ++i) { 731 int idx = (*flow->rss.queue)[i]; 732 struct mlx5_rxq_ctrl *rxq_ctrl = 733 container_of((*priv->rxqs)[idx], 734 struct mlx5_rxq_ctrl, rxq); 735 736 /* 737 * To support metadata register copy on Tx loopback, 738 * this must be always enabled (metadata may arive 739 * from other port - not from local flows only. 740 */ 741 if (priv->config.dv_flow_en && 742 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 743 mlx5_flow_ext_mreg_supported(dev)) { 744 rxq_ctrl->rxq.mark = 1; 745 rxq_ctrl->flow_mark_n = 1; 746 } else if (mark) { 747 rxq_ctrl->rxq.mark = 1; 748 rxq_ctrl->flow_mark_n++; 749 } 750 if (tunnel) { 751 unsigned int j; 752 753 /* Increase the counter matching the flow. */ 754 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 755 if ((tunnels_info[j].tunnel & 756 dev_handle->layers) == 757 tunnels_info[j].tunnel) { 758 rxq_ctrl->flow_tunnels_n[j]++; 759 break; 760 } 761 } 762 flow_rxq_tunnel_ptype_update(rxq_ctrl); 763 } 764 } 765 } 766 767 /** 768 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 769 * 770 * @param[in] dev 771 * Pointer to the Ethernet device structure. 772 * @param[in] flow 773 * Pointer to flow structure. 774 */ 775 static void 776 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 777 { 778 struct mlx5_flow_handle *dev_handle; 779 780 LIST_FOREACH(dev_handle, &flow->dev_handles, next) 781 flow_drv_rxq_flags_set(dev, flow, dev_handle); 782 } 783 784 /** 785 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 786 * device flow if no other flow uses it with the same kind of request. 787 * 788 * @param dev 789 * Pointer to Ethernet device. 790 * @param[in] flow 791 * Pointer to flow structure. 792 * @param[in] dev_handle 793 * Pointer to the device flow handle structure. 794 */ 795 static void 796 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow, 797 struct mlx5_flow_handle *dev_handle) 798 { 799 struct mlx5_priv *priv = dev->data->dev_private; 800 const int mark = !!(dev_handle->act_flags & 801 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 802 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 803 unsigned int i; 804 805 MLX5_ASSERT(dev->data->dev_started); 806 for (i = 0; i != flow->rss.queue_num; ++i) { 807 int idx = (*flow->rss.queue)[i]; 808 struct mlx5_rxq_ctrl *rxq_ctrl = 809 container_of((*priv->rxqs)[idx], 810 struct mlx5_rxq_ctrl, rxq); 811 812 if (priv->config.dv_flow_en && 813 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 814 mlx5_flow_ext_mreg_supported(dev)) { 815 rxq_ctrl->rxq.mark = 1; 816 rxq_ctrl->flow_mark_n = 1; 817 } else if (mark) { 818 rxq_ctrl->flow_mark_n--; 819 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 820 } 821 if (tunnel) { 822 unsigned int j; 823 824 /* Decrease the counter matching the flow. */ 825 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 826 if ((tunnels_info[j].tunnel & 827 dev_handle->layers) == 828 tunnels_info[j].tunnel) { 829 rxq_ctrl->flow_tunnels_n[j]--; 830 break; 831 } 832 } 833 flow_rxq_tunnel_ptype_update(rxq_ctrl); 834 } 835 } 836 } 837 838 /** 839 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 840 * @p flow if no other flow uses it with the same kind of request. 841 * 842 * @param dev 843 * Pointer to Ethernet device. 844 * @param[in] flow 845 * Pointer to the flow. 846 */ 847 static void 848 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 849 { 850 struct mlx5_flow_handle *dev_handle; 851 852 LIST_FOREACH(dev_handle, &flow->dev_handles, next) 853 flow_drv_rxq_flags_trim(dev, flow, dev_handle); 854 } 855 856 /** 857 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 858 * 859 * @param dev 860 * Pointer to Ethernet device. 861 */ 862 static void 863 flow_rxq_flags_clear(struct rte_eth_dev *dev) 864 { 865 struct mlx5_priv *priv = dev->data->dev_private; 866 unsigned int i; 867 868 for (i = 0; i != priv->rxqs_n; ++i) { 869 struct mlx5_rxq_ctrl *rxq_ctrl; 870 unsigned int j; 871 872 if (!(*priv->rxqs)[i]) 873 continue; 874 rxq_ctrl = container_of((*priv->rxqs)[i], 875 struct mlx5_rxq_ctrl, rxq); 876 rxq_ctrl->flow_mark_n = 0; 877 rxq_ctrl->rxq.mark = 0; 878 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 879 rxq_ctrl->flow_tunnels_n[j] = 0; 880 rxq_ctrl->rxq.tunnel = 0; 881 } 882 } 883 884 /* 885 * return a pointer to the desired action in the list of actions. 886 * 887 * @param[in] actions 888 * The list of actions to search the action in. 889 * @param[in] action 890 * The action to find. 891 * 892 * @return 893 * Pointer to the action in the list, if found. NULL otherwise. 894 */ 895 const struct rte_flow_action * 896 mlx5_flow_find_action(const struct rte_flow_action *actions, 897 enum rte_flow_action_type action) 898 { 899 if (actions == NULL) 900 return NULL; 901 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 902 if (actions->type == action) 903 return actions; 904 return NULL; 905 } 906 907 /* 908 * Validate the flag action. 909 * 910 * @param[in] action_flags 911 * Bit-fields that holds the actions detected until now. 912 * @param[in] attr 913 * Attributes of flow that includes this action. 914 * @param[out] error 915 * Pointer to error structure. 916 * 917 * @return 918 * 0 on success, a negative errno value otherwise and rte_errno is set. 919 */ 920 int 921 mlx5_flow_validate_action_flag(uint64_t action_flags, 922 const struct rte_flow_attr *attr, 923 struct rte_flow_error *error) 924 { 925 if (action_flags & MLX5_FLOW_ACTION_MARK) 926 return rte_flow_error_set(error, EINVAL, 927 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 928 "can't mark and flag in same flow"); 929 if (action_flags & MLX5_FLOW_ACTION_FLAG) 930 return rte_flow_error_set(error, EINVAL, 931 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 932 "can't have 2 flag" 933 " actions in same flow"); 934 if (attr->egress) 935 return rte_flow_error_set(error, ENOTSUP, 936 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 937 "flag action not supported for " 938 "egress"); 939 return 0; 940 } 941 942 /* 943 * Validate the mark action. 944 * 945 * @param[in] action 946 * Pointer to the queue action. 947 * @param[in] action_flags 948 * Bit-fields that holds the actions detected until now. 949 * @param[in] attr 950 * Attributes of flow that includes this action. 951 * @param[out] error 952 * Pointer to error structure. 953 * 954 * @return 955 * 0 on success, a negative errno value otherwise and rte_errno is set. 956 */ 957 int 958 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 959 uint64_t action_flags, 960 const struct rte_flow_attr *attr, 961 struct rte_flow_error *error) 962 { 963 const struct rte_flow_action_mark *mark = action->conf; 964 965 if (!mark) 966 return rte_flow_error_set(error, EINVAL, 967 RTE_FLOW_ERROR_TYPE_ACTION, 968 action, 969 "configuration cannot be null"); 970 if (mark->id >= MLX5_FLOW_MARK_MAX) 971 return rte_flow_error_set(error, EINVAL, 972 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 973 &mark->id, 974 "mark id must in 0 <= id < " 975 RTE_STR(MLX5_FLOW_MARK_MAX)); 976 if (action_flags & MLX5_FLOW_ACTION_FLAG) 977 return rte_flow_error_set(error, EINVAL, 978 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 979 "can't flag and mark in same flow"); 980 if (action_flags & MLX5_FLOW_ACTION_MARK) 981 return rte_flow_error_set(error, EINVAL, 982 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 983 "can't have 2 mark actions in same" 984 " flow"); 985 if (attr->egress) 986 return rte_flow_error_set(error, ENOTSUP, 987 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 988 "mark action not supported for " 989 "egress"); 990 return 0; 991 } 992 993 /* 994 * Validate the drop action. 995 * 996 * @param[in] action_flags 997 * Bit-fields that holds the actions detected until now. 998 * @param[in] attr 999 * Attributes of flow that includes this action. 1000 * @param[out] error 1001 * Pointer to error structure. 1002 * 1003 * @return 1004 * 0 on success, a negative errno value otherwise and rte_errno is set. 1005 */ 1006 int 1007 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1008 const struct rte_flow_attr *attr, 1009 struct rte_flow_error *error) 1010 { 1011 if (attr->egress) 1012 return rte_flow_error_set(error, ENOTSUP, 1013 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1014 "drop action not supported for " 1015 "egress"); 1016 return 0; 1017 } 1018 1019 /* 1020 * Validate the queue action. 1021 * 1022 * @param[in] action 1023 * Pointer to the queue action. 1024 * @param[in] action_flags 1025 * Bit-fields that holds the actions detected until now. 1026 * @param[in] dev 1027 * Pointer to the Ethernet device structure. 1028 * @param[in] attr 1029 * Attributes of flow that includes this action. 1030 * @param[out] error 1031 * Pointer to error structure. 1032 * 1033 * @return 1034 * 0 on success, a negative errno value otherwise and rte_errno is set. 1035 */ 1036 int 1037 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1038 uint64_t action_flags, 1039 struct rte_eth_dev *dev, 1040 const struct rte_flow_attr *attr, 1041 struct rte_flow_error *error) 1042 { 1043 struct mlx5_priv *priv = dev->data->dev_private; 1044 const struct rte_flow_action_queue *queue = action->conf; 1045 1046 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1047 return rte_flow_error_set(error, EINVAL, 1048 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1049 "can't have 2 fate actions in" 1050 " same flow"); 1051 if (!priv->rxqs_n) 1052 return rte_flow_error_set(error, EINVAL, 1053 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1054 NULL, "No Rx queues configured"); 1055 if (queue->index >= priv->rxqs_n) 1056 return rte_flow_error_set(error, EINVAL, 1057 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1058 &queue->index, 1059 "queue index out of range"); 1060 if (!(*priv->rxqs)[queue->index]) 1061 return rte_flow_error_set(error, EINVAL, 1062 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1063 &queue->index, 1064 "queue is not configured"); 1065 if (attr->egress) 1066 return rte_flow_error_set(error, ENOTSUP, 1067 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1068 "queue action not supported for " 1069 "egress"); 1070 return 0; 1071 } 1072 1073 /* 1074 * Validate the rss action. 1075 * 1076 * @param[in] action 1077 * Pointer to the queue action. 1078 * @param[in] action_flags 1079 * Bit-fields that holds the actions detected until now. 1080 * @param[in] dev 1081 * Pointer to the Ethernet device structure. 1082 * @param[in] attr 1083 * Attributes of flow that includes this action. 1084 * @param[in] item_flags 1085 * Items that were detected. 1086 * @param[out] error 1087 * Pointer to error structure. 1088 * 1089 * @return 1090 * 0 on success, a negative errno value otherwise and rte_errno is set. 1091 */ 1092 int 1093 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1094 uint64_t action_flags, 1095 struct rte_eth_dev *dev, 1096 const struct rte_flow_attr *attr, 1097 uint64_t item_flags, 1098 struct rte_flow_error *error) 1099 { 1100 struct mlx5_priv *priv = dev->data->dev_private; 1101 const struct rte_flow_action_rss *rss = action->conf; 1102 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1103 unsigned int i; 1104 1105 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1106 return rte_flow_error_set(error, EINVAL, 1107 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1108 "can't have 2 fate actions" 1109 " in same flow"); 1110 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1111 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1112 return rte_flow_error_set(error, ENOTSUP, 1113 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1114 &rss->func, 1115 "RSS hash function not supported"); 1116 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1117 if (rss->level > 2) 1118 #else 1119 if (rss->level > 1) 1120 #endif 1121 return rte_flow_error_set(error, ENOTSUP, 1122 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1123 &rss->level, 1124 "tunnel RSS is not supported"); 1125 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1126 if (rss->key_len == 0 && rss->key != NULL) 1127 return rte_flow_error_set(error, ENOTSUP, 1128 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1129 &rss->key_len, 1130 "RSS hash key length 0"); 1131 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1132 return rte_flow_error_set(error, ENOTSUP, 1133 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1134 &rss->key_len, 1135 "RSS hash key too small"); 1136 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1137 return rte_flow_error_set(error, ENOTSUP, 1138 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1139 &rss->key_len, 1140 "RSS hash key too large"); 1141 if (rss->queue_num > priv->config.ind_table_max_size) 1142 return rte_flow_error_set(error, ENOTSUP, 1143 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1144 &rss->queue_num, 1145 "number of queues too large"); 1146 if (rss->types & MLX5_RSS_HF_MASK) 1147 return rte_flow_error_set(error, ENOTSUP, 1148 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1149 &rss->types, 1150 "some RSS protocols are not" 1151 " supported"); 1152 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1153 !(rss->types & ETH_RSS_IP)) 1154 return rte_flow_error_set(error, EINVAL, 1155 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1156 "L3 partial RSS requested but L3 RSS" 1157 " type not specified"); 1158 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1159 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1160 return rte_flow_error_set(error, EINVAL, 1161 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1162 "L4 partial RSS requested but L4 RSS" 1163 " type not specified"); 1164 if (!priv->rxqs_n) 1165 return rte_flow_error_set(error, EINVAL, 1166 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1167 NULL, "No Rx queues configured"); 1168 if (!rss->queue_num) 1169 return rte_flow_error_set(error, EINVAL, 1170 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1171 NULL, "No queues configured"); 1172 for (i = 0; i != rss->queue_num; ++i) { 1173 if (rss->queue[i] >= priv->rxqs_n) 1174 return rte_flow_error_set 1175 (error, EINVAL, 1176 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1177 &rss->queue[i], "queue index out of range"); 1178 if (!(*priv->rxqs)[rss->queue[i]]) 1179 return rte_flow_error_set 1180 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1181 &rss->queue[i], "queue is not configured"); 1182 } 1183 if (attr->egress) 1184 return rte_flow_error_set(error, ENOTSUP, 1185 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1186 "rss action not supported for " 1187 "egress"); 1188 if (rss->level > 1 && !tunnel) 1189 return rte_flow_error_set(error, EINVAL, 1190 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1191 "inner RSS is not supported for " 1192 "non-tunnel flows"); 1193 return 0; 1194 } 1195 1196 /* 1197 * Validate the count action. 1198 * 1199 * @param[in] dev 1200 * Pointer to the Ethernet device structure. 1201 * @param[in] attr 1202 * Attributes of flow that includes this action. 1203 * @param[out] error 1204 * Pointer to error structure. 1205 * 1206 * @return 1207 * 0 on success, a negative errno value otherwise and rte_errno is set. 1208 */ 1209 int 1210 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1211 const struct rte_flow_attr *attr, 1212 struct rte_flow_error *error) 1213 { 1214 if (attr->egress) 1215 return rte_flow_error_set(error, ENOTSUP, 1216 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1217 "count action not supported for " 1218 "egress"); 1219 return 0; 1220 } 1221 1222 /** 1223 * Verify the @p attributes will be correctly understood by the NIC and store 1224 * them in the @p flow if everything is correct. 1225 * 1226 * @param[in] dev 1227 * Pointer to the Ethernet device structure. 1228 * @param[in] attributes 1229 * Pointer to flow attributes 1230 * @param[out] error 1231 * Pointer to error structure. 1232 * 1233 * @return 1234 * 0 on success, a negative errno value otherwise and rte_errno is set. 1235 */ 1236 int 1237 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1238 const struct rte_flow_attr *attributes, 1239 struct rte_flow_error *error) 1240 { 1241 struct mlx5_priv *priv = dev->data->dev_private; 1242 uint32_t priority_max = priv->config.flow_prio - 1; 1243 1244 if (attributes->group) 1245 return rte_flow_error_set(error, ENOTSUP, 1246 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1247 NULL, "groups is not supported"); 1248 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1249 attributes->priority >= priority_max) 1250 return rte_flow_error_set(error, ENOTSUP, 1251 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1252 NULL, "priority out of range"); 1253 if (attributes->egress) 1254 return rte_flow_error_set(error, ENOTSUP, 1255 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1256 "egress is not supported"); 1257 if (attributes->transfer && !priv->config.dv_esw_en) 1258 return rte_flow_error_set(error, ENOTSUP, 1259 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1260 NULL, "transfer is not supported"); 1261 if (!attributes->ingress) 1262 return rte_flow_error_set(error, EINVAL, 1263 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1264 NULL, 1265 "ingress attribute is mandatory"); 1266 return 0; 1267 } 1268 1269 /** 1270 * Validate ICMP6 item. 1271 * 1272 * @param[in] item 1273 * Item specification. 1274 * @param[in] item_flags 1275 * Bit-fields that holds the items detected until now. 1276 * @param[out] error 1277 * Pointer to error structure. 1278 * 1279 * @return 1280 * 0 on success, a negative errno value otherwise and rte_errno is set. 1281 */ 1282 int 1283 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1284 uint64_t item_flags, 1285 uint8_t target_protocol, 1286 struct rte_flow_error *error) 1287 { 1288 const struct rte_flow_item_icmp6 *mask = item->mask; 1289 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1290 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1291 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1292 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1293 MLX5_FLOW_LAYER_OUTER_L4; 1294 int ret; 1295 1296 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1297 return rte_flow_error_set(error, EINVAL, 1298 RTE_FLOW_ERROR_TYPE_ITEM, item, 1299 "protocol filtering not compatible" 1300 " with ICMP6 layer"); 1301 if (!(item_flags & l3m)) 1302 return rte_flow_error_set(error, EINVAL, 1303 RTE_FLOW_ERROR_TYPE_ITEM, item, 1304 "IPv6 is mandatory to filter on" 1305 " ICMP6"); 1306 if (item_flags & l4m) 1307 return rte_flow_error_set(error, EINVAL, 1308 RTE_FLOW_ERROR_TYPE_ITEM, item, 1309 "multiple L4 layers not supported"); 1310 if (!mask) 1311 mask = &rte_flow_item_icmp6_mask; 1312 ret = mlx5_flow_item_acceptable 1313 (item, (const uint8_t *)mask, 1314 (const uint8_t *)&rte_flow_item_icmp6_mask, 1315 sizeof(struct rte_flow_item_icmp6), error); 1316 if (ret < 0) 1317 return ret; 1318 return 0; 1319 } 1320 1321 /** 1322 * Validate ICMP item. 1323 * 1324 * @param[in] item 1325 * Item specification. 1326 * @param[in] item_flags 1327 * Bit-fields that holds the items detected until now. 1328 * @param[out] error 1329 * Pointer to error structure. 1330 * 1331 * @return 1332 * 0 on success, a negative errno value otherwise and rte_errno is set. 1333 */ 1334 int 1335 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1336 uint64_t item_flags, 1337 uint8_t target_protocol, 1338 struct rte_flow_error *error) 1339 { 1340 const struct rte_flow_item_icmp *mask = item->mask; 1341 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1342 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1343 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1344 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1345 MLX5_FLOW_LAYER_OUTER_L4; 1346 int ret; 1347 1348 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1349 return rte_flow_error_set(error, EINVAL, 1350 RTE_FLOW_ERROR_TYPE_ITEM, item, 1351 "protocol filtering not compatible" 1352 " with ICMP layer"); 1353 if (!(item_flags & l3m)) 1354 return rte_flow_error_set(error, EINVAL, 1355 RTE_FLOW_ERROR_TYPE_ITEM, item, 1356 "IPv4 is mandatory to filter" 1357 " on ICMP"); 1358 if (item_flags & l4m) 1359 return rte_flow_error_set(error, EINVAL, 1360 RTE_FLOW_ERROR_TYPE_ITEM, item, 1361 "multiple L4 layers not supported"); 1362 if (!mask) 1363 mask = &rte_flow_item_icmp_mask; 1364 ret = mlx5_flow_item_acceptable 1365 (item, (const uint8_t *)mask, 1366 (const uint8_t *)&rte_flow_item_icmp_mask, 1367 sizeof(struct rte_flow_item_icmp), error); 1368 if (ret < 0) 1369 return ret; 1370 return 0; 1371 } 1372 1373 /** 1374 * Validate Ethernet item. 1375 * 1376 * @param[in] item 1377 * Item specification. 1378 * @param[in] item_flags 1379 * Bit-fields that holds the items detected until now. 1380 * @param[out] error 1381 * Pointer to error structure. 1382 * 1383 * @return 1384 * 0 on success, a negative errno value otherwise and rte_errno is set. 1385 */ 1386 int 1387 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1388 uint64_t item_flags, 1389 struct rte_flow_error *error) 1390 { 1391 const struct rte_flow_item_eth *mask = item->mask; 1392 const struct rte_flow_item_eth nic_mask = { 1393 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1394 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1395 .type = RTE_BE16(0xffff), 1396 }; 1397 int ret; 1398 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1399 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1400 MLX5_FLOW_LAYER_OUTER_L2; 1401 1402 if (item_flags & ethm) 1403 return rte_flow_error_set(error, ENOTSUP, 1404 RTE_FLOW_ERROR_TYPE_ITEM, item, 1405 "multiple L2 layers not supported"); 1406 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1407 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1408 return rte_flow_error_set(error, EINVAL, 1409 RTE_FLOW_ERROR_TYPE_ITEM, item, 1410 "L2 layer should not follow " 1411 "L3 layers"); 1412 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1413 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1414 return rte_flow_error_set(error, EINVAL, 1415 RTE_FLOW_ERROR_TYPE_ITEM, item, 1416 "L2 layer should not follow VLAN"); 1417 if (!mask) 1418 mask = &rte_flow_item_eth_mask; 1419 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1420 (const uint8_t *)&nic_mask, 1421 sizeof(struct rte_flow_item_eth), 1422 error); 1423 return ret; 1424 } 1425 1426 /** 1427 * Validate VLAN item. 1428 * 1429 * @param[in] item 1430 * Item specification. 1431 * @param[in] item_flags 1432 * Bit-fields that holds the items detected until now. 1433 * @param[in] dev 1434 * Ethernet device flow is being created on. 1435 * @param[out] error 1436 * Pointer to error structure. 1437 * 1438 * @return 1439 * 0 on success, a negative errno value otherwise and rte_errno is set. 1440 */ 1441 int 1442 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1443 uint64_t item_flags, 1444 struct rte_eth_dev *dev, 1445 struct rte_flow_error *error) 1446 { 1447 const struct rte_flow_item_vlan *spec = item->spec; 1448 const struct rte_flow_item_vlan *mask = item->mask; 1449 const struct rte_flow_item_vlan nic_mask = { 1450 .tci = RTE_BE16(UINT16_MAX), 1451 .inner_type = RTE_BE16(UINT16_MAX), 1452 }; 1453 uint16_t vlan_tag = 0; 1454 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1455 int ret; 1456 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1457 MLX5_FLOW_LAYER_INNER_L4) : 1458 (MLX5_FLOW_LAYER_OUTER_L3 | 1459 MLX5_FLOW_LAYER_OUTER_L4); 1460 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1461 MLX5_FLOW_LAYER_OUTER_VLAN; 1462 1463 if (item_flags & vlanm) 1464 return rte_flow_error_set(error, EINVAL, 1465 RTE_FLOW_ERROR_TYPE_ITEM, item, 1466 "multiple VLAN layers not supported"); 1467 else if ((item_flags & l34m) != 0) 1468 return rte_flow_error_set(error, EINVAL, 1469 RTE_FLOW_ERROR_TYPE_ITEM, item, 1470 "VLAN cannot follow L3/L4 layer"); 1471 if (!mask) 1472 mask = &rte_flow_item_vlan_mask; 1473 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1474 (const uint8_t *)&nic_mask, 1475 sizeof(struct rte_flow_item_vlan), 1476 error); 1477 if (ret) 1478 return ret; 1479 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1480 struct mlx5_priv *priv = dev->data->dev_private; 1481 1482 if (priv->vmwa_context) { 1483 /* 1484 * Non-NULL context means we have a virtual machine 1485 * and SR-IOV enabled, we have to create VLAN interface 1486 * to make hypervisor to setup E-Switch vport 1487 * context correctly. We avoid creating the multiple 1488 * VLAN interfaces, so we cannot support VLAN tag mask. 1489 */ 1490 return rte_flow_error_set(error, EINVAL, 1491 RTE_FLOW_ERROR_TYPE_ITEM, 1492 item, 1493 "VLAN tag mask is not" 1494 " supported in virtual" 1495 " environment"); 1496 } 1497 } 1498 if (spec) { 1499 vlan_tag = spec->tci; 1500 vlan_tag &= mask->tci; 1501 } 1502 /* 1503 * From verbs perspective an empty VLAN is equivalent 1504 * to a packet without VLAN layer. 1505 */ 1506 if (!vlan_tag) 1507 return rte_flow_error_set(error, EINVAL, 1508 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1509 item->spec, 1510 "VLAN cannot be empty"); 1511 return 0; 1512 } 1513 1514 /** 1515 * Validate IPV4 item. 1516 * 1517 * @param[in] item 1518 * Item specification. 1519 * @param[in] item_flags 1520 * Bit-fields that holds the items detected until now. 1521 * @param[in] acc_mask 1522 * Acceptable mask, if NULL default internal default mask 1523 * will be used to check whether item fields are supported. 1524 * @param[out] error 1525 * Pointer to error structure. 1526 * 1527 * @return 1528 * 0 on success, a negative errno value otherwise and rte_errno is set. 1529 */ 1530 int 1531 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1532 uint64_t item_flags, 1533 uint64_t last_item, 1534 uint16_t ether_type, 1535 const struct rte_flow_item_ipv4 *acc_mask, 1536 struct rte_flow_error *error) 1537 { 1538 const struct rte_flow_item_ipv4 *mask = item->mask; 1539 const struct rte_flow_item_ipv4 *spec = item->spec; 1540 const struct rte_flow_item_ipv4 nic_mask = { 1541 .hdr = { 1542 .src_addr = RTE_BE32(0xffffffff), 1543 .dst_addr = RTE_BE32(0xffffffff), 1544 .type_of_service = 0xff, 1545 .next_proto_id = 0xff, 1546 }, 1547 }; 1548 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1549 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1550 MLX5_FLOW_LAYER_OUTER_L3; 1551 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1552 MLX5_FLOW_LAYER_OUTER_L4; 1553 int ret; 1554 uint8_t next_proto = 0xFF; 1555 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1556 MLX5_FLOW_LAYER_OUTER_VLAN | 1557 MLX5_FLOW_LAYER_INNER_VLAN); 1558 1559 if ((last_item & l2_vlan) && ether_type && 1560 ether_type != RTE_ETHER_TYPE_IPV4) 1561 return rte_flow_error_set(error, EINVAL, 1562 RTE_FLOW_ERROR_TYPE_ITEM, item, 1563 "IPv4 cannot follow L2/VLAN layer " 1564 "which ether type is not IPv4"); 1565 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1566 if (mask && spec) 1567 next_proto = mask->hdr.next_proto_id & 1568 spec->hdr.next_proto_id; 1569 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1570 return rte_flow_error_set(error, EINVAL, 1571 RTE_FLOW_ERROR_TYPE_ITEM, 1572 item, 1573 "multiple tunnel " 1574 "not supported"); 1575 } 1576 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1577 return rte_flow_error_set(error, EINVAL, 1578 RTE_FLOW_ERROR_TYPE_ITEM, item, 1579 "wrong tunnel type - IPv6 specified " 1580 "but IPv4 item provided"); 1581 if (item_flags & l3m) 1582 return rte_flow_error_set(error, ENOTSUP, 1583 RTE_FLOW_ERROR_TYPE_ITEM, item, 1584 "multiple L3 layers not supported"); 1585 else if (item_flags & l4m) 1586 return rte_flow_error_set(error, EINVAL, 1587 RTE_FLOW_ERROR_TYPE_ITEM, item, 1588 "L3 cannot follow an L4 layer."); 1589 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1590 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1591 return rte_flow_error_set(error, EINVAL, 1592 RTE_FLOW_ERROR_TYPE_ITEM, item, 1593 "L3 cannot follow an NVGRE layer."); 1594 if (!mask) 1595 mask = &rte_flow_item_ipv4_mask; 1596 else if (mask->hdr.next_proto_id != 0 && 1597 mask->hdr.next_proto_id != 0xff) 1598 return rte_flow_error_set(error, EINVAL, 1599 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1600 "partial mask is not supported" 1601 " for protocol"); 1602 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1603 acc_mask ? (const uint8_t *)acc_mask 1604 : (const uint8_t *)&nic_mask, 1605 sizeof(struct rte_flow_item_ipv4), 1606 error); 1607 if (ret < 0) 1608 return ret; 1609 return 0; 1610 } 1611 1612 /** 1613 * Validate IPV6 item. 1614 * 1615 * @param[in] item 1616 * Item specification. 1617 * @param[in] item_flags 1618 * Bit-fields that holds the items detected until now. 1619 * @param[in] acc_mask 1620 * Acceptable mask, if NULL default internal default mask 1621 * will be used to check whether item fields are supported. 1622 * @param[out] error 1623 * Pointer to error structure. 1624 * 1625 * @return 1626 * 0 on success, a negative errno value otherwise and rte_errno is set. 1627 */ 1628 int 1629 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1630 uint64_t item_flags, 1631 uint64_t last_item, 1632 uint16_t ether_type, 1633 const struct rte_flow_item_ipv6 *acc_mask, 1634 struct rte_flow_error *error) 1635 { 1636 const struct rte_flow_item_ipv6 *mask = item->mask; 1637 const struct rte_flow_item_ipv6 *spec = item->spec; 1638 const struct rte_flow_item_ipv6 nic_mask = { 1639 .hdr = { 1640 .src_addr = 1641 "\xff\xff\xff\xff\xff\xff\xff\xff" 1642 "\xff\xff\xff\xff\xff\xff\xff\xff", 1643 .dst_addr = 1644 "\xff\xff\xff\xff\xff\xff\xff\xff" 1645 "\xff\xff\xff\xff\xff\xff\xff\xff", 1646 .vtc_flow = RTE_BE32(0xffffffff), 1647 .proto = 0xff, 1648 }, 1649 }; 1650 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1651 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1652 MLX5_FLOW_LAYER_OUTER_L3; 1653 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1654 MLX5_FLOW_LAYER_OUTER_L4; 1655 int ret; 1656 uint8_t next_proto = 0xFF; 1657 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1658 MLX5_FLOW_LAYER_OUTER_VLAN | 1659 MLX5_FLOW_LAYER_INNER_VLAN); 1660 1661 if ((last_item & l2_vlan) && ether_type && 1662 ether_type != RTE_ETHER_TYPE_IPV6) 1663 return rte_flow_error_set(error, EINVAL, 1664 RTE_FLOW_ERROR_TYPE_ITEM, item, 1665 "IPv6 cannot follow L2/VLAN layer " 1666 "which ether type is not IPv6"); 1667 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1668 if (mask && spec) 1669 next_proto = mask->hdr.proto & spec->hdr.proto; 1670 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1671 return rte_flow_error_set(error, EINVAL, 1672 RTE_FLOW_ERROR_TYPE_ITEM, 1673 item, 1674 "multiple tunnel " 1675 "not supported"); 1676 } 1677 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1678 return rte_flow_error_set(error, EINVAL, 1679 RTE_FLOW_ERROR_TYPE_ITEM, item, 1680 "wrong tunnel type - IPv4 specified " 1681 "but IPv6 item provided"); 1682 if (item_flags & l3m) 1683 return rte_flow_error_set(error, ENOTSUP, 1684 RTE_FLOW_ERROR_TYPE_ITEM, item, 1685 "multiple L3 layers not supported"); 1686 else if (item_flags & l4m) 1687 return rte_flow_error_set(error, EINVAL, 1688 RTE_FLOW_ERROR_TYPE_ITEM, item, 1689 "L3 cannot follow an L4 layer."); 1690 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1691 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1692 return rte_flow_error_set(error, EINVAL, 1693 RTE_FLOW_ERROR_TYPE_ITEM, item, 1694 "L3 cannot follow an NVGRE layer."); 1695 if (!mask) 1696 mask = &rte_flow_item_ipv6_mask; 1697 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1698 acc_mask ? (const uint8_t *)acc_mask 1699 : (const uint8_t *)&nic_mask, 1700 sizeof(struct rte_flow_item_ipv6), 1701 error); 1702 if (ret < 0) 1703 return ret; 1704 return 0; 1705 } 1706 1707 /** 1708 * Validate UDP item. 1709 * 1710 * @param[in] item 1711 * Item specification. 1712 * @param[in] item_flags 1713 * Bit-fields that holds the items detected until now. 1714 * @param[in] target_protocol 1715 * The next protocol in the previous item. 1716 * @param[in] flow_mask 1717 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1718 * @param[out] error 1719 * Pointer to error structure. 1720 * 1721 * @return 1722 * 0 on success, a negative errno value otherwise and rte_errno is set. 1723 */ 1724 int 1725 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1726 uint64_t item_flags, 1727 uint8_t target_protocol, 1728 struct rte_flow_error *error) 1729 { 1730 const struct rte_flow_item_udp *mask = item->mask; 1731 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1732 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1733 MLX5_FLOW_LAYER_OUTER_L3; 1734 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1735 MLX5_FLOW_LAYER_OUTER_L4; 1736 int ret; 1737 1738 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1739 return rte_flow_error_set(error, EINVAL, 1740 RTE_FLOW_ERROR_TYPE_ITEM, item, 1741 "protocol filtering not compatible" 1742 " with UDP layer"); 1743 if (!(item_flags & l3m)) 1744 return rte_flow_error_set(error, EINVAL, 1745 RTE_FLOW_ERROR_TYPE_ITEM, item, 1746 "L3 is mandatory to filter on L4"); 1747 if (item_flags & l4m) 1748 return rte_flow_error_set(error, EINVAL, 1749 RTE_FLOW_ERROR_TYPE_ITEM, item, 1750 "multiple L4 layers not supported"); 1751 if (!mask) 1752 mask = &rte_flow_item_udp_mask; 1753 ret = mlx5_flow_item_acceptable 1754 (item, (const uint8_t *)mask, 1755 (const uint8_t *)&rte_flow_item_udp_mask, 1756 sizeof(struct rte_flow_item_udp), error); 1757 if (ret < 0) 1758 return ret; 1759 return 0; 1760 } 1761 1762 /** 1763 * Validate TCP item. 1764 * 1765 * @param[in] item 1766 * Item specification. 1767 * @param[in] item_flags 1768 * Bit-fields that holds the items detected until now. 1769 * @param[in] target_protocol 1770 * The next protocol in the previous item. 1771 * @param[out] error 1772 * Pointer to error structure. 1773 * 1774 * @return 1775 * 0 on success, a negative errno value otherwise and rte_errno is set. 1776 */ 1777 int 1778 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1779 uint64_t item_flags, 1780 uint8_t target_protocol, 1781 const struct rte_flow_item_tcp *flow_mask, 1782 struct rte_flow_error *error) 1783 { 1784 const struct rte_flow_item_tcp *mask = item->mask; 1785 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1786 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1787 MLX5_FLOW_LAYER_OUTER_L3; 1788 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1789 MLX5_FLOW_LAYER_OUTER_L4; 1790 int ret; 1791 1792 MLX5_ASSERT(flow_mask); 1793 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1794 return rte_flow_error_set(error, EINVAL, 1795 RTE_FLOW_ERROR_TYPE_ITEM, item, 1796 "protocol filtering not compatible" 1797 " with TCP layer"); 1798 if (!(item_flags & l3m)) 1799 return rte_flow_error_set(error, EINVAL, 1800 RTE_FLOW_ERROR_TYPE_ITEM, item, 1801 "L3 is mandatory to filter on L4"); 1802 if (item_flags & l4m) 1803 return rte_flow_error_set(error, EINVAL, 1804 RTE_FLOW_ERROR_TYPE_ITEM, item, 1805 "multiple L4 layers not supported"); 1806 if (!mask) 1807 mask = &rte_flow_item_tcp_mask; 1808 ret = mlx5_flow_item_acceptable 1809 (item, (const uint8_t *)mask, 1810 (const uint8_t *)flow_mask, 1811 sizeof(struct rte_flow_item_tcp), error); 1812 if (ret < 0) 1813 return ret; 1814 return 0; 1815 } 1816 1817 /** 1818 * Validate VXLAN item. 1819 * 1820 * @param[in] item 1821 * Item specification. 1822 * @param[in] item_flags 1823 * Bit-fields that holds the items detected until now. 1824 * @param[in] target_protocol 1825 * The next protocol in the previous item. 1826 * @param[out] error 1827 * Pointer to error structure. 1828 * 1829 * @return 1830 * 0 on success, a negative errno value otherwise and rte_errno is set. 1831 */ 1832 int 1833 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1834 uint64_t item_flags, 1835 struct rte_flow_error *error) 1836 { 1837 const struct rte_flow_item_vxlan *spec = item->spec; 1838 const struct rte_flow_item_vxlan *mask = item->mask; 1839 int ret; 1840 union vni { 1841 uint32_t vlan_id; 1842 uint8_t vni[4]; 1843 } id = { .vlan_id = 0, }; 1844 1845 1846 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1847 return rte_flow_error_set(error, ENOTSUP, 1848 RTE_FLOW_ERROR_TYPE_ITEM, item, 1849 "multiple tunnel layers not" 1850 " supported"); 1851 /* 1852 * Verify only UDPv4 is present as defined in 1853 * https://tools.ietf.org/html/rfc7348 1854 */ 1855 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1856 return rte_flow_error_set(error, EINVAL, 1857 RTE_FLOW_ERROR_TYPE_ITEM, item, 1858 "no outer UDP layer found"); 1859 if (!mask) 1860 mask = &rte_flow_item_vxlan_mask; 1861 ret = mlx5_flow_item_acceptable 1862 (item, (const uint8_t *)mask, 1863 (const uint8_t *)&rte_flow_item_vxlan_mask, 1864 sizeof(struct rte_flow_item_vxlan), 1865 error); 1866 if (ret < 0) 1867 return ret; 1868 if (spec) { 1869 memcpy(&id.vni[1], spec->vni, 3); 1870 memcpy(&id.vni[1], mask->vni, 3); 1871 } 1872 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1873 return rte_flow_error_set(error, ENOTSUP, 1874 RTE_FLOW_ERROR_TYPE_ITEM, item, 1875 "VXLAN tunnel must be fully defined"); 1876 return 0; 1877 } 1878 1879 /** 1880 * Validate VXLAN_GPE item. 1881 * 1882 * @param[in] item 1883 * Item specification. 1884 * @param[in] item_flags 1885 * Bit-fields that holds the items detected until now. 1886 * @param[in] priv 1887 * Pointer to the private data structure. 1888 * @param[in] target_protocol 1889 * The next protocol in the previous item. 1890 * @param[out] error 1891 * Pointer to error structure. 1892 * 1893 * @return 1894 * 0 on success, a negative errno value otherwise and rte_errno is set. 1895 */ 1896 int 1897 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1898 uint64_t item_flags, 1899 struct rte_eth_dev *dev, 1900 struct rte_flow_error *error) 1901 { 1902 struct mlx5_priv *priv = dev->data->dev_private; 1903 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1904 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1905 int ret; 1906 union vni { 1907 uint32_t vlan_id; 1908 uint8_t vni[4]; 1909 } id = { .vlan_id = 0, }; 1910 1911 if (!priv->config.l3_vxlan_en) 1912 return rte_flow_error_set(error, ENOTSUP, 1913 RTE_FLOW_ERROR_TYPE_ITEM, item, 1914 "L3 VXLAN is not enabled by device" 1915 " parameter and/or not configured in" 1916 " firmware"); 1917 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1918 return rte_flow_error_set(error, ENOTSUP, 1919 RTE_FLOW_ERROR_TYPE_ITEM, item, 1920 "multiple tunnel layers not" 1921 " supported"); 1922 /* 1923 * Verify only UDPv4 is present as defined in 1924 * https://tools.ietf.org/html/rfc7348 1925 */ 1926 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1927 return rte_flow_error_set(error, EINVAL, 1928 RTE_FLOW_ERROR_TYPE_ITEM, item, 1929 "no outer UDP layer found"); 1930 if (!mask) 1931 mask = &rte_flow_item_vxlan_gpe_mask; 1932 ret = mlx5_flow_item_acceptable 1933 (item, (const uint8_t *)mask, 1934 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1935 sizeof(struct rte_flow_item_vxlan_gpe), 1936 error); 1937 if (ret < 0) 1938 return ret; 1939 if (spec) { 1940 if (spec->protocol) 1941 return rte_flow_error_set(error, ENOTSUP, 1942 RTE_FLOW_ERROR_TYPE_ITEM, 1943 item, 1944 "VxLAN-GPE protocol" 1945 " not supported"); 1946 memcpy(&id.vni[1], spec->vni, 3); 1947 memcpy(&id.vni[1], mask->vni, 3); 1948 } 1949 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1950 return rte_flow_error_set(error, ENOTSUP, 1951 RTE_FLOW_ERROR_TYPE_ITEM, item, 1952 "VXLAN-GPE tunnel must be fully" 1953 " defined"); 1954 return 0; 1955 } 1956 /** 1957 * Validate GRE Key item. 1958 * 1959 * @param[in] item 1960 * Item specification. 1961 * @param[in] item_flags 1962 * Bit flags to mark detected items. 1963 * @param[in] gre_item 1964 * Pointer to gre_item 1965 * @param[out] error 1966 * Pointer to error structure. 1967 * 1968 * @return 1969 * 0 on success, a negative errno value otherwise and rte_errno is set. 1970 */ 1971 int 1972 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 1973 uint64_t item_flags, 1974 const struct rte_flow_item *gre_item, 1975 struct rte_flow_error *error) 1976 { 1977 const rte_be32_t *mask = item->mask; 1978 int ret = 0; 1979 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 1980 const struct rte_flow_item_gre *gre_spec; 1981 const struct rte_flow_item_gre *gre_mask; 1982 1983 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 1984 return rte_flow_error_set(error, ENOTSUP, 1985 RTE_FLOW_ERROR_TYPE_ITEM, item, 1986 "Multiple GRE key not support"); 1987 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 1988 return rte_flow_error_set(error, ENOTSUP, 1989 RTE_FLOW_ERROR_TYPE_ITEM, item, 1990 "No preceding GRE header"); 1991 if (item_flags & MLX5_FLOW_LAYER_INNER) 1992 return rte_flow_error_set(error, ENOTSUP, 1993 RTE_FLOW_ERROR_TYPE_ITEM, item, 1994 "GRE key following a wrong item"); 1995 gre_mask = gre_item->mask; 1996 if (!gre_mask) 1997 gre_mask = &rte_flow_item_gre_mask; 1998 gre_spec = gre_item->spec; 1999 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2000 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2001 return rte_flow_error_set(error, EINVAL, 2002 RTE_FLOW_ERROR_TYPE_ITEM, item, 2003 "Key bit must be on"); 2004 2005 if (!mask) 2006 mask = &gre_key_default_mask; 2007 ret = mlx5_flow_item_acceptable 2008 (item, (const uint8_t *)mask, 2009 (const uint8_t *)&gre_key_default_mask, 2010 sizeof(rte_be32_t), error); 2011 return ret; 2012 } 2013 2014 /** 2015 * Validate GRE item. 2016 * 2017 * @param[in] item 2018 * Item specification. 2019 * @param[in] item_flags 2020 * Bit flags to mark detected items. 2021 * @param[in] target_protocol 2022 * The next protocol in the previous item. 2023 * @param[out] error 2024 * Pointer to error structure. 2025 * 2026 * @return 2027 * 0 on success, a negative errno value otherwise and rte_errno is set. 2028 */ 2029 int 2030 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2031 uint64_t item_flags, 2032 uint8_t target_protocol, 2033 struct rte_flow_error *error) 2034 { 2035 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2036 const struct rte_flow_item_gre *mask = item->mask; 2037 int ret; 2038 const struct rte_flow_item_gre nic_mask = { 2039 .c_rsvd0_ver = RTE_BE16(0xB000), 2040 .protocol = RTE_BE16(UINT16_MAX), 2041 }; 2042 2043 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2044 return rte_flow_error_set(error, EINVAL, 2045 RTE_FLOW_ERROR_TYPE_ITEM, item, 2046 "protocol filtering not compatible" 2047 " with this GRE layer"); 2048 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2049 return rte_flow_error_set(error, ENOTSUP, 2050 RTE_FLOW_ERROR_TYPE_ITEM, item, 2051 "multiple tunnel layers not" 2052 " supported"); 2053 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2054 return rte_flow_error_set(error, ENOTSUP, 2055 RTE_FLOW_ERROR_TYPE_ITEM, item, 2056 "L3 Layer is missing"); 2057 if (!mask) 2058 mask = &rte_flow_item_gre_mask; 2059 ret = mlx5_flow_item_acceptable 2060 (item, (const uint8_t *)mask, 2061 (const uint8_t *)&nic_mask, 2062 sizeof(struct rte_flow_item_gre), error); 2063 if (ret < 0) 2064 return ret; 2065 #ifndef HAVE_MLX5DV_DR 2066 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2067 if (spec && (spec->protocol & mask->protocol)) 2068 return rte_flow_error_set(error, ENOTSUP, 2069 RTE_FLOW_ERROR_TYPE_ITEM, item, 2070 "without MPLS support the" 2071 " specification cannot be used for" 2072 " filtering"); 2073 #endif 2074 #endif 2075 return 0; 2076 } 2077 2078 /** 2079 * Validate Geneve item. 2080 * 2081 * @param[in] item 2082 * Item specification. 2083 * @param[in] itemFlags 2084 * Bit-fields that holds the items detected until now. 2085 * @param[in] enPriv 2086 * Pointer to the private data structure. 2087 * @param[out] error 2088 * Pointer to error structure. 2089 * 2090 * @return 2091 * 0 on success, a negative errno value otherwise and rte_errno is set. 2092 */ 2093 2094 int 2095 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2096 uint64_t item_flags, 2097 struct rte_eth_dev *dev, 2098 struct rte_flow_error *error) 2099 { 2100 struct mlx5_priv *priv = dev->data->dev_private; 2101 const struct rte_flow_item_geneve *spec = item->spec; 2102 const struct rte_flow_item_geneve *mask = item->mask; 2103 int ret; 2104 uint16_t gbhdr; 2105 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2106 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2107 const struct rte_flow_item_geneve nic_mask = { 2108 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2109 .vni = "\xff\xff\xff", 2110 .protocol = RTE_BE16(UINT16_MAX), 2111 }; 2112 2113 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2114 return rte_flow_error_set(error, ENOTSUP, 2115 RTE_FLOW_ERROR_TYPE_ITEM, item, 2116 "L3 Geneve is not enabled by device" 2117 " parameter and/or not configured in" 2118 " firmware"); 2119 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2120 return rte_flow_error_set(error, ENOTSUP, 2121 RTE_FLOW_ERROR_TYPE_ITEM, item, 2122 "multiple tunnel layers not" 2123 " supported"); 2124 /* 2125 * Verify only UDPv4 is present as defined in 2126 * https://tools.ietf.org/html/rfc7348 2127 */ 2128 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2129 return rte_flow_error_set(error, EINVAL, 2130 RTE_FLOW_ERROR_TYPE_ITEM, item, 2131 "no outer UDP layer found"); 2132 if (!mask) 2133 mask = &rte_flow_item_geneve_mask; 2134 ret = mlx5_flow_item_acceptable 2135 (item, (const uint8_t *)mask, 2136 (const uint8_t *)&nic_mask, 2137 sizeof(struct rte_flow_item_geneve), error); 2138 if (ret) 2139 return ret; 2140 if (spec) { 2141 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2142 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2143 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2144 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2145 return rte_flow_error_set(error, ENOTSUP, 2146 RTE_FLOW_ERROR_TYPE_ITEM, 2147 item, 2148 "Geneve protocol unsupported" 2149 " fields are being used"); 2150 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2151 return rte_flow_error_set 2152 (error, ENOTSUP, 2153 RTE_FLOW_ERROR_TYPE_ITEM, 2154 item, 2155 "Unsupported Geneve options length"); 2156 } 2157 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2158 return rte_flow_error_set 2159 (error, ENOTSUP, 2160 RTE_FLOW_ERROR_TYPE_ITEM, item, 2161 "Geneve tunnel must be fully defined"); 2162 return 0; 2163 } 2164 2165 /** 2166 * Validate MPLS item. 2167 * 2168 * @param[in] dev 2169 * Pointer to the rte_eth_dev structure. 2170 * @param[in] item 2171 * Item specification. 2172 * @param[in] item_flags 2173 * Bit-fields that holds the items detected until now. 2174 * @param[in] prev_layer 2175 * The protocol layer indicated in previous item. 2176 * @param[out] error 2177 * Pointer to error structure. 2178 * 2179 * @return 2180 * 0 on success, a negative errno value otherwise and rte_errno is set. 2181 */ 2182 int 2183 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2184 const struct rte_flow_item *item __rte_unused, 2185 uint64_t item_flags __rte_unused, 2186 uint64_t prev_layer __rte_unused, 2187 struct rte_flow_error *error) 2188 { 2189 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2190 const struct rte_flow_item_mpls *mask = item->mask; 2191 struct mlx5_priv *priv = dev->data->dev_private; 2192 int ret; 2193 2194 if (!priv->config.mpls_en) 2195 return rte_flow_error_set(error, ENOTSUP, 2196 RTE_FLOW_ERROR_TYPE_ITEM, item, 2197 "MPLS not supported or" 2198 " disabled in firmware" 2199 " configuration."); 2200 /* MPLS over IP, UDP, GRE is allowed */ 2201 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2202 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2203 MLX5_FLOW_LAYER_GRE))) 2204 return rte_flow_error_set(error, EINVAL, 2205 RTE_FLOW_ERROR_TYPE_ITEM, item, 2206 "protocol filtering not compatible" 2207 " with MPLS layer"); 2208 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2209 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2210 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2211 return rte_flow_error_set(error, ENOTSUP, 2212 RTE_FLOW_ERROR_TYPE_ITEM, item, 2213 "multiple tunnel layers not" 2214 " supported"); 2215 if (!mask) 2216 mask = &rte_flow_item_mpls_mask; 2217 ret = mlx5_flow_item_acceptable 2218 (item, (const uint8_t *)mask, 2219 (const uint8_t *)&rte_flow_item_mpls_mask, 2220 sizeof(struct rte_flow_item_mpls), error); 2221 if (ret < 0) 2222 return ret; 2223 return 0; 2224 #endif 2225 return rte_flow_error_set(error, ENOTSUP, 2226 RTE_FLOW_ERROR_TYPE_ITEM, item, 2227 "MPLS is not supported by Verbs, please" 2228 " update."); 2229 } 2230 2231 /** 2232 * Validate NVGRE item. 2233 * 2234 * @param[in] item 2235 * Item specification. 2236 * @param[in] item_flags 2237 * Bit flags to mark detected items. 2238 * @param[in] target_protocol 2239 * The next protocol in the previous item. 2240 * @param[out] error 2241 * Pointer to error structure. 2242 * 2243 * @return 2244 * 0 on success, a negative errno value otherwise and rte_errno is set. 2245 */ 2246 int 2247 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2248 uint64_t item_flags, 2249 uint8_t target_protocol, 2250 struct rte_flow_error *error) 2251 { 2252 const struct rte_flow_item_nvgre *mask = item->mask; 2253 int ret; 2254 2255 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2256 return rte_flow_error_set(error, EINVAL, 2257 RTE_FLOW_ERROR_TYPE_ITEM, item, 2258 "protocol filtering not compatible" 2259 " with this GRE layer"); 2260 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2261 return rte_flow_error_set(error, ENOTSUP, 2262 RTE_FLOW_ERROR_TYPE_ITEM, item, 2263 "multiple tunnel layers not" 2264 " supported"); 2265 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2266 return rte_flow_error_set(error, ENOTSUP, 2267 RTE_FLOW_ERROR_TYPE_ITEM, item, 2268 "L3 Layer is missing"); 2269 if (!mask) 2270 mask = &rte_flow_item_nvgre_mask; 2271 ret = mlx5_flow_item_acceptable 2272 (item, (const uint8_t *)mask, 2273 (const uint8_t *)&rte_flow_item_nvgre_mask, 2274 sizeof(struct rte_flow_item_nvgre), error); 2275 if (ret < 0) 2276 return ret; 2277 return 0; 2278 } 2279 2280 /* Allocate unique ID for the split Q/RSS subflows. */ 2281 static uint32_t 2282 flow_qrss_get_id(struct rte_eth_dev *dev) 2283 { 2284 struct mlx5_priv *priv = dev->data->dev_private; 2285 uint32_t qrss_id, ret; 2286 2287 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2288 if (ret) 2289 return 0; 2290 MLX5_ASSERT(qrss_id); 2291 return qrss_id; 2292 } 2293 2294 /* Free unique ID for the split Q/RSS subflows. */ 2295 static void 2296 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2297 { 2298 struct mlx5_priv *priv = dev->data->dev_private; 2299 2300 if (qrss_id) 2301 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2302 } 2303 2304 /** 2305 * Release resource related QUEUE/RSS action split. 2306 * 2307 * @param dev 2308 * Pointer to Ethernet device. 2309 * @param flow 2310 * Flow to release id's from. 2311 */ 2312 static void 2313 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2314 struct rte_flow *flow) 2315 { 2316 struct mlx5_flow_handle *dev_handle; 2317 2318 LIST_FOREACH(dev_handle, &flow->dev_handles, next) 2319 if (dev_handle->qrss_id) 2320 flow_qrss_free_id(dev, dev_handle->qrss_id); 2321 } 2322 2323 static int 2324 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2325 const struct rte_flow_attr *attr __rte_unused, 2326 const struct rte_flow_item items[] __rte_unused, 2327 const struct rte_flow_action actions[] __rte_unused, 2328 bool external __rte_unused, 2329 struct rte_flow_error *error) 2330 { 2331 return rte_flow_error_set(error, ENOTSUP, 2332 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2333 } 2334 2335 static struct mlx5_flow * 2336 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2337 const struct rte_flow_attr *attr __rte_unused, 2338 const struct rte_flow_item items[] __rte_unused, 2339 const struct rte_flow_action actions[] __rte_unused, 2340 struct rte_flow_error *error) 2341 { 2342 rte_flow_error_set(error, ENOTSUP, 2343 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2344 return NULL; 2345 } 2346 2347 static int 2348 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2349 struct mlx5_flow *dev_flow __rte_unused, 2350 const struct rte_flow_attr *attr __rte_unused, 2351 const struct rte_flow_item items[] __rte_unused, 2352 const struct rte_flow_action actions[] __rte_unused, 2353 struct rte_flow_error *error) 2354 { 2355 return rte_flow_error_set(error, ENOTSUP, 2356 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2357 } 2358 2359 static int 2360 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2361 struct rte_flow *flow __rte_unused, 2362 struct rte_flow_error *error) 2363 { 2364 return rte_flow_error_set(error, ENOTSUP, 2365 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2366 } 2367 2368 static void 2369 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2370 struct rte_flow *flow __rte_unused) 2371 { 2372 } 2373 2374 static void 2375 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2376 struct rte_flow *flow __rte_unused) 2377 { 2378 } 2379 2380 static int 2381 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2382 struct rte_flow *flow __rte_unused, 2383 const struct rte_flow_action *actions __rte_unused, 2384 void *data __rte_unused, 2385 struct rte_flow_error *error) 2386 { 2387 return rte_flow_error_set(error, ENOTSUP, 2388 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2389 } 2390 2391 /* Void driver to protect from null pointer reference. */ 2392 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2393 .validate = flow_null_validate, 2394 .prepare = flow_null_prepare, 2395 .translate = flow_null_translate, 2396 .apply = flow_null_apply, 2397 .remove = flow_null_remove, 2398 .destroy = flow_null_destroy, 2399 .query = flow_null_query, 2400 }; 2401 2402 /** 2403 * Select flow driver type according to flow attributes and device 2404 * configuration. 2405 * 2406 * @param[in] dev 2407 * Pointer to the dev structure. 2408 * @param[in] attr 2409 * Pointer to the flow attributes. 2410 * 2411 * @return 2412 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2413 */ 2414 static enum mlx5_flow_drv_type 2415 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2416 { 2417 struct mlx5_priv *priv = dev->data->dev_private; 2418 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2419 2420 if (attr->transfer && priv->config.dv_esw_en) 2421 type = MLX5_FLOW_TYPE_DV; 2422 if (!attr->transfer) 2423 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2424 MLX5_FLOW_TYPE_VERBS; 2425 return type; 2426 } 2427 2428 #define flow_get_drv_ops(type) flow_drv_ops[type] 2429 2430 /** 2431 * Flow driver validation API. This abstracts calling driver specific functions. 2432 * The type of flow driver is determined according to flow attributes. 2433 * 2434 * @param[in] dev 2435 * Pointer to the dev structure. 2436 * @param[in] attr 2437 * Pointer to the flow attributes. 2438 * @param[in] items 2439 * Pointer to the list of items. 2440 * @param[in] actions 2441 * Pointer to the list of actions. 2442 * @param[in] external 2443 * This flow rule is created by request external to PMD. 2444 * @param[out] error 2445 * Pointer to the error structure. 2446 * 2447 * @return 2448 * 0 on success, a negative errno value otherwise and rte_errno is set. 2449 */ 2450 static inline int 2451 flow_drv_validate(struct rte_eth_dev *dev, 2452 const struct rte_flow_attr *attr, 2453 const struct rte_flow_item items[], 2454 const struct rte_flow_action actions[], 2455 bool external, struct rte_flow_error *error) 2456 { 2457 const struct mlx5_flow_driver_ops *fops; 2458 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2459 2460 fops = flow_get_drv_ops(type); 2461 return fops->validate(dev, attr, items, actions, external, error); 2462 } 2463 2464 /** 2465 * Flow driver preparation API. This abstracts calling driver specific 2466 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2467 * calculates the size of memory required for device flow, allocates the memory, 2468 * initializes the device flow and returns the pointer. 2469 * 2470 * @note 2471 * This function initializes device flow structure such as dv or verbs in 2472 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2473 * rest. For example, adding returning device flow to flow->dev_flow list and 2474 * setting backward reference to the flow should be done out of this function. 2475 * layers field is not filled either. 2476 * 2477 * @param[in] dev 2478 * Pointer to the dev structure. 2479 * @param[in] attr 2480 * Pointer to the flow attributes. 2481 * @param[in] items 2482 * Pointer to the list of items. 2483 * @param[in] actions 2484 * Pointer to the list of actions. 2485 * @param[out] error 2486 * Pointer to the error structure. 2487 * 2488 * @return 2489 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2490 */ 2491 static inline struct mlx5_flow * 2492 flow_drv_prepare(struct rte_eth_dev *dev, 2493 const struct rte_flow *flow, 2494 const struct rte_flow_attr *attr, 2495 const struct rte_flow_item items[], 2496 const struct rte_flow_action actions[], 2497 struct rte_flow_error *error) 2498 { 2499 const struct mlx5_flow_driver_ops *fops; 2500 enum mlx5_flow_drv_type type = flow->drv_type; 2501 2502 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2503 fops = flow_get_drv_ops(type); 2504 return fops->prepare(dev, attr, items, actions, error); 2505 } 2506 2507 /** 2508 * Flow driver translation API. This abstracts calling driver specific 2509 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2510 * translates a generic flow into a driver flow. flow_drv_prepare() must 2511 * precede. 2512 * 2513 * @note 2514 * dev_flow->layers could be filled as a result of parsing during translation 2515 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2516 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2517 * flow->actions could be overwritten even though all the expanded dev_flows 2518 * have the same actions. 2519 * 2520 * @param[in] dev 2521 * Pointer to the rte dev structure. 2522 * @param[in, out] dev_flow 2523 * Pointer to the mlx5 flow. 2524 * @param[in] attr 2525 * Pointer to the flow attributes. 2526 * @param[in] items 2527 * Pointer to the list of items. 2528 * @param[in] actions 2529 * Pointer to the list of actions. 2530 * @param[out] error 2531 * Pointer to the error structure. 2532 * 2533 * @return 2534 * 0 on success, a negative errno value otherwise and rte_errno is set. 2535 */ 2536 static inline int 2537 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2538 const struct rte_flow_attr *attr, 2539 const struct rte_flow_item items[], 2540 const struct rte_flow_action actions[], 2541 struct rte_flow_error *error) 2542 { 2543 const struct mlx5_flow_driver_ops *fops; 2544 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2545 2546 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2547 fops = flow_get_drv_ops(type); 2548 return fops->translate(dev, dev_flow, attr, items, actions, error); 2549 } 2550 2551 /** 2552 * Flow driver apply API. This abstracts calling driver specific functions. 2553 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2554 * translated driver flows on to device. flow_drv_translate() must precede. 2555 * 2556 * @param[in] dev 2557 * Pointer to Ethernet device structure. 2558 * @param[in, out] flow 2559 * Pointer to flow structure. 2560 * @param[out] error 2561 * Pointer to error structure. 2562 * 2563 * @return 2564 * 0 on success, a negative errno value otherwise and rte_errno is set. 2565 */ 2566 static inline int 2567 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2568 struct rte_flow_error *error) 2569 { 2570 const struct mlx5_flow_driver_ops *fops; 2571 enum mlx5_flow_drv_type type = flow->drv_type; 2572 2573 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2574 fops = flow_get_drv_ops(type); 2575 return fops->apply(dev, flow, error); 2576 } 2577 2578 /** 2579 * Flow driver remove API. This abstracts calling driver specific functions. 2580 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2581 * on device. All the resources of the flow should be freed by calling 2582 * flow_drv_destroy(). 2583 * 2584 * @param[in] dev 2585 * Pointer to Ethernet device. 2586 * @param[in, out] flow 2587 * Pointer to flow structure. 2588 */ 2589 static inline void 2590 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2591 { 2592 const struct mlx5_flow_driver_ops *fops; 2593 enum mlx5_flow_drv_type type = flow->drv_type; 2594 2595 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2596 fops = flow_get_drv_ops(type); 2597 fops->remove(dev, flow); 2598 } 2599 2600 /** 2601 * Flow driver destroy API. This abstracts calling driver specific functions. 2602 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2603 * on device and releases resources of the flow. 2604 * 2605 * @param[in] dev 2606 * Pointer to Ethernet device. 2607 * @param[in, out] flow 2608 * Pointer to flow structure. 2609 */ 2610 static inline void 2611 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2612 { 2613 const struct mlx5_flow_driver_ops *fops; 2614 enum mlx5_flow_drv_type type = flow->drv_type; 2615 2616 flow_mreg_split_qrss_release(dev, flow); 2617 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2618 fops = flow_get_drv_ops(type); 2619 fops->destroy(dev, flow); 2620 } 2621 2622 /** 2623 * Validate a flow supported by the NIC. 2624 * 2625 * @see rte_flow_validate() 2626 * @see rte_flow_ops 2627 */ 2628 int 2629 mlx5_flow_validate(struct rte_eth_dev *dev, 2630 const struct rte_flow_attr *attr, 2631 const struct rte_flow_item items[], 2632 const struct rte_flow_action actions[], 2633 struct rte_flow_error *error) 2634 { 2635 int ret; 2636 2637 ret = flow_drv_validate(dev, attr, items, actions, true, error); 2638 if (ret < 0) 2639 return ret; 2640 return 0; 2641 } 2642 2643 /** 2644 * Get RSS action from the action list. 2645 * 2646 * @param[in] actions 2647 * Pointer to the list of actions. 2648 * 2649 * @return 2650 * Pointer to the RSS action if exist, else return NULL. 2651 */ 2652 static const struct rte_flow_action_rss* 2653 flow_get_rss_action(const struct rte_flow_action actions[]) 2654 { 2655 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2656 switch (actions->type) { 2657 case RTE_FLOW_ACTION_TYPE_RSS: 2658 return (const struct rte_flow_action_rss *) 2659 actions->conf; 2660 default: 2661 break; 2662 } 2663 } 2664 return NULL; 2665 } 2666 2667 static unsigned int 2668 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2669 { 2670 const struct rte_flow_item *item; 2671 unsigned int has_vlan = 0; 2672 2673 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2674 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2675 has_vlan = 1; 2676 break; 2677 } 2678 } 2679 if (has_vlan) 2680 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2681 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2682 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2683 MLX5_EXPANSION_ROOT_OUTER; 2684 } 2685 2686 /** 2687 * Get layer flags from the prefix flow. 2688 * 2689 * Some flows may be split to several subflows, the prefix subflow gets the 2690 * match items and the suffix sub flow gets the actions. 2691 * Some actions need the user defined match item flags to get the detail for 2692 * the action. 2693 * This function helps the suffix flow to get the item layer flags from prefix 2694 * subflow. 2695 * 2696 * @param[in] dev_flow 2697 * Pointer the created preifx subflow. 2698 * 2699 * @return 2700 * The layers get from prefix subflow. 2701 */ 2702 static inline uint64_t 2703 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 2704 { 2705 uint64_t layers = 0; 2706 2707 /* 2708 * Layers bits could be localization, but usually the compiler will 2709 * help to do the optimization work for source code. 2710 * If no decap actions, use the layers directly. 2711 */ 2712 if (!(dev_flow->handle->act_flags & MLX5_FLOW_ACTION_DECAP)) 2713 return dev_flow->handle->layers; 2714 /* Convert L3 layers with decap action. */ 2715 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 2716 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2717 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 2718 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2719 /* Convert L4 layers with decap action. */ 2720 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 2721 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 2722 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 2723 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 2724 return layers; 2725 } 2726 2727 /** 2728 * Get metadata split action information. 2729 * 2730 * @param[in] actions 2731 * Pointer to the list of actions. 2732 * @param[out] qrss 2733 * Pointer to the return pointer. 2734 * @param[out] qrss_type 2735 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2736 * if no QUEUE/RSS is found. 2737 * @param[out] encap_idx 2738 * Pointer to the index of the encap action if exists, otherwise the last 2739 * action index. 2740 * 2741 * @return 2742 * Total number of actions. 2743 */ 2744 static int 2745 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 2746 const struct rte_flow_action **qrss, 2747 int *encap_idx) 2748 { 2749 const struct rte_flow_action_raw_encap *raw_encap; 2750 int actions_n = 0; 2751 int raw_decap_idx = -1; 2752 2753 *encap_idx = -1; 2754 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2755 switch (actions->type) { 2756 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2757 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2758 *encap_idx = actions_n; 2759 break; 2760 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 2761 raw_decap_idx = actions_n; 2762 break; 2763 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2764 raw_encap = actions->conf; 2765 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2766 *encap_idx = raw_decap_idx != -1 ? 2767 raw_decap_idx : actions_n; 2768 break; 2769 case RTE_FLOW_ACTION_TYPE_QUEUE: 2770 case RTE_FLOW_ACTION_TYPE_RSS: 2771 *qrss = actions; 2772 break; 2773 default: 2774 break; 2775 } 2776 actions_n++; 2777 } 2778 if (*encap_idx == -1) 2779 *encap_idx = actions_n; 2780 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2781 return actions_n + 1; 2782 } 2783 2784 /** 2785 * Check meter action from the action list. 2786 * 2787 * @param[in] actions 2788 * Pointer to the list of actions. 2789 * @param[out] mtr 2790 * Pointer to the meter exist flag. 2791 * 2792 * @return 2793 * Total number of actions. 2794 */ 2795 static int 2796 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2797 { 2798 int actions_n = 0; 2799 2800 MLX5_ASSERT(mtr); 2801 *mtr = 0; 2802 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2803 switch (actions->type) { 2804 case RTE_FLOW_ACTION_TYPE_METER: 2805 *mtr = 1; 2806 break; 2807 default: 2808 break; 2809 } 2810 actions_n++; 2811 } 2812 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2813 return actions_n + 1; 2814 } 2815 2816 /** 2817 * Check if the flow should be splited due to hairpin. 2818 * The reason for the split is that in current HW we can't 2819 * support encap on Rx, so if a flow have encap we move it 2820 * to Tx. 2821 * 2822 * @param dev 2823 * Pointer to Ethernet device. 2824 * @param[in] attr 2825 * Flow rule attributes. 2826 * @param[in] actions 2827 * Associated actions (list terminated by the END action). 2828 * 2829 * @return 2830 * > 0 the number of actions and the flow should be split, 2831 * 0 when no split required. 2832 */ 2833 static int 2834 flow_check_hairpin_split(struct rte_eth_dev *dev, 2835 const struct rte_flow_attr *attr, 2836 const struct rte_flow_action actions[]) 2837 { 2838 int queue_action = 0; 2839 int action_n = 0; 2840 int encap = 0; 2841 const struct rte_flow_action_queue *queue; 2842 const struct rte_flow_action_rss *rss; 2843 const struct rte_flow_action_raw_encap *raw_encap; 2844 2845 if (!attr->ingress) 2846 return 0; 2847 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2848 switch (actions->type) { 2849 case RTE_FLOW_ACTION_TYPE_QUEUE: 2850 queue = actions->conf; 2851 if (queue == NULL) 2852 return 0; 2853 if (mlx5_rxq_get_type(dev, queue->index) != 2854 MLX5_RXQ_TYPE_HAIRPIN) 2855 return 0; 2856 queue_action = 1; 2857 action_n++; 2858 break; 2859 case RTE_FLOW_ACTION_TYPE_RSS: 2860 rss = actions->conf; 2861 if (rss == NULL || rss->queue_num == 0) 2862 return 0; 2863 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2864 MLX5_RXQ_TYPE_HAIRPIN) 2865 return 0; 2866 queue_action = 1; 2867 action_n++; 2868 break; 2869 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2870 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2871 encap = 1; 2872 action_n++; 2873 break; 2874 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2875 raw_encap = actions->conf; 2876 if (raw_encap->size > 2877 (sizeof(struct rte_flow_item_eth) + 2878 sizeof(struct rte_flow_item_ipv4))) 2879 encap = 1; 2880 action_n++; 2881 break; 2882 default: 2883 action_n++; 2884 break; 2885 } 2886 } 2887 if (encap == 1 && queue_action) 2888 return action_n; 2889 return 0; 2890 } 2891 2892 /* Declare flow create/destroy prototype in advance. */ 2893 static struct rte_flow * 2894 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 2895 const struct rte_flow_attr *attr, 2896 const struct rte_flow_item items[], 2897 const struct rte_flow_action actions[], 2898 bool external, struct rte_flow_error *error); 2899 2900 static void 2901 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2902 struct rte_flow *flow); 2903 2904 /** 2905 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2906 * 2907 * As mark_id is unique, if there's already a registered flow for the mark_id, 2908 * return by increasing the reference counter of the resource. Otherwise, create 2909 * the resource (mcp_res) and flow. 2910 * 2911 * Flow looks like, 2912 * - If ingress port is ANY and reg_c[1] is mark_id, 2913 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2914 * 2915 * For default flow (zero mark_id), flow is like, 2916 * - If ingress port is ANY, 2917 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2918 * 2919 * @param dev 2920 * Pointer to Ethernet device. 2921 * @param mark_id 2922 * ID of MARK action, zero means default flow for META. 2923 * @param[out] error 2924 * Perform verbose error reporting if not NULL. 2925 * 2926 * @return 2927 * Associated resource on success, NULL otherwise and rte_errno is set. 2928 */ 2929 static struct mlx5_flow_mreg_copy_resource * 2930 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2931 struct rte_flow_error *error) 2932 { 2933 struct mlx5_priv *priv = dev->data->dev_private; 2934 struct rte_flow_attr attr = { 2935 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2936 .ingress = 1, 2937 }; 2938 struct mlx5_rte_flow_item_tag tag_spec = { 2939 .data = mark_id, 2940 }; 2941 struct rte_flow_item items[] = { 2942 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2943 }; 2944 struct rte_flow_action_mark ftag = { 2945 .id = mark_id, 2946 }; 2947 struct mlx5_flow_action_copy_mreg cp_mreg = { 2948 .dst = REG_B, 2949 .src = 0, 2950 }; 2951 struct rte_flow_action_jump jump = { 2952 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2953 }; 2954 struct rte_flow_action actions[] = { 2955 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2956 }; 2957 struct mlx5_flow_mreg_copy_resource *mcp_res; 2958 int ret; 2959 2960 /* Fill the register fileds in the flow. */ 2961 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2962 if (ret < 0) 2963 return NULL; 2964 tag_spec.id = ret; 2965 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 2966 if (ret < 0) 2967 return NULL; 2968 cp_mreg.src = ret; 2969 /* Check if already registered. */ 2970 MLX5_ASSERT(priv->mreg_cp_tbl); 2971 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 2972 if (mcp_res) { 2973 /* For non-default rule. */ 2974 if (mark_id != MLX5_DEFAULT_COPY_ID) 2975 mcp_res->refcnt++; 2976 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 2977 mcp_res->refcnt == 1); 2978 return mcp_res; 2979 } 2980 /* Provide the full width of FLAG specific value. */ 2981 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 2982 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 2983 /* Build a new flow. */ 2984 if (mark_id != MLX5_DEFAULT_COPY_ID) { 2985 items[0] = (struct rte_flow_item){ 2986 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 2987 .spec = &tag_spec, 2988 }; 2989 items[1] = (struct rte_flow_item){ 2990 .type = RTE_FLOW_ITEM_TYPE_END, 2991 }; 2992 actions[0] = (struct rte_flow_action){ 2993 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK, 2994 .conf = &ftag, 2995 }; 2996 actions[1] = (struct rte_flow_action){ 2997 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2998 .conf = &cp_mreg, 2999 }; 3000 actions[2] = (struct rte_flow_action){ 3001 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3002 .conf = &jump, 3003 }; 3004 actions[3] = (struct rte_flow_action){ 3005 .type = RTE_FLOW_ACTION_TYPE_END, 3006 }; 3007 } else { 3008 /* Default rule, wildcard match. */ 3009 attr.priority = MLX5_FLOW_PRIO_RSVD; 3010 items[0] = (struct rte_flow_item){ 3011 .type = RTE_FLOW_ITEM_TYPE_END, 3012 }; 3013 actions[0] = (struct rte_flow_action){ 3014 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3015 .conf = &cp_mreg, 3016 }; 3017 actions[1] = (struct rte_flow_action){ 3018 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3019 .conf = &jump, 3020 }; 3021 actions[2] = (struct rte_flow_action){ 3022 .type = RTE_FLOW_ACTION_TYPE_END, 3023 }; 3024 } 3025 /* Build a new entry. */ 3026 mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0); 3027 if (!mcp_res) { 3028 rte_errno = ENOMEM; 3029 return NULL; 3030 } 3031 /* 3032 * The copy Flows are not included in any list. There 3033 * ones are referenced from other Flows and can not 3034 * be applied, removed, deleted in ardbitrary order 3035 * by list traversing. 3036 */ 3037 mcp_res->flow = flow_list_create(dev, NULL, &attr, items, 3038 actions, false, error); 3039 if (!mcp_res->flow) 3040 goto error; 3041 mcp_res->refcnt++; 3042 mcp_res->hlist_ent.key = mark_id; 3043 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3044 &mcp_res->hlist_ent); 3045 MLX5_ASSERT(!ret); 3046 if (ret) 3047 goto error; 3048 return mcp_res; 3049 error: 3050 if (mcp_res->flow) 3051 flow_list_destroy(dev, NULL, mcp_res->flow); 3052 rte_free(mcp_res); 3053 return NULL; 3054 } 3055 3056 /** 3057 * Release flow in RX_CP_TBL. 3058 * 3059 * @param dev 3060 * Pointer to Ethernet device. 3061 * @flow 3062 * Parent flow for wich copying is provided. 3063 */ 3064 static void 3065 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3066 struct rte_flow *flow) 3067 { 3068 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3069 struct mlx5_priv *priv = dev->data->dev_private; 3070 3071 if (!mcp_res || !priv->mreg_cp_tbl) 3072 return; 3073 if (flow->copy_applied) { 3074 MLX5_ASSERT(mcp_res->appcnt); 3075 flow->copy_applied = 0; 3076 --mcp_res->appcnt; 3077 if (!mcp_res->appcnt) 3078 flow_drv_remove(dev, mcp_res->flow); 3079 } 3080 /* 3081 * We do not check availability of metadata registers here, 3082 * because copy resources are not allocated in this case. 3083 */ 3084 if (--mcp_res->refcnt) 3085 return; 3086 MLX5_ASSERT(mcp_res->flow); 3087 flow_list_destroy(dev, NULL, mcp_res->flow); 3088 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3089 rte_free(mcp_res); 3090 flow->mreg_copy = NULL; 3091 } 3092 3093 /** 3094 * Start flow in RX_CP_TBL. 3095 * 3096 * @param dev 3097 * Pointer to Ethernet device. 3098 * @flow 3099 * Parent flow for wich copying is provided. 3100 * 3101 * @return 3102 * 0 on success, a negative errno value otherwise and rte_errno is set. 3103 */ 3104 static int 3105 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3106 struct rte_flow *flow) 3107 { 3108 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3109 int ret; 3110 3111 if (!mcp_res || flow->copy_applied) 3112 return 0; 3113 if (!mcp_res->appcnt) { 3114 ret = flow_drv_apply(dev, mcp_res->flow, NULL); 3115 if (ret) 3116 return ret; 3117 } 3118 ++mcp_res->appcnt; 3119 flow->copy_applied = 1; 3120 return 0; 3121 } 3122 3123 /** 3124 * Stop flow in RX_CP_TBL. 3125 * 3126 * @param dev 3127 * Pointer to Ethernet device. 3128 * @flow 3129 * Parent flow for wich copying is provided. 3130 */ 3131 static void 3132 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3133 struct rte_flow *flow) 3134 { 3135 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3136 3137 if (!mcp_res || !flow->copy_applied) 3138 return; 3139 MLX5_ASSERT(mcp_res->appcnt); 3140 --mcp_res->appcnt; 3141 flow->copy_applied = 0; 3142 if (!mcp_res->appcnt) 3143 flow_drv_remove(dev, mcp_res->flow); 3144 } 3145 3146 /** 3147 * Remove the default copy action from RX_CP_TBL. 3148 * 3149 * @param dev 3150 * Pointer to Ethernet device. 3151 */ 3152 static void 3153 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3154 { 3155 struct mlx5_flow_mreg_copy_resource *mcp_res; 3156 struct mlx5_priv *priv = dev->data->dev_private; 3157 3158 /* Check if default flow is registered. */ 3159 if (!priv->mreg_cp_tbl) 3160 return; 3161 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3162 MLX5_DEFAULT_COPY_ID); 3163 if (!mcp_res) 3164 return; 3165 MLX5_ASSERT(mcp_res->flow); 3166 flow_list_destroy(dev, NULL, mcp_res->flow); 3167 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3168 rte_free(mcp_res); 3169 } 3170 3171 /** 3172 * Add the default copy action in in RX_CP_TBL. 3173 * 3174 * @param dev 3175 * Pointer to Ethernet device. 3176 * @param[out] error 3177 * Perform verbose error reporting if not NULL. 3178 * 3179 * @return 3180 * 0 for success, negative value otherwise and rte_errno is set. 3181 */ 3182 static int 3183 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3184 struct rte_flow_error *error) 3185 { 3186 struct mlx5_priv *priv = dev->data->dev_private; 3187 struct mlx5_flow_mreg_copy_resource *mcp_res; 3188 3189 /* Check whether extensive metadata feature is engaged. */ 3190 if (!priv->config.dv_flow_en || 3191 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3192 !mlx5_flow_ext_mreg_supported(dev) || 3193 !priv->sh->dv_regc0_mask) 3194 return 0; 3195 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3196 if (!mcp_res) 3197 return -rte_errno; 3198 return 0; 3199 } 3200 3201 /** 3202 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3203 * 3204 * All the flow having Q/RSS action should be split by 3205 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3206 * performs the following, 3207 * - CQE->flow_tag := reg_c[1] (MARK) 3208 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3209 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3210 * but there should be a flow per each MARK ID set by MARK action. 3211 * 3212 * For the aforementioned reason, if there's a MARK action in flow's action 3213 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3214 * the MARK ID to CQE's flow_tag like, 3215 * - If reg_c[1] is mark_id, 3216 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3217 * 3218 * For SET_META action which stores value in reg_c[0], as the destination is 3219 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3220 * MARK ID means the default flow. The default flow looks like, 3221 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3222 * 3223 * @param dev 3224 * Pointer to Ethernet device. 3225 * @param flow 3226 * Pointer to flow structure. 3227 * @param[in] actions 3228 * Pointer to the list of actions. 3229 * @param[out] error 3230 * Perform verbose error reporting if not NULL. 3231 * 3232 * @return 3233 * 0 on success, negative value otherwise and rte_errno is set. 3234 */ 3235 static int 3236 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3237 struct rte_flow *flow, 3238 const struct rte_flow_action *actions, 3239 struct rte_flow_error *error) 3240 { 3241 struct mlx5_priv *priv = dev->data->dev_private; 3242 struct mlx5_dev_config *config = &priv->config; 3243 struct mlx5_flow_mreg_copy_resource *mcp_res; 3244 const struct rte_flow_action_mark *mark; 3245 3246 /* Check whether extensive metadata feature is engaged. */ 3247 if (!config->dv_flow_en || 3248 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3249 !mlx5_flow_ext_mreg_supported(dev) || 3250 !priv->sh->dv_regc0_mask) 3251 return 0; 3252 /* Find MARK action. */ 3253 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3254 switch (actions->type) { 3255 case RTE_FLOW_ACTION_TYPE_FLAG: 3256 mcp_res = flow_mreg_add_copy_action 3257 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3258 if (!mcp_res) 3259 return -rte_errno; 3260 flow->mreg_copy = mcp_res; 3261 if (dev->data->dev_started) { 3262 mcp_res->appcnt++; 3263 flow->copy_applied = 1; 3264 } 3265 return 0; 3266 case RTE_FLOW_ACTION_TYPE_MARK: 3267 mark = (const struct rte_flow_action_mark *) 3268 actions->conf; 3269 mcp_res = 3270 flow_mreg_add_copy_action(dev, mark->id, error); 3271 if (!mcp_res) 3272 return -rte_errno; 3273 flow->mreg_copy = mcp_res; 3274 if (dev->data->dev_started) { 3275 mcp_res->appcnt++; 3276 flow->copy_applied = 1; 3277 } 3278 return 0; 3279 default: 3280 break; 3281 } 3282 } 3283 return 0; 3284 } 3285 3286 #define MLX5_MAX_SPLIT_ACTIONS 24 3287 #define MLX5_MAX_SPLIT_ITEMS 24 3288 3289 /** 3290 * Split the hairpin flow. 3291 * Since HW can't support encap on Rx we move the encap to Tx. 3292 * If the count action is after the encap then we also 3293 * move the count action. in this case the count will also measure 3294 * the outer bytes. 3295 * 3296 * @param dev 3297 * Pointer to Ethernet device. 3298 * @param[in] actions 3299 * Associated actions (list terminated by the END action). 3300 * @param[out] actions_rx 3301 * Rx flow actions. 3302 * @param[out] actions_tx 3303 * Tx flow actions.. 3304 * @param[out] pattern_tx 3305 * The pattern items for the Tx flow. 3306 * @param[out] flow_id 3307 * The flow ID connected to this flow. 3308 * 3309 * @return 3310 * 0 on success. 3311 */ 3312 static int 3313 flow_hairpin_split(struct rte_eth_dev *dev, 3314 const struct rte_flow_action actions[], 3315 struct rte_flow_action actions_rx[], 3316 struct rte_flow_action actions_tx[], 3317 struct rte_flow_item pattern_tx[], 3318 uint32_t *flow_id) 3319 { 3320 struct mlx5_priv *priv = dev->data->dev_private; 3321 const struct rte_flow_action_raw_encap *raw_encap; 3322 const struct rte_flow_action_raw_decap *raw_decap; 3323 struct mlx5_rte_flow_action_set_tag *set_tag; 3324 struct rte_flow_action *tag_action; 3325 struct mlx5_rte_flow_item_tag *tag_item; 3326 struct rte_flow_item *item; 3327 char *addr; 3328 int encap = 0; 3329 3330 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3331 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3332 switch (actions->type) { 3333 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3334 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3335 rte_memcpy(actions_tx, actions, 3336 sizeof(struct rte_flow_action)); 3337 actions_tx++; 3338 break; 3339 case RTE_FLOW_ACTION_TYPE_COUNT: 3340 if (encap) { 3341 rte_memcpy(actions_tx, actions, 3342 sizeof(struct rte_flow_action)); 3343 actions_tx++; 3344 } else { 3345 rte_memcpy(actions_rx, actions, 3346 sizeof(struct rte_flow_action)); 3347 actions_rx++; 3348 } 3349 break; 3350 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3351 raw_encap = actions->conf; 3352 if (raw_encap->size > 3353 (sizeof(struct rte_flow_item_eth) + 3354 sizeof(struct rte_flow_item_ipv4))) { 3355 memcpy(actions_tx, actions, 3356 sizeof(struct rte_flow_action)); 3357 actions_tx++; 3358 encap = 1; 3359 } else { 3360 rte_memcpy(actions_rx, actions, 3361 sizeof(struct rte_flow_action)); 3362 actions_rx++; 3363 } 3364 break; 3365 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3366 raw_decap = actions->conf; 3367 if (raw_decap->size < 3368 (sizeof(struct rte_flow_item_eth) + 3369 sizeof(struct rte_flow_item_ipv4))) { 3370 memcpy(actions_tx, actions, 3371 sizeof(struct rte_flow_action)); 3372 actions_tx++; 3373 } else { 3374 rte_memcpy(actions_rx, actions, 3375 sizeof(struct rte_flow_action)); 3376 actions_rx++; 3377 } 3378 break; 3379 default: 3380 rte_memcpy(actions_rx, actions, 3381 sizeof(struct rte_flow_action)); 3382 actions_rx++; 3383 break; 3384 } 3385 } 3386 /* Add set meta action and end action for the Rx flow. */ 3387 tag_action = actions_rx; 3388 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3389 actions_rx++; 3390 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3391 actions_rx++; 3392 set_tag = (void *)actions_rx; 3393 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3394 MLX5_ASSERT(set_tag->id > REG_NONE); 3395 set_tag->data = *flow_id; 3396 tag_action->conf = set_tag; 3397 /* Create Tx item list. */ 3398 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3399 addr = (void *)&pattern_tx[2]; 3400 item = pattern_tx; 3401 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3402 tag_item = (void *)addr; 3403 tag_item->data = *flow_id; 3404 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3405 MLX5_ASSERT(set_tag->id > REG_NONE); 3406 item->spec = tag_item; 3407 addr += sizeof(struct mlx5_rte_flow_item_tag); 3408 tag_item = (void *)addr; 3409 tag_item->data = UINT32_MAX; 3410 tag_item->id = UINT16_MAX; 3411 item->mask = tag_item; 3412 addr += sizeof(struct mlx5_rte_flow_item_tag); 3413 item->last = NULL; 3414 item++; 3415 item->type = RTE_FLOW_ITEM_TYPE_END; 3416 return 0; 3417 } 3418 3419 /** 3420 * The last stage of splitting chain, just creates the subflow 3421 * without any modification. 3422 * 3423 * @param[in] dev 3424 * Pointer to Ethernet device. 3425 * @param[in] flow 3426 * Parent flow structure pointer. 3427 * @param[in, out] sub_flow 3428 * Pointer to return the created subflow, may be NULL. 3429 * @param[in] prefix_layers 3430 * Prefix subflow layers, may be 0. 3431 * @param[in] attr 3432 * Flow rule attributes. 3433 * @param[in] items 3434 * Pattern specification (list terminated by the END pattern item). 3435 * @param[in] actions 3436 * Associated actions (list terminated by the END action). 3437 * @param[in] external 3438 * This flow rule is created by request external to PMD. 3439 * @param[out] error 3440 * Perform verbose error reporting if not NULL. 3441 * @return 3442 * 0 on success, negative value otherwise 3443 */ 3444 static int 3445 flow_create_split_inner(struct rte_eth_dev *dev, 3446 struct rte_flow *flow, 3447 struct mlx5_flow **sub_flow, 3448 uint64_t prefix_layers, 3449 const struct rte_flow_attr *attr, 3450 const struct rte_flow_item items[], 3451 const struct rte_flow_action actions[], 3452 bool external, struct rte_flow_error *error) 3453 { 3454 struct mlx5_flow *dev_flow; 3455 3456 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, error); 3457 if (!dev_flow) 3458 return -rte_errno; 3459 dev_flow->flow = flow; 3460 dev_flow->external = external; 3461 /* Subflow object was created, we must include one in the list. */ 3462 LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next); 3463 /* 3464 * If dev_flow is as one of the suffix flow, some actions in suffix 3465 * flow may need some user defined item layer flags. 3466 */ 3467 if (prefix_layers) 3468 dev_flow->handle->layers = prefix_layers; 3469 if (sub_flow) 3470 *sub_flow = dev_flow; 3471 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3472 } 3473 3474 /** 3475 * Split the meter flow. 3476 * 3477 * As meter flow will split to three sub flow, other than meter 3478 * action, the other actions make sense to only meter accepts 3479 * the packet. If it need to be dropped, no other additional 3480 * actions should be take. 3481 * 3482 * One kind of special action which decapsulates the L3 tunnel 3483 * header will be in the prefix sub flow, as not to take the 3484 * L3 tunnel header into account. 3485 * 3486 * @param dev 3487 * Pointer to Ethernet device. 3488 * @param[in] items 3489 * Pattern specification (list terminated by the END pattern item). 3490 * @param[out] sfx_items 3491 * Suffix flow match items (list terminated by the END pattern item). 3492 * @param[in] actions 3493 * Associated actions (list terminated by the END action). 3494 * @param[out] actions_sfx 3495 * Suffix flow actions. 3496 * @param[out] actions_pre 3497 * Prefix flow actions. 3498 * @param[out] pattern_sfx 3499 * The pattern items for the suffix flow. 3500 * @param[out] tag_sfx 3501 * Pointer to suffix flow tag. 3502 * 3503 * @return 3504 * 0 on success. 3505 */ 3506 static int 3507 flow_meter_split_prep(struct rte_eth_dev *dev, 3508 const struct rte_flow_item items[], 3509 struct rte_flow_item sfx_items[], 3510 const struct rte_flow_action actions[], 3511 struct rte_flow_action actions_sfx[], 3512 struct rte_flow_action actions_pre[]) 3513 { 3514 struct rte_flow_action *tag_action = NULL; 3515 struct rte_flow_item *tag_item; 3516 struct mlx5_rte_flow_action_set_tag *set_tag; 3517 struct rte_flow_error error; 3518 const struct rte_flow_action_raw_encap *raw_encap; 3519 const struct rte_flow_action_raw_decap *raw_decap; 3520 struct mlx5_rte_flow_item_tag *tag_spec; 3521 struct mlx5_rte_flow_item_tag *tag_mask; 3522 uint32_t tag_id; 3523 bool copy_vlan = false; 3524 3525 /* Prepare the actions for prefix and suffix flow. */ 3526 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3527 struct rte_flow_action **action_cur = NULL; 3528 3529 switch (actions->type) { 3530 case RTE_FLOW_ACTION_TYPE_METER: 3531 /* Add the extra tag action first. */ 3532 tag_action = actions_pre; 3533 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3534 actions_pre++; 3535 action_cur = &actions_pre; 3536 break; 3537 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3538 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3539 action_cur = &actions_pre; 3540 break; 3541 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3542 raw_encap = actions->conf; 3543 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 3544 action_cur = &actions_pre; 3545 break; 3546 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3547 raw_decap = actions->conf; 3548 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3549 action_cur = &actions_pre; 3550 break; 3551 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3552 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3553 copy_vlan = true; 3554 break; 3555 default: 3556 break; 3557 } 3558 if (!action_cur) 3559 action_cur = &actions_sfx; 3560 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 3561 (*action_cur)++; 3562 } 3563 /* Add end action to the actions. */ 3564 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3565 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3566 actions_pre++; 3567 /* Set the tag. */ 3568 set_tag = (void *)actions_pre; 3569 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3570 /* 3571 * Get the id from the qrss_pool to make qrss share the id with meter. 3572 */ 3573 tag_id = flow_qrss_get_id(dev); 3574 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 3575 assert(tag_action); 3576 tag_action->conf = set_tag; 3577 /* Prepare the suffix subflow items. */ 3578 tag_item = sfx_items++; 3579 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 3580 int item_type = items->type; 3581 3582 switch (item_type) { 3583 case RTE_FLOW_ITEM_TYPE_PORT_ID: 3584 memcpy(sfx_items, items, sizeof(*sfx_items)); 3585 sfx_items++; 3586 break; 3587 case RTE_FLOW_ITEM_TYPE_VLAN: 3588 if (copy_vlan) { 3589 memcpy(sfx_items, items, sizeof(*sfx_items)); 3590 /* 3591 * Convert to internal match item, it is used 3592 * for vlan push and set vid. 3593 */ 3594 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 3595 sfx_items++; 3596 } 3597 break; 3598 default: 3599 break; 3600 } 3601 } 3602 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 3603 sfx_items++; 3604 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 3605 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 3606 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3607 tag_mask = tag_spec + 1; 3608 tag_mask->data = 0xffffff00; 3609 tag_item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3610 tag_item->spec = tag_spec; 3611 tag_item->last = NULL; 3612 tag_item->mask = tag_mask; 3613 return tag_id; 3614 } 3615 3616 /** 3617 * Split action list having QUEUE/RSS for metadata register copy. 3618 * 3619 * Once Q/RSS action is detected in user's action list, the flow action 3620 * should be split in order to copy metadata registers, which will happen in 3621 * RX_CP_TBL like, 3622 * - CQE->flow_tag := reg_c[1] (MARK) 3623 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3624 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3625 * This is because the last action of each flow must be a terminal action 3626 * (QUEUE, RSS or DROP). 3627 * 3628 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3629 * stored and kept in the mlx5_flow structure per each sub_flow. 3630 * 3631 * The Q/RSS action is replaced with, 3632 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3633 * And the following JUMP action is added at the end, 3634 * - JUMP, to RX_CP_TBL. 3635 * 3636 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3637 * flow_create_split_metadata() routine. The flow will look like, 3638 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3639 * 3640 * @param dev 3641 * Pointer to Ethernet device. 3642 * @param[out] split_actions 3643 * Pointer to store split actions to jump to CP_TBL. 3644 * @param[in] actions 3645 * Pointer to the list of original flow actions. 3646 * @param[in] qrss 3647 * Pointer to the Q/RSS action. 3648 * @param[in] actions_n 3649 * Number of original actions. 3650 * @param[out] error 3651 * Perform verbose error reporting if not NULL. 3652 * 3653 * @return 3654 * non-zero unique flow_id on success, otherwise 0 and 3655 * error/rte_error are set. 3656 */ 3657 static uint32_t 3658 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3659 struct rte_flow_action *split_actions, 3660 const struct rte_flow_action *actions, 3661 const struct rte_flow_action *qrss, 3662 int actions_n, struct rte_flow_error *error) 3663 { 3664 struct mlx5_rte_flow_action_set_tag *set_tag; 3665 struct rte_flow_action_jump *jump; 3666 const int qrss_idx = qrss - actions; 3667 uint32_t flow_id = 0; 3668 int ret = 0; 3669 3670 /* 3671 * Given actions will be split 3672 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3673 * - Add jump to mreg CP_TBL. 3674 * As a result, there will be one more action. 3675 */ 3676 ++actions_n; 3677 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3678 set_tag = (void *)(split_actions + actions_n); 3679 /* 3680 * If tag action is not set to void(it means we are not the meter 3681 * suffix flow), add the tag action. Since meter suffix flow already 3682 * has the tag added. 3683 */ 3684 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3685 /* 3686 * Allocate the new subflow ID. This one is unique within 3687 * device and not shared with representors. Otherwise, 3688 * we would have to resolve multi-thread access synch 3689 * issue. Each flow on the shared device is appended 3690 * with source vport identifier, so the resulting 3691 * flows will be unique in the shared (by master and 3692 * representors) domain even if they have coinciding 3693 * IDs. 3694 */ 3695 flow_id = flow_qrss_get_id(dev); 3696 if (!flow_id) 3697 return rte_flow_error_set(error, ENOMEM, 3698 RTE_FLOW_ERROR_TYPE_ACTION, 3699 NULL, "can't allocate id " 3700 "for split Q/RSS subflow"); 3701 /* Internal SET_TAG action to set flow ID. */ 3702 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3703 .data = flow_id, 3704 }; 3705 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3706 if (ret < 0) 3707 return ret; 3708 set_tag->id = ret; 3709 /* Construct new actions array. */ 3710 /* Replace QUEUE/RSS action. */ 3711 split_actions[qrss_idx] = (struct rte_flow_action){ 3712 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3713 .conf = set_tag, 3714 }; 3715 } 3716 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3717 jump = (void *)(set_tag + 1); 3718 *jump = (struct rte_flow_action_jump){ 3719 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3720 }; 3721 split_actions[actions_n - 2] = (struct rte_flow_action){ 3722 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3723 .conf = jump, 3724 }; 3725 split_actions[actions_n - 1] = (struct rte_flow_action){ 3726 .type = RTE_FLOW_ACTION_TYPE_END, 3727 }; 3728 return flow_id; 3729 } 3730 3731 /** 3732 * Extend the given action list for Tx metadata copy. 3733 * 3734 * Copy the given action list to the ext_actions and add flow metadata register 3735 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3736 * 3737 * @param[out] ext_actions 3738 * Pointer to the extended action list. 3739 * @param[in] actions 3740 * Pointer to the list of actions. 3741 * @param[in] actions_n 3742 * Number of actions in the list. 3743 * @param[out] error 3744 * Perform verbose error reporting if not NULL. 3745 * @param[in] encap_idx 3746 * The encap action inndex. 3747 * 3748 * @return 3749 * 0 on success, negative value otherwise 3750 */ 3751 static int 3752 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3753 struct rte_flow_action *ext_actions, 3754 const struct rte_flow_action *actions, 3755 int actions_n, struct rte_flow_error *error, 3756 int encap_idx) 3757 { 3758 struct mlx5_flow_action_copy_mreg *cp_mreg = 3759 (struct mlx5_flow_action_copy_mreg *) 3760 (ext_actions + actions_n + 1); 3761 int ret; 3762 3763 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3764 if (ret < 0) 3765 return ret; 3766 cp_mreg->dst = ret; 3767 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3768 if (ret < 0) 3769 return ret; 3770 cp_mreg->src = ret; 3771 if (encap_idx != 0) 3772 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 3773 if (encap_idx == actions_n - 1) { 3774 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3775 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3776 .conf = cp_mreg, 3777 }; 3778 ext_actions[actions_n] = (struct rte_flow_action){ 3779 .type = RTE_FLOW_ACTION_TYPE_END, 3780 }; 3781 } else { 3782 ext_actions[encap_idx] = (struct rte_flow_action){ 3783 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3784 .conf = cp_mreg, 3785 }; 3786 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 3787 sizeof(*ext_actions) * (actions_n - encap_idx)); 3788 } 3789 return 0; 3790 } 3791 3792 /** 3793 * The splitting for metadata feature. 3794 * 3795 * - Q/RSS action on NIC Rx should be split in order to pass by 3796 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3797 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3798 * 3799 * - All the actions on NIC Tx should have a mreg copy action to 3800 * copy reg_a from WQE to reg_c[0]. 3801 * 3802 * @param dev 3803 * Pointer to Ethernet device. 3804 * @param[in] flow 3805 * Parent flow structure pointer. 3806 * @param[in] prefix_layers 3807 * Prefix flow layer flags. 3808 * @param[in] attr 3809 * Flow rule attributes. 3810 * @param[in] items 3811 * Pattern specification (list terminated by the END pattern item). 3812 * @param[in] actions 3813 * Associated actions (list terminated by the END action). 3814 * @param[in] external 3815 * This flow rule is created by request external to PMD. 3816 * @param[out] error 3817 * Perform verbose error reporting if not NULL. 3818 * @return 3819 * 0 on success, negative value otherwise 3820 */ 3821 static int 3822 flow_create_split_metadata(struct rte_eth_dev *dev, 3823 struct rte_flow *flow, 3824 uint64_t prefix_layers, 3825 const struct rte_flow_attr *attr, 3826 const struct rte_flow_item items[], 3827 const struct rte_flow_action actions[], 3828 bool external, struct rte_flow_error *error) 3829 { 3830 struct mlx5_priv *priv = dev->data->dev_private; 3831 struct mlx5_dev_config *config = &priv->config; 3832 const struct rte_flow_action *qrss = NULL; 3833 struct rte_flow_action *ext_actions = NULL; 3834 struct mlx5_flow *dev_flow = NULL; 3835 uint32_t qrss_id = 0; 3836 int mtr_sfx = 0; 3837 size_t act_size; 3838 int actions_n; 3839 int encap_idx; 3840 int ret; 3841 3842 /* Check whether extensive metadata feature is engaged. */ 3843 if (!config->dv_flow_en || 3844 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3845 !mlx5_flow_ext_mreg_supported(dev)) 3846 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 3847 attr, items, actions, external, 3848 error); 3849 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 3850 &encap_idx); 3851 if (qrss) { 3852 /* Exclude hairpin flows from splitting. */ 3853 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3854 const struct rte_flow_action_queue *queue; 3855 3856 queue = qrss->conf; 3857 if (mlx5_rxq_get_type(dev, queue->index) == 3858 MLX5_RXQ_TYPE_HAIRPIN) 3859 qrss = NULL; 3860 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3861 const struct rte_flow_action_rss *rss; 3862 3863 rss = qrss->conf; 3864 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3865 MLX5_RXQ_TYPE_HAIRPIN) 3866 qrss = NULL; 3867 } 3868 } 3869 if (qrss) { 3870 /* Check if it is in meter suffix table. */ 3871 mtr_sfx = attr->group == (attr->transfer ? 3872 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3873 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3874 /* 3875 * Q/RSS action on NIC Rx should be split in order to pass by 3876 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3877 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3878 */ 3879 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3880 sizeof(struct rte_flow_action_set_tag) + 3881 sizeof(struct rte_flow_action_jump); 3882 ext_actions = rte_zmalloc(__func__, act_size, 0); 3883 if (!ext_actions) 3884 return rte_flow_error_set(error, ENOMEM, 3885 RTE_FLOW_ERROR_TYPE_ACTION, 3886 NULL, "no memory to split " 3887 "metadata flow"); 3888 /* 3889 * If we are the suffix flow of meter, tag already exist. 3890 * Set the tag action to void. 3891 */ 3892 if (mtr_sfx) 3893 ext_actions[qrss - actions].type = 3894 RTE_FLOW_ACTION_TYPE_VOID; 3895 else 3896 ext_actions[qrss - actions].type = 3897 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3898 /* 3899 * Create the new actions list with removed Q/RSS action 3900 * and appended set tag and jump to register copy table 3901 * (RX_CP_TBL). We should preallocate unique tag ID here 3902 * in advance, because it is needed for set tag action. 3903 */ 3904 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3905 qrss, actions_n, error); 3906 if (!mtr_sfx && !qrss_id) { 3907 ret = -rte_errno; 3908 goto exit; 3909 } 3910 } else if (attr->egress && !attr->transfer) { 3911 /* 3912 * All the actions on NIC Tx should have a metadata register 3913 * copy action to copy reg_a from WQE to reg_c[meta] 3914 */ 3915 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3916 sizeof(struct mlx5_flow_action_copy_mreg); 3917 ext_actions = rte_zmalloc(__func__, act_size, 0); 3918 if (!ext_actions) 3919 return rte_flow_error_set(error, ENOMEM, 3920 RTE_FLOW_ERROR_TYPE_ACTION, 3921 NULL, "no memory to split " 3922 "metadata flow"); 3923 /* Create the action list appended with copy register. */ 3924 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 3925 actions_n, error, encap_idx); 3926 if (ret < 0) 3927 goto exit; 3928 } 3929 /* Add the unmodified original or prefix subflow. */ 3930 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr, 3931 items, ext_actions ? ext_actions : 3932 actions, external, error); 3933 if (ret < 0) 3934 goto exit; 3935 MLX5_ASSERT(dev_flow); 3936 if (qrss) { 3937 const struct rte_flow_attr q_attr = { 3938 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3939 .ingress = 1, 3940 }; 3941 /* Internal PMD action to set register. */ 3942 struct mlx5_rte_flow_item_tag q_tag_spec = { 3943 .data = qrss_id, 3944 .id = 0, 3945 }; 3946 struct rte_flow_item q_items[] = { 3947 { 3948 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3949 .spec = &q_tag_spec, 3950 .last = NULL, 3951 .mask = NULL, 3952 }, 3953 { 3954 .type = RTE_FLOW_ITEM_TYPE_END, 3955 }, 3956 }; 3957 struct rte_flow_action q_actions[] = { 3958 { 3959 .type = qrss->type, 3960 .conf = qrss->conf, 3961 }, 3962 { 3963 .type = RTE_FLOW_ACTION_TYPE_END, 3964 }, 3965 }; 3966 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 3967 3968 /* 3969 * Configure the tag item only if there is no meter subflow. 3970 * Since tag is already marked in the meter suffix subflow 3971 * we can just use the meter suffix items as is. 3972 */ 3973 if (qrss_id) { 3974 /* Not meter subflow. */ 3975 MLX5_ASSERT(!mtr_sfx); 3976 /* 3977 * Put unique id in prefix flow due to it is destroyed 3978 * after suffix flow and id will be freed after there 3979 * is no actual flows with this id and identifier 3980 * reallocation becomes possible (for example, for 3981 * other flows in other threads). 3982 */ 3983 dev_flow->handle->qrss_id = qrss_id; 3984 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 3985 error); 3986 if (ret < 0) 3987 goto exit; 3988 q_tag_spec.id = ret; 3989 } 3990 dev_flow = NULL; 3991 /* Add suffix subflow to execute Q/RSS. */ 3992 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 3993 &q_attr, mtr_sfx ? items : 3994 q_items, q_actions, 3995 external, error); 3996 if (ret < 0) 3997 goto exit; 3998 /* qrss ID should be freed if failed. */ 3999 qrss_id = 0; 4000 MLX5_ASSERT(dev_flow); 4001 } 4002 4003 exit: 4004 /* 4005 * We do not destroy the partially created sub_flows in case of error. 4006 * These ones are included into parent flow list and will be destroyed 4007 * by flow_drv_destroy. 4008 */ 4009 flow_qrss_free_id(dev, qrss_id); 4010 rte_free(ext_actions); 4011 return ret; 4012 } 4013 4014 /** 4015 * The splitting for meter feature. 4016 * 4017 * - The meter flow will be split to two flows as prefix and 4018 * suffix flow. The packets make sense only it pass the prefix 4019 * meter action. 4020 * 4021 * - Reg_C_5 is used for the packet to match betweend prefix and 4022 * suffix flow. 4023 * 4024 * @param dev 4025 * Pointer to Ethernet device. 4026 * @param[in] flow 4027 * Parent flow structure pointer. 4028 * @param[in] attr 4029 * Flow rule attributes. 4030 * @param[in] items 4031 * Pattern specification (list terminated by the END pattern item). 4032 * @param[in] actions 4033 * Associated actions (list terminated by the END action). 4034 * @param[in] external 4035 * This flow rule is created by request external to PMD. 4036 * @param[out] error 4037 * Perform verbose error reporting if not NULL. 4038 * @return 4039 * 0 on success, negative value otherwise 4040 */ 4041 static int 4042 flow_create_split_meter(struct rte_eth_dev *dev, 4043 struct rte_flow *flow, 4044 const struct rte_flow_attr *attr, 4045 const struct rte_flow_item items[], 4046 const struct rte_flow_action actions[], 4047 bool external, struct rte_flow_error *error) 4048 { 4049 struct mlx5_priv *priv = dev->data->dev_private; 4050 struct rte_flow_action *sfx_actions = NULL; 4051 struct rte_flow_action *pre_actions = NULL; 4052 struct rte_flow_item *sfx_items = NULL; 4053 struct mlx5_flow *dev_flow = NULL; 4054 struct rte_flow_attr sfx_attr = *attr; 4055 uint32_t mtr = 0; 4056 uint32_t mtr_tag_id = 0; 4057 size_t act_size; 4058 size_t item_size; 4059 int actions_n = 0; 4060 int ret; 4061 4062 if (priv->mtr_en) 4063 actions_n = flow_check_meter_action(actions, &mtr); 4064 if (mtr) { 4065 /* The five prefix actions: meter, decap, encap, tag, end. */ 4066 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4067 sizeof(struct mlx5_rte_flow_action_set_tag); 4068 /* tag, vlan, port id, end. */ 4069 #define METER_SUFFIX_ITEM 4 4070 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4071 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4072 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 4073 if (!sfx_actions) 4074 return rte_flow_error_set(error, ENOMEM, 4075 RTE_FLOW_ERROR_TYPE_ACTION, 4076 NULL, "no memory to split " 4077 "meter flow"); 4078 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4079 act_size); 4080 pre_actions = sfx_actions + actions_n; 4081 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4082 actions, sfx_actions, 4083 pre_actions); 4084 if (!mtr_tag_id) { 4085 ret = -rte_errno; 4086 goto exit; 4087 } 4088 /* Add the prefix subflow. */ 4089 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr, 4090 items, pre_actions, external, 4091 error); 4092 if (ret) { 4093 ret = -rte_errno; 4094 goto exit; 4095 } 4096 dev_flow->handle->mtr_flow_id = mtr_tag_id; 4097 /* Setting the sfx group atrr. */ 4098 sfx_attr.group = sfx_attr.transfer ? 4099 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4100 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4101 } 4102 /* Add the prefix subflow. */ 4103 ret = flow_create_split_metadata(dev, flow, dev_flow ? 4104 flow_get_prefix_layer_flags(dev_flow) : 4105 0, &sfx_attr, 4106 sfx_items ? sfx_items : items, 4107 sfx_actions ? sfx_actions : actions, 4108 external, error); 4109 exit: 4110 if (sfx_actions) 4111 rte_free(sfx_actions); 4112 return ret; 4113 } 4114 4115 /** 4116 * Split the flow to subflow set. The splitters might be linked 4117 * in the chain, like this: 4118 * flow_create_split_outer() calls: 4119 * flow_create_split_meter() calls: 4120 * flow_create_split_metadata(meter_subflow_0) calls: 4121 * flow_create_split_inner(metadata_subflow_0) 4122 * flow_create_split_inner(metadata_subflow_1) 4123 * flow_create_split_inner(metadata_subflow_2) 4124 * flow_create_split_metadata(meter_subflow_1) calls: 4125 * flow_create_split_inner(metadata_subflow_0) 4126 * flow_create_split_inner(metadata_subflow_1) 4127 * flow_create_split_inner(metadata_subflow_2) 4128 * 4129 * This provide flexible way to add new levels of flow splitting. 4130 * The all of successfully created subflows are included to the 4131 * parent flow dev_flow list. 4132 * 4133 * @param dev 4134 * Pointer to Ethernet device. 4135 * @param[in] flow 4136 * Parent flow structure pointer. 4137 * @param[in] attr 4138 * Flow rule attributes. 4139 * @param[in] items 4140 * Pattern specification (list terminated by the END pattern item). 4141 * @param[in] actions 4142 * Associated actions (list terminated by the END action). 4143 * @param[in] external 4144 * This flow rule is created by request external to PMD. 4145 * @param[out] error 4146 * Perform verbose error reporting if not NULL. 4147 * @return 4148 * 0 on success, negative value otherwise 4149 */ 4150 static int 4151 flow_create_split_outer(struct rte_eth_dev *dev, 4152 struct rte_flow *flow, 4153 const struct rte_flow_attr *attr, 4154 const struct rte_flow_item items[], 4155 const struct rte_flow_action actions[], 4156 bool external, struct rte_flow_error *error) 4157 { 4158 int ret; 4159 4160 ret = flow_create_split_meter(dev, flow, attr, items, 4161 actions, external, error); 4162 MLX5_ASSERT(ret <= 0); 4163 return ret; 4164 } 4165 4166 /** 4167 * Create a flow and add it to @p list. 4168 * 4169 * @param dev 4170 * Pointer to Ethernet device. 4171 * @param list 4172 * Pointer to a TAILQ flow list. If this parameter NULL, 4173 * no list insertion occurred, flow is just created, 4174 * this is caller's responsibility to track the 4175 * created flow. 4176 * @param[in] attr 4177 * Flow rule attributes. 4178 * @param[in] items 4179 * Pattern specification (list terminated by the END pattern item). 4180 * @param[in] actions 4181 * Associated actions (list terminated by the END action). 4182 * @param[in] external 4183 * This flow rule is created by request external to PMD. 4184 * @param[out] error 4185 * Perform verbose error reporting if not NULL. 4186 * 4187 * @return 4188 * A flow on success, NULL otherwise and rte_errno is set. 4189 */ 4190 static struct rte_flow * 4191 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 4192 const struct rte_flow_attr *attr, 4193 const struct rte_flow_item items[], 4194 const struct rte_flow_action actions[], 4195 bool external, struct rte_flow_error *error) 4196 { 4197 struct mlx5_priv *priv = dev->data->dev_private; 4198 struct rte_flow *flow = NULL; 4199 struct mlx5_flow *dev_flow; 4200 const struct rte_flow_action_rss *rss; 4201 union { 4202 struct rte_flow_expand_rss buf; 4203 uint8_t buffer[2048]; 4204 } expand_buffer; 4205 union { 4206 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4207 uint8_t buffer[2048]; 4208 } actions_rx; 4209 union { 4210 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4211 uint8_t buffer[2048]; 4212 } actions_hairpin_tx; 4213 union { 4214 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4215 uint8_t buffer[2048]; 4216 } items_tx; 4217 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4218 const struct rte_flow_action *p_actions_rx = actions; 4219 uint32_t i; 4220 uint32_t flow_size; 4221 int hairpin_flow = 0; 4222 uint32_t hairpin_id = 0; 4223 struct rte_flow_attr attr_tx = { .priority = 0 }; 4224 int ret = flow_drv_validate(dev, attr, items, p_actions_rx, external, 4225 error); 4226 4227 if (ret < 0) 4228 return NULL; 4229 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4230 if (hairpin_flow > 0) { 4231 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4232 rte_errno = EINVAL; 4233 return NULL; 4234 } 4235 flow_hairpin_split(dev, actions, actions_rx.actions, 4236 actions_hairpin_tx.actions, items_tx.items, 4237 &hairpin_id); 4238 p_actions_rx = actions_rx.actions; 4239 } 4240 flow_size = sizeof(struct rte_flow); 4241 rss = flow_get_rss_action(p_actions_rx); 4242 if (rss) 4243 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t), 4244 sizeof(void *)); 4245 else 4246 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *)); 4247 flow = rte_calloc(__func__, 1, flow_size, 0); 4248 if (!flow) { 4249 rte_errno = ENOMEM; 4250 goto error_before_flow; 4251 } 4252 flow->drv_type = flow_get_drv_type(dev, attr); 4253 if (hairpin_id != 0) 4254 flow->hairpin_flow_id = hairpin_id; 4255 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4256 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4257 flow->rss.queue = (void *)(flow + 1); 4258 if (rss) { 4259 /* 4260 * The following information is required by 4261 * mlx5_flow_hashfields_adjust() in advance. 4262 */ 4263 flow->rss.level = rss->level; 4264 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4265 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; 4266 } 4267 LIST_INIT(&flow->dev_handles); 4268 if (rss && rss->types) { 4269 unsigned int graph_root; 4270 4271 graph_root = find_graph_root(items, rss->level); 4272 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4273 items, rss->types, 4274 mlx5_support_expansion, 4275 graph_root); 4276 MLX5_ASSERT(ret > 0 && 4277 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4278 } else { 4279 buf->entries = 1; 4280 buf->entry[0].pattern = (void *)(uintptr_t)items; 4281 } 4282 /* Reset device flow index to 0. */ 4283 priv->flow_idx = 0; 4284 for (i = 0; i < buf->entries; ++i) { 4285 /* 4286 * The splitter may create multiple dev_flows, 4287 * depending on configuration. In the simplest 4288 * case it just creates unmodified original flow. 4289 */ 4290 ret = flow_create_split_outer(dev, flow, attr, 4291 buf->entry[i].pattern, 4292 p_actions_rx, external, 4293 error); 4294 if (ret < 0) 4295 goto error; 4296 } 4297 /* Create the tx flow. */ 4298 if (hairpin_flow) { 4299 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4300 attr_tx.ingress = 0; 4301 attr_tx.egress = 1; 4302 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 4303 actions_hairpin_tx.actions, error); 4304 if (!dev_flow) 4305 goto error; 4306 dev_flow->flow = flow; 4307 dev_flow->external = 0; 4308 LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next); 4309 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4310 items_tx.items, 4311 actions_hairpin_tx.actions, error); 4312 if (ret < 0) 4313 goto error; 4314 } 4315 /* 4316 * Update the metadata register copy table. If extensive 4317 * metadata feature is enabled and registers are supported 4318 * we might create the extra rte_flow for each unique 4319 * MARK/FLAG action ID. 4320 * 4321 * The table is updated for ingress Flows only, because 4322 * the egress Flows belong to the different device and 4323 * copy table should be updated in peer NIC Rx domain. 4324 */ 4325 if (attr->ingress && 4326 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4327 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4328 if (ret) 4329 goto error; 4330 } 4331 /* 4332 * If the flow is external (from application) OR device is started, then 4333 * the flow will be applied immediately. 4334 */ 4335 if (external || dev->data->dev_started) { 4336 ret = flow_drv_apply(dev, flow, error); 4337 if (ret < 0) 4338 goto error; 4339 } 4340 if (list) 4341 TAILQ_INSERT_TAIL(list, flow, next); 4342 flow_rxq_flags_set(dev, flow); 4343 return flow; 4344 error_before_flow: 4345 if (hairpin_id) 4346 mlx5_flow_id_release(priv->sh->flow_id_pool, 4347 hairpin_id); 4348 return NULL; 4349 error: 4350 MLX5_ASSERT(flow); 4351 flow_mreg_del_copy_action(dev, flow); 4352 ret = rte_errno; /* Save rte_errno before cleanup. */ 4353 if (flow->hairpin_flow_id) 4354 mlx5_flow_id_release(priv->sh->flow_id_pool, 4355 flow->hairpin_flow_id); 4356 MLX5_ASSERT(flow); 4357 flow_drv_destroy(dev, flow); 4358 rte_free(flow); 4359 rte_errno = ret; /* Restore rte_errno. */ 4360 return NULL; 4361 } 4362 4363 /** 4364 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4365 * incoming packets to table 1. 4366 * 4367 * Other flow rules, requested for group n, will be created in 4368 * e-switch table n+1. 4369 * Jump action to e-switch group n will be created to group n+1. 4370 * 4371 * Used when working in switchdev mode, to utilise advantages of table 1 4372 * and above. 4373 * 4374 * @param dev 4375 * Pointer to Ethernet device. 4376 * 4377 * @return 4378 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4379 */ 4380 struct rte_flow * 4381 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4382 { 4383 const struct rte_flow_attr attr = { 4384 .group = 0, 4385 .priority = 0, 4386 .ingress = 1, 4387 .egress = 0, 4388 .transfer = 1, 4389 }; 4390 const struct rte_flow_item pattern = { 4391 .type = RTE_FLOW_ITEM_TYPE_END, 4392 }; 4393 struct rte_flow_action_jump jump = { 4394 .group = 1, 4395 }; 4396 const struct rte_flow_action actions[] = { 4397 { 4398 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4399 .conf = &jump, 4400 }, 4401 { 4402 .type = RTE_FLOW_ACTION_TYPE_END, 4403 }, 4404 }; 4405 struct mlx5_priv *priv = dev->data->dev_private; 4406 struct rte_flow_error error; 4407 4408 return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern, 4409 actions, false, &error); 4410 } 4411 4412 /** 4413 * Create a flow. 4414 * 4415 * @see rte_flow_create() 4416 * @see rte_flow_ops 4417 */ 4418 struct rte_flow * 4419 mlx5_flow_create(struct rte_eth_dev *dev, 4420 const struct rte_flow_attr *attr, 4421 const struct rte_flow_item items[], 4422 const struct rte_flow_action actions[], 4423 struct rte_flow_error *error) 4424 { 4425 struct mlx5_priv *priv = dev->data->dev_private; 4426 4427 /* 4428 * If the device is not started yet, it is not allowed to created a 4429 * flow from application. PMD default flows and traffic control flows 4430 * are not affected. 4431 */ 4432 if (unlikely(!dev->data->dev_started)) { 4433 rte_errno = ENODEV; 4434 DRV_LOG(DEBUG, "port %u is not started when " 4435 "inserting a flow", dev->data->port_id); 4436 return NULL; 4437 } 4438 return flow_list_create(dev, &priv->flows, 4439 attr, items, actions, true, error); 4440 } 4441 4442 /** 4443 * Destroy a flow in a list. 4444 * 4445 * @param dev 4446 * Pointer to Ethernet device. 4447 * @param list 4448 * Pointer to a TAILQ flow list. If this parameter NULL, 4449 * there is no flow removal from the list. 4450 * @param[in] flow 4451 * Flow to destroy. 4452 */ 4453 static void 4454 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 4455 struct rte_flow *flow) 4456 { 4457 struct mlx5_priv *priv = dev->data->dev_private; 4458 4459 /* 4460 * Update RX queue flags only if port is started, otherwise it is 4461 * already clean. 4462 */ 4463 if (dev->data->dev_started) 4464 flow_rxq_flags_trim(dev, flow); 4465 if (flow->hairpin_flow_id) 4466 mlx5_flow_id_release(priv->sh->flow_id_pool, 4467 flow->hairpin_flow_id); 4468 flow_drv_destroy(dev, flow); 4469 if (list) 4470 TAILQ_REMOVE(list, flow, next); 4471 flow_mreg_del_copy_action(dev, flow); 4472 rte_free(flow->fdir); 4473 rte_free(flow); 4474 } 4475 4476 /** 4477 * Destroy all flows. 4478 * 4479 * @param dev 4480 * Pointer to Ethernet device. 4481 * @param list 4482 * Pointer to a TAILQ flow list. 4483 * @param active 4484 * If flushing is called avtively. 4485 */ 4486 void 4487 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list, 4488 bool active) 4489 { 4490 uint32_t num_flushed = 0; 4491 4492 while (!TAILQ_EMPTY(list)) { 4493 struct rte_flow *flow; 4494 4495 flow = TAILQ_FIRST(list); 4496 flow_list_destroy(dev, list, flow); 4497 num_flushed++; 4498 } 4499 if (active) { 4500 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 4501 dev->data->port_id, num_flushed); 4502 } 4503 } 4504 4505 /** 4506 * Remove all flows. 4507 * 4508 * @param dev 4509 * Pointer to Ethernet device. 4510 * @param list 4511 * Pointer to a TAILQ flow list. 4512 */ 4513 void 4514 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 4515 { 4516 struct rte_flow *flow; 4517 4518 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 4519 flow_drv_remove(dev, flow); 4520 flow_mreg_stop_copy_action(dev, flow); 4521 } 4522 flow_mreg_del_default_copy_action(dev); 4523 flow_rxq_flags_clear(dev); 4524 } 4525 4526 /** 4527 * Add all flows. 4528 * 4529 * @param dev 4530 * Pointer to Ethernet device. 4531 * @param list 4532 * Pointer to a TAILQ flow list. 4533 * 4534 * @return 4535 * 0 on success, a negative errno value otherwise and rte_errno is set. 4536 */ 4537 int 4538 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 4539 { 4540 struct rte_flow *flow; 4541 struct rte_flow_error error; 4542 int ret = 0; 4543 4544 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4545 ret = flow_mreg_add_default_copy_action(dev, &error); 4546 if (ret < 0) 4547 return -rte_errno; 4548 /* Apply Flows created by application. */ 4549 TAILQ_FOREACH(flow, list, next) { 4550 ret = flow_mreg_start_copy_action(dev, flow); 4551 if (ret < 0) 4552 goto error; 4553 ret = flow_drv_apply(dev, flow, &error); 4554 if (ret < 0) 4555 goto error; 4556 flow_rxq_flags_set(dev, flow); 4557 } 4558 return 0; 4559 error: 4560 ret = rte_errno; /* Save rte_errno before cleanup. */ 4561 mlx5_flow_stop(dev, list); 4562 rte_errno = ret; /* Restore rte_errno. */ 4563 return -rte_errno; 4564 } 4565 4566 /** 4567 * Stop all default actions for flows. 4568 * 4569 * @param dev 4570 * Pointer to Ethernet device. 4571 */ 4572 void 4573 mlx5_flow_stop_default(struct rte_eth_dev *dev) 4574 { 4575 flow_mreg_del_default_copy_action(dev); 4576 } 4577 4578 /** 4579 * Start all default actions for flows. 4580 * 4581 * @param dev 4582 * Pointer to Ethernet device. 4583 * @return 4584 * 0 on success, a negative errno value otherwise and rte_errno is set. 4585 */ 4586 int 4587 mlx5_flow_start_default(struct rte_eth_dev *dev) 4588 { 4589 struct rte_flow_error error; 4590 4591 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4592 return flow_mreg_add_default_copy_action(dev, &error); 4593 } 4594 4595 /** 4596 * Allocate intermediate resources for flow creation. 4597 * 4598 * @param dev 4599 * Pointer to Ethernet device. 4600 */ 4601 void 4602 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) 4603 { 4604 struct mlx5_priv *priv = dev->data->dev_private; 4605 4606 if (!priv->inter_flows) 4607 priv->inter_flows = rte_calloc(__func__, MLX5_NUM_MAX_DEV_FLOWS, 4608 sizeof(struct mlx5_flow), 0); 4609 } 4610 4611 /** 4612 * Free intermediate resources for flows. 4613 * 4614 * @param dev 4615 * Pointer to Ethernet device. 4616 */ 4617 void 4618 mlx5_flow_free_intermediate(struct rte_eth_dev *dev) 4619 { 4620 struct mlx5_priv *priv = dev->data->dev_private; 4621 4622 rte_free(priv->inter_flows); 4623 priv->inter_flows = NULL; 4624 } 4625 4626 /** 4627 * Verify the flow list is empty 4628 * 4629 * @param dev 4630 * Pointer to Ethernet device. 4631 * 4632 * @return the number of flows not released. 4633 */ 4634 int 4635 mlx5_flow_verify(struct rte_eth_dev *dev) 4636 { 4637 struct mlx5_priv *priv = dev->data->dev_private; 4638 struct rte_flow *flow; 4639 int ret = 0; 4640 4641 TAILQ_FOREACH(flow, &priv->flows, next) { 4642 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4643 dev->data->port_id, (void *)flow); 4644 ++ret; 4645 } 4646 return ret; 4647 } 4648 4649 /** 4650 * Enable default hairpin egress flow. 4651 * 4652 * @param dev 4653 * Pointer to Ethernet device. 4654 * @param queue 4655 * The queue index. 4656 * 4657 * @return 4658 * 0 on success, a negative errno value otherwise and rte_errno is set. 4659 */ 4660 int 4661 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4662 uint32_t queue) 4663 { 4664 struct mlx5_priv *priv = dev->data->dev_private; 4665 const struct rte_flow_attr attr = { 4666 .egress = 1, 4667 .priority = 0, 4668 }; 4669 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4670 .queue = queue, 4671 }; 4672 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4673 .queue = UINT32_MAX, 4674 }; 4675 struct rte_flow_item items[] = { 4676 { 4677 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4678 .spec = &queue_spec, 4679 .last = NULL, 4680 .mask = &queue_mask, 4681 }, 4682 { 4683 .type = RTE_FLOW_ITEM_TYPE_END, 4684 }, 4685 }; 4686 struct rte_flow_action_jump jump = { 4687 .group = MLX5_HAIRPIN_TX_TABLE, 4688 }; 4689 struct rte_flow_action actions[2]; 4690 struct rte_flow *flow; 4691 struct rte_flow_error error; 4692 4693 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4694 actions[0].conf = &jump; 4695 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4696 flow = flow_list_create(dev, &priv->ctrl_flows, 4697 &attr, items, actions, false, &error); 4698 if (!flow) { 4699 DRV_LOG(DEBUG, 4700 "Failed to create ctrl flow: rte_errno(%d)," 4701 " type(%d), message(%s)", 4702 rte_errno, error.type, 4703 error.message ? error.message : " (no stated reason)"); 4704 return -rte_errno; 4705 } 4706 return 0; 4707 } 4708 4709 /** 4710 * Enable a control flow configured from the control plane. 4711 * 4712 * @param dev 4713 * Pointer to Ethernet device. 4714 * @param eth_spec 4715 * An Ethernet flow spec to apply. 4716 * @param eth_mask 4717 * An Ethernet flow mask to apply. 4718 * @param vlan_spec 4719 * A VLAN flow spec to apply. 4720 * @param vlan_mask 4721 * A VLAN flow mask to apply. 4722 * 4723 * @return 4724 * 0 on success, a negative errno value otherwise and rte_errno is set. 4725 */ 4726 int 4727 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4728 struct rte_flow_item_eth *eth_spec, 4729 struct rte_flow_item_eth *eth_mask, 4730 struct rte_flow_item_vlan *vlan_spec, 4731 struct rte_flow_item_vlan *vlan_mask) 4732 { 4733 struct mlx5_priv *priv = dev->data->dev_private; 4734 const struct rte_flow_attr attr = { 4735 .ingress = 1, 4736 .priority = MLX5_FLOW_PRIO_RSVD, 4737 }; 4738 struct rte_flow_item items[] = { 4739 { 4740 .type = RTE_FLOW_ITEM_TYPE_ETH, 4741 .spec = eth_spec, 4742 .last = NULL, 4743 .mask = eth_mask, 4744 }, 4745 { 4746 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4747 RTE_FLOW_ITEM_TYPE_END, 4748 .spec = vlan_spec, 4749 .last = NULL, 4750 .mask = vlan_mask, 4751 }, 4752 { 4753 .type = RTE_FLOW_ITEM_TYPE_END, 4754 }, 4755 }; 4756 uint16_t queue[priv->reta_idx_n]; 4757 struct rte_flow_action_rss action_rss = { 4758 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4759 .level = 0, 4760 .types = priv->rss_conf.rss_hf, 4761 .key_len = priv->rss_conf.rss_key_len, 4762 .queue_num = priv->reta_idx_n, 4763 .key = priv->rss_conf.rss_key, 4764 .queue = queue, 4765 }; 4766 struct rte_flow_action actions[] = { 4767 { 4768 .type = RTE_FLOW_ACTION_TYPE_RSS, 4769 .conf = &action_rss, 4770 }, 4771 { 4772 .type = RTE_FLOW_ACTION_TYPE_END, 4773 }, 4774 }; 4775 struct rte_flow *flow; 4776 struct rte_flow_error error; 4777 unsigned int i; 4778 4779 if (!priv->reta_idx_n || !priv->rxqs_n) { 4780 return 0; 4781 } 4782 for (i = 0; i != priv->reta_idx_n; ++i) 4783 queue[i] = (*priv->reta_idx)[i]; 4784 flow = flow_list_create(dev, &priv->ctrl_flows, 4785 &attr, items, actions, false, &error); 4786 if (!flow) 4787 return -rte_errno; 4788 return 0; 4789 } 4790 4791 /** 4792 * Enable a flow control configured from the control plane. 4793 * 4794 * @param dev 4795 * Pointer to Ethernet device. 4796 * @param eth_spec 4797 * An Ethernet flow spec to apply. 4798 * @param eth_mask 4799 * An Ethernet flow mask to apply. 4800 * 4801 * @return 4802 * 0 on success, a negative errno value otherwise and rte_errno is set. 4803 */ 4804 int 4805 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4806 struct rte_flow_item_eth *eth_spec, 4807 struct rte_flow_item_eth *eth_mask) 4808 { 4809 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4810 } 4811 4812 /** 4813 * Destroy a flow. 4814 * 4815 * @see rte_flow_destroy() 4816 * @see rte_flow_ops 4817 */ 4818 int 4819 mlx5_flow_destroy(struct rte_eth_dev *dev, 4820 struct rte_flow *flow, 4821 struct rte_flow_error *error __rte_unused) 4822 { 4823 struct mlx5_priv *priv = dev->data->dev_private; 4824 4825 flow_list_destroy(dev, &priv->flows, flow); 4826 return 0; 4827 } 4828 4829 /** 4830 * Destroy all flows. 4831 * 4832 * @see rte_flow_flush() 4833 * @see rte_flow_ops 4834 */ 4835 int 4836 mlx5_flow_flush(struct rte_eth_dev *dev, 4837 struct rte_flow_error *error __rte_unused) 4838 { 4839 struct mlx5_priv *priv = dev->data->dev_private; 4840 4841 mlx5_flow_list_flush(dev, &priv->flows, false); 4842 return 0; 4843 } 4844 4845 /** 4846 * Isolated mode. 4847 * 4848 * @see rte_flow_isolate() 4849 * @see rte_flow_ops 4850 */ 4851 int 4852 mlx5_flow_isolate(struct rte_eth_dev *dev, 4853 int enable, 4854 struct rte_flow_error *error) 4855 { 4856 struct mlx5_priv *priv = dev->data->dev_private; 4857 4858 if (dev->data->dev_started) { 4859 rte_flow_error_set(error, EBUSY, 4860 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4861 NULL, 4862 "port must be stopped first"); 4863 return -rte_errno; 4864 } 4865 priv->isolated = !!enable; 4866 if (enable) 4867 dev->dev_ops = &mlx5_dev_ops_isolate; 4868 else 4869 dev->dev_ops = &mlx5_dev_ops; 4870 return 0; 4871 } 4872 4873 /** 4874 * Query a flow. 4875 * 4876 * @see rte_flow_query() 4877 * @see rte_flow_ops 4878 */ 4879 static int 4880 flow_drv_query(struct rte_eth_dev *dev, 4881 struct rte_flow *flow, 4882 const struct rte_flow_action *actions, 4883 void *data, 4884 struct rte_flow_error *error) 4885 { 4886 const struct mlx5_flow_driver_ops *fops; 4887 enum mlx5_flow_drv_type ftype = flow->drv_type; 4888 4889 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 4890 fops = flow_get_drv_ops(ftype); 4891 4892 return fops->query(dev, flow, actions, data, error); 4893 } 4894 4895 /** 4896 * Query a flow. 4897 * 4898 * @see rte_flow_query() 4899 * @see rte_flow_ops 4900 */ 4901 int 4902 mlx5_flow_query(struct rte_eth_dev *dev, 4903 struct rte_flow *flow, 4904 const struct rte_flow_action *actions, 4905 void *data, 4906 struct rte_flow_error *error) 4907 { 4908 int ret; 4909 4910 ret = flow_drv_query(dev, flow, actions, data, error); 4911 if (ret < 0) 4912 return ret; 4913 return 0; 4914 } 4915 4916 /** 4917 * Convert a flow director filter to a generic flow. 4918 * 4919 * @param dev 4920 * Pointer to Ethernet device. 4921 * @param fdir_filter 4922 * Flow director filter to add. 4923 * @param attributes 4924 * Generic flow parameters structure. 4925 * 4926 * @return 4927 * 0 on success, a negative errno value otherwise and rte_errno is set. 4928 */ 4929 static int 4930 flow_fdir_filter_convert(struct rte_eth_dev *dev, 4931 const struct rte_eth_fdir_filter *fdir_filter, 4932 struct mlx5_fdir *attributes) 4933 { 4934 struct mlx5_priv *priv = dev->data->dev_private; 4935 const struct rte_eth_fdir_input *input = &fdir_filter->input; 4936 const struct rte_eth_fdir_masks *mask = 4937 &dev->data->dev_conf.fdir_conf.mask; 4938 4939 /* Validate queue number. */ 4940 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 4941 DRV_LOG(ERR, "port %u invalid queue number %d", 4942 dev->data->port_id, fdir_filter->action.rx_queue); 4943 rte_errno = EINVAL; 4944 return -rte_errno; 4945 } 4946 attributes->attr.ingress = 1; 4947 attributes->items[0] = (struct rte_flow_item) { 4948 .type = RTE_FLOW_ITEM_TYPE_ETH, 4949 .spec = &attributes->l2, 4950 .mask = &attributes->l2_mask, 4951 }; 4952 switch (fdir_filter->action.behavior) { 4953 case RTE_ETH_FDIR_ACCEPT: 4954 attributes->actions[0] = (struct rte_flow_action){ 4955 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 4956 .conf = &attributes->queue, 4957 }; 4958 break; 4959 case RTE_ETH_FDIR_REJECT: 4960 attributes->actions[0] = (struct rte_flow_action){ 4961 .type = RTE_FLOW_ACTION_TYPE_DROP, 4962 }; 4963 break; 4964 default: 4965 DRV_LOG(ERR, "port %u invalid behavior %d", 4966 dev->data->port_id, 4967 fdir_filter->action.behavior); 4968 rte_errno = ENOTSUP; 4969 return -rte_errno; 4970 } 4971 attributes->queue.index = fdir_filter->action.rx_queue; 4972 /* Handle L3. */ 4973 switch (fdir_filter->input.flow_type) { 4974 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4975 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4976 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4977 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 4978 .src_addr = input->flow.ip4_flow.src_ip, 4979 .dst_addr = input->flow.ip4_flow.dst_ip, 4980 .time_to_live = input->flow.ip4_flow.ttl, 4981 .type_of_service = input->flow.ip4_flow.tos, 4982 }; 4983 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 4984 .src_addr = mask->ipv4_mask.src_ip, 4985 .dst_addr = mask->ipv4_mask.dst_ip, 4986 .time_to_live = mask->ipv4_mask.ttl, 4987 .type_of_service = mask->ipv4_mask.tos, 4988 .next_proto_id = mask->ipv4_mask.proto, 4989 }; 4990 attributes->items[1] = (struct rte_flow_item){ 4991 .type = RTE_FLOW_ITEM_TYPE_IPV4, 4992 .spec = &attributes->l3, 4993 .mask = &attributes->l3_mask, 4994 }; 4995 break; 4996 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4997 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4998 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4999 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 5000 .hop_limits = input->flow.ipv6_flow.hop_limits, 5001 .proto = input->flow.ipv6_flow.proto, 5002 }; 5003 5004 memcpy(attributes->l3.ipv6.hdr.src_addr, 5005 input->flow.ipv6_flow.src_ip, 5006 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5007 memcpy(attributes->l3.ipv6.hdr.dst_addr, 5008 input->flow.ipv6_flow.dst_ip, 5009 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5010 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 5011 mask->ipv6_mask.src_ip, 5012 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5013 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 5014 mask->ipv6_mask.dst_ip, 5015 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5016 attributes->items[1] = (struct rte_flow_item){ 5017 .type = RTE_FLOW_ITEM_TYPE_IPV6, 5018 .spec = &attributes->l3, 5019 .mask = &attributes->l3_mask, 5020 }; 5021 break; 5022 default: 5023 DRV_LOG(ERR, "port %u invalid flow type%d", 5024 dev->data->port_id, fdir_filter->input.flow_type); 5025 rte_errno = ENOTSUP; 5026 return -rte_errno; 5027 } 5028 /* Handle L4. */ 5029 switch (fdir_filter->input.flow_type) { 5030 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5031 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5032 .src_port = input->flow.udp4_flow.src_port, 5033 .dst_port = input->flow.udp4_flow.dst_port, 5034 }; 5035 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5036 .src_port = mask->src_port_mask, 5037 .dst_port = mask->dst_port_mask, 5038 }; 5039 attributes->items[2] = (struct rte_flow_item){ 5040 .type = RTE_FLOW_ITEM_TYPE_UDP, 5041 .spec = &attributes->l4, 5042 .mask = &attributes->l4_mask, 5043 }; 5044 break; 5045 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5046 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5047 .src_port = input->flow.tcp4_flow.src_port, 5048 .dst_port = input->flow.tcp4_flow.dst_port, 5049 }; 5050 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5051 .src_port = mask->src_port_mask, 5052 .dst_port = mask->dst_port_mask, 5053 }; 5054 attributes->items[2] = (struct rte_flow_item){ 5055 .type = RTE_FLOW_ITEM_TYPE_TCP, 5056 .spec = &attributes->l4, 5057 .mask = &attributes->l4_mask, 5058 }; 5059 break; 5060 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5061 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5062 .src_port = input->flow.udp6_flow.src_port, 5063 .dst_port = input->flow.udp6_flow.dst_port, 5064 }; 5065 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5066 .src_port = mask->src_port_mask, 5067 .dst_port = mask->dst_port_mask, 5068 }; 5069 attributes->items[2] = (struct rte_flow_item){ 5070 .type = RTE_FLOW_ITEM_TYPE_UDP, 5071 .spec = &attributes->l4, 5072 .mask = &attributes->l4_mask, 5073 }; 5074 break; 5075 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5076 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5077 .src_port = input->flow.tcp6_flow.src_port, 5078 .dst_port = input->flow.tcp6_flow.dst_port, 5079 }; 5080 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5081 .src_port = mask->src_port_mask, 5082 .dst_port = mask->dst_port_mask, 5083 }; 5084 attributes->items[2] = (struct rte_flow_item){ 5085 .type = RTE_FLOW_ITEM_TYPE_TCP, 5086 .spec = &attributes->l4, 5087 .mask = &attributes->l4_mask, 5088 }; 5089 break; 5090 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5091 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5092 break; 5093 default: 5094 DRV_LOG(ERR, "port %u invalid flow type%d", 5095 dev->data->port_id, fdir_filter->input.flow_type); 5096 rte_errno = ENOTSUP; 5097 return -rte_errno; 5098 } 5099 return 0; 5100 } 5101 5102 #define FLOW_FDIR_CMP(f1, f2, fld) \ 5103 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 5104 5105 /** 5106 * Compare two FDIR flows. If items and actions are identical, the two flows are 5107 * regarded as same. 5108 * 5109 * @param dev 5110 * Pointer to Ethernet device. 5111 * @param f1 5112 * FDIR flow to compare. 5113 * @param f2 5114 * FDIR flow to compare. 5115 * 5116 * @return 5117 * Zero on match, 1 otherwise. 5118 */ 5119 static int 5120 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 5121 { 5122 if (FLOW_FDIR_CMP(f1, f2, attr) || 5123 FLOW_FDIR_CMP(f1, f2, l2) || 5124 FLOW_FDIR_CMP(f1, f2, l2_mask) || 5125 FLOW_FDIR_CMP(f1, f2, l3) || 5126 FLOW_FDIR_CMP(f1, f2, l3_mask) || 5127 FLOW_FDIR_CMP(f1, f2, l4) || 5128 FLOW_FDIR_CMP(f1, f2, l4_mask) || 5129 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 5130 return 1; 5131 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 5132 FLOW_FDIR_CMP(f1, f2, queue)) 5133 return 1; 5134 return 0; 5135 } 5136 5137 /** 5138 * Search device flow list to find out a matched FDIR flow. 5139 * 5140 * @param dev 5141 * Pointer to Ethernet device. 5142 * @param fdir_flow 5143 * FDIR flow to lookup. 5144 * 5145 * @return 5146 * Pointer of flow if found, NULL otherwise. 5147 */ 5148 static struct rte_flow * 5149 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5150 { 5151 struct mlx5_priv *priv = dev->data->dev_private; 5152 struct rte_flow *flow = NULL; 5153 5154 MLX5_ASSERT(fdir_flow); 5155 TAILQ_FOREACH(flow, &priv->flows, next) { 5156 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) { 5157 DRV_LOG(DEBUG, "port %u found FDIR flow %p", 5158 dev->data->port_id, (void *)flow); 5159 break; 5160 } 5161 } 5162 return flow; 5163 } 5164 5165 /** 5166 * Add new flow director filter and store it in list. 5167 * 5168 * @param dev 5169 * Pointer to Ethernet device. 5170 * @param fdir_filter 5171 * Flow director filter to add. 5172 * 5173 * @return 5174 * 0 on success, a negative errno value otherwise and rte_errno is set. 5175 */ 5176 static int 5177 flow_fdir_filter_add(struct rte_eth_dev *dev, 5178 const struct rte_eth_fdir_filter *fdir_filter) 5179 { 5180 struct mlx5_priv *priv = dev->data->dev_private; 5181 struct mlx5_fdir *fdir_flow; 5182 struct rte_flow *flow; 5183 int ret; 5184 5185 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5186 if (!fdir_flow) { 5187 rte_errno = ENOMEM; 5188 return -rte_errno; 5189 } 5190 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5191 if (ret) 5192 goto error; 5193 flow = flow_fdir_filter_lookup(dev, fdir_flow); 5194 if (flow) { 5195 rte_errno = EEXIST; 5196 goto error; 5197 } 5198 flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5199 fdir_flow->items, fdir_flow->actions, true, 5200 NULL); 5201 if (!flow) 5202 goto error; 5203 MLX5_ASSERT(!flow->fdir); 5204 flow->fdir = fdir_flow; 5205 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5206 dev->data->port_id, (void *)flow); 5207 return 0; 5208 error: 5209 rte_free(fdir_flow); 5210 return -rte_errno; 5211 } 5212 5213 /** 5214 * Delete specific filter. 5215 * 5216 * @param dev 5217 * Pointer to Ethernet device. 5218 * @param fdir_filter 5219 * Filter to be deleted. 5220 * 5221 * @return 5222 * 0 on success, a negative errno value otherwise and rte_errno is set. 5223 */ 5224 static int 5225 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5226 const struct rte_eth_fdir_filter *fdir_filter) 5227 { 5228 struct mlx5_priv *priv = dev->data->dev_private; 5229 struct rte_flow *flow; 5230 struct mlx5_fdir fdir_flow = { 5231 .attr.group = 0, 5232 }; 5233 int ret; 5234 5235 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5236 if (ret) 5237 return -rte_errno; 5238 flow = flow_fdir_filter_lookup(dev, &fdir_flow); 5239 if (!flow) { 5240 rte_errno = ENOENT; 5241 return -rte_errno; 5242 } 5243 flow_list_destroy(dev, &priv->flows, flow); 5244 DRV_LOG(DEBUG, "port %u deleted FDIR flow %p", 5245 dev->data->port_id, (void *)flow); 5246 return 0; 5247 } 5248 5249 /** 5250 * Update queue for specific filter. 5251 * 5252 * @param dev 5253 * Pointer to Ethernet device. 5254 * @param fdir_filter 5255 * Filter to be updated. 5256 * 5257 * @return 5258 * 0 on success, a negative errno value otherwise and rte_errno is set. 5259 */ 5260 static int 5261 flow_fdir_filter_update(struct rte_eth_dev *dev, 5262 const struct rte_eth_fdir_filter *fdir_filter) 5263 { 5264 int ret; 5265 5266 ret = flow_fdir_filter_delete(dev, fdir_filter); 5267 if (ret) 5268 return ret; 5269 return flow_fdir_filter_add(dev, fdir_filter); 5270 } 5271 5272 /** 5273 * Flush all filters. 5274 * 5275 * @param dev 5276 * Pointer to Ethernet device. 5277 */ 5278 static void 5279 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5280 { 5281 struct mlx5_priv *priv = dev->data->dev_private; 5282 5283 mlx5_flow_list_flush(dev, &priv->flows, false); 5284 } 5285 5286 /** 5287 * Get flow director information. 5288 * 5289 * @param dev 5290 * Pointer to Ethernet device. 5291 * @param[out] fdir_info 5292 * Resulting flow director information. 5293 */ 5294 static void 5295 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5296 { 5297 struct rte_eth_fdir_masks *mask = 5298 &dev->data->dev_conf.fdir_conf.mask; 5299 5300 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5301 fdir_info->guarant_spc = 0; 5302 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5303 fdir_info->max_flexpayload = 0; 5304 fdir_info->flow_types_mask[0] = 0; 5305 fdir_info->flex_payload_unit = 0; 5306 fdir_info->max_flex_payload_segment_num = 0; 5307 fdir_info->flex_payload_limit = 0; 5308 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5309 } 5310 5311 /** 5312 * Deal with flow director operations. 5313 * 5314 * @param dev 5315 * Pointer to Ethernet device. 5316 * @param filter_op 5317 * Operation to perform. 5318 * @param arg 5319 * Pointer to operation-specific structure. 5320 * 5321 * @return 5322 * 0 on success, a negative errno value otherwise and rte_errno is set. 5323 */ 5324 static int 5325 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5326 void *arg) 5327 { 5328 enum rte_fdir_mode fdir_mode = 5329 dev->data->dev_conf.fdir_conf.mode; 5330 5331 if (filter_op == RTE_ETH_FILTER_NOP) 5332 return 0; 5333 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5334 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5335 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5336 dev->data->port_id, fdir_mode); 5337 rte_errno = EINVAL; 5338 return -rte_errno; 5339 } 5340 switch (filter_op) { 5341 case RTE_ETH_FILTER_ADD: 5342 return flow_fdir_filter_add(dev, arg); 5343 case RTE_ETH_FILTER_UPDATE: 5344 return flow_fdir_filter_update(dev, arg); 5345 case RTE_ETH_FILTER_DELETE: 5346 return flow_fdir_filter_delete(dev, arg); 5347 case RTE_ETH_FILTER_FLUSH: 5348 flow_fdir_filter_flush(dev); 5349 break; 5350 case RTE_ETH_FILTER_INFO: 5351 flow_fdir_info_get(dev, arg); 5352 break; 5353 default: 5354 DRV_LOG(DEBUG, "port %u unknown operation %u", 5355 dev->data->port_id, filter_op); 5356 rte_errno = EINVAL; 5357 return -rte_errno; 5358 } 5359 return 0; 5360 } 5361 5362 /** 5363 * Manage filter operations. 5364 * 5365 * @param dev 5366 * Pointer to Ethernet device structure. 5367 * @param filter_type 5368 * Filter type. 5369 * @param filter_op 5370 * Operation to perform. 5371 * @param arg 5372 * Pointer to operation-specific structure. 5373 * 5374 * @return 5375 * 0 on success, a negative errno value otherwise and rte_errno is set. 5376 */ 5377 int 5378 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5379 enum rte_filter_type filter_type, 5380 enum rte_filter_op filter_op, 5381 void *arg) 5382 { 5383 switch (filter_type) { 5384 case RTE_ETH_FILTER_GENERIC: 5385 if (filter_op != RTE_ETH_FILTER_GET) { 5386 rte_errno = EINVAL; 5387 return -rte_errno; 5388 } 5389 *(const void **)arg = &mlx5_flow_ops; 5390 return 0; 5391 case RTE_ETH_FILTER_FDIR: 5392 return flow_fdir_ctrl_func(dev, filter_op, arg); 5393 default: 5394 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5395 dev->data->port_id, filter_type); 5396 rte_errno = ENOTSUP; 5397 return -rte_errno; 5398 } 5399 return 0; 5400 } 5401 5402 /** 5403 * Create the needed meter and suffix tables. 5404 * 5405 * @param[in] dev 5406 * Pointer to Ethernet device. 5407 * @param[in] fm 5408 * Pointer to the flow meter. 5409 * 5410 * @return 5411 * Pointer to table set on success, NULL otherwise. 5412 */ 5413 struct mlx5_meter_domains_infos * 5414 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5415 const struct mlx5_flow_meter *fm) 5416 { 5417 const struct mlx5_flow_driver_ops *fops; 5418 5419 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5420 return fops->create_mtr_tbls(dev, fm); 5421 } 5422 5423 /** 5424 * Destroy the meter table set. 5425 * 5426 * @param[in] dev 5427 * Pointer to Ethernet device. 5428 * @param[in] tbl 5429 * Pointer to the meter table set. 5430 * 5431 * @return 5432 * 0 on success. 5433 */ 5434 int 5435 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5436 struct mlx5_meter_domains_infos *tbls) 5437 { 5438 const struct mlx5_flow_driver_ops *fops; 5439 5440 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5441 return fops->destroy_mtr_tbls(dev, tbls); 5442 } 5443 5444 /** 5445 * Create policer rules. 5446 * 5447 * @param[in] dev 5448 * Pointer to Ethernet device. 5449 * @param[in] fm 5450 * Pointer to flow meter structure. 5451 * @param[in] attr 5452 * Pointer to flow attributes. 5453 * 5454 * @return 5455 * 0 on success, -1 otherwise. 5456 */ 5457 int 5458 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5459 struct mlx5_flow_meter *fm, 5460 const struct rte_flow_attr *attr) 5461 { 5462 const struct mlx5_flow_driver_ops *fops; 5463 5464 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5465 return fops->create_policer_rules(dev, fm, attr); 5466 } 5467 5468 /** 5469 * Destroy policer rules. 5470 * 5471 * @param[in] fm 5472 * Pointer to flow meter structure. 5473 * @param[in] attr 5474 * Pointer to flow attributes. 5475 * 5476 * @return 5477 * 0 on success, -1 otherwise. 5478 */ 5479 int 5480 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5481 struct mlx5_flow_meter *fm, 5482 const struct rte_flow_attr *attr) 5483 { 5484 const struct mlx5_flow_driver_ops *fops; 5485 5486 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5487 return fops->destroy_policer_rules(dev, fm, attr); 5488 } 5489 5490 /** 5491 * Allocate a counter. 5492 * 5493 * @param[in] dev 5494 * Pointer to Ethernet device structure. 5495 * 5496 * @return 5497 * Index to allocated counter on success, 0 otherwise. 5498 */ 5499 uint32_t 5500 mlx5_counter_alloc(struct rte_eth_dev *dev) 5501 { 5502 const struct mlx5_flow_driver_ops *fops; 5503 struct rte_flow_attr attr = { .transfer = 0 }; 5504 5505 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5506 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5507 return fops->counter_alloc(dev); 5508 } 5509 DRV_LOG(ERR, 5510 "port %u counter allocate is not supported.", 5511 dev->data->port_id); 5512 return 0; 5513 } 5514 5515 /** 5516 * Free a counter. 5517 * 5518 * @param[in] dev 5519 * Pointer to Ethernet device structure. 5520 * @param[in] cnt 5521 * Index to counter to be free. 5522 */ 5523 void 5524 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 5525 { 5526 const struct mlx5_flow_driver_ops *fops; 5527 struct rte_flow_attr attr = { .transfer = 0 }; 5528 5529 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5530 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5531 fops->counter_free(dev, cnt); 5532 return; 5533 } 5534 DRV_LOG(ERR, 5535 "port %u counter free is not supported.", 5536 dev->data->port_id); 5537 } 5538 5539 /** 5540 * Query counter statistics. 5541 * 5542 * @param[in] dev 5543 * Pointer to Ethernet device structure. 5544 * @param[in] cnt 5545 * Index to counter to query. 5546 * @param[in] clear 5547 * Set to clear counter statistics. 5548 * @param[out] pkts 5549 * The counter hits packets number to save. 5550 * @param[out] bytes 5551 * The counter hits bytes number to save. 5552 * 5553 * @return 5554 * 0 on success, a negative errno value otherwise. 5555 */ 5556 int 5557 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 5558 bool clear, uint64_t *pkts, uint64_t *bytes) 5559 { 5560 const struct mlx5_flow_driver_ops *fops; 5561 struct rte_flow_attr attr = { .transfer = 0 }; 5562 5563 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5564 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5565 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5566 } 5567 DRV_LOG(ERR, 5568 "port %u counter query is not supported.", 5569 dev->data->port_id); 5570 return -ENOTSUP; 5571 } 5572 5573 #define MLX5_POOL_QUERY_FREQ_US 1000000 5574 5575 /** 5576 * Set the periodic procedure for triggering asynchronous batch queries for all 5577 * the counter pools. 5578 * 5579 * @param[in] sh 5580 * Pointer to mlx5_ibv_shared object. 5581 */ 5582 void 5583 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5584 { 5585 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0); 5586 uint32_t pools_n = rte_atomic16_read(&cont->n_valid); 5587 uint32_t us; 5588 5589 cont = MLX5_CNT_CONTAINER(sh, 1, 0); 5590 pools_n += rte_atomic16_read(&cont->n_valid); 5591 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5592 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5593 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5594 sh->cmng.query_thread_on = 0; 5595 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5596 } else { 5597 sh->cmng.query_thread_on = 1; 5598 } 5599 } 5600 5601 /** 5602 * The periodic procedure for triggering asynchronous batch queries for all the 5603 * counter pools. This function is probably called by the host thread. 5604 * 5605 * @param[in] arg 5606 * The parameter for the alarm process. 5607 */ 5608 void 5609 mlx5_flow_query_alarm(void *arg) 5610 { 5611 struct mlx5_ibv_shared *sh = arg; 5612 struct mlx5_devx_obj *dcs; 5613 uint16_t offset; 5614 int ret; 5615 uint8_t batch = sh->cmng.batch; 5616 uint16_t pool_index = sh->cmng.pool_index; 5617 struct mlx5_pools_container *cont; 5618 struct mlx5_pools_container *mcont; 5619 struct mlx5_flow_counter_pool *pool; 5620 5621 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5622 goto set_alarm; 5623 next_container: 5624 cont = MLX5_CNT_CONTAINER(sh, batch, 1); 5625 mcont = MLX5_CNT_CONTAINER(sh, batch, 0); 5626 /* Check if resize was done and need to flip a container. */ 5627 if (cont != mcont) { 5628 if (cont->pools) { 5629 /* Clean the old container. */ 5630 rte_free(cont->pools); 5631 memset(cont, 0, sizeof(*cont)); 5632 } 5633 rte_cio_wmb(); 5634 /* Flip the host container. */ 5635 sh->cmng.mhi[batch] ^= (uint8_t)2; 5636 cont = mcont; 5637 } 5638 if (!cont->pools) { 5639 /* 2 empty containers case is unexpected. */ 5640 if (unlikely(batch != sh->cmng.batch)) 5641 goto set_alarm; 5642 batch ^= 0x1; 5643 pool_index = 0; 5644 goto next_container; 5645 } 5646 pool = cont->pools[pool_index]; 5647 if (pool->raw_hw) 5648 /* There is a pool query in progress. */ 5649 goto set_alarm; 5650 pool->raw_hw = 5651 LIST_FIRST(&sh->cmng.free_stat_raws); 5652 if (!pool->raw_hw) 5653 /* No free counter statistics raw memory. */ 5654 goto set_alarm; 5655 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5656 (&pool->a64_dcs); 5657 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5658 /* 5659 * Identify the counters released between query trigger and query 5660 * handle more effiecntly. The counter released in this gap period 5661 * should wait for a new round of query as the new arrived packets 5662 * will not be taken into account. 5663 */ 5664 rte_atomic64_add(&pool->start_query_gen, 1); 5665 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5666 offset, NULL, NULL, 5667 pool->raw_hw->mem_mng->dm->id, 5668 (void *)(uintptr_t) 5669 (pool->raw_hw->data + offset), 5670 sh->devx_comp, 5671 (uint64_t)(uintptr_t)pool); 5672 if (ret) { 5673 rte_atomic64_sub(&pool->start_query_gen, 1); 5674 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5675 " %d", pool->min_dcs->id); 5676 pool->raw_hw = NULL; 5677 goto set_alarm; 5678 } 5679 pool->raw_hw->min_dcs_id = dcs->id; 5680 LIST_REMOVE(pool->raw_hw, next); 5681 sh->cmng.pending_queries++; 5682 pool_index++; 5683 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5684 batch ^= 0x1; 5685 pool_index = 0; 5686 } 5687 set_alarm: 5688 sh->cmng.batch = batch; 5689 sh->cmng.pool_index = pool_index; 5690 mlx5_set_query_alarm(sh); 5691 } 5692 5693 /** 5694 * Handler for the HW respond about ready values from an asynchronous batch 5695 * query. This function is probably called by the host thread. 5696 * 5697 * @param[in] sh 5698 * The pointer to the shared IB device context. 5699 * @param[in] async_id 5700 * The Devx async ID. 5701 * @param[in] status 5702 * The status of the completion. 5703 */ 5704 void 5705 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 5706 uint64_t async_id, int status) 5707 { 5708 struct mlx5_flow_counter_pool *pool = 5709 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 5710 struct mlx5_counter_stats_raw *raw_to_free; 5711 5712 if (unlikely(status)) { 5713 rte_atomic64_sub(&pool->start_query_gen, 1); 5714 raw_to_free = pool->raw_hw; 5715 } else { 5716 raw_to_free = pool->raw; 5717 rte_spinlock_lock(&pool->sl); 5718 pool->raw = pool->raw_hw; 5719 rte_spinlock_unlock(&pool->sl); 5720 MLX5_ASSERT(rte_atomic64_read(&pool->end_query_gen) + 1 == 5721 rte_atomic64_read(&pool->start_query_gen)); 5722 rte_atomic64_set(&pool->end_query_gen, 5723 rte_atomic64_read(&pool->start_query_gen)); 5724 /* Be sure the new raw counters data is updated in memory. */ 5725 rte_cio_wmb(); 5726 } 5727 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 5728 pool->raw_hw = NULL; 5729 sh->cmng.pending_queries--; 5730 } 5731 5732 /** 5733 * Translate the rte_flow group index to HW table value. 5734 * 5735 * @param[in] attributes 5736 * Pointer to flow attributes 5737 * @param[in] external 5738 * Value is part of flow rule created by request external to PMD. 5739 * @param[in] group 5740 * rte_flow group index value. 5741 * @param[out] fdb_def_rule 5742 * Whether fdb jump to table 1 is configured. 5743 * @param[out] table 5744 * HW table value. 5745 * @param[out] error 5746 * Pointer to error structure. 5747 * 5748 * @return 5749 * 0 on success, a negative errno value otherwise and rte_errno is set. 5750 */ 5751 int 5752 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 5753 uint32_t group, bool fdb_def_rule, uint32_t *table, 5754 struct rte_flow_error *error) 5755 { 5756 if (attributes->transfer && external && fdb_def_rule) { 5757 if (group == UINT32_MAX) 5758 return rte_flow_error_set 5759 (error, EINVAL, 5760 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 5761 NULL, 5762 "group index not supported"); 5763 *table = group + 1; 5764 } else { 5765 *table = group; 5766 } 5767 return 0; 5768 } 5769 5770 /** 5771 * Discover availability of metadata reg_c's. 5772 * 5773 * Iteratively use test flows to check availability. 5774 * 5775 * @param[in] dev 5776 * Pointer to the Ethernet device structure. 5777 * 5778 * @return 5779 * 0 on success, a negative errno value otherwise and rte_errno is set. 5780 */ 5781 int 5782 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 5783 { 5784 struct mlx5_priv *priv = dev->data->dev_private; 5785 struct mlx5_dev_config *config = &priv->config; 5786 enum modify_reg idx; 5787 int n = 0; 5788 5789 /* reg_c[0] and reg_c[1] are reserved. */ 5790 config->flow_mreg_c[n++] = REG_C_0; 5791 config->flow_mreg_c[n++] = REG_C_1; 5792 /* Discover availability of other reg_c's. */ 5793 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 5794 struct rte_flow_attr attr = { 5795 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5796 .priority = MLX5_FLOW_PRIO_RSVD, 5797 .ingress = 1, 5798 }; 5799 struct rte_flow_item items[] = { 5800 [0] = { 5801 .type = RTE_FLOW_ITEM_TYPE_END, 5802 }, 5803 }; 5804 struct rte_flow_action actions[] = { 5805 [0] = { 5806 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5807 .conf = &(struct mlx5_flow_action_copy_mreg){ 5808 .src = REG_C_1, 5809 .dst = idx, 5810 }, 5811 }, 5812 [1] = { 5813 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5814 .conf = &(struct rte_flow_action_jump){ 5815 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5816 }, 5817 }, 5818 [2] = { 5819 .type = RTE_FLOW_ACTION_TYPE_END, 5820 }, 5821 }; 5822 struct rte_flow *flow; 5823 struct rte_flow_error error; 5824 5825 if (!config->dv_flow_en) 5826 break; 5827 /* Create internal flow, validation skips copy action. */ 5828 flow = flow_list_create(dev, NULL, &attr, items, 5829 actions, false, &error); 5830 if (!flow) 5831 continue; 5832 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 5833 config->flow_mreg_c[n++] = idx; 5834 flow_list_destroy(dev, NULL, flow); 5835 } 5836 for (; n < MLX5_MREG_C_NUM; ++n) 5837 config->flow_mreg_c[n] = REG_NONE; 5838 return 0; 5839 } 5840 5841 /** 5842 * Dump flow raw hw data to file 5843 * 5844 * @param[in] dev 5845 * The pointer to Ethernet device. 5846 * @param[in] file 5847 * A pointer to a file for output. 5848 * @param[out] error 5849 * Perform verbose error reporting if not NULL. PMDs initialize this 5850 * structure in case of error only. 5851 * @return 5852 * 0 on success, a nagative value otherwise. 5853 */ 5854 int 5855 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 5856 FILE *file, 5857 struct rte_flow_error *error __rte_unused) 5858 { 5859 struct mlx5_priv *priv = dev->data->dev_private; 5860 struct mlx5_ibv_shared *sh = priv->sh; 5861 5862 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 5863 sh->tx_domain, file); 5864 } 5865