1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 12 /* Verbs header. */ 13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 14 #ifdef PEDANTIC 15 #pragma GCC diagnostic ignored "-Wpedantic" 16 #endif 17 #include <infiniband/verbs.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_common.h> 23 #include <rte_ether.h> 24 #include <rte_ethdev_driver.h> 25 #include <rte_flow.h> 26 #include <rte_flow_driver.h> 27 #include <rte_malloc.h> 28 #include <rte_ip.h> 29 30 #include "mlx5.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_flow.h" 33 #include "mlx5_glue.h" 34 #include "mlx5_prm.h" 35 #include "mlx5_rxtx.h" 36 37 /* Dev ops structure defined in mlx5.c */ 38 extern const struct eth_dev_ops mlx5_dev_ops; 39 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 40 41 /** Device flow drivers. */ 42 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 43 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 44 #endif 45 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 46 47 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 48 49 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 50 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 51 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 52 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 53 #endif 54 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 55 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 56 }; 57 58 enum mlx5_expansion { 59 MLX5_EXPANSION_ROOT, 60 MLX5_EXPANSION_ROOT_OUTER, 61 MLX5_EXPANSION_ROOT_ETH_VLAN, 62 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 63 MLX5_EXPANSION_OUTER_ETH, 64 MLX5_EXPANSION_OUTER_ETH_VLAN, 65 MLX5_EXPANSION_OUTER_VLAN, 66 MLX5_EXPANSION_OUTER_IPV4, 67 MLX5_EXPANSION_OUTER_IPV4_UDP, 68 MLX5_EXPANSION_OUTER_IPV4_TCP, 69 MLX5_EXPANSION_OUTER_IPV6, 70 MLX5_EXPANSION_OUTER_IPV6_UDP, 71 MLX5_EXPANSION_OUTER_IPV6_TCP, 72 MLX5_EXPANSION_VXLAN, 73 MLX5_EXPANSION_VXLAN_GPE, 74 MLX5_EXPANSION_GRE, 75 MLX5_EXPANSION_MPLS, 76 MLX5_EXPANSION_ETH, 77 MLX5_EXPANSION_ETH_VLAN, 78 MLX5_EXPANSION_VLAN, 79 MLX5_EXPANSION_IPV4, 80 MLX5_EXPANSION_IPV4_UDP, 81 MLX5_EXPANSION_IPV4_TCP, 82 MLX5_EXPANSION_IPV6, 83 MLX5_EXPANSION_IPV6_UDP, 84 MLX5_EXPANSION_IPV6_TCP, 85 }; 86 87 /** Supported expansion of items. */ 88 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 89 [MLX5_EXPANSION_ROOT] = { 90 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 91 MLX5_EXPANSION_IPV4, 92 MLX5_EXPANSION_IPV6), 93 .type = RTE_FLOW_ITEM_TYPE_END, 94 }, 95 [MLX5_EXPANSION_ROOT_OUTER] = { 96 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 97 MLX5_EXPANSION_OUTER_IPV4, 98 MLX5_EXPANSION_OUTER_IPV6), 99 .type = RTE_FLOW_ITEM_TYPE_END, 100 }, 101 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 102 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 103 .type = RTE_FLOW_ITEM_TYPE_END, 104 }, 105 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 107 .type = RTE_FLOW_ITEM_TYPE_END, 108 }, 109 [MLX5_EXPANSION_OUTER_ETH] = { 110 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 111 MLX5_EXPANSION_OUTER_IPV6, 112 MLX5_EXPANSION_MPLS), 113 .type = RTE_FLOW_ITEM_TYPE_ETH, 114 .rss_types = 0, 115 }, 116 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 117 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 118 .type = RTE_FLOW_ITEM_TYPE_ETH, 119 .rss_types = 0, 120 }, 121 [MLX5_EXPANSION_OUTER_VLAN] = { 122 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 123 MLX5_EXPANSION_OUTER_IPV6), 124 .type = RTE_FLOW_ITEM_TYPE_VLAN, 125 }, 126 [MLX5_EXPANSION_OUTER_IPV4] = { 127 .next = RTE_FLOW_EXPAND_RSS_NEXT 128 (MLX5_EXPANSION_OUTER_IPV4_UDP, 129 MLX5_EXPANSION_OUTER_IPV4_TCP, 130 MLX5_EXPANSION_GRE, 131 MLX5_EXPANSION_IPV4, 132 MLX5_EXPANSION_IPV6), 133 .type = RTE_FLOW_ITEM_TYPE_IPV4, 134 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 135 ETH_RSS_NONFRAG_IPV4_OTHER, 136 }, 137 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 138 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 139 MLX5_EXPANSION_VXLAN_GPE), 140 .type = RTE_FLOW_ITEM_TYPE_UDP, 141 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 142 }, 143 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 144 .type = RTE_FLOW_ITEM_TYPE_TCP, 145 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 146 }, 147 [MLX5_EXPANSION_OUTER_IPV6] = { 148 .next = RTE_FLOW_EXPAND_RSS_NEXT 149 (MLX5_EXPANSION_OUTER_IPV6_UDP, 150 MLX5_EXPANSION_OUTER_IPV6_TCP, 151 MLX5_EXPANSION_IPV4, 152 MLX5_EXPANSION_IPV6), 153 .type = RTE_FLOW_ITEM_TYPE_IPV6, 154 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 155 ETH_RSS_NONFRAG_IPV6_OTHER, 156 }, 157 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 158 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 159 MLX5_EXPANSION_VXLAN_GPE), 160 .type = RTE_FLOW_ITEM_TYPE_UDP, 161 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 162 }, 163 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 164 .type = RTE_FLOW_ITEM_TYPE_TCP, 165 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 166 }, 167 [MLX5_EXPANSION_VXLAN] = { 168 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 169 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 170 }, 171 [MLX5_EXPANSION_VXLAN_GPE] = { 172 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 173 MLX5_EXPANSION_IPV4, 174 MLX5_EXPANSION_IPV6), 175 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 176 }, 177 [MLX5_EXPANSION_GRE] = { 178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 179 .type = RTE_FLOW_ITEM_TYPE_GRE, 180 }, 181 [MLX5_EXPANSION_MPLS] = { 182 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 183 MLX5_EXPANSION_IPV6), 184 .type = RTE_FLOW_ITEM_TYPE_MPLS, 185 }, 186 [MLX5_EXPANSION_ETH] = { 187 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 188 MLX5_EXPANSION_IPV6), 189 .type = RTE_FLOW_ITEM_TYPE_ETH, 190 }, 191 [MLX5_EXPANSION_ETH_VLAN] = { 192 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 193 .type = RTE_FLOW_ITEM_TYPE_ETH, 194 }, 195 [MLX5_EXPANSION_VLAN] = { 196 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 197 MLX5_EXPANSION_IPV6), 198 .type = RTE_FLOW_ITEM_TYPE_VLAN, 199 }, 200 [MLX5_EXPANSION_IPV4] = { 201 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 202 MLX5_EXPANSION_IPV4_TCP), 203 .type = RTE_FLOW_ITEM_TYPE_IPV4, 204 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 205 ETH_RSS_NONFRAG_IPV4_OTHER, 206 }, 207 [MLX5_EXPANSION_IPV4_UDP] = { 208 .type = RTE_FLOW_ITEM_TYPE_UDP, 209 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 210 }, 211 [MLX5_EXPANSION_IPV4_TCP] = { 212 .type = RTE_FLOW_ITEM_TYPE_TCP, 213 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 214 }, 215 [MLX5_EXPANSION_IPV6] = { 216 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 217 MLX5_EXPANSION_IPV6_TCP), 218 .type = RTE_FLOW_ITEM_TYPE_IPV6, 219 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 220 ETH_RSS_NONFRAG_IPV6_OTHER, 221 }, 222 [MLX5_EXPANSION_IPV6_UDP] = { 223 .type = RTE_FLOW_ITEM_TYPE_UDP, 224 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 225 }, 226 [MLX5_EXPANSION_IPV6_TCP] = { 227 .type = RTE_FLOW_ITEM_TYPE_TCP, 228 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 229 }, 230 }; 231 232 static const struct rte_flow_ops mlx5_flow_ops = { 233 .validate = mlx5_flow_validate, 234 .create = mlx5_flow_create, 235 .destroy = mlx5_flow_destroy, 236 .flush = mlx5_flow_flush, 237 .isolate = mlx5_flow_isolate, 238 .query = mlx5_flow_query, 239 }; 240 241 /* Convert FDIR request to Generic flow. */ 242 struct mlx5_fdir { 243 struct rte_flow_attr attr; 244 struct rte_flow_item items[4]; 245 struct rte_flow_item_eth l2; 246 struct rte_flow_item_eth l2_mask; 247 union { 248 struct rte_flow_item_ipv4 ipv4; 249 struct rte_flow_item_ipv6 ipv6; 250 } l3; 251 union { 252 struct rte_flow_item_ipv4 ipv4; 253 struct rte_flow_item_ipv6 ipv6; 254 } l3_mask; 255 union { 256 struct rte_flow_item_udp udp; 257 struct rte_flow_item_tcp tcp; 258 } l4; 259 union { 260 struct rte_flow_item_udp udp; 261 struct rte_flow_item_tcp tcp; 262 } l4_mask; 263 struct rte_flow_action actions[2]; 264 struct rte_flow_action_queue queue; 265 }; 266 267 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 268 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 269 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 270 }; 271 272 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 273 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 274 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 275 { 9, 10, 11 }, { 12, 13, 14 }, 276 }; 277 278 /* Tunnel information. */ 279 struct mlx5_flow_tunnel_info { 280 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 281 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 282 }; 283 284 static struct mlx5_flow_tunnel_info tunnels_info[] = { 285 { 286 .tunnel = MLX5_FLOW_LAYER_VXLAN, 287 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 288 }, 289 { 290 .tunnel = MLX5_FLOW_LAYER_GENEVE, 291 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 292 }, 293 { 294 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 295 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 296 }, 297 { 298 .tunnel = MLX5_FLOW_LAYER_GRE, 299 .ptype = RTE_PTYPE_TUNNEL_GRE, 300 }, 301 { 302 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 303 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 304 }, 305 { 306 .tunnel = MLX5_FLOW_LAYER_MPLS, 307 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 308 }, 309 { 310 .tunnel = MLX5_FLOW_LAYER_NVGRE, 311 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 312 }, 313 { 314 .tunnel = MLX5_FLOW_LAYER_IPIP, 315 .ptype = RTE_PTYPE_TUNNEL_IP, 316 }, 317 { 318 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 319 .ptype = RTE_PTYPE_TUNNEL_IP, 320 }, 321 }; 322 323 /** 324 * Translate tag ID to register. 325 * 326 * @param[in] dev 327 * Pointer to the Ethernet device structure. 328 * @param[in] feature 329 * The feature that request the register. 330 * @param[in] id 331 * The request register ID. 332 * @param[out] error 333 * Error description in case of any. 334 * 335 * @return 336 * The request register on success, a negative errno 337 * value otherwise and rte_errno is set. 338 */ 339 enum modify_reg 340 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 341 enum mlx5_feature_name feature, 342 uint32_t id, 343 struct rte_flow_error *error) 344 { 345 struct mlx5_priv *priv = dev->data->dev_private; 346 struct mlx5_dev_config *config = &priv->config; 347 enum modify_reg start_reg; 348 349 switch (feature) { 350 case MLX5_HAIRPIN_RX: 351 return REG_B; 352 case MLX5_HAIRPIN_TX: 353 return REG_A; 354 case MLX5_METADATA_RX: 355 switch (config->dv_xmeta_en) { 356 case MLX5_XMETA_MODE_LEGACY: 357 return REG_B; 358 case MLX5_XMETA_MODE_META16: 359 return REG_C_0; 360 case MLX5_XMETA_MODE_META32: 361 return REG_C_1; 362 } 363 break; 364 case MLX5_METADATA_TX: 365 return REG_A; 366 case MLX5_METADATA_FDB: 367 return REG_C_0; 368 case MLX5_FLOW_MARK: 369 switch (config->dv_xmeta_en) { 370 case MLX5_XMETA_MODE_LEGACY: 371 return REG_NONE; 372 case MLX5_XMETA_MODE_META16: 373 return REG_C_1; 374 case MLX5_XMETA_MODE_META32: 375 return REG_C_0; 376 } 377 break; 378 case MLX5_COPY_MARK: 379 case MLX5_MTR_SFX: 380 /* 381 * Metadata COPY_MARK register using is in meter suffix sub 382 * flow while with meter. It's safe to share the same register. 383 */ 384 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 385 case MLX5_MTR_COLOR: 386 RTE_ASSERT(priv->mtr_color_reg != REG_NONE); 387 return priv->mtr_color_reg; 388 case MLX5_APP_TAG: 389 /* 390 * If meter is enable, it will engage two registers for color 391 * match and flow match. If meter color match is not using the 392 * REG_C_2, need to skip the REG_C_x be used by meter color 393 * match. 394 * If meter is disable, free to use all available registers. 395 */ 396 if (priv->mtr_color_reg != REG_NONE) 397 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_3 : 398 REG_C_4; 399 else 400 start_reg = REG_C_2; 401 if (id > (REG_C_7 - start_reg)) 402 return rte_flow_error_set(error, EINVAL, 403 RTE_FLOW_ERROR_TYPE_ITEM, 404 NULL, "invalid tag id"); 405 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 406 return rte_flow_error_set(error, ENOTSUP, 407 RTE_FLOW_ERROR_TYPE_ITEM, 408 NULL, "unsupported tag id"); 409 /* 410 * This case means meter is using the REG_C_x great than 2. 411 * Take care not to conflict with meter color REG_C_x. 412 * If the available index REG_C_y >= REG_C_x, skip the 413 * color register. 414 */ 415 if (start_reg == REG_C_3 && config->flow_mreg_c 416 [id + REG_C_3 - REG_C_0] >= priv->mtr_color_reg) { 417 if (config->flow_mreg_c[id + 1 + REG_C_3 - REG_C_0] != 418 REG_NONE) 419 return config->flow_mreg_c 420 [id + 1 + REG_C_3 - REG_C_0]; 421 return rte_flow_error_set(error, ENOTSUP, 422 RTE_FLOW_ERROR_TYPE_ITEM, 423 NULL, "unsupported tag id"); 424 } 425 return config->flow_mreg_c[id + start_reg - REG_C_0]; 426 } 427 assert(false); 428 return rte_flow_error_set(error, EINVAL, 429 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 430 NULL, "invalid feature name"); 431 } 432 433 /** 434 * Check extensive flow metadata register support. 435 * 436 * @param dev 437 * Pointer to rte_eth_dev structure. 438 * 439 * @return 440 * True if device supports extensive flow metadata register, otherwise false. 441 */ 442 bool 443 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 444 { 445 struct mlx5_priv *priv = dev->data->dev_private; 446 struct mlx5_dev_config *config = &priv->config; 447 448 /* 449 * Having available reg_c can be regarded inclusively as supporting 450 * extensive flow metadata register, which could mean, 451 * - metadata register copy action by modify header. 452 * - 16 modify header actions is supported. 453 * - reg_c's are preserved across different domain (FDB and NIC) on 454 * packet loopback by flow lookup miss. 455 */ 456 return config->flow_mreg_c[2] != REG_NONE; 457 } 458 459 /** 460 * Discover the maximum number of priority available. 461 * 462 * @param[in] dev 463 * Pointer to the Ethernet device structure. 464 * 465 * @return 466 * number of supported flow priority on success, a negative errno 467 * value otherwise and rte_errno is set. 468 */ 469 int 470 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 471 { 472 struct mlx5_priv *priv = dev->data->dev_private; 473 struct { 474 struct ibv_flow_attr attr; 475 struct ibv_flow_spec_eth eth; 476 struct ibv_flow_spec_action_drop drop; 477 } flow_attr = { 478 .attr = { 479 .num_of_specs = 2, 480 .port = (uint8_t)priv->ibv_port, 481 }, 482 .eth = { 483 .type = IBV_FLOW_SPEC_ETH, 484 .size = sizeof(struct ibv_flow_spec_eth), 485 }, 486 .drop = { 487 .size = sizeof(struct ibv_flow_spec_action_drop), 488 .type = IBV_FLOW_SPEC_ACTION_DROP, 489 }, 490 }; 491 struct ibv_flow *flow; 492 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 493 uint16_t vprio[] = { 8, 16 }; 494 int i; 495 int priority = 0; 496 497 if (!drop) { 498 rte_errno = ENOTSUP; 499 return -rte_errno; 500 } 501 for (i = 0; i != RTE_DIM(vprio); i++) { 502 flow_attr.attr.priority = vprio[i] - 1; 503 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 504 if (!flow) 505 break; 506 claim_zero(mlx5_glue->destroy_flow(flow)); 507 priority = vprio[i]; 508 } 509 mlx5_hrxq_drop_release(dev); 510 switch (priority) { 511 case 8: 512 priority = RTE_DIM(priority_map_3); 513 break; 514 case 16: 515 priority = RTE_DIM(priority_map_5); 516 break; 517 default: 518 rte_errno = ENOTSUP; 519 DRV_LOG(ERR, 520 "port %u verbs maximum priority: %d expected 8/16", 521 dev->data->port_id, priority); 522 return -rte_errno; 523 } 524 DRV_LOG(INFO, "port %u flow maximum priority: %d", 525 dev->data->port_id, priority); 526 return priority; 527 } 528 529 /** 530 * Adjust flow priority based on the highest layer and the request priority. 531 * 532 * @param[in] dev 533 * Pointer to the Ethernet device structure. 534 * @param[in] priority 535 * The rule base priority. 536 * @param[in] subpriority 537 * The priority based on the items. 538 * 539 * @return 540 * The new priority. 541 */ 542 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 543 uint32_t subpriority) 544 { 545 uint32_t res = 0; 546 struct mlx5_priv *priv = dev->data->dev_private; 547 548 switch (priv->config.flow_prio) { 549 case RTE_DIM(priority_map_3): 550 res = priority_map_3[priority][subpriority]; 551 break; 552 case RTE_DIM(priority_map_5): 553 res = priority_map_5[priority][subpriority]; 554 break; 555 } 556 return res; 557 } 558 559 /** 560 * Verify the @p item specifications (spec, last, mask) are compatible with the 561 * NIC capabilities. 562 * 563 * @param[in] item 564 * Item specification. 565 * @param[in] mask 566 * @p item->mask or flow default bit-masks. 567 * @param[in] nic_mask 568 * Bit-masks covering supported fields by the NIC to compare with user mask. 569 * @param[in] size 570 * Bit-masks size in bytes. 571 * @param[out] error 572 * Pointer to error structure. 573 * 574 * @return 575 * 0 on success, a negative errno value otherwise and rte_errno is set. 576 */ 577 int 578 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 579 const uint8_t *mask, 580 const uint8_t *nic_mask, 581 unsigned int size, 582 struct rte_flow_error *error) 583 { 584 unsigned int i; 585 586 assert(nic_mask); 587 for (i = 0; i < size; ++i) 588 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 589 return rte_flow_error_set(error, ENOTSUP, 590 RTE_FLOW_ERROR_TYPE_ITEM, 591 item, 592 "mask enables non supported" 593 " bits"); 594 if (!item->spec && (item->mask || item->last)) 595 return rte_flow_error_set(error, EINVAL, 596 RTE_FLOW_ERROR_TYPE_ITEM, item, 597 "mask/last without a spec is not" 598 " supported"); 599 if (item->spec && item->last) { 600 uint8_t spec[size]; 601 uint8_t last[size]; 602 unsigned int i; 603 int ret; 604 605 for (i = 0; i < size; ++i) { 606 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 607 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 608 } 609 ret = memcmp(spec, last, size); 610 if (ret != 0) 611 return rte_flow_error_set(error, EINVAL, 612 RTE_FLOW_ERROR_TYPE_ITEM, 613 item, 614 "range is not valid"); 615 } 616 return 0; 617 } 618 619 /** 620 * Adjust the hash fields according to the @p flow information. 621 * 622 * @param[in] dev_flow. 623 * Pointer to the mlx5_flow. 624 * @param[in] tunnel 625 * 1 when the hash field is for a tunnel item. 626 * @param[in] layer_types 627 * ETH_RSS_* types. 628 * @param[in] hash_fields 629 * Item hash fields. 630 * 631 * @return 632 * The hash fields that should be used. 633 */ 634 uint64_t 635 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, 636 int tunnel __rte_unused, uint64_t layer_types, 637 uint64_t hash_fields) 638 { 639 struct rte_flow *flow = dev_flow->flow; 640 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 641 int rss_request_inner = flow->rss.level >= 2; 642 643 /* Check RSS hash level for tunnel. */ 644 if (tunnel && rss_request_inner) 645 hash_fields |= IBV_RX_HASH_INNER; 646 else if (tunnel || rss_request_inner) 647 return 0; 648 #endif 649 /* Check if requested layer matches RSS hash fields. */ 650 if (!(flow->rss.types & layer_types)) 651 return 0; 652 return hash_fields; 653 } 654 655 /** 656 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 657 * if several tunnel rules are used on this queue, the tunnel ptype will be 658 * cleared. 659 * 660 * @param rxq_ctrl 661 * Rx queue to update. 662 */ 663 static void 664 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 665 { 666 unsigned int i; 667 uint32_t tunnel_ptype = 0; 668 669 /* Look up for the ptype to use. */ 670 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 671 if (!rxq_ctrl->flow_tunnels_n[i]) 672 continue; 673 if (!tunnel_ptype) { 674 tunnel_ptype = tunnels_info[i].ptype; 675 } else { 676 tunnel_ptype = 0; 677 break; 678 } 679 } 680 rxq_ctrl->rxq.tunnel = tunnel_ptype; 681 } 682 683 /** 684 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 685 * flow. 686 * 687 * @param[in] dev 688 * Pointer to the Ethernet device structure. 689 * @param[in] dev_flow 690 * Pointer to device flow structure. 691 */ 692 static void 693 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 694 { 695 struct mlx5_priv *priv = dev->data->dev_private; 696 struct rte_flow *flow = dev_flow->flow; 697 const int mark = !!(dev_flow->actions & 698 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 699 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 700 unsigned int i; 701 702 for (i = 0; i != flow->rss.queue_num; ++i) { 703 int idx = (*flow->rss.queue)[i]; 704 struct mlx5_rxq_ctrl *rxq_ctrl = 705 container_of((*priv->rxqs)[idx], 706 struct mlx5_rxq_ctrl, rxq); 707 708 /* 709 * To support metadata register copy on Tx loopback, 710 * this must be always enabled (metadata may arive 711 * from other port - not from local flows only. 712 */ 713 if (priv->config.dv_flow_en && 714 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 715 mlx5_flow_ext_mreg_supported(dev)) { 716 rxq_ctrl->rxq.mark = 1; 717 rxq_ctrl->flow_mark_n = 1; 718 } else if (mark) { 719 rxq_ctrl->rxq.mark = 1; 720 rxq_ctrl->flow_mark_n++; 721 } 722 if (tunnel) { 723 unsigned int j; 724 725 /* Increase the counter matching the flow. */ 726 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 727 if ((tunnels_info[j].tunnel & 728 dev_flow->layers) == 729 tunnels_info[j].tunnel) { 730 rxq_ctrl->flow_tunnels_n[j]++; 731 break; 732 } 733 } 734 flow_rxq_tunnel_ptype_update(rxq_ctrl); 735 } 736 } 737 } 738 739 /** 740 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 741 * 742 * @param[in] dev 743 * Pointer to the Ethernet device structure. 744 * @param[in] flow 745 * Pointer to flow structure. 746 */ 747 static void 748 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 749 { 750 struct mlx5_flow *dev_flow; 751 752 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 753 flow_drv_rxq_flags_set(dev, dev_flow); 754 } 755 756 /** 757 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 758 * device flow if no other flow uses it with the same kind of request. 759 * 760 * @param dev 761 * Pointer to Ethernet device. 762 * @param[in] dev_flow 763 * Pointer to the device flow. 764 */ 765 static void 766 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 767 { 768 struct mlx5_priv *priv = dev->data->dev_private; 769 struct rte_flow *flow = dev_flow->flow; 770 const int mark = !!(dev_flow->actions & 771 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 772 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 773 unsigned int i; 774 775 assert(dev->data->dev_started); 776 for (i = 0; i != flow->rss.queue_num; ++i) { 777 int idx = (*flow->rss.queue)[i]; 778 struct mlx5_rxq_ctrl *rxq_ctrl = 779 container_of((*priv->rxqs)[idx], 780 struct mlx5_rxq_ctrl, rxq); 781 782 if (priv->config.dv_flow_en && 783 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 784 mlx5_flow_ext_mreg_supported(dev)) { 785 rxq_ctrl->rxq.mark = 1; 786 rxq_ctrl->flow_mark_n = 1; 787 } else if (mark) { 788 rxq_ctrl->flow_mark_n--; 789 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 790 } 791 if (tunnel) { 792 unsigned int j; 793 794 /* Decrease the counter matching the flow. */ 795 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 796 if ((tunnels_info[j].tunnel & 797 dev_flow->layers) == 798 tunnels_info[j].tunnel) { 799 rxq_ctrl->flow_tunnels_n[j]--; 800 break; 801 } 802 } 803 flow_rxq_tunnel_ptype_update(rxq_ctrl); 804 } 805 } 806 } 807 808 /** 809 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 810 * @p flow if no other flow uses it with the same kind of request. 811 * 812 * @param dev 813 * Pointer to Ethernet device. 814 * @param[in] flow 815 * Pointer to the flow. 816 */ 817 static void 818 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 819 { 820 struct mlx5_flow *dev_flow; 821 822 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 823 flow_drv_rxq_flags_trim(dev, dev_flow); 824 } 825 826 /** 827 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 828 * 829 * @param dev 830 * Pointer to Ethernet device. 831 */ 832 static void 833 flow_rxq_flags_clear(struct rte_eth_dev *dev) 834 { 835 struct mlx5_priv *priv = dev->data->dev_private; 836 unsigned int i; 837 838 for (i = 0; i != priv->rxqs_n; ++i) { 839 struct mlx5_rxq_ctrl *rxq_ctrl; 840 unsigned int j; 841 842 if (!(*priv->rxqs)[i]) 843 continue; 844 rxq_ctrl = container_of((*priv->rxqs)[i], 845 struct mlx5_rxq_ctrl, rxq); 846 rxq_ctrl->flow_mark_n = 0; 847 rxq_ctrl->rxq.mark = 0; 848 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 849 rxq_ctrl->flow_tunnels_n[j] = 0; 850 rxq_ctrl->rxq.tunnel = 0; 851 } 852 } 853 854 /* 855 * return a pointer to the desired action in the list of actions. 856 * 857 * @param[in] actions 858 * The list of actions to search the action in. 859 * @param[in] action 860 * The action to find. 861 * 862 * @return 863 * Pointer to the action in the list, if found. NULL otherwise. 864 */ 865 const struct rte_flow_action * 866 mlx5_flow_find_action(const struct rte_flow_action *actions, 867 enum rte_flow_action_type action) 868 { 869 if (actions == NULL) 870 return NULL; 871 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 872 if (actions->type == action) 873 return actions; 874 return NULL; 875 } 876 877 /* 878 * Validate the flag action. 879 * 880 * @param[in] action_flags 881 * Bit-fields that holds the actions detected until now. 882 * @param[in] attr 883 * Attributes of flow that includes this action. 884 * @param[out] error 885 * Pointer to error structure. 886 * 887 * @return 888 * 0 on success, a negative errno value otherwise and rte_errno is set. 889 */ 890 int 891 mlx5_flow_validate_action_flag(uint64_t action_flags, 892 const struct rte_flow_attr *attr, 893 struct rte_flow_error *error) 894 { 895 896 if (action_flags & MLX5_FLOW_ACTION_DROP) 897 return rte_flow_error_set(error, EINVAL, 898 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 899 "can't drop and flag in same flow"); 900 if (action_flags & MLX5_FLOW_ACTION_MARK) 901 return rte_flow_error_set(error, EINVAL, 902 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 903 "can't mark and flag in same flow"); 904 if (action_flags & MLX5_FLOW_ACTION_FLAG) 905 return rte_flow_error_set(error, EINVAL, 906 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 907 "can't have 2 flag" 908 " actions in same flow"); 909 if (attr->egress) 910 return rte_flow_error_set(error, ENOTSUP, 911 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 912 "flag action not supported for " 913 "egress"); 914 return 0; 915 } 916 917 /* 918 * Validate the mark action. 919 * 920 * @param[in] action 921 * Pointer to the queue action. 922 * @param[in] action_flags 923 * Bit-fields that holds the actions detected until now. 924 * @param[in] attr 925 * Attributes of flow that includes this action. 926 * @param[out] error 927 * Pointer to error structure. 928 * 929 * @return 930 * 0 on success, a negative errno value otherwise and rte_errno is set. 931 */ 932 int 933 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 934 uint64_t action_flags, 935 const struct rte_flow_attr *attr, 936 struct rte_flow_error *error) 937 { 938 const struct rte_flow_action_mark *mark = action->conf; 939 940 if (!mark) 941 return rte_flow_error_set(error, EINVAL, 942 RTE_FLOW_ERROR_TYPE_ACTION, 943 action, 944 "configuration cannot be null"); 945 if (mark->id >= MLX5_FLOW_MARK_MAX) 946 return rte_flow_error_set(error, EINVAL, 947 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 948 &mark->id, 949 "mark id must in 0 <= id < " 950 RTE_STR(MLX5_FLOW_MARK_MAX)); 951 if (action_flags & MLX5_FLOW_ACTION_DROP) 952 return rte_flow_error_set(error, EINVAL, 953 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 954 "can't drop and mark in same flow"); 955 if (action_flags & MLX5_FLOW_ACTION_FLAG) 956 return rte_flow_error_set(error, EINVAL, 957 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 958 "can't flag and mark in same flow"); 959 if (action_flags & MLX5_FLOW_ACTION_MARK) 960 return rte_flow_error_set(error, EINVAL, 961 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 962 "can't have 2 mark actions in same" 963 " flow"); 964 if (attr->egress) 965 return rte_flow_error_set(error, ENOTSUP, 966 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 967 "mark action not supported for " 968 "egress"); 969 return 0; 970 } 971 972 /* 973 * Validate the drop action. 974 * 975 * @param[in] action_flags 976 * Bit-fields that holds the actions detected until now. 977 * @param[in] attr 978 * Attributes of flow that includes this action. 979 * @param[out] error 980 * Pointer to error structure. 981 * 982 * @return 983 * 0 on success, a negative errno value otherwise and rte_errno is set. 984 */ 985 int 986 mlx5_flow_validate_action_drop(uint64_t action_flags, 987 const struct rte_flow_attr *attr, 988 struct rte_flow_error *error) 989 { 990 if (action_flags & MLX5_FLOW_ACTION_FLAG) 991 return rte_flow_error_set(error, EINVAL, 992 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 993 "can't drop and flag in same flow"); 994 if (action_flags & MLX5_FLOW_ACTION_MARK) 995 return rte_flow_error_set(error, EINVAL, 996 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 997 "can't drop and mark in same flow"); 998 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 999 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 1000 return rte_flow_error_set(error, EINVAL, 1001 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1002 "can't have 2 fate actions in" 1003 " same flow"); 1004 if (attr->egress) 1005 return rte_flow_error_set(error, ENOTSUP, 1006 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1007 "drop action not supported for " 1008 "egress"); 1009 return 0; 1010 } 1011 1012 /* 1013 * Validate the queue action. 1014 * 1015 * @param[in] action 1016 * Pointer to the queue action. 1017 * @param[in] action_flags 1018 * Bit-fields that holds the actions detected until now. 1019 * @param[in] dev 1020 * Pointer to the Ethernet device structure. 1021 * @param[in] attr 1022 * Attributes of flow that includes this action. 1023 * @param[out] error 1024 * Pointer to error structure. 1025 * 1026 * @return 1027 * 0 on success, a negative errno value otherwise and rte_errno is set. 1028 */ 1029 int 1030 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1031 uint64_t action_flags, 1032 struct rte_eth_dev *dev, 1033 const struct rte_flow_attr *attr, 1034 struct rte_flow_error *error) 1035 { 1036 struct mlx5_priv *priv = dev->data->dev_private; 1037 const struct rte_flow_action_queue *queue = action->conf; 1038 1039 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1040 return rte_flow_error_set(error, EINVAL, 1041 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1042 "can't have 2 fate actions in" 1043 " same flow"); 1044 if (!priv->rxqs_n) 1045 return rte_flow_error_set(error, EINVAL, 1046 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1047 NULL, "No Rx queues configured"); 1048 if (queue->index >= priv->rxqs_n) 1049 return rte_flow_error_set(error, EINVAL, 1050 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1051 &queue->index, 1052 "queue index out of range"); 1053 if (!(*priv->rxqs)[queue->index]) 1054 return rte_flow_error_set(error, EINVAL, 1055 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1056 &queue->index, 1057 "queue is not configured"); 1058 if (attr->egress) 1059 return rte_flow_error_set(error, ENOTSUP, 1060 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1061 "queue action not supported for " 1062 "egress"); 1063 return 0; 1064 } 1065 1066 /* 1067 * Validate the rss action. 1068 * 1069 * @param[in] action 1070 * Pointer to the queue action. 1071 * @param[in] action_flags 1072 * Bit-fields that holds the actions detected until now. 1073 * @param[in] dev 1074 * Pointer to the Ethernet device structure. 1075 * @param[in] attr 1076 * Attributes of flow that includes this action. 1077 * @param[in] item_flags 1078 * Items that were detected. 1079 * @param[out] error 1080 * Pointer to error structure. 1081 * 1082 * @return 1083 * 0 on success, a negative errno value otherwise and rte_errno is set. 1084 */ 1085 int 1086 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1087 uint64_t action_flags, 1088 struct rte_eth_dev *dev, 1089 const struct rte_flow_attr *attr, 1090 uint64_t item_flags, 1091 struct rte_flow_error *error) 1092 { 1093 struct mlx5_priv *priv = dev->data->dev_private; 1094 const struct rte_flow_action_rss *rss = action->conf; 1095 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1096 unsigned int i; 1097 1098 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1099 return rte_flow_error_set(error, EINVAL, 1100 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1101 "can't have 2 fate actions" 1102 " in same flow"); 1103 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1104 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1105 return rte_flow_error_set(error, ENOTSUP, 1106 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1107 &rss->func, 1108 "RSS hash function not supported"); 1109 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1110 if (rss->level > 2) 1111 #else 1112 if (rss->level > 1) 1113 #endif 1114 return rte_flow_error_set(error, ENOTSUP, 1115 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1116 &rss->level, 1117 "tunnel RSS is not supported"); 1118 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1119 if (rss->key_len == 0 && rss->key != NULL) 1120 return rte_flow_error_set(error, ENOTSUP, 1121 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1122 &rss->key_len, 1123 "RSS hash key length 0"); 1124 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1125 return rte_flow_error_set(error, ENOTSUP, 1126 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1127 &rss->key_len, 1128 "RSS hash key too small"); 1129 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1130 return rte_flow_error_set(error, ENOTSUP, 1131 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1132 &rss->key_len, 1133 "RSS hash key too large"); 1134 if (rss->queue_num > priv->config.ind_table_max_size) 1135 return rte_flow_error_set(error, ENOTSUP, 1136 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1137 &rss->queue_num, 1138 "number of queues too large"); 1139 if (rss->types & MLX5_RSS_HF_MASK) 1140 return rte_flow_error_set(error, ENOTSUP, 1141 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1142 &rss->types, 1143 "some RSS protocols are not" 1144 " supported"); 1145 if (!priv->rxqs_n) 1146 return rte_flow_error_set(error, EINVAL, 1147 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1148 NULL, "No Rx queues configured"); 1149 if (!rss->queue_num) 1150 return rte_flow_error_set(error, EINVAL, 1151 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1152 NULL, "No queues configured"); 1153 for (i = 0; i != rss->queue_num; ++i) { 1154 if (!(*priv->rxqs)[rss->queue[i]]) 1155 return rte_flow_error_set 1156 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1157 &rss->queue[i], "queue is not configured"); 1158 } 1159 if (attr->egress) 1160 return rte_flow_error_set(error, ENOTSUP, 1161 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1162 "rss action not supported for " 1163 "egress"); 1164 if (rss->level > 1 && !tunnel) 1165 return rte_flow_error_set(error, EINVAL, 1166 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1167 "inner RSS is not supported for " 1168 "non-tunnel flows"); 1169 return 0; 1170 } 1171 1172 /* 1173 * Validate the count action. 1174 * 1175 * @param[in] dev 1176 * Pointer to the Ethernet device structure. 1177 * @param[in] attr 1178 * Attributes of flow that includes this action. 1179 * @param[out] error 1180 * Pointer to error structure. 1181 * 1182 * @return 1183 * 0 on success, a negative errno value otherwise and rte_errno is set. 1184 */ 1185 int 1186 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1187 const struct rte_flow_attr *attr, 1188 struct rte_flow_error *error) 1189 { 1190 if (attr->egress) 1191 return rte_flow_error_set(error, ENOTSUP, 1192 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1193 "count action not supported for " 1194 "egress"); 1195 return 0; 1196 } 1197 1198 /** 1199 * Verify the @p attributes will be correctly understood by the NIC and store 1200 * them in the @p flow if everything is correct. 1201 * 1202 * @param[in] dev 1203 * Pointer to the Ethernet device structure. 1204 * @param[in] attributes 1205 * Pointer to flow attributes 1206 * @param[out] error 1207 * Pointer to error structure. 1208 * 1209 * @return 1210 * 0 on success, a negative errno value otherwise and rte_errno is set. 1211 */ 1212 int 1213 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1214 const struct rte_flow_attr *attributes, 1215 struct rte_flow_error *error) 1216 { 1217 struct mlx5_priv *priv = dev->data->dev_private; 1218 uint32_t priority_max = priv->config.flow_prio - 1; 1219 1220 if (attributes->group) 1221 return rte_flow_error_set(error, ENOTSUP, 1222 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1223 NULL, "groups is not supported"); 1224 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1225 attributes->priority >= priority_max) 1226 return rte_flow_error_set(error, ENOTSUP, 1227 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1228 NULL, "priority out of range"); 1229 if (attributes->egress) 1230 return rte_flow_error_set(error, ENOTSUP, 1231 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1232 "egress is not supported"); 1233 if (attributes->transfer && !priv->config.dv_esw_en) 1234 return rte_flow_error_set(error, ENOTSUP, 1235 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1236 NULL, "transfer is not supported"); 1237 if (!attributes->ingress) 1238 return rte_flow_error_set(error, EINVAL, 1239 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1240 NULL, 1241 "ingress attribute is mandatory"); 1242 return 0; 1243 } 1244 1245 /** 1246 * Validate ICMP6 item. 1247 * 1248 * @param[in] item 1249 * Item specification. 1250 * @param[in] item_flags 1251 * Bit-fields that holds the items detected until now. 1252 * @param[out] error 1253 * Pointer to error structure. 1254 * 1255 * @return 1256 * 0 on success, a negative errno value otherwise and rte_errno is set. 1257 */ 1258 int 1259 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1260 uint64_t item_flags, 1261 uint8_t target_protocol, 1262 struct rte_flow_error *error) 1263 { 1264 const struct rte_flow_item_icmp6 *mask = item->mask; 1265 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1266 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1267 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1268 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1269 MLX5_FLOW_LAYER_OUTER_L4; 1270 int ret; 1271 1272 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1273 return rte_flow_error_set(error, EINVAL, 1274 RTE_FLOW_ERROR_TYPE_ITEM, item, 1275 "protocol filtering not compatible" 1276 " with ICMP6 layer"); 1277 if (!(item_flags & l3m)) 1278 return rte_flow_error_set(error, EINVAL, 1279 RTE_FLOW_ERROR_TYPE_ITEM, item, 1280 "IPv6 is mandatory to filter on" 1281 " ICMP6"); 1282 if (item_flags & l4m) 1283 return rte_flow_error_set(error, EINVAL, 1284 RTE_FLOW_ERROR_TYPE_ITEM, item, 1285 "multiple L4 layers not supported"); 1286 if (!mask) 1287 mask = &rte_flow_item_icmp6_mask; 1288 ret = mlx5_flow_item_acceptable 1289 (item, (const uint8_t *)mask, 1290 (const uint8_t *)&rte_flow_item_icmp6_mask, 1291 sizeof(struct rte_flow_item_icmp6), error); 1292 if (ret < 0) 1293 return ret; 1294 return 0; 1295 } 1296 1297 /** 1298 * Validate ICMP item. 1299 * 1300 * @param[in] item 1301 * Item specification. 1302 * @param[in] item_flags 1303 * Bit-fields that holds the items detected until now. 1304 * @param[out] error 1305 * Pointer to error structure. 1306 * 1307 * @return 1308 * 0 on success, a negative errno value otherwise and rte_errno is set. 1309 */ 1310 int 1311 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1312 uint64_t item_flags, 1313 uint8_t target_protocol, 1314 struct rte_flow_error *error) 1315 { 1316 const struct rte_flow_item_icmp *mask = item->mask; 1317 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1318 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1319 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1320 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1321 MLX5_FLOW_LAYER_OUTER_L4; 1322 int ret; 1323 1324 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1325 return rte_flow_error_set(error, EINVAL, 1326 RTE_FLOW_ERROR_TYPE_ITEM, item, 1327 "protocol filtering not compatible" 1328 " with ICMP layer"); 1329 if (!(item_flags & l3m)) 1330 return rte_flow_error_set(error, EINVAL, 1331 RTE_FLOW_ERROR_TYPE_ITEM, item, 1332 "IPv4 is mandatory to filter" 1333 " on ICMP"); 1334 if (item_flags & l4m) 1335 return rte_flow_error_set(error, EINVAL, 1336 RTE_FLOW_ERROR_TYPE_ITEM, item, 1337 "multiple L4 layers not supported"); 1338 if (!mask) 1339 mask = &rte_flow_item_icmp_mask; 1340 ret = mlx5_flow_item_acceptable 1341 (item, (const uint8_t *)mask, 1342 (const uint8_t *)&rte_flow_item_icmp_mask, 1343 sizeof(struct rte_flow_item_icmp), error); 1344 if (ret < 0) 1345 return ret; 1346 return 0; 1347 } 1348 1349 /** 1350 * Validate Ethernet item. 1351 * 1352 * @param[in] item 1353 * Item specification. 1354 * @param[in] item_flags 1355 * Bit-fields that holds the items detected until now. 1356 * @param[out] error 1357 * Pointer to error structure. 1358 * 1359 * @return 1360 * 0 on success, a negative errno value otherwise and rte_errno is set. 1361 */ 1362 int 1363 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1364 uint64_t item_flags, 1365 struct rte_flow_error *error) 1366 { 1367 const struct rte_flow_item_eth *mask = item->mask; 1368 const struct rte_flow_item_eth nic_mask = { 1369 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1370 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1371 .type = RTE_BE16(0xffff), 1372 }; 1373 int ret; 1374 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1375 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1376 MLX5_FLOW_LAYER_OUTER_L2; 1377 1378 if (item_flags & ethm) 1379 return rte_flow_error_set(error, ENOTSUP, 1380 RTE_FLOW_ERROR_TYPE_ITEM, item, 1381 "multiple L2 layers not supported"); 1382 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1383 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1384 return rte_flow_error_set(error, EINVAL, 1385 RTE_FLOW_ERROR_TYPE_ITEM, item, 1386 "L2 layer should not follow " 1387 "L3 layers"); 1388 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1389 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1390 return rte_flow_error_set(error, EINVAL, 1391 RTE_FLOW_ERROR_TYPE_ITEM, item, 1392 "L2 layer should not follow VLAN"); 1393 if (!mask) 1394 mask = &rte_flow_item_eth_mask; 1395 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1396 (const uint8_t *)&nic_mask, 1397 sizeof(struct rte_flow_item_eth), 1398 error); 1399 return ret; 1400 } 1401 1402 /** 1403 * Validate VLAN item. 1404 * 1405 * @param[in] item 1406 * Item specification. 1407 * @param[in] item_flags 1408 * Bit-fields that holds the items detected until now. 1409 * @param[in] dev 1410 * Ethernet device flow is being created on. 1411 * @param[out] error 1412 * Pointer to error structure. 1413 * 1414 * @return 1415 * 0 on success, a negative errno value otherwise and rte_errno is set. 1416 */ 1417 int 1418 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1419 uint64_t item_flags, 1420 struct rte_eth_dev *dev, 1421 struct rte_flow_error *error) 1422 { 1423 const struct rte_flow_item_vlan *spec = item->spec; 1424 const struct rte_flow_item_vlan *mask = item->mask; 1425 const struct rte_flow_item_vlan nic_mask = { 1426 .tci = RTE_BE16(UINT16_MAX), 1427 .inner_type = RTE_BE16(UINT16_MAX), 1428 }; 1429 uint16_t vlan_tag = 0; 1430 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1431 int ret; 1432 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1433 MLX5_FLOW_LAYER_INNER_L4) : 1434 (MLX5_FLOW_LAYER_OUTER_L3 | 1435 MLX5_FLOW_LAYER_OUTER_L4); 1436 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1437 MLX5_FLOW_LAYER_OUTER_VLAN; 1438 1439 if (item_flags & vlanm) 1440 return rte_flow_error_set(error, EINVAL, 1441 RTE_FLOW_ERROR_TYPE_ITEM, item, 1442 "multiple VLAN layers not supported"); 1443 else if ((item_flags & l34m) != 0) 1444 return rte_flow_error_set(error, EINVAL, 1445 RTE_FLOW_ERROR_TYPE_ITEM, item, 1446 "VLAN cannot follow L3/L4 layer"); 1447 if (!mask) 1448 mask = &rte_flow_item_vlan_mask; 1449 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1450 (const uint8_t *)&nic_mask, 1451 sizeof(struct rte_flow_item_vlan), 1452 error); 1453 if (ret) 1454 return ret; 1455 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1456 struct mlx5_priv *priv = dev->data->dev_private; 1457 1458 if (priv->vmwa_context) { 1459 /* 1460 * Non-NULL context means we have a virtual machine 1461 * and SR-IOV enabled, we have to create VLAN interface 1462 * to make hypervisor to setup E-Switch vport 1463 * context correctly. We avoid creating the multiple 1464 * VLAN interfaces, so we cannot support VLAN tag mask. 1465 */ 1466 return rte_flow_error_set(error, EINVAL, 1467 RTE_FLOW_ERROR_TYPE_ITEM, 1468 item, 1469 "VLAN tag mask is not" 1470 " supported in virtual" 1471 " environment"); 1472 } 1473 } 1474 if (spec) { 1475 vlan_tag = spec->tci; 1476 vlan_tag &= mask->tci; 1477 } 1478 /* 1479 * From verbs perspective an empty VLAN is equivalent 1480 * to a packet without VLAN layer. 1481 */ 1482 if (!vlan_tag) 1483 return rte_flow_error_set(error, EINVAL, 1484 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1485 item->spec, 1486 "VLAN cannot be empty"); 1487 return 0; 1488 } 1489 1490 /** 1491 * Validate IPV4 item. 1492 * 1493 * @param[in] item 1494 * Item specification. 1495 * @param[in] item_flags 1496 * Bit-fields that holds the items detected until now. 1497 * @param[in] acc_mask 1498 * Acceptable mask, if NULL default internal default mask 1499 * will be used to check whether item fields are supported. 1500 * @param[out] error 1501 * Pointer to error structure. 1502 * 1503 * @return 1504 * 0 on success, a negative errno value otherwise and rte_errno is set. 1505 */ 1506 int 1507 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1508 uint64_t item_flags, 1509 uint64_t last_item, 1510 uint16_t ether_type, 1511 const struct rte_flow_item_ipv4 *acc_mask, 1512 struct rte_flow_error *error) 1513 { 1514 const struct rte_flow_item_ipv4 *mask = item->mask; 1515 const struct rte_flow_item_ipv4 *spec = item->spec; 1516 const struct rte_flow_item_ipv4 nic_mask = { 1517 .hdr = { 1518 .src_addr = RTE_BE32(0xffffffff), 1519 .dst_addr = RTE_BE32(0xffffffff), 1520 .type_of_service = 0xff, 1521 .next_proto_id = 0xff, 1522 }, 1523 }; 1524 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1525 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1526 MLX5_FLOW_LAYER_OUTER_L3; 1527 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1528 MLX5_FLOW_LAYER_OUTER_L4; 1529 int ret; 1530 uint8_t next_proto = 0xFF; 1531 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1532 MLX5_FLOW_LAYER_OUTER_VLAN | 1533 MLX5_FLOW_LAYER_INNER_VLAN); 1534 1535 if ((last_item & l2_vlan) && ether_type && 1536 ether_type != RTE_ETHER_TYPE_IPV4) 1537 return rte_flow_error_set(error, EINVAL, 1538 RTE_FLOW_ERROR_TYPE_ITEM, item, 1539 "IPv4 cannot follow L2/VLAN layer " 1540 "which ether type is not IPv4"); 1541 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1542 if (mask && spec) 1543 next_proto = mask->hdr.next_proto_id & 1544 spec->hdr.next_proto_id; 1545 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1546 return rte_flow_error_set(error, EINVAL, 1547 RTE_FLOW_ERROR_TYPE_ITEM, 1548 item, 1549 "multiple tunnel " 1550 "not supported"); 1551 } 1552 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1553 return rte_flow_error_set(error, EINVAL, 1554 RTE_FLOW_ERROR_TYPE_ITEM, item, 1555 "wrong tunnel type - IPv6 specified " 1556 "but IPv4 item provided"); 1557 if (item_flags & l3m) 1558 return rte_flow_error_set(error, ENOTSUP, 1559 RTE_FLOW_ERROR_TYPE_ITEM, item, 1560 "multiple L3 layers not supported"); 1561 else if (item_flags & l4m) 1562 return rte_flow_error_set(error, EINVAL, 1563 RTE_FLOW_ERROR_TYPE_ITEM, item, 1564 "L3 cannot follow an L4 layer."); 1565 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1566 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1567 return rte_flow_error_set(error, EINVAL, 1568 RTE_FLOW_ERROR_TYPE_ITEM, item, 1569 "L3 cannot follow an NVGRE layer."); 1570 if (!mask) 1571 mask = &rte_flow_item_ipv4_mask; 1572 else if (mask->hdr.next_proto_id != 0 && 1573 mask->hdr.next_proto_id != 0xff) 1574 return rte_flow_error_set(error, EINVAL, 1575 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1576 "partial mask is not supported" 1577 " for protocol"); 1578 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1579 acc_mask ? (const uint8_t *)acc_mask 1580 : (const uint8_t *)&nic_mask, 1581 sizeof(struct rte_flow_item_ipv4), 1582 error); 1583 if (ret < 0) 1584 return ret; 1585 return 0; 1586 } 1587 1588 /** 1589 * Validate IPV6 item. 1590 * 1591 * @param[in] item 1592 * Item specification. 1593 * @param[in] item_flags 1594 * Bit-fields that holds the items detected until now. 1595 * @param[in] acc_mask 1596 * Acceptable mask, if NULL default internal default mask 1597 * will be used to check whether item fields are supported. 1598 * @param[out] error 1599 * Pointer to error structure. 1600 * 1601 * @return 1602 * 0 on success, a negative errno value otherwise and rte_errno is set. 1603 */ 1604 int 1605 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1606 uint64_t item_flags, 1607 uint64_t last_item, 1608 uint16_t ether_type, 1609 const struct rte_flow_item_ipv6 *acc_mask, 1610 struct rte_flow_error *error) 1611 { 1612 const struct rte_flow_item_ipv6 *mask = item->mask; 1613 const struct rte_flow_item_ipv6 *spec = item->spec; 1614 const struct rte_flow_item_ipv6 nic_mask = { 1615 .hdr = { 1616 .src_addr = 1617 "\xff\xff\xff\xff\xff\xff\xff\xff" 1618 "\xff\xff\xff\xff\xff\xff\xff\xff", 1619 .dst_addr = 1620 "\xff\xff\xff\xff\xff\xff\xff\xff" 1621 "\xff\xff\xff\xff\xff\xff\xff\xff", 1622 .vtc_flow = RTE_BE32(0xffffffff), 1623 .proto = 0xff, 1624 .hop_limits = 0xff, 1625 }, 1626 }; 1627 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1628 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1629 MLX5_FLOW_LAYER_OUTER_L3; 1630 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1631 MLX5_FLOW_LAYER_OUTER_L4; 1632 int ret; 1633 uint8_t next_proto = 0xFF; 1634 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1635 MLX5_FLOW_LAYER_OUTER_VLAN | 1636 MLX5_FLOW_LAYER_INNER_VLAN); 1637 1638 if ((last_item & l2_vlan) && ether_type && 1639 ether_type != RTE_ETHER_TYPE_IPV6) 1640 return rte_flow_error_set(error, EINVAL, 1641 RTE_FLOW_ERROR_TYPE_ITEM, item, 1642 "IPv6 cannot follow L2/VLAN layer " 1643 "which ether type is not IPv6"); 1644 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1645 if (mask && spec) 1646 next_proto = mask->hdr.proto & spec->hdr.proto; 1647 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1648 return rte_flow_error_set(error, EINVAL, 1649 RTE_FLOW_ERROR_TYPE_ITEM, 1650 item, 1651 "multiple tunnel " 1652 "not supported"); 1653 } 1654 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1655 return rte_flow_error_set(error, EINVAL, 1656 RTE_FLOW_ERROR_TYPE_ITEM, item, 1657 "wrong tunnel type - IPv4 specified " 1658 "but IPv6 item provided"); 1659 if (item_flags & l3m) 1660 return rte_flow_error_set(error, ENOTSUP, 1661 RTE_FLOW_ERROR_TYPE_ITEM, item, 1662 "multiple L3 layers not supported"); 1663 else if (item_flags & l4m) 1664 return rte_flow_error_set(error, EINVAL, 1665 RTE_FLOW_ERROR_TYPE_ITEM, item, 1666 "L3 cannot follow an L4 layer."); 1667 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1668 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1669 return rte_flow_error_set(error, EINVAL, 1670 RTE_FLOW_ERROR_TYPE_ITEM, item, 1671 "L3 cannot follow an NVGRE layer."); 1672 if (!mask) 1673 mask = &rte_flow_item_ipv6_mask; 1674 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1675 acc_mask ? (const uint8_t *)acc_mask 1676 : (const uint8_t *)&nic_mask, 1677 sizeof(struct rte_flow_item_ipv6), 1678 error); 1679 if (ret < 0) 1680 return ret; 1681 return 0; 1682 } 1683 1684 /** 1685 * Validate UDP item. 1686 * 1687 * @param[in] item 1688 * Item specification. 1689 * @param[in] item_flags 1690 * Bit-fields that holds the items detected until now. 1691 * @param[in] target_protocol 1692 * The next protocol in the previous item. 1693 * @param[in] flow_mask 1694 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1695 * @param[out] error 1696 * Pointer to error structure. 1697 * 1698 * @return 1699 * 0 on success, a negative errno value otherwise and rte_errno is set. 1700 */ 1701 int 1702 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1703 uint64_t item_flags, 1704 uint8_t target_protocol, 1705 struct rte_flow_error *error) 1706 { 1707 const struct rte_flow_item_udp *mask = item->mask; 1708 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1709 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1710 MLX5_FLOW_LAYER_OUTER_L3; 1711 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1712 MLX5_FLOW_LAYER_OUTER_L4; 1713 int ret; 1714 1715 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1716 return rte_flow_error_set(error, EINVAL, 1717 RTE_FLOW_ERROR_TYPE_ITEM, item, 1718 "protocol filtering not compatible" 1719 " with UDP layer"); 1720 if (!(item_flags & l3m)) 1721 return rte_flow_error_set(error, EINVAL, 1722 RTE_FLOW_ERROR_TYPE_ITEM, item, 1723 "L3 is mandatory to filter on L4"); 1724 if (item_flags & l4m) 1725 return rte_flow_error_set(error, EINVAL, 1726 RTE_FLOW_ERROR_TYPE_ITEM, item, 1727 "multiple L4 layers not supported"); 1728 if (!mask) 1729 mask = &rte_flow_item_udp_mask; 1730 ret = mlx5_flow_item_acceptable 1731 (item, (const uint8_t *)mask, 1732 (const uint8_t *)&rte_flow_item_udp_mask, 1733 sizeof(struct rte_flow_item_udp), error); 1734 if (ret < 0) 1735 return ret; 1736 return 0; 1737 } 1738 1739 /** 1740 * Validate TCP item. 1741 * 1742 * @param[in] item 1743 * Item specification. 1744 * @param[in] item_flags 1745 * Bit-fields that holds the items detected until now. 1746 * @param[in] target_protocol 1747 * The next protocol in the previous item. 1748 * @param[out] error 1749 * Pointer to error structure. 1750 * 1751 * @return 1752 * 0 on success, a negative errno value otherwise and rte_errno is set. 1753 */ 1754 int 1755 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1756 uint64_t item_flags, 1757 uint8_t target_protocol, 1758 const struct rte_flow_item_tcp *flow_mask, 1759 struct rte_flow_error *error) 1760 { 1761 const struct rte_flow_item_tcp *mask = item->mask; 1762 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1763 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1764 MLX5_FLOW_LAYER_OUTER_L3; 1765 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1766 MLX5_FLOW_LAYER_OUTER_L4; 1767 int ret; 1768 1769 assert(flow_mask); 1770 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1771 return rte_flow_error_set(error, EINVAL, 1772 RTE_FLOW_ERROR_TYPE_ITEM, item, 1773 "protocol filtering not compatible" 1774 " with TCP layer"); 1775 if (!(item_flags & l3m)) 1776 return rte_flow_error_set(error, EINVAL, 1777 RTE_FLOW_ERROR_TYPE_ITEM, item, 1778 "L3 is mandatory to filter on L4"); 1779 if (item_flags & l4m) 1780 return rte_flow_error_set(error, EINVAL, 1781 RTE_FLOW_ERROR_TYPE_ITEM, item, 1782 "multiple L4 layers not supported"); 1783 if (!mask) 1784 mask = &rte_flow_item_tcp_mask; 1785 ret = mlx5_flow_item_acceptable 1786 (item, (const uint8_t *)mask, 1787 (const uint8_t *)flow_mask, 1788 sizeof(struct rte_flow_item_tcp), error); 1789 if (ret < 0) 1790 return ret; 1791 return 0; 1792 } 1793 1794 /** 1795 * Validate VXLAN item. 1796 * 1797 * @param[in] item 1798 * Item specification. 1799 * @param[in] item_flags 1800 * Bit-fields that holds the items detected until now. 1801 * @param[in] target_protocol 1802 * The next protocol in the previous item. 1803 * @param[out] error 1804 * Pointer to error structure. 1805 * 1806 * @return 1807 * 0 on success, a negative errno value otherwise and rte_errno is set. 1808 */ 1809 int 1810 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1811 uint64_t item_flags, 1812 struct rte_flow_error *error) 1813 { 1814 const struct rte_flow_item_vxlan *spec = item->spec; 1815 const struct rte_flow_item_vxlan *mask = item->mask; 1816 int ret; 1817 union vni { 1818 uint32_t vlan_id; 1819 uint8_t vni[4]; 1820 } id = { .vlan_id = 0, }; 1821 uint32_t vlan_id = 0; 1822 1823 1824 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1825 return rte_flow_error_set(error, ENOTSUP, 1826 RTE_FLOW_ERROR_TYPE_ITEM, item, 1827 "multiple tunnel layers not" 1828 " supported"); 1829 /* 1830 * Verify only UDPv4 is present as defined in 1831 * https://tools.ietf.org/html/rfc7348 1832 */ 1833 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1834 return rte_flow_error_set(error, EINVAL, 1835 RTE_FLOW_ERROR_TYPE_ITEM, item, 1836 "no outer UDP layer found"); 1837 if (!mask) 1838 mask = &rte_flow_item_vxlan_mask; 1839 ret = mlx5_flow_item_acceptable 1840 (item, (const uint8_t *)mask, 1841 (const uint8_t *)&rte_flow_item_vxlan_mask, 1842 sizeof(struct rte_flow_item_vxlan), 1843 error); 1844 if (ret < 0) 1845 return ret; 1846 if (spec) { 1847 memcpy(&id.vni[1], spec->vni, 3); 1848 vlan_id = id.vlan_id; 1849 memcpy(&id.vni[1], mask->vni, 3); 1850 vlan_id &= id.vlan_id; 1851 } 1852 /* 1853 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if 1854 * only this layer is defined in the Verbs specification it is 1855 * interpreted as wildcard and all packets will match this 1856 * rule, if it follows a full stack layer (ex: eth / ipv4 / 1857 * udp), all packets matching the layers before will also 1858 * match this rule. To avoid such situation, VNI 0 is 1859 * currently refused. 1860 */ 1861 if (!vlan_id) 1862 return rte_flow_error_set(error, ENOTSUP, 1863 RTE_FLOW_ERROR_TYPE_ITEM, item, 1864 "VXLAN vni cannot be 0"); 1865 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1866 return rte_flow_error_set(error, ENOTSUP, 1867 RTE_FLOW_ERROR_TYPE_ITEM, item, 1868 "VXLAN tunnel must be fully defined"); 1869 return 0; 1870 } 1871 1872 /** 1873 * Validate VXLAN_GPE item. 1874 * 1875 * @param[in] item 1876 * Item specification. 1877 * @param[in] item_flags 1878 * Bit-fields that holds the items detected until now. 1879 * @param[in] priv 1880 * Pointer to the private data structure. 1881 * @param[in] target_protocol 1882 * The next protocol in the previous item. 1883 * @param[out] error 1884 * Pointer to error structure. 1885 * 1886 * @return 1887 * 0 on success, a negative errno value otherwise and rte_errno is set. 1888 */ 1889 int 1890 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1891 uint64_t item_flags, 1892 struct rte_eth_dev *dev, 1893 struct rte_flow_error *error) 1894 { 1895 struct mlx5_priv *priv = dev->data->dev_private; 1896 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1897 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1898 int ret; 1899 union vni { 1900 uint32_t vlan_id; 1901 uint8_t vni[4]; 1902 } id = { .vlan_id = 0, }; 1903 uint32_t vlan_id = 0; 1904 1905 if (!priv->config.l3_vxlan_en) 1906 return rte_flow_error_set(error, ENOTSUP, 1907 RTE_FLOW_ERROR_TYPE_ITEM, item, 1908 "L3 VXLAN is not enabled by device" 1909 " parameter and/or not configured in" 1910 " firmware"); 1911 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1912 return rte_flow_error_set(error, ENOTSUP, 1913 RTE_FLOW_ERROR_TYPE_ITEM, item, 1914 "multiple tunnel layers not" 1915 " supported"); 1916 /* 1917 * Verify only UDPv4 is present as defined in 1918 * https://tools.ietf.org/html/rfc7348 1919 */ 1920 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1921 return rte_flow_error_set(error, EINVAL, 1922 RTE_FLOW_ERROR_TYPE_ITEM, item, 1923 "no outer UDP layer found"); 1924 if (!mask) 1925 mask = &rte_flow_item_vxlan_gpe_mask; 1926 ret = mlx5_flow_item_acceptable 1927 (item, (const uint8_t *)mask, 1928 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1929 sizeof(struct rte_flow_item_vxlan_gpe), 1930 error); 1931 if (ret < 0) 1932 return ret; 1933 if (spec) { 1934 if (spec->protocol) 1935 return rte_flow_error_set(error, ENOTSUP, 1936 RTE_FLOW_ERROR_TYPE_ITEM, 1937 item, 1938 "VxLAN-GPE protocol" 1939 " not supported"); 1940 memcpy(&id.vni[1], spec->vni, 3); 1941 vlan_id = id.vlan_id; 1942 memcpy(&id.vni[1], mask->vni, 3); 1943 vlan_id &= id.vlan_id; 1944 } 1945 /* 1946 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1947 * layer is defined in the Verbs specification it is interpreted as 1948 * wildcard and all packets will match this rule, if it follows a full 1949 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1950 * before will also match this rule. To avoid such situation, VNI 0 1951 * is currently refused. 1952 */ 1953 if (!vlan_id) 1954 return rte_flow_error_set(error, ENOTSUP, 1955 RTE_FLOW_ERROR_TYPE_ITEM, item, 1956 "VXLAN-GPE vni cannot be 0"); 1957 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1958 return rte_flow_error_set(error, ENOTSUP, 1959 RTE_FLOW_ERROR_TYPE_ITEM, item, 1960 "VXLAN-GPE tunnel must be fully" 1961 " defined"); 1962 return 0; 1963 } 1964 /** 1965 * Validate GRE Key item. 1966 * 1967 * @param[in] item 1968 * Item specification. 1969 * @param[in] item_flags 1970 * Bit flags to mark detected items. 1971 * @param[in] gre_item 1972 * Pointer to gre_item 1973 * @param[out] error 1974 * Pointer to error structure. 1975 * 1976 * @return 1977 * 0 on success, a negative errno value otherwise and rte_errno is set. 1978 */ 1979 int 1980 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 1981 uint64_t item_flags, 1982 const struct rte_flow_item *gre_item, 1983 struct rte_flow_error *error) 1984 { 1985 const rte_be32_t *mask = item->mask; 1986 int ret = 0; 1987 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 1988 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 1989 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 1990 1991 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 1992 return rte_flow_error_set(error, ENOTSUP, 1993 RTE_FLOW_ERROR_TYPE_ITEM, item, 1994 "Multiple GRE key not support"); 1995 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 1996 return rte_flow_error_set(error, ENOTSUP, 1997 RTE_FLOW_ERROR_TYPE_ITEM, item, 1998 "No preceding GRE header"); 1999 if (item_flags & MLX5_FLOW_LAYER_INNER) 2000 return rte_flow_error_set(error, ENOTSUP, 2001 RTE_FLOW_ERROR_TYPE_ITEM, item, 2002 "GRE key following a wrong item"); 2003 if (!gre_mask) 2004 gre_mask = &rte_flow_item_gre_mask; 2005 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2006 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2007 return rte_flow_error_set(error, EINVAL, 2008 RTE_FLOW_ERROR_TYPE_ITEM, item, 2009 "Key bit must be on"); 2010 2011 if (!mask) 2012 mask = &gre_key_default_mask; 2013 ret = mlx5_flow_item_acceptable 2014 (item, (const uint8_t *)mask, 2015 (const uint8_t *)&gre_key_default_mask, 2016 sizeof(rte_be32_t), error); 2017 return ret; 2018 } 2019 2020 /** 2021 * Validate GRE item. 2022 * 2023 * @param[in] item 2024 * Item specification. 2025 * @param[in] item_flags 2026 * Bit flags to mark detected items. 2027 * @param[in] target_protocol 2028 * The next protocol in the previous item. 2029 * @param[out] error 2030 * Pointer to error structure. 2031 * 2032 * @return 2033 * 0 on success, a negative errno value otherwise and rte_errno is set. 2034 */ 2035 int 2036 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2037 uint64_t item_flags, 2038 uint8_t target_protocol, 2039 struct rte_flow_error *error) 2040 { 2041 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2042 const struct rte_flow_item_gre *mask = item->mask; 2043 int ret; 2044 const struct rte_flow_item_gre nic_mask = { 2045 .c_rsvd0_ver = RTE_BE16(0xB000), 2046 .protocol = RTE_BE16(UINT16_MAX), 2047 }; 2048 2049 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2050 return rte_flow_error_set(error, EINVAL, 2051 RTE_FLOW_ERROR_TYPE_ITEM, item, 2052 "protocol filtering not compatible" 2053 " with this GRE layer"); 2054 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2055 return rte_flow_error_set(error, ENOTSUP, 2056 RTE_FLOW_ERROR_TYPE_ITEM, item, 2057 "multiple tunnel layers not" 2058 " supported"); 2059 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2060 return rte_flow_error_set(error, ENOTSUP, 2061 RTE_FLOW_ERROR_TYPE_ITEM, item, 2062 "L3 Layer is missing"); 2063 if (!mask) 2064 mask = &rte_flow_item_gre_mask; 2065 ret = mlx5_flow_item_acceptable 2066 (item, (const uint8_t *)mask, 2067 (const uint8_t *)&nic_mask, 2068 sizeof(struct rte_flow_item_gre), error); 2069 if (ret < 0) 2070 return ret; 2071 #ifndef HAVE_MLX5DV_DR 2072 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2073 if (spec && (spec->protocol & mask->protocol)) 2074 return rte_flow_error_set(error, ENOTSUP, 2075 RTE_FLOW_ERROR_TYPE_ITEM, item, 2076 "without MPLS support the" 2077 " specification cannot be used for" 2078 " filtering"); 2079 #endif 2080 #endif 2081 return 0; 2082 } 2083 2084 /** 2085 * Validate Geneve item. 2086 * 2087 * @param[in] item 2088 * Item specification. 2089 * @param[in] itemFlags 2090 * Bit-fields that holds the items detected until now. 2091 * @param[in] enPriv 2092 * Pointer to the private data structure. 2093 * @param[out] error 2094 * Pointer to error structure. 2095 * 2096 * @return 2097 * 0 on success, a negative errno value otherwise and rte_errno is set. 2098 */ 2099 2100 int 2101 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2102 uint64_t item_flags, 2103 struct rte_eth_dev *dev, 2104 struct rte_flow_error *error) 2105 { 2106 struct mlx5_priv *priv = dev->data->dev_private; 2107 const struct rte_flow_item_geneve *spec = item->spec; 2108 const struct rte_flow_item_geneve *mask = item->mask; 2109 int ret; 2110 uint16_t gbhdr; 2111 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2112 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2113 const struct rte_flow_item_geneve nic_mask = { 2114 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2115 .vni = "\xff\xff\xff", 2116 .protocol = RTE_BE16(UINT16_MAX), 2117 }; 2118 2119 if (!(priv->config.hca_attr.flex_parser_protocols & 2120 MLX5_HCA_FLEX_GENEVE_ENABLED) || 2121 !priv->config.hca_attr.tunnel_stateless_geneve_rx) 2122 return rte_flow_error_set(error, ENOTSUP, 2123 RTE_FLOW_ERROR_TYPE_ITEM, item, 2124 "L3 Geneve is not enabled by device" 2125 " parameter and/or not configured in" 2126 " firmware"); 2127 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2128 return rte_flow_error_set(error, ENOTSUP, 2129 RTE_FLOW_ERROR_TYPE_ITEM, item, 2130 "multiple tunnel layers not" 2131 " supported"); 2132 /* 2133 * Verify only UDPv4 is present as defined in 2134 * https://tools.ietf.org/html/rfc7348 2135 */ 2136 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2137 return rte_flow_error_set(error, EINVAL, 2138 RTE_FLOW_ERROR_TYPE_ITEM, item, 2139 "no outer UDP layer found"); 2140 if (!mask) 2141 mask = &rte_flow_item_geneve_mask; 2142 ret = mlx5_flow_item_acceptable 2143 (item, (const uint8_t *)mask, 2144 (const uint8_t *)&nic_mask, 2145 sizeof(struct rte_flow_item_geneve), error); 2146 if (ret) 2147 return ret; 2148 if (spec) { 2149 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2150 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2151 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2152 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2153 return rte_flow_error_set(error, ENOTSUP, 2154 RTE_FLOW_ERROR_TYPE_ITEM, 2155 item, 2156 "Geneve protocol unsupported" 2157 " fields are being used"); 2158 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2159 return rte_flow_error_set 2160 (error, ENOTSUP, 2161 RTE_FLOW_ERROR_TYPE_ITEM, 2162 item, 2163 "Unsupported Geneve options length"); 2164 } 2165 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2166 return rte_flow_error_set 2167 (error, ENOTSUP, 2168 RTE_FLOW_ERROR_TYPE_ITEM, item, 2169 "Geneve tunnel must be fully defined"); 2170 return 0; 2171 } 2172 2173 /** 2174 * Validate MPLS item. 2175 * 2176 * @param[in] dev 2177 * Pointer to the rte_eth_dev structure. 2178 * @param[in] item 2179 * Item specification. 2180 * @param[in] item_flags 2181 * Bit-fields that holds the items detected until now. 2182 * @param[in] prev_layer 2183 * The protocol layer indicated in previous item. 2184 * @param[out] error 2185 * Pointer to error structure. 2186 * 2187 * @return 2188 * 0 on success, a negative errno value otherwise and rte_errno is set. 2189 */ 2190 int 2191 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2192 const struct rte_flow_item *item __rte_unused, 2193 uint64_t item_flags __rte_unused, 2194 uint64_t prev_layer __rte_unused, 2195 struct rte_flow_error *error) 2196 { 2197 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2198 const struct rte_flow_item_mpls *mask = item->mask; 2199 struct mlx5_priv *priv = dev->data->dev_private; 2200 int ret; 2201 2202 if (!priv->config.mpls_en) 2203 return rte_flow_error_set(error, ENOTSUP, 2204 RTE_FLOW_ERROR_TYPE_ITEM, item, 2205 "MPLS not supported or" 2206 " disabled in firmware" 2207 " configuration."); 2208 /* MPLS over IP, UDP, GRE is allowed */ 2209 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2210 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2211 MLX5_FLOW_LAYER_GRE))) 2212 return rte_flow_error_set(error, EINVAL, 2213 RTE_FLOW_ERROR_TYPE_ITEM, item, 2214 "protocol filtering not compatible" 2215 " with MPLS layer"); 2216 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2217 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2218 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2219 return rte_flow_error_set(error, ENOTSUP, 2220 RTE_FLOW_ERROR_TYPE_ITEM, item, 2221 "multiple tunnel layers not" 2222 " supported"); 2223 if (!mask) 2224 mask = &rte_flow_item_mpls_mask; 2225 ret = mlx5_flow_item_acceptable 2226 (item, (const uint8_t *)mask, 2227 (const uint8_t *)&rte_flow_item_mpls_mask, 2228 sizeof(struct rte_flow_item_mpls), error); 2229 if (ret < 0) 2230 return ret; 2231 return 0; 2232 #endif 2233 return rte_flow_error_set(error, ENOTSUP, 2234 RTE_FLOW_ERROR_TYPE_ITEM, item, 2235 "MPLS is not supported by Verbs, please" 2236 " update."); 2237 } 2238 2239 /** 2240 * Validate NVGRE item. 2241 * 2242 * @param[in] item 2243 * Item specification. 2244 * @param[in] item_flags 2245 * Bit flags to mark detected items. 2246 * @param[in] target_protocol 2247 * The next protocol in the previous item. 2248 * @param[out] error 2249 * Pointer to error structure. 2250 * 2251 * @return 2252 * 0 on success, a negative errno value otherwise and rte_errno is set. 2253 */ 2254 int 2255 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2256 uint64_t item_flags, 2257 uint8_t target_protocol, 2258 struct rte_flow_error *error) 2259 { 2260 const struct rte_flow_item_nvgre *mask = item->mask; 2261 int ret; 2262 2263 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2264 return rte_flow_error_set(error, EINVAL, 2265 RTE_FLOW_ERROR_TYPE_ITEM, item, 2266 "protocol filtering not compatible" 2267 " with this GRE layer"); 2268 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2269 return rte_flow_error_set(error, ENOTSUP, 2270 RTE_FLOW_ERROR_TYPE_ITEM, item, 2271 "multiple tunnel layers not" 2272 " supported"); 2273 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2274 return rte_flow_error_set(error, ENOTSUP, 2275 RTE_FLOW_ERROR_TYPE_ITEM, item, 2276 "L3 Layer is missing"); 2277 if (!mask) 2278 mask = &rte_flow_item_nvgre_mask; 2279 ret = mlx5_flow_item_acceptable 2280 (item, (const uint8_t *)mask, 2281 (const uint8_t *)&rte_flow_item_nvgre_mask, 2282 sizeof(struct rte_flow_item_nvgre), error); 2283 if (ret < 0) 2284 return ret; 2285 return 0; 2286 } 2287 2288 /* Allocate unique ID for the split Q/RSS subflows. */ 2289 static uint32_t 2290 flow_qrss_get_id(struct rte_eth_dev *dev) 2291 { 2292 struct mlx5_priv *priv = dev->data->dev_private; 2293 uint32_t qrss_id, ret; 2294 2295 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2296 if (ret) 2297 return 0; 2298 assert(qrss_id); 2299 return qrss_id; 2300 } 2301 2302 /* Free unique ID for the split Q/RSS subflows. */ 2303 static void 2304 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2305 { 2306 struct mlx5_priv *priv = dev->data->dev_private; 2307 2308 if (qrss_id) 2309 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2310 } 2311 2312 /** 2313 * Release resource related QUEUE/RSS action split. 2314 * 2315 * @param dev 2316 * Pointer to Ethernet device. 2317 * @param flow 2318 * Flow to release id's from. 2319 */ 2320 static void 2321 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2322 struct rte_flow *flow) 2323 { 2324 struct mlx5_flow *dev_flow; 2325 2326 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 2327 if (dev_flow->qrss_id) 2328 flow_qrss_free_id(dev, dev_flow->qrss_id); 2329 } 2330 2331 static int 2332 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2333 const struct rte_flow_attr *attr __rte_unused, 2334 const struct rte_flow_item items[] __rte_unused, 2335 const struct rte_flow_action actions[] __rte_unused, 2336 bool external __rte_unused, 2337 struct rte_flow_error *error) 2338 { 2339 return rte_flow_error_set(error, ENOTSUP, 2340 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2341 } 2342 2343 static struct mlx5_flow * 2344 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, 2345 const struct rte_flow_item items[] __rte_unused, 2346 const struct rte_flow_action actions[] __rte_unused, 2347 struct rte_flow_error *error) 2348 { 2349 rte_flow_error_set(error, ENOTSUP, 2350 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2351 return NULL; 2352 } 2353 2354 static int 2355 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2356 struct mlx5_flow *dev_flow __rte_unused, 2357 const struct rte_flow_attr *attr __rte_unused, 2358 const struct rte_flow_item items[] __rte_unused, 2359 const struct rte_flow_action actions[] __rte_unused, 2360 struct rte_flow_error *error) 2361 { 2362 return rte_flow_error_set(error, ENOTSUP, 2363 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2364 } 2365 2366 static int 2367 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2368 struct rte_flow *flow __rte_unused, 2369 struct rte_flow_error *error) 2370 { 2371 return rte_flow_error_set(error, ENOTSUP, 2372 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2373 } 2374 2375 static void 2376 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2377 struct rte_flow *flow __rte_unused) 2378 { 2379 } 2380 2381 static void 2382 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2383 struct rte_flow *flow __rte_unused) 2384 { 2385 } 2386 2387 static int 2388 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2389 struct rte_flow *flow __rte_unused, 2390 const struct rte_flow_action *actions __rte_unused, 2391 void *data __rte_unused, 2392 struct rte_flow_error *error) 2393 { 2394 return rte_flow_error_set(error, ENOTSUP, 2395 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2396 } 2397 2398 /* Void driver to protect from null pointer reference. */ 2399 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2400 .validate = flow_null_validate, 2401 .prepare = flow_null_prepare, 2402 .translate = flow_null_translate, 2403 .apply = flow_null_apply, 2404 .remove = flow_null_remove, 2405 .destroy = flow_null_destroy, 2406 .query = flow_null_query, 2407 }; 2408 2409 /** 2410 * Select flow driver type according to flow attributes and device 2411 * configuration. 2412 * 2413 * @param[in] dev 2414 * Pointer to the dev structure. 2415 * @param[in] attr 2416 * Pointer to the flow attributes. 2417 * 2418 * @return 2419 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2420 */ 2421 static enum mlx5_flow_drv_type 2422 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2423 { 2424 struct mlx5_priv *priv = dev->data->dev_private; 2425 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2426 2427 if (attr->transfer && priv->config.dv_esw_en) 2428 type = MLX5_FLOW_TYPE_DV; 2429 if (!attr->transfer) 2430 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2431 MLX5_FLOW_TYPE_VERBS; 2432 return type; 2433 } 2434 2435 #define flow_get_drv_ops(type) flow_drv_ops[type] 2436 2437 /** 2438 * Flow driver validation API. This abstracts calling driver specific functions. 2439 * The type of flow driver is determined according to flow attributes. 2440 * 2441 * @param[in] dev 2442 * Pointer to the dev structure. 2443 * @param[in] attr 2444 * Pointer to the flow attributes. 2445 * @param[in] items 2446 * Pointer to the list of items. 2447 * @param[in] actions 2448 * Pointer to the list of actions. 2449 * @param[in] external 2450 * This flow rule is created by request external to PMD. 2451 * @param[out] error 2452 * Pointer to the error structure. 2453 * 2454 * @return 2455 * 0 on success, a negative errno value otherwise and rte_errno is set. 2456 */ 2457 static inline int 2458 flow_drv_validate(struct rte_eth_dev *dev, 2459 const struct rte_flow_attr *attr, 2460 const struct rte_flow_item items[], 2461 const struct rte_flow_action actions[], 2462 bool external, struct rte_flow_error *error) 2463 { 2464 const struct mlx5_flow_driver_ops *fops; 2465 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2466 2467 fops = flow_get_drv_ops(type); 2468 return fops->validate(dev, attr, items, actions, external, error); 2469 } 2470 2471 /** 2472 * Flow driver preparation API. This abstracts calling driver specific 2473 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2474 * calculates the size of memory required for device flow, allocates the memory, 2475 * initializes the device flow and returns the pointer. 2476 * 2477 * @note 2478 * This function initializes device flow structure such as dv or verbs in 2479 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2480 * rest. For example, adding returning device flow to flow->dev_flow list and 2481 * setting backward reference to the flow should be done out of this function. 2482 * layers field is not filled either. 2483 * 2484 * @param[in] attr 2485 * Pointer to the flow attributes. 2486 * @param[in] items 2487 * Pointer to the list of items. 2488 * @param[in] actions 2489 * Pointer to the list of actions. 2490 * @param[out] error 2491 * Pointer to the error structure. 2492 * 2493 * @return 2494 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2495 */ 2496 static inline struct mlx5_flow * 2497 flow_drv_prepare(const struct rte_flow *flow, 2498 const struct rte_flow_attr *attr, 2499 const struct rte_flow_item items[], 2500 const struct rte_flow_action actions[], 2501 struct rte_flow_error *error) 2502 { 2503 const struct mlx5_flow_driver_ops *fops; 2504 enum mlx5_flow_drv_type type = flow->drv_type; 2505 2506 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2507 fops = flow_get_drv_ops(type); 2508 return fops->prepare(attr, items, actions, error); 2509 } 2510 2511 /** 2512 * Flow driver translation API. This abstracts calling driver specific 2513 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2514 * translates a generic flow into a driver flow. flow_drv_prepare() must 2515 * precede. 2516 * 2517 * @note 2518 * dev_flow->layers could be filled as a result of parsing during translation 2519 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2520 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2521 * flow->actions could be overwritten even though all the expanded dev_flows 2522 * have the same actions. 2523 * 2524 * @param[in] dev 2525 * Pointer to the rte dev structure. 2526 * @param[in, out] dev_flow 2527 * Pointer to the mlx5 flow. 2528 * @param[in] attr 2529 * Pointer to the flow attributes. 2530 * @param[in] items 2531 * Pointer to the list of items. 2532 * @param[in] actions 2533 * Pointer to the list of actions. 2534 * @param[out] error 2535 * Pointer to the error structure. 2536 * 2537 * @return 2538 * 0 on success, a negative errno value otherwise and rte_errno is set. 2539 */ 2540 static inline int 2541 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2542 const struct rte_flow_attr *attr, 2543 const struct rte_flow_item items[], 2544 const struct rte_flow_action actions[], 2545 struct rte_flow_error *error) 2546 { 2547 const struct mlx5_flow_driver_ops *fops; 2548 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2549 2550 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2551 fops = flow_get_drv_ops(type); 2552 return fops->translate(dev, dev_flow, attr, items, actions, error); 2553 } 2554 2555 /** 2556 * Flow driver apply API. This abstracts calling driver specific functions. 2557 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2558 * translated driver flows on to device. flow_drv_translate() must precede. 2559 * 2560 * @param[in] dev 2561 * Pointer to Ethernet device structure. 2562 * @param[in, out] flow 2563 * Pointer to flow structure. 2564 * @param[out] error 2565 * Pointer to error structure. 2566 * 2567 * @return 2568 * 0 on success, a negative errno value otherwise and rte_errno is set. 2569 */ 2570 static inline int 2571 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2572 struct rte_flow_error *error) 2573 { 2574 const struct mlx5_flow_driver_ops *fops; 2575 enum mlx5_flow_drv_type type = flow->drv_type; 2576 2577 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2578 fops = flow_get_drv_ops(type); 2579 return fops->apply(dev, flow, error); 2580 } 2581 2582 /** 2583 * Flow driver remove API. This abstracts calling driver specific functions. 2584 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2585 * on device. All the resources of the flow should be freed by calling 2586 * flow_drv_destroy(). 2587 * 2588 * @param[in] dev 2589 * Pointer to Ethernet device. 2590 * @param[in, out] flow 2591 * Pointer to flow structure. 2592 */ 2593 static inline void 2594 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2595 { 2596 const struct mlx5_flow_driver_ops *fops; 2597 enum mlx5_flow_drv_type type = flow->drv_type; 2598 2599 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2600 fops = flow_get_drv_ops(type); 2601 fops->remove(dev, flow); 2602 } 2603 2604 /** 2605 * Flow driver destroy API. This abstracts calling driver specific functions. 2606 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2607 * on device and releases resources of the flow. 2608 * 2609 * @param[in] dev 2610 * Pointer to Ethernet device. 2611 * @param[in, out] flow 2612 * Pointer to flow structure. 2613 */ 2614 static inline void 2615 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2616 { 2617 const struct mlx5_flow_driver_ops *fops; 2618 enum mlx5_flow_drv_type type = flow->drv_type; 2619 2620 flow_mreg_split_qrss_release(dev, flow); 2621 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2622 fops = flow_get_drv_ops(type); 2623 fops->destroy(dev, flow); 2624 } 2625 2626 /** 2627 * Validate a flow supported by the NIC. 2628 * 2629 * @see rte_flow_validate() 2630 * @see rte_flow_ops 2631 */ 2632 int 2633 mlx5_flow_validate(struct rte_eth_dev *dev, 2634 const struct rte_flow_attr *attr, 2635 const struct rte_flow_item items[], 2636 const struct rte_flow_action actions[], 2637 struct rte_flow_error *error) 2638 { 2639 int ret; 2640 2641 ret = flow_drv_validate(dev, attr, items, actions, true, error); 2642 if (ret < 0) 2643 return ret; 2644 return 0; 2645 } 2646 2647 /** 2648 * Get port id item from the item list. 2649 * 2650 * @param[in] item 2651 * Pointer to the list of items. 2652 * 2653 * @return 2654 * Pointer to the port id item if exist, else return NULL. 2655 */ 2656 static const struct rte_flow_item * 2657 find_port_id_item(const struct rte_flow_item *item) 2658 { 2659 assert(item); 2660 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2661 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) 2662 return item; 2663 } 2664 return NULL; 2665 } 2666 2667 /** 2668 * Get RSS action from the action list. 2669 * 2670 * @param[in] actions 2671 * Pointer to the list of actions. 2672 * 2673 * @return 2674 * Pointer to the RSS action if exist, else return NULL. 2675 */ 2676 static const struct rte_flow_action_rss* 2677 flow_get_rss_action(const struct rte_flow_action actions[]) 2678 { 2679 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2680 switch (actions->type) { 2681 case RTE_FLOW_ACTION_TYPE_RSS: 2682 return (const struct rte_flow_action_rss *) 2683 actions->conf; 2684 default: 2685 break; 2686 } 2687 } 2688 return NULL; 2689 } 2690 2691 static unsigned int 2692 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2693 { 2694 const struct rte_flow_item *item; 2695 unsigned int has_vlan = 0; 2696 2697 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2698 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2699 has_vlan = 1; 2700 break; 2701 } 2702 } 2703 if (has_vlan) 2704 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2705 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2706 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2707 MLX5_EXPANSION_ROOT_OUTER; 2708 } 2709 2710 /** 2711 * Get QUEUE/RSS action from the action list. 2712 * 2713 * @param[in] actions 2714 * Pointer to the list of actions. 2715 * @param[out] qrss 2716 * Pointer to the return pointer. 2717 * @param[out] qrss_type 2718 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2719 * if no QUEUE/RSS is found. 2720 * 2721 * @return 2722 * Total number of actions. 2723 */ 2724 static int 2725 flow_parse_qrss_action(const struct rte_flow_action actions[], 2726 const struct rte_flow_action **qrss) 2727 { 2728 int actions_n = 0; 2729 2730 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2731 switch (actions->type) { 2732 case RTE_FLOW_ACTION_TYPE_QUEUE: 2733 case RTE_FLOW_ACTION_TYPE_RSS: 2734 *qrss = actions; 2735 break; 2736 default: 2737 break; 2738 } 2739 actions_n++; 2740 } 2741 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2742 return actions_n + 1; 2743 } 2744 2745 /** 2746 * Check meter action from the action list. 2747 * 2748 * @param[in] actions 2749 * Pointer to the list of actions. 2750 * @param[out] mtr 2751 * Pointer to the meter exist flag. 2752 * 2753 * @return 2754 * Total number of actions. 2755 */ 2756 static int 2757 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2758 { 2759 int actions_n = 0; 2760 2761 assert(mtr); 2762 *mtr = 0; 2763 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2764 switch (actions->type) { 2765 case RTE_FLOW_ACTION_TYPE_METER: 2766 *mtr = 1; 2767 break; 2768 default: 2769 break; 2770 } 2771 actions_n++; 2772 } 2773 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2774 return actions_n + 1; 2775 } 2776 2777 /** 2778 * Check if the flow should be splited due to hairpin. 2779 * The reason for the split is that in current HW we can't 2780 * support encap on Rx, so if a flow have encap we move it 2781 * to Tx. 2782 * 2783 * @param dev 2784 * Pointer to Ethernet device. 2785 * @param[in] attr 2786 * Flow rule attributes. 2787 * @param[in] actions 2788 * Associated actions (list terminated by the END action). 2789 * 2790 * @return 2791 * > 0 the number of actions and the flow should be split, 2792 * 0 when no split required. 2793 */ 2794 static int 2795 flow_check_hairpin_split(struct rte_eth_dev *dev, 2796 const struct rte_flow_attr *attr, 2797 const struct rte_flow_action actions[]) 2798 { 2799 int queue_action = 0; 2800 int action_n = 0; 2801 int encap = 0; 2802 const struct rte_flow_action_queue *queue; 2803 const struct rte_flow_action_rss *rss; 2804 const struct rte_flow_action_raw_encap *raw_encap; 2805 2806 if (!attr->ingress) 2807 return 0; 2808 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2809 switch (actions->type) { 2810 case RTE_FLOW_ACTION_TYPE_QUEUE: 2811 queue = actions->conf; 2812 if (mlx5_rxq_get_type(dev, queue->index) != 2813 MLX5_RXQ_TYPE_HAIRPIN) 2814 return 0; 2815 queue_action = 1; 2816 action_n++; 2817 break; 2818 case RTE_FLOW_ACTION_TYPE_RSS: 2819 rss = actions->conf; 2820 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2821 MLX5_RXQ_TYPE_HAIRPIN) 2822 return 0; 2823 queue_action = 1; 2824 action_n++; 2825 break; 2826 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2827 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2828 encap = 1; 2829 action_n++; 2830 break; 2831 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2832 raw_encap = actions->conf; 2833 if (raw_encap->size > 2834 (sizeof(struct rte_flow_item_eth) + 2835 sizeof(struct rte_flow_item_ipv4))) 2836 encap = 1; 2837 action_n++; 2838 break; 2839 default: 2840 action_n++; 2841 break; 2842 } 2843 } 2844 if (encap == 1 && queue_action) 2845 return action_n; 2846 return 0; 2847 } 2848 2849 /* Declare flow create/destroy prototype in advance. */ 2850 static struct rte_flow * 2851 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 2852 const struct rte_flow_attr *attr, 2853 const struct rte_flow_item items[], 2854 const struct rte_flow_action actions[], 2855 bool external, struct rte_flow_error *error); 2856 2857 static void 2858 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2859 struct rte_flow *flow); 2860 2861 /** 2862 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2863 * 2864 * As mark_id is unique, if there's already a registered flow for the mark_id, 2865 * return by increasing the reference counter of the resource. Otherwise, create 2866 * the resource (mcp_res) and flow. 2867 * 2868 * Flow looks like, 2869 * - If ingress port is ANY and reg_c[1] is mark_id, 2870 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2871 * 2872 * For default flow (zero mark_id), flow is like, 2873 * - If ingress port is ANY, 2874 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2875 * 2876 * @param dev 2877 * Pointer to Ethernet device. 2878 * @param mark_id 2879 * ID of MARK action, zero means default flow for META. 2880 * @param[out] error 2881 * Perform verbose error reporting if not NULL. 2882 * 2883 * @return 2884 * Associated resource on success, NULL otherwise and rte_errno is set. 2885 */ 2886 static struct mlx5_flow_mreg_copy_resource * 2887 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2888 struct rte_flow_error *error) 2889 { 2890 struct mlx5_priv *priv = dev->data->dev_private; 2891 struct rte_flow_attr attr = { 2892 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2893 .ingress = 1, 2894 }; 2895 struct mlx5_rte_flow_item_tag tag_spec = { 2896 .data = mark_id, 2897 }; 2898 struct rte_flow_item items[] = { 2899 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2900 }; 2901 struct rte_flow_action_mark ftag = { 2902 .id = mark_id, 2903 }; 2904 struct mlx5_flow_action_copy_mreg cp_mreg = { 2905 .dst = REG_B, 2906 .src = 0, 2907 }; 2908 struct rte_flow_action_jump jump = { 2909 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2910 }; 2911 struct rte_flow_action actions[] = { 2912 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2913 }; 2914 struct mlx5_flow_mreg_copy_resource *mcp_res; 2915 int ret; 2916 2917 /* Fill the register fileds in the flow. */ 2918 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2919 if (ret < 0) 2920 return NULL; 2921 tag_spec.id = ret; 2922 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 2923 if (ret < 0) 2924 return NULL; 2925 cp_mreg.src = ret; 2926 /* Check if already registered. */ 2927 assert(priv->mreg_cp_tbl); 2928 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 2929 if (mcp_res) { 2930 /* For non-default rule. */ 2931 if (mark_id) 2932 mcp_res->refcnt++; 2933 assert(mark_id || mcp_res->refcnt == 1); 2934 return mcp_res; 2935 } 2936 /* Provide the full width of FLAG specific value. */ 2937 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 2938 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 2939 /* Build a new flow. */ 2940 if (mark_id) { 2941 items[0] = (struct rte_flow_item){ 2942 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 2943 .spec = &tag_spec, 2944 }; 2945 items[1] = (struct rte_flow_item){ 2946 .type = RTE_FLOW_ITEM_TYPE_END, 2947 }; 2948 actions[0] = (struct rte_flow_action){ 2949 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK, 2950 .conf = &ftag, 2951 }; 2952 actions[1] = (struct rte_flow_action){ 2953 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2954 .conf = &cp_mreg, 2955 }; 2956 actions[2] = (struct rte_flow_action){ 2957 .type = RTE_FLOW_ACTION_TYPE_JUMP, 2958 .conf = &jump, 2959 }; 2960 actions[3] = (struct rte_flow_action){ 2961 .type = RTE_FLOW_ACTION_TYPE_END, 2962 }; 2963 } else { 2964 /* Default rule, wildcard match. */ 2965 attr.priority = MLX5_FLOW_PRIO_RSVD; 2966 items[0] = (struct rte_flow_item){ 2967 .type = RTE_FLOW_ITEM_TYPE_END, 2968 }; 2969 actions[0] = (struct rte_flow_action){ 2970 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2971 .conf = &cp_mreg, 2972 }; 2973 actions[1] = (struct rte_flow_action){ 2974 .type = RTE_FLOW_ACTION_TYPE_JUMP, 2975 .conf = &jump, 2976 }; 2977 actions[2] = (struct rte_flow_action){ 2978 .type = RTE_FLOW_ACTION_TYPE_END, 2979 }; 2980 } 2981 /* Build a new entry. */ 2982 mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0); 2983 if (!mcp_res) { 2984 rte_errno = ENOMEM; 2985 return NULL; 2986 } 2987 /* 2988 * The copy Flows are not included in any list. There 2989 * ones are referenced from other Flows and can not 2990 * be applied, removed, deleted in ardbitrary order 2991 * by list traversing. 2992 */ 2993 mcp_res->flow = flow_list_create(dev, NULL, &attr, items, 2994 actions, false, error); 2995 if (!mcp_res->flow) 2996 goto error; 2997 mcp_res->refcnt++; 2998 mcp_res->hlist_ent.key = mark_id; 2999 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3000 &mcp_res->hlist_ent); 3001 assert(!ret); 3002 if (ret) 3003 goto error; 3004 return mcp_res; 3005 error: 3006 if (mcp_res->flow) 3007 flow_list_destroy(dev, NULL, mcp_res->flow); 3008 rte_free(mcp_res); 3009 return NULL; 3010 } 3011 3012 /** 3013 * Release flow in RX_CP_TBL. 3014 * 3015 * @param dev 3016 * Pointer to Ethernet device. 3017 * @flow 3018 * Parent flow for wich copying is provided. 3019 */ 3020 static void 3021 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3022 struct rte_flow *flow) 3023 { 3024 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3025 struct mlx5_priv *priv = dev->data->dev_private; 3026 3027 if (!mcp_res || !priv->mreg_cp_tbl) 3028 return; 3029 if (flow->copy_applied) { 3030 assert(mcp_res->appcnt); 3031 flow->copy_applied = 0; 3032 --mcp_res->appcnt; 3033 if (!mcp_res->appcnt) 3034 flow_drv_remove(dev, mcp_res->flow); 3035 } 3036 /* 3037 * We do not check availability of metadata registers here, 3038 * because copy resources are allocated in this case. 3039 */ 3040 if (--mcp_res->refcnt) 3041 return; 3042 assert(mcp_res->flow); 3043 flow_list_destroy(dev, NULL, mcp_res->flow); 3044 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3045 rte_free(mcp_res); 3046 flow->mreg_copy = NULL; 3047 } 3048 3049 /** 3050 * Start flow in RX_CP_TBL. 3051 * 3052 * @param dev 3053 * Pointer to Ethernet device. 3054 * @flow 3055 * Parent flow for wich copying is provided. 3056 * 3057 * @return 3058 * 0 on success, a negative errno value otherwise and rte_errno is set. 3059 */ 3060 static int 3061 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3062 struct rte_flow *flow) 3063 { 3064 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3065 int ret; 3066 3067 if (!mcp_res || flow->copy_applied) 3068 return 0; 3069 if (!mcp_res->appcnt) { 3070 ret = flow_drv_apply(dev, mcp_res->flow, NULL); 3071 if (ret) 3072 return ret; 3073 } 3074 ++mcp_res->appcnt; 3075 flow->copy_applied = 1; 3076 return 0; 3077 } 3078 3079 /** 3080 * Stop flow in RX_CP_TBL. 3081 * 3082 * @param dev 3083 * Pointer to Ethernet device. 3084 * @flow 3085 * Parent flow for wich copying is provided. 3086 */ 3087 static void 3088 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3089 struct rte_flow *flow) 3090 { 3091 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3092 3093 if (!mcp_res || !flow->copy_applied) 3094 return; 3095 assert(mcp_res->appcnt); 3096 --mcp_res->appcnt; 3097 flow->copy_applied = 0; 3098 if (!mcp_res->appcnt) 3099 flow_drv_remove(dev, mcp_res->flow); 3100 } 3101 3102 /** 3103 * Remove the default copy action from RX_CP_TBL. 3104 * 3105 * @param dev 3106 * Pointer to Ethernet device. 3107 */ 3108 static void 3109 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3110 { 3111 struct mlx5_flow_mreg_copy_resource *mcp_res; 3112 struct mlx5_priv *priv = dev->data->dev_private; 3113 3114 /* Check if default flow is registered. */ 3115 if (!priv->mreg_cp_tbl) 3116 return; 3117 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 0ULL); 3118 if (!mcp_res) 3119 return; 3120 assert(mcp_res->flow); 3121 flow_list_destroy(dev, NULL, mcp_res->flow); 3122 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3123 rte_free(mcp_res); 3124 } 3125 3126 /** 3127 * Add the default copy action in in RX_CP_TBL. 3128 * 3129 * @param dev 3130 * Pointer to Ethernet device. 3131 * @param[out] error 3132 * Perform verbose error reporting if not NULL. 3133 * 3134 * @return 3135 * 0 for success, negative value otherwise and rte_errno is set. 3136 */ 3137 static int 3138 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3139 struct rte_flow_error *error) 3140 { 3141 struct mlx5_priv *priv = dev->data->dev_private; 3142 struct mlx5_flow_mreg_copy_resource *mcp_res; 3143 3144 /* Check whether extensive metadata feature is engaged. */ 3145 if (!priv->config.dv_flow_en || 3146 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3147 !mlx5_flow_ext_mreg_supported(dev) || 3148 !priv->sh->dv_regc0_mask) 3149 return 0; 3150 mcp_res = flow_mreg_add_copy_action(dev, 0, error); 3151 if (!mcp_res) 3152 return -rte_errno; 3153 return 0; 3154 } 3155 3156 /** 3157 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3158 * 3159 * All the flow having Q/RSS action should be split by 3160 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3161 * performs the following, 3162 * - CQE->flow_tag := reg_c[1] (MARK) 3163 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3164 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3165 * but there should be a flow per each MARK ID set by MARK action. 3166 * 3167 * For the aforementioned reason, if there's a MARK action in flow's action 3168 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3169 * the MARK ID to CQE's flow_tag like, 3170 * - If reg_c[1] is mark_id, 3171 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3172 * 3173 * For SET_META action which stores value in reg_c[0], as the destination is 3174 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3175 * MARK ID means the default flow. The default flow looks like, 3176 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3177 * 3178 * @param dev 3179 * Pointer to Ethernet device. 3180 * @param flow 3181 * Pointer to flow structure. 3182 * @param[in] actions 3183 * Pointer to the list of actions. 3184 * @param[out] error 3185 * Perform verbose error reporting if not NULL. 3186 * 3187 * @return 3188 * 0 on success, negative value otherwise and rte_errno is set. 3189 */ 3190 static int 3191 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3192 struct rte_flow *flow, 3193 const struct rte_flow_action *actions, 3194 struct rte_flow_error *error) 3195 { 3196 struct mlx5_priv *priv = dev->data->dev_private; 3197 struct mlx5_dev_config *config = &priv->config; 3198 struct mlx5_flow_mreg_copy_resource *mcp_res; 3199 const struct rte_flow_action_mark *mark; 3200 3201 /* Check whether extensive metadata feature is engaged. */ 3202 if (!config->dv_flow_en || 3203 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3204 !mlx5_flow_ext_mreg_supported(dev) || 3205 !priv->sh->dv_regc0_mask) 3206 return 0; 3207 /* Find MARK action. */ 3208 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3209 switch (actions->type) { 3210 case RTE_FLOW_ACTION_TYPE_FLAG: 3211 mcp_res = flow_mreg_add_copy_action 3212 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3213 if (!mcp_res) 3214 return -rte_errno; 3215 flow->mreg_copy = mcp_res; 3216 if (dev->data->dev_started) { 3217 mcp_res->appcnt++; 3218 flow->copy_applied = 1; 3219 } 3220 return 0; 3221 case RTE_FLOW_ACTION_TYPE_MARK: 3222 mark = (const struct rte_flow_action_mark *) 3223 actions->conf; 3224 mcp_res = 3225 flow_mreg_add_copy_action(dev, mark->id, error); 3226 if (!mcp_res) 3227 return -rte_errno; 3228 flow->mreg_copy = mcp_res; 3229 if (dev->data->dev_started) { 3230 mcp_res->appcnt++; 3231 flow->copy_applied = 1; 3232 } 3233 return 0; 3234 default: 3235 break; 3236 } 3237 } 3238 return 0; 3239 } 3240 3241 #define MLX5_MAX_SPLIT_ACTIONS 24 3242 #define MLX5_MAX_SPLIT_ITEMS 24 3243 3244 /** 3245 * Split the hairpin flow. 3246 * Since HW can't support encap on Rx we move the encap to Tx. 3247 * If the count action is after the encap then we also 3248 * move the count action. in this case the count will also measure 3249 * the outer bytes. 3250 * 3251 * @param dev 3252 * Pointer to Ethernet device. 3253 * @param[in] actions 3254 * Associated actions (list terminated by the END action). 3255 * @param[out] actions_rx 3256 * Rx flow actions. 3257 * @param[out] actions_tx 3258 * Tx flow actions.. 3259 * @param[out] pattern_tx 3260 * The pattern items for the Tx flow. 3261 * @param[out] flow_id 3262 * The flow ID connected to this flow. 3263 * 3264 * @return 3265 * 0 on success. 3266 */ 3267 static int 3268 flow_hairpin_split(struct rte_eth_dev *dev, 3269 const struct rte_flow_action actions[], 3270 struct rte_flow_action actions_rx[], 3271 struct rte_flow_action actions_tx[], 3272 struct rte_flow_item pattern_tx[], 3273 uint32_t *flow_id) 3274 { 3275 struct mlx5_priv *priv = dev->data->dev_private; 3276 const struct rte_flow_action_raw_encap *raw_encap; 3277 const struct rte_flow_action_raw_decap *raw_decap; 3278 struct mlx5_rte_flow_action_set_tag *set_tag; 3279 struct rte_flow_action *tag_action; 3280 struct mlx5_rte_flow_item_tag *tag_item; 3281 struct rte_flow_item *item; 3282 char *addr; 3283 int encap = 0; 3284 3285 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3286 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3287 switch (actions->type) { 3288 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3289 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3290 rte_memcpy(actions_tx, actions, 3291 sizeof(struct rte_flow_action)); 3292 actions_tx++; 3293 break; 3294 case RTE_FLOW_ACTION_TYPE_COUNT: 3295 if (encap) { 3296 rte_memcpy(actions_tx, actions, 3297 sizeof(struct rte_flow_action)); 3298 actions_tx++; 3299 } else { 3300 rte_memcpy(actions_rx, actions, 3301 sizeof(struct rte_flow_action)); 3302 actions_rx++; 3303 } 3304 break; 3305 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3306 raw_encap = actions->conf; 3307 if (raw_encap->size > 3308 (sizeof(struct rte_flow_item_eth) + 3309 sizeof(struct rte_flow_item_ipv4))) { 3310 memcpy(actions_tx, actions, 3311 sizeof(struct rte_flow_action)); 3312 actions_tx++; 3313 encap = 1; 3314 } else { 3315 rte_memcpy(actions_rx, actions, 3316 sizeof(struct rte_flow_action)); 3317 actions_rx++; 3318 } 3319 break; 3320 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3321 raw_decap = actions->conf; 3322 if (raw_decap->size < 3323 (sizeof(struct rte_flow_item_eth) + 3324 sizeof(struct rte_flow_item_ipv4))) { 3325 memcpy(actions_tx, actions, 3326 sizeof(struct rte_flow_action)); 3327 actions_tx++; 3328 } else { 3329 rte_memcpy(actions_rx, actions, 3330 sizeof(struct rte_flow_action)); 3331 actions_rx++; 3332 } 3333 break; 3334 default: 3335 rte_memcpy(actions_rx, actions, 3336 sizeof(struct rte_flow_action)); 3337 actions_rx++; 3338 break; 3339 } 3340 } 3341 /* Add set meta action and end action for the Rx flow. */ 3342 tag_action = actions_rx; 3343 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3344 actions_rx++; 3345 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3346 actions_rx++; 3347 set_tag = (void *)actions_rx; 3348 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3349 assert(set_tag->id > REG_NONE); 3350 set_tag->data = *flow_id; 3351 tag_action->conf = set_tag; 3352 /* Create Tx item list. */ 3353 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3354 addr = (void *)&pattern_tx[2]; 3355 item = pattern_tx; 3356 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3357 tag_item = (void *)addr; 3358 tag_item->data = *flow_id; 3359 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3360 assert(set_tag->id > REG_NONE); 3361 item->spec = tag_item; 3362 addr += sizeof(struct mlx5_rte_flow_item_tag); 3363 tag_item = (void *)addr; 3364 tag_item->data = UINT32_MAX; 3365 tag_item->id = UINT16_MAX; 3366 item->mask = tag_item; 3367 addr += sizeof(struct mlx5_rte_flow_item_tag); 3368 item->last = NULL; 3369 item++; 3370 item->type = RTE_FLOW_ITEM_TYPE_END; 3371 return 0; 3372 } 3373 3374 /** 3375 * The last stage of splitting chain, just creates the subflow 3376 * without any modification. 3377 * 3378 * @param dev 3379 * Pointer to Ethernet device. 3380 * @param[in] flow 3381 * Parent flow structure pointer. 3382 * @param[in, out] sub_flow 3383 * Pointer to return the created subflow, may be NULL. 3384 * @param[in] attr 3385 * Flow rule attributes. 3386 * @param[in] items 3387 * Pattern specification (list terminated by the END pattern item). 3388 * @param[in] actions 3389 * Associated actions (list terminated by the END action). 3390 * @param[in] external 3391 * This flow rule is created by request external to PMD. 3392 * @param[out] error 3393 * Perform verbose error reporting if not NULL. 3394 * @return 3395 * 0 on success, negative value otherwise 3396 */ 3397 static int 3398 flow_create_split_inner(struct rte_eth_dev *dev, 3399 struct rte_flow *flow, 3400 struct mlx5_flow **sub_flow, 3401 const struct rte_flow_attr *attr, 3402 const struct rte_flow_item items[], 3403 const struct rte_flow_action actions[], 3404 bool external, struct rte_flow_error *error) 3405 { 3406 struct mlx5_flow *dev_flow; 3407 3408 dev_flow = flow_drv_prepare(flow, attr, items, actions, error); 3409 if (!dev_flow) 3410 return -rte_errno; 3411 dev_flow->flow = flow; 3412 dev_flow->external = external; 3413 /* Subflow object was created, we must include one in the list. */ 3414 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 3415 if (sub_flow) 3416 *sub_flow = dev_flow; 3417 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3418 } 3419 3420 /** 3421 * Split the meter flow. 3422 * 3423 * As meter flow will split to three sub flow, other than meter 3424 * action, the other actions make sense to only meter accepts 3425 * the packet. If it need to be dropped, no other additional 3426 * actions should be take. 3427 * 3428 * One kind of special action which decapsulates the L3 tunnel 3429 * header will be in the prefix sub flow, as not to take the 3430 * L3 tunnel header into account. 3431 * 3432 * @param dev 3433 * Pointer to Ethernet device. 3434 * @param[in] actions 3435 * Associated actions (list terminated by the END action). 3436 * @param[out] actions_sfx 3437 * Suffix flow actions. 3438 * @param[out] actions_pre 3439 * Prefix flow actions. 3440 * @param[out] pattern_sfx 3441 * The pattern items for the suffix flow. 3442 * @param[out] tag_sfx 3443 * Pointer to suffix flow tag. 3444 * 3445 * @return 3446 * 0 on success. 3447 */ 3448 static int 3449 flow_meter_split_prep(struct rte_eth_dev *dev, 3450 const struct rte_flow_action actions[], 3451 struct rte_flow_action actions_sfx[], 3452 struct rte_flow_action actions_pre[]) 3453 { 3454 struct rte_flow_action *tag_action; 3455 struct mlx5_rte_flow_action_set_tag *set_tag; 3456 struct rte_flow_error error; 3457 const struct rte_flow_action_raw_encap *raw_encap; 3458 const struct rte_flow_action_raw_decap *raw_decap; 3459 uint32_t tag_id; 3460 3461 /* Add the extra tag action first. */ 3462 tag_action = actions_pre; 3463 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3464 actions_pre++; 3465 /* Prepare the actions for prefix and suffix flow. */ 3466 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3467 switch (actions->type) { 3468 case RTE_FLOW_ACTION_TYPE_METER: 3469 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3470 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3471 memcpy(actions_pre, actions, 3472 sizeof(struct rte_flow_action)); 3473 actions_pre++; 3474 break; 3475 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3476 raw_encap = actions->conf; 3477 if (raw_encap->size > 3478 (sizeof(struct rte_flow_item_eth) + 3479 sizeof(struct rte_flow_item_ipv4))) { 3480 memcpy(actions_sfx, actions, 3481 sizeof(struct rte_flow_action)); 3482 actions_sfx++; 3483 } else { 3484 rte_memcpy(actions_pre, actions, 3485 sizeof(struct rte_flow_action)); 3486 actions_pre++; 3487 } 3488 break; 3489 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3490 raw_decap = actions->conf; 3491 /* Size 0 decap means 50 bytes as vxlan decap. */ 3492 if (raw_decap->size && (raw_decap->size < 3493 (sizeof(struct rte_flow_item_eth) + 3494 sizeof(struct rte_flow_item_ipv4)))) { 3495 memcpy(actions_sfx, actions, 3496 sizeof(struct rte_flow_action)); 3497 actions_sfx++; 3498 } else { 3499 rte_memcpy(actions_pre, actions, 3500 sizeof(struct rte_flow_action)); 3501 actions_pre++; 3502 } 3503 break; 3504 default: 3505 memcpy(actions_sfx, actions, 3506 sizeof(struct rte_flow_action)); 3507 actions_sfx++; 3508 break; 3509 } 3510 } 3511 /* Add end action to the actions. */ 3512 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3513 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3514 actions_pre++; 3515 /* Set the tag. */ 3516 set_tag = (void *)actions_pre; 3517 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3518 /* 3519 * Get the id from the qrss_pool to make qrss share the id with meter. 3520 */ 3521 tag_id = flow_qrss_get_id(dev); 3522 set_tag->data = rte_cpu_to_be_32(tag_id); 3523 tag_action->conf = set_tag; 3524 return tag_id; 3525 } 3526 3527 /** 3528 * Split action list having QUEUE/RSS for metadata register copy. 3529 * 3530 * Once Q/RSS action is detected in user's action list, the flow action 3531 * should be split in order to copy metadata registers, which will happen in 3532 * RX_CP_TBL like, 3533 * - CQE->flow_tag := reg_c[1] (MARK) 3534 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3535 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3536 * This is because the last action of each flow must be a terminal action 3537 * (QUEUE, RSS or DROP). 3538 * 3539 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3540 * stored and kept in the mlx5_flow structure per each sub_flow. 3541 * 3542 * The Q/RSS action is replaced with, 3543 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3544 * And the following JUMP action is added at the end, 3545 * - JUMP, to RX_CP_TBL. 3546 * 3547 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3548 * flow_create_split_metadata() routine. The flow will look like, 3549 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3550 * 3551 * @param dev 3552 * Pointer to Ethernet device. 3553 * @param[out] split_actions 3554 * Pointer to store split actions to jump to CP_TBL. 3555 * @param[in] actions 3556 * Pointer to the list of original flow actions. 3557 * @param[in] qrss 3558 * Pointer to the Q/RSS action. 3559 * @param[in] actions_n 3560 * Number of original actions. 3561 * @param[out] error 3562 * Perform verbose error reporting if not NULL. 3563 * 3564 * @return 3565 * non-zero unique flow_id on success, otherwise 0 and 3566 * error/rte_error are set. 3567 */ 3568 static uint32_t 3569 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3570 struct rte_flow_action *split_actions, 3571 const struct rte_flow_action *actions, 3572 const struct rte_flow_action *qrss, 3573 int actions_n, struct rte_flow_error *error) 3574 { 3575 struct mlx5_rte_flow_action_set_tag *set_tag; 3576 struct rte_flow_action_jump *jump; 3577 const int qrss_idx = qrss - actions; 3578 uint32_t flow_id = 0; 3579 int ret = 0; 3580 3581 /* 3582 * Given actions will be split 3583 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3584 * - Add jump to mreg CP_TBL. 3585 * As a result, there will be one more action. 3586 */ 3587 ++actions_n; 3588 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3589 set_tag = (void *)(split_actions + actions_n); 3590 /* 3591 * If tag action is not set to void(it means we are not the meter 3592 * suffix flow), add the tag action. Since meter suffix flow already 3593 * has the tag added. 3594 */ 3595 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3596 /* 3597 * Allocate the new subflow ID. This one is unique within 3598 * device and not shared with representors. Otherwise, 3599 * we would have to resolve multi-thread access synch 3600 * issue. Each flow on the shared device is appended 3601 * with source vport identifier, so the resulting 3602 * flows will be unique in the shared (by master and 3603 * representors) domain even if they have coinciding 3604 * IDs. 3605 */ 3606 flow_id = flow_qrss_get_id(dev); 3607 if (!flow_id) 3608 return rte_flow_error_set(error, ENOMEM, 3609 RTE_FLOW_ERROR_TYPE_ACTION, 3610 NULL, "can't allocate id " 3611 "for split Q/RSS subflow"); 3612 /* Internal SET_TAG action to set flow ID. */ 3613 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3614 .data = flow_id, 3615 }; 3616 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3617 if (ret < 0) 3618 return ret; 3619 set_tag->id = ret; 3620 /* Construct new actions array. */ 3621 /* Replace QUEUE/RSS action. */ 3622 split_actions[qrss_idx] = (struct rte_flow_action){ 3623 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3624 .conf = set_tag, 3625 }; 3626 } 3627 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3628 jump = (void *)(set_tag + 1); 3629 *jump = (struct rte_flow_action_jump){ 3630 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3631 }; 3632 split_actions[actions_n - 2] = (struct rte_flow_action){ 3633 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3634 .conf = jump, 3635 }; 3636 split_actions[actions_n - 1] = (struct rte_flow_action){ 3637 .type = RTE_FLOW_ACTION_TYPE_END, 3638 }; 3639 return flow_id; 3640 } 3641 3642 /** 3643 * Extend the given action list for Tx metadata copy. 3644 * 3645 * Copy the given action list to the ext_actions and add flow metadata register 3646 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3647 * 3648 * @param[out] ext_actions 3649 * Pointer to the extended action list. 3650 * @param[in] actions 3651 * Pointer to the list of actions. 3652 * @param[in] actions_n 3653 * Number of actions in the list. 3654 * @param[out] error 3655 * Perform verbose error reporting if not NULL. 3656 * 3657 * @return 3658 * 0 on success, negative value otherwise 3659 */ 3660 static int 3661 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3662 struct rte_flow_action *ext_actions, 3663 const struct rte_flow_action *actions, 3664 int actions_n, struct rte_flow_error *error) 3665 { 3666 struct mlx5_flow_action_copy_mreg *cp_mreg = 3667 (struct mlx5_flow_action_copy_mreg *) 3668 (ext_actions + actions_n + 1); 3669 int ret; 3670 3671 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3672 if (ret < 0) 3673 return ret; 3674 cp_mreg->dst = ret; 3675 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3676 if (ret < 0) 3677 return ret; 3678 cp_mreg->src = ret; 3679 memcpy(ext_actions, actions, 3680 sizeof(*ext_actions) * actions_n); 3681 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3682 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3683 .conf = cp_mreg, 3684 }; 3685 ext_actions[actions_n] = (struct rte_flow_action){ 3686 .type = RTE_FLOW_ACTION_TYPE_END, 3687 }; 3688 return 0; 3689 } 3690 3691 /** 3692 * The splitting for metadata feature. 3693 * 3694 * - Q/RSS action on NIC Rx should be split in order to pass by 3695 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3696 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3697 * 3698 * - All the actions on NIC Tx should have a mreg copy action to 3699 * copy reg_a from WQE to reg_c[0]. 3700 * 3701 * @param dev 3702 * Pointer to Ethernet device. 3703 * @param[in] flow 3704 * Parent flow structure pointer. 3705 * @param[in] attr 3706 * Flow rule attributes. 3707 * @param[in] items 3708 * Pattern specification (list terminated by the END pattern item). 3709 * @param[in] actions 3710 * Associated actions (list terminated by the END action). 3711 * @param[in] external 3712 * This flow rule is created by request external to PMD. 3713 * @param[out] error 3714 * Perform verbose error reporting if not NULL. 3715 * @return 3716 * 0 on success, negative value otherwise 3717 */ 3718 static int 3719 flow_create_split_metadata(struct rte_eth_dev *dev, 3720 struct rte_flow *flow, 3721 const struct rte_flow_attr *attr, 3722 const struct rte_flow_item items[], 3723 const struct rte_flow_action actions[], 3724 bool external, struct rte_flow_error *error) 3725 { 3726 struct mlx5_priv *priv = dev->data->dev_private; 3727 struct mlx5_dev_config *config = &priv->config; 3728 const struct rte_flow_action *qrss = NULL; 3729 struct rte_flow_action *ext_actions = NULL; 3730 struct mlx5_flow *dev_flow = NULL; 3731 uint32_t qrss_id = 0; 3732 int mtr_sfx = 0; 3733 size_t act_size; 3734 int actions_n; 3735 int ret; 3736 3737 /* Check whether extensive metadata feature is engaged. */ 3738 if (!config->dv_flow_en || 3739 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3740 !mlx5_flow_ext_mreg_supported(dev)) 3741 return flow_create_split_inner(dev, flow, NULL, attr, items, 3742 actions, external, error); 3743 actions_n = flow_parse_qrss_action(actions, &qrss); 3744 if (qrss) { 3745 /* Exclude hairpin flows from splitting. */ 3746 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3747 const struct rte_flow_action_queue *queue; 3748 3749 queue = qrss->conf; 3750 if (mlx5_rxq_get_type(dev, queue->index) == 3751 MLX5_RXQ_TYPE_HAIRPIN) 3752 qrss = NULL; 3753 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3754 const struct rte_flow_action_rss *rss; 3755 3756 rss = qrss->conf; 3757 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3758 MLX5_RXQ_TYPE_HAIRPIN) 3759 qrss = NULL; 3760 } 3761 } 3762 if (qrss) { 3763 /* Check if it is in meter suffix table. */ 3764 mtr_sfx = attr->group == (attr->transfer ? 3765 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3766 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3767 /* 3768 * Q/RSS action on NIC Rx should be split in order to pass by 3769 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3770 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3771 */ 3772 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3773 sizeof(struct rte_flow_action_set_tag) + 3774 sizeof(struct rte_flow_action_jump); 3775 ext_actions = rte_zmalloc(__func__, act_size, 0); 3776 if (!ext_actions) 3777 return rte_flow_error_set(error, ENOMEM, 3778 RTE_FLOW_ERROR_TYPE_ACTION, 3779 NULL, "no memory to split " 3780 "metadata flow"); 3781 /* 3782 * If we are the suffix flow of meter, tag already exist. 3783 * Set the tag action to void. 3784 */ 3785 if (mtr_sfx) 3786 ext_actions[qrss - actions].type = 3787 RTE_FLOW_ACTION_TYPE_VOID; 3788 else 3789 ext_actions[qrss - actions].type = 3790 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3791 /* 3792 * Create the new actions list with removed Q/RSS action 3793 * and appended set tag and jump to register copy table 3794 * (RX_CP_TBL). We should preallocate unique tag ID here 3795 * in advance, because it is needed for set tag action. 3796 */ 3797 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3798 qrss, actions_n, error); 3799 if (!mtr_sfx && !qrss_id) { 3800 ret = -rte_errno; 3801 goto exit; 3802 } 3803 } else if (attr->egress && !attr->transfer) { 3804 /* 3805 * All the actions on NIC Tx should have a metadata register 3806 * copy action to copy reg_a from WQE to reg_c[meta] 3807 */ 3808 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3809 sizeof(struct mlx5_flow_action_copy_mreg); 3810 ext_actions = rte_zmalloc(__func__, act_size, 0); 3811 if (!ext_actions) 3812 return rte_flow_error_set(error, ENOMEM, 3813 RTE_FLOW_ERROR_TYPE_ACTION, 3814 NULL, "no memory to split " 3815 "metadata flow"); 3816 /* Create the action list appended with copy register. */ 3817 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 3818 actions_n, error); 3819 if (ret < 0) 3820 goto exit; 3821 } 3822 /* Add the unmodified original or prefix subflow. */ 3823 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, 3824 ext_actions ? ext_actions : actions, 3825 external, error); 3826 if (ret < 0) 3827 goto exit; 3828 assert(dev_flow); 3829 if (qrss) { 3830 const struct rte_flow_attr q_attr = { 3831 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3832 .ingress = 1, 3833 }; 3834 /* Internal PMD action to set register. */ 3835 struct mlx5_rte_flow_item_tag q_tag_spec = { 3836 .data = qrss_id, 3837 .id = 0, 3838 }; 3839 struct rte_flow_item q_items[] = { 3840 { 3841 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3842 .spec = &q_tag_spec, 3843 .last = NULL, 3844 .mask = NULL, 3845 }, 3846 { 3847 .type = RTE_FLOW_ITEM_TYPE_END, 3848 }, 3849 }; 3850 struct rte_flow_action q_actions[] = { 3851 { 3852 .type = qrss->type, 3853 .conf = qrss->conf, 3854 }, 3855 { 3856 .type = RTE_FLOW_ACTION_TYPE_END, 3857 }, 3858 }; 3859 uint64_t hash_fields = dev_flow->hash_fields; 3860 dev_flow = NULL; 3861 /* 3862 * Configure the tag action only if we are not the meter sub 3863 * flow. Since tag is already marked in the meter suffix sub 3864 * flow. 3865 */ 3866 if (qrss_id) { 3867 /* 3868 * Put unique id in prefix flow due to it is destroyed 3869 * after prefix flow and id will be freed after there 3870 * is no actual flows with this id and identifier 3871 * reallocation becomes possible (for example, for 3872 * other flows in other threads). 3873 */ 3874 dev_flow->qrss_id = qrss_id; 3875 qrss_id = 0; 3876 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 3877 error); 3878 if (ret < 0) 3879 goto exit; 3880 q_tag_spec.id = ret; 3881 } 3882 /* Add suffix subflow to execute Q/RSS. */ 3883 ret = flow_create_split_inner(dev, flow, &dev_flow, 3884 &q_attr, mtr_sfx ? items : 3885 q_items, q_actions, 3886 external, error); 3887 if (ret < 0) 3888 goto exit; 3889 assert(dev_flow); 3890 dev_flow->hash_fields = hash_fields; 3891 } 3892 3893 exit: 3894 /* 3895 * We do not destroy the partially created sub_flows in case of error. 3896 * These ones are included into parent flow list and will be destroyed 3897 * by flow_drv_destroy. 3898 */ 3899 flow_qrss_free_id(dev, qrss_id); 3900 rte_free(ext_actions); 3901 return ret; 3902 } 3903 3904 /** 3905 * The splitting for meter feature. 3906 * 3907 * - The meter flow will be split to two flows as prefix and 3908 * suffix flow. The packets make sense only it pass the prefix 3909 * meter action. 3910 * 3911 * - Reg_C_5 is used for the packet to match betweend prefix and 3912 * suffix flow. 3913 * 3914 * @param dev 3915 * Pointer to Ethernet device. 3916 * @param[in] flow 3917 * Parent flow structure pointer. 3918 * @param[in] attr 3919 * Flow rule attributes. 3920 * @param[in] items 3921 * Pattern specification (list terminated by the END pattern item). 3922 * @param[in] actions 3923 * Associated actions (list terminated by the END action). 3924 * @param[in] external 3925 * This flow rule is created by request external to PMD. 3926 * @param[out] error 3927 * Perform verbose error reporting if not NULL. 3928 * @return 3929 * 0 on success, negative value otherwise 3930 */ 3931 static int 3932 flow_create_split_meter(struct rte_eth_dev *dev, 3933 struct rte_flow *flow, 3934 const struct rte_flow_attr *attr, 3935 const struct rte_flow_item items[], 3936 const struct rte_flow_action actions[], 3937 bool external, struct rte_flow_error *error) 3938 { 3939 struct mlx5_priv *priv = dev->data->dev_private; 3940 struct rte_flow_action *sfx_actions = NULL; 3941 struct rte_flow_action *pre_actions = NULL; 3942 struct rte_flow_item *sfx_items = NULL; 3943 const struct rte_flow_item *sfx_port_id_item; 3944 struct mlx5_flow *dev_flow = NULL; 3945 struct rte_flow_attr sfx_attr = *attr; 3946 uint32_t mtr = 0; 3947 uint32_t mtr_tag_id = 0; 3948 size_t act_size; 3949 size_t item_size; 3950 int actions_n = 0; 3951 int ret; 3952 3953 if (priv->mtr_en) 3954 actions_n = flow_check_meter_action(actions, &mtr); 3955 if (mtr) { 3956 struct mlx5_rte_flow_item_tag *tag_spec; 3957 /* The five prefix actions: meter, decap, encap, tag, end. */ 3958 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 3959 sizeof(struct rte_flow_action_set_tag); 3960 /* tag, end. */ 3961 #define METER_SUFFIX_ITEM 3 3962 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 3963 sizeof(struct mlx5_rte_flow_item_tag); 3964 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 3965 if (!sfx_actions) 3966 return rte_flow_error_set(error, ENOMEM, 3967 RTE_FLOW_ERROR_TYPE_ACTION, 3968 NULL, "no memory to split " 3969 "meter flow"); 3970 pre_actions = sfx_actions + actions_n; 3971 mtr_tag_id = flow_meter_split_prep(dev, actions, sfx_actions, 3972 pre_actions); 3973 if (!mtr_tag_id) { 3974 ret = -rte_errno; 3975 goto exit; 3976 } 3977 /* Add the prefix subflow. */ 3978 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, 3979 pre_actions, external, error); 3980 if (ret) { 3981 ret = -rte_errno; 3982 goto exit; 3983 } 3984 dev_flow->mtr_flow_id = mtr_tag_id; 3985 /* Prepare the suffix flow match pattern. */ 3986 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 3987 act_size); 3988 tag_spec = (struct mlx5_rte_flow_item_tag *)(sfx_items + 3989 METER_SUFFIX_ITEM); 3990 tag_spec->data = rte_cpu_to_be_32(dev_flow->mtr_flow_id); 3991 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, 3992 error); 3993 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3994 sfx_items->spec = tag_spec; 3995 sfx_items->last = NULL; 3996 sfx_items->mask = NULL; 3997 sfx_items++; 3998 sfx_port_id_item = find_port_id_item(items); 3999 if (sfx_port_id_item) { 4000 memcpy(sfx_items, sfx_port_id_item, 4001 sizeof(*sfx_items)); 4002 sfx_items++; 4003 } 4004 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4005 sfx_items -= METER_SUFFIX_ITEM; 4006 /* Setting the sfx group atrr. */ 4007 sfx_attr.group = sfx_attr.transfer ? 4008 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4009 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4010 } 4011 /* Add the prefix subflow. */ 4012 ret = flow_create_split_metadata(dev, flow, &sfx_attr, 4013 sfx_items ? sfx_items : items, 4014 sfx_actions ? sfx_actions : actions, 4015 external, error); 4016 exit: 4017 if (sfx_actions) 4018 rte_free(sfx_actions); 4019 return ret; 4020 } 4021 4022 /** 4023 * Split the flow to subflow set. The splitters might be linked 4024 * in the chain, like this: 4025 * flow_create_split_outer() calls: 4026 * flow_create_split_meter() calls: 4027 * flow_create_split_metadata(meter_subflow_0) calls: 4028 * flow_create_split_inner(metadata_subflow_0) 4029 * flow_create_split_inner(metadata_subflow_1) 4030 * flow_create_split_inner(metadata_subflow_2) 4031 * flow_create_split_metadata(meter_subflow_1) calls: 4032 * flow_create_split_inner(metadata_subflow_0) 4033 * flow_create_split_inner(metadata_subflow_1) 4034 * flow_create_split_inner(metadata_subflow_2) 4035 * 4036 * This provide flexible way to add new levels of flow splitting. 4037 * The all of successfully created subflows are included to the 4038 * parent flow dev_flow list. 4039 * 4040 * @param dev 4041 * Pointer to Ethernet device. 4042 * @param[in] flow 4043 * Parent flow structure pointer. 4044 * @param[in] attr 4045 * Flow rule attributes. 4046 * @param[in] items 4047 * Pattern specification (list terminated by the END pattern item). 4048 * @param[in] actions 4049 * Associated actions (list terminated by the END action). 4050 * @param[in] external 4051 * This flow rule is created by request external to PMD. 4052 * @param[out] error 4053 * Perform verbose error reporting if not NULL. 4054 * @return 4055 * 0 on success, negative value otherwise 4056 */ 4057 static int 4058 flow_create_split_outer(struct rte_eth_dev *dev, 4059 struct rte_flow *flow, 4060 const struct rte_flow_attr *attr, 4061 const struct rte_flow_item items[], 4062 const struct rte_flow_action actions[], 4063 bool external, struct rte_flow_error *error) 4064 { 4065 int ret; 4066 4067 ret = flow_create_split_meter(dev, flow, attr, items, 4068 actions, external, error); 4069 assert(ret <= 0); 4070 return ret; 4071 } 4072 4073 /** 4074 * Create a flow and add it to @p list. 4075 * 4076 * @param dev 4077 * Pointer to Ethernet device. 4078 * @param list 4079 * Pointer to a TAILQ flow list. If this parameter NULL, 4080 * no list insertion occurred, flow is just created, 4081 * this is caller's responsibility to track the 4082 * created flow. 4083 * @param[in] attr 4084 * Flow rule attributes. 4085 * @param[in] items 4086 * Pattern specification (list terminated by the END pattern item). 4087 * @param[in] actions 4088 * Associated actions (list terminated by the END action). 4089 * @param[in] external 4090 * This flow rule is created by request external to PMD. 4091 * @param[out] error 4092 * Perform verbose error reporting if not NULL. 4093 * 4094 * @return 4095 * A flow on success, NULL otherwise and rte_errno is set. 4096 */ 4097 static struct rte_flow * 4098 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 4099 const struct rte_flow_attr *attr, 4100 const struct rte_flow_item items[], 4101 const struct rte_flow_action actions[], 4102 bool external, struct rte_flow_error *error) 4103 { 4104 struct mlx5_priv *priv = dev->data->dev_private; 4105 struct rte_flow *flow = NULL; 4106 struct mlx5_flow *dev_flow; 4107 const struct rte_flow_action_rss *rss; 4108 union { 4109 struct rte_flow_expand_rss buf; 4110 uint8_t buffer[2048]; 4111 } expand_buffer; 4112 union { 4113 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4114 uint8_t buffer[2048]; 4115 } actions_rx; 4116 union { 4117 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4118 uint8_t buffer[2048]; 4119 } actions_hairpin_tx; 4120 union { 4121 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4122 uint8_t buffer[2048]; 4123 } items_tx; 4124 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4125 const struct rte_flow_action *p_actions_rx = actions; 4126 int ret; 4127 uint32_t i; 4128 uint32_t flow_size; 4129 int hairpin_flow = 0; 4130 uint32_t hairpin_id = 0; 4131 struct rte_flow_attr attr_tx = { .priority = 0 }; 4132 4133 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4134 if (hairpin_flow > 0) { 4135 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4136 rte_errno = EINVAL; 4137 return NULL; 4138 } 4139 flow_hairpin_split(dev, actions, actions_rx.actions, 4140 actions_hairpin_tx.actions, items_tx.items, 4141 &hairpin_id); 4142 p_actions_rx = actions_rx.actions; 4143 } 4144 ret = flow_drv_validate(dev, attr, items, p_actions_rx, external, 4145 error); 4146 if (ret < 0) 4147 goto error_before_flow; 4148 flow_size = sizeof(struct rte_flow); 4149 rss = flow_get_rss_action(p_actions_rx); 4150 if (rss) 4151 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t), 4152 sizeof(void *)); 4153 else 4154 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *)); 4155 flow = rte_calloc(__func__, 1, flow_size, 0); 4156 if (!flow) { 4157 rte_errno = ENOMEM; 4158 goto error_before_flow; 4159 } 4160 flow->drv_type = flow_get_drv_type(dev, attr); 4161 if (hairpin_id != 0) 4162 flow->hairpin_flow_id = hairpin_id; 4163 assert(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4164 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4165 flow->rss.queue = (void *)(flow + 1); 4166 if (rss) { 4167 /* 4168 * The following information is required by 4169 * mlx5_flow_hashfields_adjust() in advance. 4170 */ 4171 flow->rss.level = rss->level; 4172 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4173 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; 4174 } 4175 LIST_INIT(&flow->dev_flows); 4176 if (rss && rss->types) { 4177 unsigned int graph_root; 4178 4179 graph_root = find_graph_root(items, rss->level); 4180 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4181 items, rss->types, 4182 mlx5_support_expansion, 4183 graph_root); 4184 assert(ret > 0 && 4185 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4186 } else { 4187 buf->entries = 1; 4188 buf->entry[0].pattern = (void *)(uintptr_t)items; 4189 } 4190 for (i = 0; i < buf->entries; ++i) { 4191 /* 4192 * The splitter may create multiple dev_flows, 4193 * depending on configuration. In the simplest 4194 * case it just creates unmodified original flow. 4195 */ 4196 ret = flow_create_split_outer(dev, flow, attr, 4197 buf->entry[i].pattern, 4198 p_actions_rx, external, 4199 error); 4200 if (ret < 0) 4201 goto error; 4202 } 4203 /* Create the tx flow. */ 4204 if (hairpin_flow) { 4205 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4206 attr_tx.ingress = 0; 4207 attr_tx.egress = 1; 4208 dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items, 4209 actions_hairpin_tx.actions, error); 4210 if (!dev_flow) 4211 goto error; 4212 dev_flow->flow = flow; 4213 dev_flow->external = 0; 4214 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 4215 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4216 items_tx.items, 4217 actions_hairpin_tx.actions, error); 4218 if (ret < 0) 4219 goto error; 4220 } 4221 /* 4222 * Update the metadata register copy table. If extensive 4223 * metadata feature is enabled and registers are supported 4224 * we might create the extra rte_flow for each unique 4225 * MARK/FLAG action ID. 4226 * 4227 * The table is updated for ingress Flows only, because 4228 * the egress Flows belong to the different device and 4229 * copy table should be updated in peer NIC Rx domain. 4230 */ 4231 if (attr->ingress && 4232 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4233 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4234 if (ret) 4235 goto error; 4236 } 4237 if (dev->data->dev_started) { 4238 ret = flow_drv_apply(dev, flow, error); 4239 if (ret < 0) 4240 goto error; 4241 } 4242 if (list) 4243 TAILQ_INSERT_TAIL(list, flow, next); 4244 flow_rxq_flags_set(dev, flow); 4245 return flow; 4246 error_before_flow: 4247 if (hairpin_id) 4248 mlx5_flow_id_release(priv->sh->flow_id_pool, 4249 hairpin_id); 4250 return NULL; 4251 error: 4252 assert(flow); 4253 flow_mreg_del_copy_action(dev, flow); 4254 ret = rte_errno; /* Save rte_errno before cleanup. */ 4255 if (flow->hairpin_flow_id) 4256 mlx5_flow_id_release(priv->sh->flow_id_pool, 4257 flow->hairpin_flow_id); 4258 assert(flow); 4259 flow_drv_destroy(dev, flow); 4260 rte_free(flow); 4261 rte_errno = ret; /* Restore rte_errno. */ 4262 return NULL; 4263 } 4264 4265 /** 4266 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4267 * incoming packets to table 1. 4268 * 4269 * Other flow rules, requested for group n, will be created in 4270 * e-switch table n+1. 4271 * Jump action to e-switch group n will be created to group n+1. 4272 * 4273 * Used when working in switchdev mode, to utilise advantages of table 1 4274 * and above. 4275 * 4276 * @param dev 4277 * Pointer to Ethernet device. 4278 * 4279 * @return 4280 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4281 */ 4282 struct rte_flow * 4283 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4284 { 4285 const struct rte_flow_attr attr = { 4286 .group = 0, 4287 .priority = 0, 4288 .ingress = 1, 4289 .egress = 0, 4290 .transfer = 1, 4291 }; 4292 const struct rte_flow_item pattern = { 4293 .type = RTE_FLOW_ITEM_TYPE_END, 4294 }; 4295 struct rte_flow_action_jump jump = { 4296 .group = 1, 4297 }; 4298 const struct rte_flow_action actions[] = { 4299 { 4300 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4301 .conf = &jump, 4302 }, 4303 { 4304 .type = RTE_FLOW_ACTION_TYPE_END, 4305 }, 4306 }; 4307 struct mlx5_priv *priv = dev->data->dev_private; 4308 struct rte_flow_error error; 4309 4310 return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern, 4311 actions, false, &error); 4312 } 4313 4314 /** 4315 * Create a flow. 4316 * 4317 * @see rte_flow_create() 4318 * @see rte_flow_ops 4319 */ 4320 struct rte_flow * 4321 mlx5_flow_create(struct rte_eth_dev *dev, 4322 const struct rte_flow_attr *attr, 4323 const struct rte_flow_item items[], 4324 const struct rte_flow_action actions[], 4325 struct rte_flow_error *error) 4326 { 4327 struct mlx5_priv *priv = dev->data->dev_private; 4328 4329 return flow_list_create(dev, &priv->flows, 4330 attr, items, actions, true, error); 4331 } 4332 4333 /** 4334 * Destroy a flow in a list. 4335 * 4336 * @param dev 4337 * Pointer to Ethernet device. 4338 * @param list 4339 * Pointer to a TAILQ flow list. If this parameter NULL, 4340 * there is no flow removal from the list. 4341 * @param[in] flow 4342 * Flow to destroy. 4343 */ 4344 static void 4345 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 4346 struct rte_flow *flow) 4347 { 4348 struct mlx5_priv *priv = dev->data->dev_private; 4349 4350 /* 4351 * Update RX queue flags only if port is started, otherwise it is 4352 * already clean. 4353 */ 4354 if (dev->data->dev_started) 4355 flow_rxq_flags_trim(dev, flow); 4356 if (flow->hairpin_flow_id) 4357 mlx5_flow_id_release(priv->sh->flow_id_pool, 4358 flow->hairpin_flow_id); 4359 flow_drv_destroy(dev, flow); 4360 if (list) 4361 TAILQ_REMOVE(list, flow, next); 4362 flow_mreg_del_copy_action(dev, flow); 4363 rte_free(flow->fdir); 4364 rte_free(flow); 4365 } 4366 4367 /** 4368 * Destroy all flows. 4369 * 4370 * @param dev 4371 * Pointer to Ethernet device. 4372 * @param list 4373 * Pointer to a TAILQ flow list. 4374 */ 4375 void 4376 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 4377 { 4378 while (!TAILQ_EMPTY(list)) { 4379 struct rte_flow *flow; 4380 4381 flow = TAILQ_FIRST(list); 4382 flow_list_destroy(dev, list, flow); 4383 } 4384 } 4385 4386 /** 4387 * Remove all flows. 4388 * 4389 * @param dev 4390 * Pointer to Ethernet device. 4391 * @param list 4392 * Pointer to a TAILQ flow list. 4393 */ 4394 void 4395 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 4396 { 4397 struct rte_flow *flow; 4398 4399 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 4400 flow_drv_remove(dev, flow); 4401 flow_mreg_stop_copy_action(dev, flow); 4402 } 4403 flow_mreg_del_default_copy_action(dev); 4404 flow_rxq_flags_clear(dev); 4405 } 4406 4407 /** 4408 * Add all flows. 4409 * 4410 * @param dev 4411 * Pointer to Ethernet device. 4412 * @param list 4413 * Pointer to a TAILQ flow list. 4414 * 4415 * @return 4416 * 0 on success, a negative errno value otherwise and rte_errno is set. 4417 */ 4418 int 4419 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 4420 { 4421 struct rte_flow *flow; 4422 struct rte_flow_error error; 4423 int ret = 0; 4424 4425 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4426 ret = flow_mreg_add_default_copy_action(dev, &error); 4427 if (ret < 0) 4428 return -rte_errno; 4429 /* Apply Flows created by application. */ 4430 TAILQ_FOREACH(flow, list, next) { 4431 ret = flow_mreg_start_copy_action(dev, flow); 4432 if (ret < 0) 4433 goto error; 4434 ret = flow_drv_apply(dev, flow, &error); 4435 if (ret < 0) 4436 goto error; 4437 flow_rxq_flags_set(dev, flow); 4438 } 4439 return 0; 4440 error: 4441 ret = rte_errno; /* Save rte_errno before cleanup. */ 4442 mlx5_flow_stop(dev, list); 4443 rte_errno = ret; /* Restore rte_errno. */ 4444 return -rte_errno; 4445 } 4446 4447 /** 4448 * Verify the flow list is empty 4449 * 4450 * @param dev 4451 * Pointer to Ethernet device. 4452 * 4453 * @return the number of flows not released. 4454 */ 4455 int 4456 mlx5_flow_verify(struct rte_eth_dev *dev) 4457 { 4458 struct mlx5_priv *priv = dev->data->dev_private; 4459 struct rte_flow *flow; 4460 int ret = 0; 4461 4462 TAILQ_FOREACH(flow, &priv->flows, next) { 4463 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4464 dev->data->port_id, (void *)flow); 4465 ++ret; 4466 } 4467 return ret; 4468 } 4469 4470 /** 4471 * Enable default hairpin egress flow. 4472 * 4473 * @param dev 4474 * Pointer to Ethernet device. 4475 * @param queue 4476 * The queue index. 4477 * 4478 * @return 4479 * 0 on success, a negative errno value otherwise and rte_errno is set. 4480 */ 4481 int 4482 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4483 uint32_t queue) 4484 { 4485 struct mlx5_priv *priv = dev->data->dev_private; 4486 const struct rte_flow_attr attr = { 4487 .egress = 1, 4488 .priority = 0, 4489 }; 4490 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4491 .queue = queue, 4492 }; 4493 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4494 .queue = UINT32_MAX, 4495 }; 4496 struct rte_flow_item items[] = { 4497 { 4498 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4499 .spec = &queue_spec, 4500 .last = NULL, 4501 .mask = &queue_mask, 4502 }, 4503 { 4504 .type = RTE_FLOW_ITEM_TYPE_END, 4505 }, 4506 }; 4507 struct rte_flow_action_jump jump = { 4508 .group = MLX5_HAIRPIN_TX_TABLE, 4509 }; 4510 struct rte_flow_action actions[2]; 4511 struct rte_flow *flow; 4512 struct rte_flow_error error; 4513 4514 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4515 actions[0].conf = &jump; 4516 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4517 flow = flow_list_create(dev, &priv->ctrl_flows, 4518 &attr, items, actions, false, &error); 4519 if (!flow) { 4520 DRV_LOG(DEBUG, 4521 "Failed to create ctrl flow: rte_errno(%d)," 4522 " type(%d), message(%s)", 4523 rte_errno, error.type, 4524 error.message ? error.message : " (no stated reason)"); 4525 return -rte_errno; 4526 } 4527 return 0; 4528 } 4529 4530 /** 4531 * Enable a control flow configured from the control plane. 4532 * 4533 * @param dev 4534 * Pointer to Ethernet device. 4535 * @param eth_spec 4536 * An Ethernet flow spec to apply. 4537 * @param eth_mask 4538 * An Ethernet flow mask to apply. 4539 * @param vlan_spec 4540 * A VLAN flow spec to apply. 4541 * @param vlan_mask 4542 * A VLAN flow mask to apply. 4543 * 4544 * @return 4545 * 0 on success, a negative errno value otherwise and rte_errno is set. 4546 */ 4547 int 4548 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4549 struct rte_flow_item_eth *eth_spec, 4550 struct rte_flow_item_eth *eth_mask, 4551 struct rte_flow_item_vlan *vlan_spec, 4552 struct rte_flow_item_vlan *vlan_mask) 4553 { 4554 struct mlx5_priv *priv = dev->data->dev_private; 4555 const struct rte_flow_attr attr = { 4556 .ingress = 1, 4557 .priority = MLX5_FLOW_PRIO_RSVD, 4558 }; 4559 struct rte_flow_item items[] = { 4560 { 4561 .type = RTE_FLOW_ITEM_TYPE_ETH, 4562 .spec = eth_spec, 4563 .last = NULL, 4564 .mask = eth_mask, 4565 }, 4566 { 4567 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4568 RTE_FLOW_ITEM_TYPE_END, 4569 .spec = vlan_spec, 4570 .last = NULL, 4571 .mask = vlan_mask, 4572 }, 4573 { 4574 .type = RTE_FLOW_ITEM_TYPE_END, 4575 }, 4576 }; 4577 uint16_t queue[priv->reta_idx_n]; 4578 struct rte_flow_action_rss action_rss = { 4579 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4580 .level = 0, 4581 .types = priv->rss_conf.rss_hf, 4582 .key_len = priv->rss_conf.rss_key_len, 4583 .queue_num = priv->reta_idx_n, 4584 .key = priv->rss_conf.rss_key, 4585 .queue = queue, 4586 }; 4587 struct rte_flow_action actions[] = { 4588 { 4589 .type = RTE_FLOW_ACTION_TYPE_RSS, 4590 .conf = &action_rss, 4591 }, 4592 { 4593 .type = RTE_FLOW_ACTION_TYPE_END, 4594 }, 4595 }; 4596 struct rte_flow *flow; 4597 struct rte_flow_error error; 4598 unsigned int i; 4599 4600 if (!priv->reta_idx_n || !priv->rxqs_n) { 4601 return 0; 4602 } 4603 for (i = 0; i != priv->reta_idx_n; ++i) 4604 queue[i] = (*priv->reta_idx)[i]; 4605 flow = flow_list_create(dev, &priv->ctrl_flows, 4606 &attr, items, actions, false, &error); 4607 if (!flow) 4608 return -rte_errno; 4609 return 0; 4610 } 4611 4612 /** 4613 * Enable a flow control configured from the control plane. 4614 * 4615 * @param dev 4616 * Pointer to Ethernet device. 4617 * @param eth_spec 4618 * An Ethernet flow spec to apply. 4619 * @param eth_mask 4620 * An Ethernet flow mask to apply. 4621 * 4622 * @return 4623 * 0 on success, a negative errno value otherwise and rte_errno is set. 4624 */ 4625 int 4626 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4627 struct rte_flow_item_eth *eth_spec, 4628 struct rte_flow_item_eth *eth_mask) 4629 { 4630 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4631 } 4632 4633 /** 4634 * Destroy a flow. 4635 * 4636 * @see rte_flow_destroy() 4637 * @see rte_flow_ops 4638 */ 4639 int 4640 mlx5_flow_destroy(struct rte_eth_dev *dev, 4641 struct rte_flow *flow, 4642 struct rte_flow_error *error __rte_unused) 4643 { 4644 struct mlx5_priv *priv = dev->data->dev_private; 4645 4646 flow_list_destroy(dev, &priv->flows, flow); 4647 return 0; 4648 } 4649 4650 /** 4651 * Destroy all flows. 4652 * 4653 * @see rte_flow_flush() 4654 * @see rte_flow_ops 4655 */ 4656 int 4657 mlx5_flow_flush(struct rte_eth_dev *dev, 4658 struct rte_flow_error *error __rte_unused) 4659 { 4660 struct mlx5_priv *priv = dev->data->dev_private; 4661 4662 mlx5_flow_list_flush(dev, &priv->flows); 4663 return 0; 4664 } 4665 4666 /** 4667 * Isolated mode. 4668 * 4669 * @see rte_flow_isolate() 4670 * @see rte_flow_ops 4671 */ 4672 int 4673 mlx5_flow_isolate(struct rte_eth_dev *dev, 4674 int enable, 4675 struct rte_flow_error *error) 4676 { 4677 struct mlx5_priv *priv = dev->data->dev_private; 4678 4679 if (dev->data->dev_started) { 4680 rte_flow_error_set(error, EBUSY, 4681 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4682 NULL, 4683 "port must be stopped first"); 4684 return -rte_errno; 4685 } 4686 priv->isolated = !!enable; 4687 if (enable) 4688 dev->dev_ops = &mlx5_dev_ops_isolate; 4689 else 4690 dev->dev_ops = &mlx5_dev_ops; 4691 return 0; 4692 } 4693 4694 /** 4695 * Query a flow. 4696 * 4697 * @see rte_flow_query() 4698 * @see rte_flow_ops 4699 */ 4700 static int 4701 flow_drv_query(struct rte_eth_dev *dev, 4702 struct rte_flow *flow, 4703 const struct rte_flow_action *actions, 4704 void *data, 4705 struct rte_flow_error *error) 4706 { 4707 const struct mlx5_flow_driver_ops *fops; 4708 enum mlx5_flow_drv_type ftype = flow->drv_type; 4709 4710 assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 4711 fops = flow_get_drv_ops(ftype); 4712 4713 return fops->query(dev, flow, actions, data, error); 4714 } 4715 4716 /** 4717 * Query a flow. 4718 * 4719 * @see rte_flow_query() 4720 * @see rte_flow_ops 4721 */ 4722 int 4723 mlx5_flow_query(struct rte_eth_dev *dev, 4724 struct rte_flow *flow, 4725 const struct rte_flow_action *actions, 4726 void *data, 4727 struct rte_flow_error *error) 4728 { 4729 int ret; 4730 4731 ret = flow_drv_query(dev, flow, actions, data, error); 4732 if (ret < 0) 4733 return ret; 4734 return 0; 4735 } 4736 4737 /** 4738 * Convert a flow director filter to a generic flow. 4739 * 4740 * @param dev 4741 * Pointer to Ethernet device. 4742 * @param fdir_filter 4743 * Flow director filter to add. 4744 * @param attributes 4745 * Generic flow parameters structure. 4746 * 4747 * @return 4748 * 0 on success, a negative errno value otherwise and rte_errno is set. 4749 */ 4750 static int 4751 flow_fdir_filter_convert(struct rte_eth_dev *dev, 4752 const struct rte_eth_fdir_filter *fdir_filter, 4753 struct mlx5_fdir *attributes) 4754 { 4755 struct mlx5_priv *priv = dev->data->dev_private; 4756 const struct rte_eth_fdir_input *input = &fdir_filter->input; 4757 const struct rte_eth_fdir_masks *mask = 4758 &dev->data->dev_conf.fdir_conf.mask; 4759 4760 /* Validate queue number. */ 4761 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 4762 DRV_LOG(ERR, "port %u invalid queue number %d", 4763 dev->data->port_id, fdir_filter->action.rx_queue); 4764 rte_errno = EINVAL; 4765 return -rte_errno; 4766 } 4767 attributes->attr.ingress = 1; 4768 attributes->items[0] = (struct rte_flow_item) { 4769 .type = RTE_FLOW_ITEM_TYPE_ETH, 4770 .spec = &attributes->l2, 4771 .mask = &attributes->l2_mask, 4772 }; 4773 switch (fdir_filter->action.behavior) { 4774 case RTE_ETH_FDIR_ACCEPT: 4775 attributes->actions[0] = (struct rte_flow_action){ 4776 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 4777 .conf = &attributes->queue, 4778 }; 4779 break; 4780 case RTE_ETH_FDIR_REJECT: 4781 attributes->actions[0] = (struct rte_flow_action){ 4782 .type = RTE_FLOW_ACTION_TYPE_DROP, 4783 }; 4784 break; 4785 default: 4786 DRV_LOG(ERR, "port %u invalid behavior %d", 4787 dev->data->port_id, 4788 fdir_filter->action.behavior); 4789 rte_errno = ENOTSUP; 4790 return -rte_errno; 4791 } 4792 attributes->queue.index = fdir_filter->action.rx_queue; 4793 /* Handle L3. */ 4794 switch (fdir_filter->input.flow_type) { 4795 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4796 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4797 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4798 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 4799 .src_addr = input->flow.ip4_flow.src_ip, 4800 .dst_addr = input->flow.ip4_flow.dst_ip, 4801 .time_to_live = input->flow.ip4_flow.ttl, 4802 .type_of_service = input->flow.ip4_flow.tos, 4803 }; 4804 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 4805 .src_addr = mask->ipv4_mask.src_ip, 4806 .dst_addr = mask->ipv4_mask.dst_ip, 4807 .time_to_live = mask->ipv4_mask.ttl, 4808 .type_of_service = mask->ipv4_mask.tos, 4809 .next_proto_id = mask->ipv4_mask.proto, 4810 }; 4811 attributes->items[1] = (struct rte_flow_item){ 4812 .type = RTE_FLOW_ITEM_TYPE_IPV4, 4813 .spec = &attributes->l3, 4814 .mask = &attributes->l3_mask, 4815 }; 4816 break; 4817 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4818 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4819 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4820 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 4821 .hop_limits = input->flow.ipv6_flow.hop_limits, 4822 .proto = input->flow.ipv6_flow.proto, 4823 }; 4824 4825 memcpy(attributes->l3.ipv6.hdr.src_addr, 4826 input->flow.ipv6_flow.src_ip, 4827 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4828 memcpy(attributes->l3.ipv6.hdr.dst_addr, 4829 input->flow.ipv6_flow.dst_ip, 4830 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4831 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 4832 mask->ipv6_mask.src_ip, 4833 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4834 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 4835 mask->ipv6_mask.dst_ip, 4836 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4837 attributes->items[1] = (struct rte_flow_item){ 4838 .type = RTE_FLOW_ITEM_TYPE_IPV6, 4839 .spec = &attributes->l3, 4840 .mask = &attributes->l3_mask, 4841 }; 4842 break; 4843 default: 4844 DRV_LOG(ERR, "port %u invalid flow type%d", 4845 dev->data->port_id, fdir_filter->input.flow_type); 4846 rte_errno = ENOTSUP; 4847 return -rte_errno; 4848 } 4849 /* Handle L4. */ 4850 switch (fdir_filter->input.flow_type) { 4851 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4852 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4853 .src_port = input->flow.udp4_flow.src_port, 4854 .dst_port = input->flow.udp4_flow.dst_port, 4855 }; 4856 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4857 .src_port = mask->src_port_mask, 4858 .dst_port = mask->dst_port_mask, 4859 }; 4860 attributes->items[2] = (struct rte_flow_item){ 4861 .type = RTE_FLOW_ITEM_TYPE_UDP, 4862 .spec = &attributes->l4, 4863 .mask = &attributes->l4_mask, 4864 }; 4865 break; 4866 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4867 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4868 .src_port = input->flow.tcp4_flow.src_port, 4869 .dst_port = input->flow.tcp4_flow.dst_port, 4870 }; 4871 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4872 .src_port = mask->src_port_mask, 4873 .dst_port = mask->dst_port_mask, 4874 }; 4875 attributes->items[2] = (struct rte_flow_item){ 4876 .type = RTE_FLOW_ITEM_TYPE_TCP, 4877 .spec = &attributes->l4, 4878 .mask = &attributes->l4_mask, 4879 }; 4880 break; 4881 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4882 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4883 .src_port = input->flow.udp6_flow.src_port, 4884 .dst_port = input->flow.udp6_flow.dst_port, 4885 }; 4886 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4887 .src_port = mask->src_port_mask, 4888 .dst_port = mask->dst_port_mask, 4889 }; 4890 attributes->items[2] = (struct rte_flow_item){ 4891 .type = RTE_FLOW_ITEM_TYPE_UDP, 4892 .spec = &attributes->l4, 4893 .mask = &attributes->l4_mask, 4894 }; 4895 break; 4896 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4897 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4898 .src_port = input->flow.tcp6_flow.src_port, 4899 .dst_port = input->flow.tcp6_flow.dst_port, 4900 }; 4901 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4902 .src_port = mask->src_port_mask, 4903 .dst_port = mask->dst_port_mask, 4904 }; 4905 attributes->items[2] = (struct rte_flow_item){ 4906 .type = RTE_FLOW_ITEM_TYPE_TCP, 4907 .spec = &attributes->l4, 4908 .mask = &attributes->l4_mask, 4909 }; 4910 break; 4911 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4912 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4913 break; 4914 default: 4915 DRV_LOG(ERR, "port %u invalid flow type%d", 4916 dev->data->port_id, fdir_filter->input.flow_type); 4917 rte_errno = ENOTSUP; 4918 return -rte_errno; 4919 } 4920 return 0; 4921 } 4922 4923 #define FLOW_FDIR_CMP(f1, f2, fld) \ 4924 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 4925 4926 /** 4927 * Compare two FDIR flows. If items and actions are identical, the two flows are 4928 * regarded as same. 4929 * 4930 * @param dev 4931 * Pointer to Ethernet device. 4932 * @param f1 4933 * FDIR flow to compare. 4934 * @param f2 4935 * FDIR flow to compare. 4936 * 4937 * @return 4938 * Zero on match, 1 otherwise. 4939 */ 4940 static int 4941 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 4942 { 4943 if (FLOW_FDIR_CMP(f1, f2, attr) || 4944 FLOW_FDIR_CMP(f1, f2, l2) || 4945 FLOW_FDIR_CMP(f1, f2, l2_mask) || 4946 FLOW_FDIR_CMP(f1, f2, l3) || 4947 FLOW_FDIR_CMP(f1, f2, l3_mask) || 4948 FLOW_FDIR_CMP(f1, f2, l4) || 4949 FLOW_FDIR_CMP(f1, f2, l4_mask) || 4950 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 4951 return 1; 4952 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 4953 FLOW_FDIR_CMP(f1, f2, queue)) 4954 return 1; 4955 return 0; 4956 } 4957 4958 /** 4959 * Search device flow list to find out a matched FDIR flow. 4960 * 4961 * @param dev 4962 * Pointer to Ethernet device. 4963 * @param fdir_flow 4964 * FDIR flow to lookup. 4965 * 4966 * @return 4967 * Pointer of flow if found, NULL otherwise. 4968 */ 4969 static struct rte_flow * 4970 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 4971 { 4972 struct mlx5_priv *priv = dev->data->dev_private; 4973 struct rte_flow *flow = NULL; 4974 4975 assert(fdir_flow); 4976 TAILQ_FOREACH(flow, &priv->flows, next) { 4977 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) { 4978 DRV_LOG(DEBUG, "port %u found FDIR flow %p", 4979 dev->data->port_id, (void *)flow); 4980 break; 4981 } 4982 } 4983 return flow; 4984 } 4985 4986 /** 4987 * Add new flow director filter and store it in list. 4988 * 4989 * @param dev 4990 * Pointer to Ethernet device. 4991 * @param fdir_filter 4992 * Flow director filter to add. 4993 * 4994 * @return 4995 * 0 on success, a negative errno value otherwise and rte_errno is set. 4996 */ 4997 static int 4998 flow_fdir_filter_add(struct rte_eth_dev *dev, 4999 const struct rte_eth_fdir_filter *fdir_filter) 5000 { 5001 struct mlx5_priv *priv = dev->data->dev_private; 5002 struct mlx5_fdir *fdir_flow; 5003 struct rte_flow *flow; 5004 int ret; 5005 5006 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5007 if (!fdir_flow) { 5008 rte_errno = ENOMEM; 5009 return -rte_errno; 5010 } 5011 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5012 if (ret) 5013 goto error; 5014 flow = flow_fdir_filter_lookup(dev, fdir_flow); 5015 if (flow) { 5016 rte_errno = EEXIST; 5017 goto error; 5018 } 5019 flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5020 fdir_flow->items, fdir_flow->actions, true, 5021 NULL); 5022 if (!flow) 5023 goto error; 5024 assert(!flow->fdir); 5025 flow->fdir = fdir_flow; 5026 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5027 dev->data->port_id, (void *)flow); 5028 return 0; 5029 error: 5030 rte_free(fdir_flow); 5031 return -rte_errno; 5032 } 5033 5034 /** 5035 * Delete specific filter. 5036 * 5037 * @param dev 5038 * Pointer to Ethernet device. 5039 * @param fdir_filter 5040 * Filter to be deleted. 5041 * 5042 * @return 5043 * 0 on success, a negative errno value otherwise and rte_errno is set. 5044 */ 5045 static int 5046 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5047 const struct rte_eth_fdir_filter *fdir_filter) 5048 { 5049 struct mlx5_priv *priv = dev->data->dev_private; 5050 struct rte_flow *flow; 5051 struct mlx5_fdir fdir_flow = { 5052 .attr.group = 0, 5053 }; 5054 int ret; 5055 5056 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5057 if (ret) 5058 return -rte_errno; 5059 flow = flow_fdir_filter_lookup(dev, &fdir_flow); 5060 if (!flow) { 5061 rte_errno = ENOENT; 5062 return -rte_errno; 5063 } 5064 flow_list_destroy(dev, &priv->flows, flow); 5065 DRV_LOG(DEBUG, "port %u deleted FDIR flow %p", 5066 dev->data->port_id, (void *)flow); 5067 return 0; 5068 } 5069 5070 /** 5071 * Update queue for specific filter. 5072 * 5073 * @param dev 5074 * Pointer to Ethernet device. 5075 * @param fdir_filter 5076 * Filter to be updated. 5077 * 5078 * @return 5079 * 0 on success, a negative errno value otherwise and rte_errno is set. 5080 */ 5081 static int 5082 flow_fdir_filter_update(struct rte_eth_dev *dev, 5083 const struct rte_eth_fdir_filter *fdir_filter) 5084 { 5085 int ret; 5086 5087 ret = flow_fdir_filter_delete(dev, fdir_filter); 5088 if (ret) 5089 return ret; 5090 return flow_fdir_filter_add(dev, fdir_filter); 5091 } 5092 5093 /** 5094 * Flush all filters. 5095 * 5096 * @param dev 5097 * Pointer to Ethernet device. 5098 */ 5099 static void 5100 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5101 { 5102 struct mlx5_priv *priv = dev->data->dev_private; 5103 5104 mlx5_flow_list_flush(dev, &priv->flows); 5105 } 5106 5107 /** 5108 * Get flow director information. 5109 * 5110 * @param dev 5111 * Pointer to Ethernet device. 5112 * @param[out] fdir_info 5113 * Resulting flow director information. 5114 */ 5115 static void 5116 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5117 { 5118 struct rte_eth_fdir_masks *mask = 5119 &dev->data->dev_conf.fdir_conf.mask; 5120 5121 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5122 fdir_info->guarant_spc = 0; 5123 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5124 fdir_info->max_flexpayload = 0; 5125 fdir_info->flow_types_mask[0] = 0; 5126 fdir_info->flex_payload_unit = 0; 5127 fdir_info->max_flex_payload_segment_num = 0; 5128 fdir_info->flex_payload_limit = 0; 5129 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5130 } 5131 5132 /** 5133 * Deal with flow director operations. 5134 * 5135 * @param dev 5136 * Pointer to Ethernet device. 5137 * @param filter_op 5138 * Operation to perform. 5139 * @param arg 5140 * Pointer to operation-specific structure. 5141 * 5142 * @return 5143 * 0 on success, a negative errno value otherwise and rte_errno is set. 5144 */ 5145 static int 5146 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5147 void *arg) 5148 { 5149 enum rte_fdir_mode fdir_mode = 5150 dev->data->dev_conf.fdir_conf.mode; 5151 5152 if (filter_op == RTE_ETH_FILTER_NOP) 5153 return 0; 5154 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5155 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5156 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5157 dev->data->port_id, fdir_mode); 5158 rte_errno = EINVAL; 5159 return -rte_errno; 5160 } 5161 switch (filter_op) { 5162 case RTE_ETH_FILTER_ADD: 5163 return flow_fdir_filter_add(dev, arg); 5164 case RTE_ETH_FILTER_UPDATE: 5165 return flow_fdir_filter_update(dev, arg); 5166 case RTE_ETH_FILTER_DELETE: 5167 return flow_fdir_filter_delete(dev, arg); 5168 case RTE_ETH_FILTER_FLUSH: 5169 flow_fdir_filter_flush(dev); 5170 break; 5171 case RTE_ETH_FILTER_INFO: 5172 flow_fdir_info_get(dev, arg); 5173 break; 5174 default: 5175 DRV_LOG(DEBUG, "port %u unknown operation %u", 5176 dev->data->port_id, filter_op); 5177 rte_errno = EINVAL; 5178 return -rte_errno; 5179 } 5180 return 0; 5181 } 5182 5183 /** 5184 * Manage filter operations. 5185 * 5186 * @param dev 5187 * Pointer to Ethernet device structure. 5188 * @param filter_type 5189 * Filter type. 5190 * @param filter_op 5191 * Operation to perform. 5192 * @param arg 5193 * Pointer to operation-specific structure. 5194 * 5195 * @return 5196 * 0 on success, a negative errno value otherwise and rte_errno is set. 5197 */ 5198 int 5199 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5200 enum rte_filter_type filter_type, 5201 enum rte_filter_op filter_op, 5202 void *arg) 5203 { 5204 switch (filter_type) { 5205 case RTE_ETH_FILTER_GENERIC: 5206 if (filter_op != RTE_ETH_FILTER_GET) { 5207 rte_errno = EINVAL; 5208 return -rte_errno; 5209 } 5210 *(const void **)arg = &mlx5_flow_ops; 5211 return 0; 5212 case RTE_ETH_FILTER_FDIR: 5213 return flow_fdir_ctrl_func(dev, filter_op, arg); 5214 default: 5215 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5216 dev->data->port_id, filter_type); 5217 rte_errno = ENOTSUP; 5218 return -rte_errno; 5219 } 5220 return 0; 5221 } 5222 5223 /** 5224 * Create the needed meter and suffix tables. 5225 * 5226 * @param[in] dev 5227 * Pointer to Ethernet device. 5228 * @param[in] fm 5229 * Pointer to the flow meter. 5230 * 5231 * @return 5232 * Pointer to table set on success, NULL otherwise. 5233 */ 5234 struct mlx5_meter_domains_infos * 5235 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5236 const struct mlx5_flow_meter *fm) 5237 { 5238 const struct mlx5_flow_driver_ops *fops; 5239 5240 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5241 return fops->create_mtr_tbls(dev, fm); 5242 } 5243 5244 /** 5245 * Destroy the meter table set. 5246 * 5247 * @param[in] dev 5248 * Pointer to Ethernet device. 5249 * @param[in] tbl 5250 * Pointer to the meter table set. 5251 * 5252 * @return 5253 * 0 on success. 5254 */ 5255 int 5256 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5257 struct mlx5_meter_domains_infos *tbls) 5258 { 5259 const struct mlx5_flow_driver_ops *fops; 5260 5261 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5262 return fops->destroy_mtr_tbls(dev, tbls); 5263 } 5264 5265 /** 5266 * Create policer rules. 5267 * 5268 * @param[in] dev 5269 * Pointer to Ethernet device. 5270 * @param[in] fm 5271 * Pointer to flow meter structure. 5272 * @param[in] attr 5273 * Pointer to flow attributes. 5274 * 5275 * @return 5276 * 0 on success, -1 otherwise. 5277 */ 5278 int 5279 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5280 struct mlx5_flow_meter *fm, 5281 const struct rte_flow_attr *attr) 5282 { 5283 const struct mlx5_flow_driver_ops *fops; 5284 5285 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5286 return fops->create_policer_rules(dev, fm, attr); 5287 } 5288 5289 /** 5290 * Destroy policer rules. 5291 * 5292 * @param[in] fm 5293 * Pointer to flow meter structure. 5294 * @param[in] attr 5295 * Pointer to flow attributes. 5296 * 5297 * @return 5298 * 0 on success, -1 otherwise. 5299 */ 5300 int 5301 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5302 struct mlx5_flow_meter *fm, 5303 const struct rte_flow_attr *attr) 5304 { 5305 const struct mlx5_flow_driver_ops *fops; 5306 5307 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5308 return fops->destroy_policer_rules(dev, fm, attr); 5309 } 5310 5311 /** 5312 * Allocate a counter. 5313 * 5314 * @param[in] dev 5315 * Pointer to Ethernet device structure. 5316 * 5317 * @return 5318 * Pointer to allocated counter on success, NULL otherwise. 5319 */ 5320 struct mlx5_flow_counter * 5321 mlx5_counter_alloc(struct rte_eth_dev *dev) 5322 { 5323 const struct mlx5_flow_driver_ops *fops; 5324 struct rte_flow_attr attr = { .transfer = 0 }; 5325 5326 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5327 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5328 return fops->counter_alloc(dev); 5329 } 5330 DRV_LOG(ERR, 5331 "port %u counter allocate is not supported.", 5332 dev->data->port_id); 5333 return NULL; 5334 } 5335 5336 /** 5337 * Free a counter. 5338 * 5339 * @param[in] dev 5340 * Pointer to Ethernet device structure. 5341 * @param[in] cnt 5342 * Pointer to counter to be free. 5343 */ 5344 void 5345 mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) 5346 { 5347 const struct mlx5_flow_driver_ops *fops; 5348 struct rte_flow_attr attr = { .transfer = 0 }; 5349 5350 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5351 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5352 fops->counter_free(dev, cnt); 5353 return; 5354 } 5355 DRV_LOG(ERR, 5356 "port %u counter free is not supported.", 5357 dev->data->port_id); 5358 } 5359 5360 /** 5361 * Query counter statistics. 5362 * 5363 * @param[in] dev 5364 * Pointer to Ethernet device structure. 5365 * @param[in] cnt 5366 * Pointer to counter to query. 5367 * @param[in] clear 5368 * Set to clear counter statistics. 5369 * @param[out] pkts 5370 * The counter hits packets number to save. 5371 * @param[out] bytes 5372 * The counter hits bytes number to save. 5373 * 5374 * @return 5375 * 0 on success, a negative errno value otherwise. 5376 */ 5377 int 5378 mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt, 5379 bool clear, uint64_t *pkts, uint64_t *bytes) 5380 { 5381 const struct mlx5_flow_driver_ops *fops; 5382 struct rte_flow_attr attr = { .transfer = 0 }; 5383 5384 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5385 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5386 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5387 } 5388 DRV_LOG(ERR, 5389 "port %u counter query is not supported.", 5390 dev->data->port_id); 5391 return -ENOTSUP; 5392 } 5393 5394 #define MLX5_POOL_QUERY_FREQ_US 1000000 5395 5396 /** 5397 * Set the periodic procedure for triggering asynchronous batch queries for all 5398 * the counter pools. 5399 * 5400 * @param[in] sh 5401 * Pointer to mlx5_ibv_shared object. 5402 */ 5403 void 5404 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5405 { 5406 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0); 5407 uint32_t pools_n = rte_atomic16_read(&cont->n_valid); 5408 uint32_t us; 5409 5410 cont = MLX5_CNT_CONTAINER(sh, 1, 0); 5411 pools_n += rte_atomic16_read(&cont->n_valid); 5412 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5413 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5414 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5415 sh->cmng.query_thread_on = 0; 5416 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5417 } else { 5418 sh->cmng.query_thread_on = 1; 5419 } 5420 } 5421 5422 /** 5423 * The periodic procedure for triggering asynchronous batch queries for all the 5424 * counter pools. This function is probably called by the host thread. 5425 * 5426 * @param[in] arg 5427 * The parameter for the alarm process. 5428 */ 5429 void 5430 mlx5_flow_query_alarm(void *arg) 5431 { 5432 struct mlx5_ibv_shared *sh = arg; 5433 struct mlx5_devx_obj *dcs; 5434 uint16_t offset; 5435 int ret; 5436 uint8_t batch = sh->cmng.batch; 5437 uint16_t pool_index = sh->cmng.pool_index; 5438 struct mlx5_pools_container *cont; 5439 struct mlx5_pools_container *mcont; 5440 struct mlx5_flow_counter_pool *pool; 5441 5442 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5443 goto set_alarm; 5444 next_container: 5445 cont = MLX5_CNT_CONTAINER(sh, batch, 1); 5446 mcont = MLX5_CNT_CONTAINER(sh, batch, 0); 5447 /* Check if resize was done and need to flip a container. */ 5448 if (cont != mcont) { 5449 if (cont->pools) { 5450 /* Clean the old container. */ 5451 rte_free(cont->pools); 5452 memset(cont, 0, sizeof(*cont)); 5453 } 5454 rte_cio_wmb(); 5455 /* Flip the host container. */ 5456 sh->cmng.mhi[batch] ^= (uint8_t)2; 5457 cont = mcont; 5458 } 5459 if (!cont->pools) { 5460 /* 2 empty containers case is unexpected. */ 5461 if (unlikely(batch != sh->cmng.batch)) 5462 goto set_alarm; 5463 batch ^= 0x1; 5464 pool_index = 0; 5465 goto next_container; 5466 } 5467 pool = cont->pools[pool_index]; 5468 if (pool->raw_hw) 5469 /* There is a pool query in progress. */ 5470 goto set_alarm; 5471 pool->raw_hw = 5472 LIST_FIRST(&sh->cmng.free_stat_raws); 5473 if (!pool->raw_hw) 5474 /* No free counter statistics raw memory. */ 5475 goto set_alarm; 5476 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5477 (&pool->a64_dcs); 5478 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5479 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5480 offset, NULL, NULL, 5481 pool->raw_hw->mem_mng->dm->id, 5482 (void *)(uintptr_t) 5483 (pool->raw_hw->data + offset), 5484 sh->devx_comp, 5485 (uint64_t)(uintptr_t)pool); 5486 if (ret) { 5487 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5488 " %d", pool->min_dcs->id); 5489 pool->raw_hw = NULL; 5490 goto set_alarm; 5491 } 5492 pool->raw_hw->min_dcs_id = dcs->id; 5493 LIST_REMOVE(pool->raw_hw, next); 5494 sh->cmng.pending_queries++; 5495 pool_index++; 5496 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5497 batch ^= 0x1; 5498 pool_index = 0; 5499 } 5500 set_alarm: 5501 sh->cmng.batch = batch; 5502 sh->cmng.pool_index = pool_index; 5503 mlx5_set_query_alarm(sh); 5504 } 5505 5506 /** 5507 * Handler for the HW respond about ready values from an asynchronous batch 5508 * query. This function is probably called by the host thread. 5509 * 5510 * @param[in] sh 5511 * The pointer to the shared IB device context. 5512 * @param[in] async_id 5513 * The Devx async ID. 5514 * @param[in] status 5515 * The status of the completion. 5516 */ 5517 void 5518 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 5519 uint64_t async_id, int status) 5520 { 5521 struct mlx5_flow_counter_pool *pool = 5522 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 5523 struct mlx5_counter_stats_raw *raw_to_free; 5524 5525 if (unlikely(status)) { 5526 raw_to_free = pool->raw_hw; 5527 } else { 5528 raw_to_free = pool->raw; 5529 rte_spinlock_lock(&pool->sl); 5530 pool->raw = pool->raw_hw; 5531 rte_spinlock_unlock(&pool->sl); 5532 rte_atomic64_add(&pool->query_gen, 1); 5533 /* Be sure the new raw counters data is updated in memory. */ 5534 rte_cio_wmb(); 5535 } 5536 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 5537 pool->raw_hw = NULL; 5538 sh->cmng.pending_queries--; 5539 } 5540 5541 /** 5542 * Translate the rte_flow group index to HW table value. 5543 * 5544 * @param[in] attributes 5545 * Pointer to flow attributes 5546 * @param[in] external 5547 * Value is part of flow rule created by request external to PMD. 5548 * @param[in] group 5549 * rte_flow group index value. 5550 * @param[out] table 5551 * HW table value. 5552 * @param[out] error 5553 * Pointer to error structure. 5554 * 5555 * @return 5556 * 0 on success, a negative errno value otherwise and rte_errno is set. 5557 */ 5558 int 5559 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 5560 uint32_t group, uint32_t *table, 5561 struct rte_flow_error *error) 5562 { 5563 if (attributes->transfer && external) { 5564 if (group == UINT32_MAX) 5565 return rte_flow_error_set 5566 (error, EINVAL, 5567 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 5568 NULL, 5569 "group index not supported"); 5570 *table = group + 1; 5571 } else { 5572 *table = group; 5573 } 5574 return 0; 5575 } 5576 5577 /** 5578 * Discover availability of metadata reg_c's. 5579 * 5580 * Iteratively use test flows to check availability. 5581 * 5582 * @param[in] dev 5583 * Pointer to the Ethernet device structure. 5584 * 5585 * @return 5586 * 0 on success, a negative errno value otherwise and rte_errno is set. 5587 */ 5588 int 5589 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 5590 { 5591 struct mlx5_priv *priv = dev->data->dev_private; 5592 struct mlx5_dev_config *config = &priv->config; 5593 enum modify_reg idx; 5594 int n = 0; 5595 5596 /* reg_c[0] and reg_c[1] are reserved. */ 5597 config->flow_mreg_c[n++] = REG_C_0; 5598 config->flow_mreg_c[n++] = REG_C_1; 5599 /* Discover availability of other reg_c's. */ 5600 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 5601 struct rte_flow_attr attr = { 5602 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5603 .priority = MLX5_FLOW_PRIO_RSVD, 5604 .ingress = 1, 5605 }; 5606 struct rte_flow_item items[] = { 5607 [0] = { 5608 .type = RTE_FLOW_ITEM_TYPE_END, 5609 }, 5610 }; 5611 struct rte_flow_action actions[] = { 5612 [0] = { 5613 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5614 .conf = &(struct mlx5_flow_action_copy_mreg){ 5615 .src = REG_C_1, 5616 .dst = idx, 5617 }, 5618 }, 5619 [1] = { 5620 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5621 .conf = &(struct rte_flow_action_jump){ 5622 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5623 }, 5624 }, 5625 [2] = { 5626 .type = RTE_FLOW_ACTION_TYPE_END, 5627 }, 5628 }; 5629 struct rte_flow *flow; 5630 struct rte_flow_error error; 5631 5632 if (!config->dv_flow_en) 5633 break; 5634 /* Create internal flow, validation skips copy action. */ 5635 flow = flow_list_create(dev, NULL, &attr, items, 5636 actions, false, &error); 5637 if (!flow) 5638 continue; 5639 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 5640 config->flow_mreg_c[n++] = idx; 5641 flow_list_destroy(dev, NULL, flow); 5642 } 5643 for (; n < MLX5_MREG_C_NUM; ++n) 5644 config->flow_mreg_c[n] = REG_NONE; 5645 return 0; 5646 } 5647