1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 12 /* Verbs header. */ 13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 14 #ifdef PEDANTIC 15 #pragma GCC diagnostic ignored "-Wpedantic" 16 #endif 17 #include <infiniband/verbs.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_common.h> 23 #include <rte_ether.h> 24 #include <rte_ethdev_driver.h> 25 #include <rte_flow.h> 26 #include <rte_flow_driver.h> 27 #include <rte_malloc.h> 28 #include <rte_ip.h> 29 30 #include "mlx5.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_flow.h" 33 #include "mlx5_glue.h" 34 #include "mlx5_prm.h" 35 #include "mlx5_rxtx.h" 36 37 /* Dev ops structure defined in mlx5.c */ 38 extern const struct eth_dev_ops mlx5_dev_ops; 39 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 40 41 /** Device flow drivers. */ 42 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 43 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 44 #endif 45 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 46 47 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 48 49 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 50 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 51 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 52 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 53 #endif 54 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 55 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 56 }; 57 58 enum mlx5_expansion { 59 MLX5_EXPANSION_ROOT, 60 MLX5_EXPANSION_ROOT_OUTER, 61 MLX5_EXPANSION_ROOT_ETH_VLAN, 62 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 63 MLX5_EXPANSION_OUTER_ETH, 64 MLX5_EXPANSION_OUTER_ETH_VLAN, 65 MLX5_EXPANSION_OUTER_VLAN, 66 MLX5_EXPANSION_OUTER_IPV4, 67 MLX5_EXPANSION_OUTER_IPV4_UDP, 68 MLX5_EXPANSION_OUTER_IPV4_TCP, 69 MLX5_EXPANSION_OUTER_IPV6, 70 MLX5_EXPANSION_OUTER_IPV6_UDP, 71 MLX5_EXPANSION_OUTER_IPV6_TCP, 72 MLX5_EXPANSION_VXLAN, 73 MLX5_EXPANSION_VXLAN_GPE, 74 MLX5_EXPANSION_GRE, 75 MLX5_EXPANSION_MPLS, 76 MLX5_EXPANSION_ETH, 77 MLX5_EXPANSION_ETH_VLAN, 78 MLX5_EXPANSION_VLAN, 79 MLX5_EXPANSION_IPV4, 80 MLX5_EXPANSION_IPV4_UDP, 81 MLX5_EXPANSION_IPV4_TCP, 82 MLX5_EXPANSION_IPV6, 83 MLX5_EXPANSION_IPV6_UDP, 84 MLX5_EXPANSION_IPV6_TCP, 85 }; 86 87 /** Supported expansion of items. */ 88 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 89 [MLX5_EXPANSION_ROOT] = { 90 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 91 MLX5_EXPANSION_IPV4, 92 MLX5_EXPANSION_IPV6), 93 .type = RTE_FLOW_ITEM_TYPE_END, 94 }, 95 [MLX5_EXPANSION_ROOT_OUTER] = { 96 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 97 MLX5_EXPANSION_OUTER_IPV4, 98 MLX5_EXPANSION_OUTER_IPV6), 99 .type = RTE_FLOW_ITEM_TYPE_END, 100 }, 101 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 102 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 103 .type = RTE_FLOW_ITEM_TYPE_END, 104 }, 105 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 107 .type = RTE_FLOW_ITEM_TYPE_END, 108 }, 109 [MLX5_EXPANSION_OUTER_ETH] = { 110 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 111 MLX5_EXPANSION_OUTER_IPV6, 112 MLX5_EXPANSION_MPLS), 113 .type = RTE_FLOW_ITEM_TYPE_ETH, 114 .rss_types = 0, 115 }, 116 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 117 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 118 .type = RTE_FLOW_ITEM_TYPE_ETH, 119 .rss_types = 0, 120 }, 121 [MLX5_EXPANSION_OUTER_VLAN] = { 122 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 123 MLX5_EXPANSION_OUTER_IPV6), 124 .type = RTE_FLOW_ITEM_TYPE_VLAN, 125 }, 126 [MLX5_EXPANSION_OUTER_IPV4] = { 127 .next = RTE_FLOW_EXPAND_RSS_NEXT 128 (MLX5_EXPANSION_OUTER_IPV4_UDP, 129 MLX5_EXPANSION_OUTER_IPV4_TCP, 130 MLX5_EXPANSION_GRE, 131 MLX5_EXPANSION_IPV4, 132 MLX5_EXPANSION_IPV6), 133 .type = RTE_FLOW_ITEM_TYPE_IPV4, 134 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 135 ETH_RSS_NONFRAG_IPV4_OTHER, 136 }, 137 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 138 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 139 MLX5_EXPANSION_VXLAN_GPE), 140 .type = RTE_FLOW_ITEM_TYPE_UDP, 141 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 142 }, 143 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 144 .type = RTE_FLOW_ITEM_TYPE_TCP, 145 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 146 }, 147 [MLX5_EXPANSION_OUTER_IPV6] = { 148 .next = RTE_FLOW_EXPAND_RSS_NEXT 149 (MLX5_EXPANSION_OUTER_IPV6_UDP, 150 MLX5_EXPANSION_OUTER_IPV6_TCP, 151 MLX5_EXPANSION_IPV4, 152 MLX5_EXPANSION_IPV6), 153 .type = RTE_FLOW_ITEM_TYPE_IPV6, 154 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 155 ETH_RSS_NONFRAG_IPV6_OTHER, 156 }, 157 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 158 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 159 MLX5_EXPANSION_VXLAN_GPE), 160 .type = RTE_FLOW_ITEM_TYPE_UDP, 161 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 162 }, 163 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 164 .type = RTE_FLOW_ITEM_TYPE_TCP, 165 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 166 }, 167 [MLX5_EXPANSION_VXLAN] = { 168 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 169 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 170 }, 171 [MLX5_EXPANSION_VXLAN_GPE] = { 172 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 173 MLX5_EXPANSION_IPV4, 174 MLX5_EXPANSION_IPV6), 175 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 176 }, 177 [MLX5_EXPANSION_GRE] = { 178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 179 .type = RTE_FLOW_ITEM_TYPE_GRE, 180 }, 181 [MLX5_EXPANSION_MPLS] = { 182 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 183 MLX5_EXPANSION_IPV6), 184 .type = RTE_FLOW_ITEM_TYPE_MPLS, 185 }, 186 [MLX5_EXPANSION_ETH] = { 187 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 188 MLX5_EXPANSION_IPV6), 189 .type = RTE_FLOW_ITEM_TYPE_ETH, 190 }, 191 [MLX5_EXPANSION_ETH_VLAN] = { 192 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 193 .type = RTE_FLOW_ITEM_TYPE_ETH, 194 }, 195 [MLX5_EXPANSION_VLAN] = { 196 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 197 MLX5_EXPANSION_IPV6), 198 .type = RTE_FLOW_ITEM_TYPE_VLAN, 199 }, 200 [MLX5_EXPANSION_IPV4] = { 201 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 202 MLX5_EXPANSION_IPV4_TCP), 203 .type = RTE_FLOW_ITEM_TYPE_IPV4, 204 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 205 ETH_RSS_NONFRAG_IPV4_OTHER, 206 }, 207 [MLX5_EXPANSION_IPV4_UDP] = { 208 .type = RTE_FLOW_ITEM_TYPE_UDP, 209 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 210 }, 211 [MLX5_EXPANSION_IPV4_TCP] = { 212 .type = RTE_FLOW_ITEM_TYPE_TCP, 213 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 214 }, 215 [MLX5_EXPANSION_IPV6] = { 216 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 217 MLX5_EXPANSION_IPV6_TCP), 218 .type = RTE_FLOW_ITEM_TYPE_IPV6, 219 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 220 ETH_RSS_NONFRAG_IPV6_OTHER, 221 }, 222 [MLX5_EXPANSION_IPV6_UDP] = { 223 .type = RTE_FLOW_ITEM_TYPE_UDP, 224 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 225 }, 226 [MLX5_EXPANSION_IPV6_TCP] = { 227 .type = RTE_FLOW_ITEM_TYPE_TCP, 228 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 229 }, 230 }; 231 232 static const struct rte_flow_ops mlx5_flow_ops = { 233 .validate = mlx5_flow_validate, 234 .create = mlx5_flow_create, 235 .destroy = mlx5_flow_destroy, 236 .flush = mlx5_flow_flush, 237 .isolate = mlx5_flow_isolate, 238 .query = mlx5_flow_query, 239 }; 240 241 /* Convert FDIR request to Generic flow. */ 242 struct mlx5_fdir { 243 struct rte_flow_attr attr; 244 struct rte_flow_item items[4]; 245 struct rte_flow_item_eth l2; 246 struct rte_flow_item_eth l2_mask; 247 union { 248 struct rte_flow_item_ipv4 ipv4; 249 struct rte_flow_item_ipv6 ipv6; 250 } l3; 251 union { 252 struct rte_flow_item_ipv4 ipv4; 253 struct rte_flow_item_ipv6 ipv6; 254 } l3_mask; 255 union { 256 struct rte_flow_item_udp udp; 257 struct rte_flow_item_tcp tcp; 258 } l4; 259 union { 260 struct rte_flow_item_udp udp; 261 struct rte_flow_item_tcp tcp; 262 } l4_mask; 263 struct rte_flow_action actions[2]; 264 struct rte_flow_action_queue queue; 265 }; 266 267 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 268 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 269 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 270 }; 271 272 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 273 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 274 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 275 { 9, 10, 11 }, { 12, 13, 14 }, 276 }; 277 278 /* Tunnel information. */ 279 struct mlx5_flow_tunnel_info { 280 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 281 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 282 }; 283 284 static struct mlx5_flow_tunnel_info tunnels_info[] = { 285 { 286 .tunnel = MLX5_FLOW_LAYER_VXLAN, 287 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 288 }, 289 { 290 .tunnel = MLX5_FLOW_LAYER_GENEVE, 291 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 292 }, 293 { 294 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 295 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 296 }, 297 { 298 .tunnel = MLX5_FLOW_LAYER_GRE, 299 .ptype = RTE_PTYPE_TUNNEL_GRE, 300 }, 301 { 302 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 303 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 304 }, 305 { 306 .tunnel = MLX5_FLOW_LAYER_MPLS, 307 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 308 }, 309 { 310 .tunnel = MLX5_FLOW_LAYER_NVGRE, 311 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 312 }, 313 { 314 .tunnel = MLX5_FLOW_LAYER_IPIP, 315 .ptype = RTE_PTYPE_TUNNEL_IP, 316 }, 317 { 318 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 319 .ptype = RTE_PTYPE_TUNNEL_IP, 320 }, 321 }; 322 323 /** 324 * Translate tag ID to register. 325 * 326 * @param[in] dev 327 * Pointer to the Ethernet device structure. 328 * @param[in] feature 329 * The feature that request the register. 330 * @param[in] id 331 * The request register ID. 332 * @param[out] error 333 * Error description in case of any. 334 * 335 * @return 336 * The request register on success, a negative errno 337 * value otherwise and rte_errno is set. 338 */ 339 enum modify_reg 340 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 341 enum mlx5_feature_name feature, 342 uint32_t id, 343 struct rte_flow_error *error) 344 { 345 struct mlx5_priv *priv = dev->data->dev_private; 346 struct mlx5_dev_config *config = &priv->config; 347 enum modify_reg start_reg; 348 349 switch (feature) { 350 case MLX5_HAIRPIN_RX: 351 return REG_B; 352 case MLX5_HAIRPIN_TX: 353 return REG_A; 354 case MLX5_METADATA_RX: 355 switch (config->dv_xmeta_en) { 356 case MLX5_XMETA_MODE_LEGACY: 357 return REG_B; 358 case MLX5_XMETA_MODE_META16: 359 return REG_C_0; 360 case MLX5_XMETA_MODE_META32: 361 return REG_C_1; 362 } 363 break; 364 case MLX5_METADATA_TX: 365 return REG_A; 366 case MLX5_METADATA_FDB: 367 return REG_C_0; 368 case MLX5_FLOW_MARK: 369 switch (config->dv_xmeta_en) { 370 case MLX5_XMETA_MODE_LEGACY: 371 return REG_NONE; 372 case MLX5_XMETA_MODE_META16: 373 return REG_C_1; 374 case MLX5_XMETA_MODE_META32: 375 return REG_C_0; 376 } 377 break; 378 case MLX5_COPY_MARK: 379 case MLX5_MTR_SFX: 380 /* 381 * Metadata COPY_MARK register using is in meter suffix sub 382 * flow while with meter. It's safe to share the same register. 383 */ 384 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 385 case MLX5_MTR_COLOR: 386 RTE_ASSERT(priv->mtr_color_reg != REG_NONE); 387 return priv->mtr_color_reg; 388 case MLX5_APP_TAG: 389 /* 390 * If meter is enable, it will engage two registers for color 391 * match and flow match. If meter color match is not using the 392 * REG_C_2, need to skip the REG_C_x be used by meter color 393 * match. 394 * If meter is disable, free to use all available registers. 395 */ 396 if (priv->mtr_color_reg != REG_NONE) 397 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_3 : 398 REG_C_4; 399 else 400 start_reg = REG_C_2; 401 if (id > (REG_C_7 - start_reg)) 402 return rte_flow_error_set(error, EINVAL, 403 RTE_FLOW_ERROR_TYPE_ITEM, 404 NULL, "invalid tag id"); 405 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 406 return rte_flow_error_set(error, ENOTSUP, 407 RTE_FLOW_ERROR_TYPE_ITEM, 408 NULL, "unsupported tag id"); 409 /* 410 * This case means meter is using the REG_C_x great than 2. 411 * Take care not to conflict with meter color REG_C_x. 412 * If the available index REG_C_y >= REG_C_x, skip the 413 * color register. 414 */ 415 if (start_reg == REG_C_3 && config->flow_mreg_c 416 [id + REG_C_3 - REG_C_0] >= priv->mtr_color_reg) { 417 if (config->flow_mreg_c[id + 1 + REG_C_3 - REG_C_0] != 418 REG_NONE) 419 return config->flow_mreg_c 420 [id + 1 + REG_C_3 - REG_C_0]; 421 return rte_flow_error_set(error, ENOTSUP, 422 RTE_FLOW_ERROR_TYPE_ITEM, 423 NULL, "unsupported tag id"); 424 } 425 return config->flow_mreg_c[id + start_reg - REG_C_0]; 426 } 427 assert(false); 428 return rte_flow_error_set(error, EINVAL, 429 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 430 NULL, "invalid feature name"); 431 } 432 433 /** 434 * Check extensive flow metadata register support. 435 * 436 * @param dev 437 * Pointer to rte_eth_dev structure. 438 * 439 * @return 440 * True if device supports extensive flow metadata register, otherwise false. 441 */ 442 bool 443 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 444 { 445 struct mlx5_priv *priv = dev->data->dev_private; 446 struct mlx5_dev_config *config = &priv->config; 447 448 /* 449 * Having available reg_c can be regarded inclusively as supporting 450 * extensive flow metadata register, which could mean, 451 * - metadata register copy action by modify header. 452 * - 16 modify header actions is supported. 453 * - reg_c's are preserved across different domain (FDB and NIC) on 454 * packet loopback by flow lookup miss. 455 */ 456 return config->flow_mreg_c[2] != REG_NONE; 457 } 458 459 /** 460 * Discover the maximum number of priority available. 461 * 462 * @param[in] dev 463 * Pointer to the Ethernet device structure. 464 * 465 * @return 466 * number of supported flow priority on success, a negative errno 467 * value otherwise and rte_errno is set. 468 */ 469 int 470 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 471 { 472 struct mlx5_priv *priv = dev->data->dev_private; 473 struct { 474 struct ibv_flow_attr attr; 475 struct ibv_flow_spec_eth eth; 476 struct ibv_flow_spec_action_drop drop; 477 } flow_attr = { 478 .attr = { 479 .num_of_specs = 2, 480 .port = (uint8_t)priv->ibv_port, 481 }, 482 .eth = { 483 .type = IBV_FLOW_SPEC_ETH, 484 .size = sizeof(struct ibv_flow_spec_eth), 485 }, 486 .drop = { 487 .size = sizeof(struct ibv_flow_spec_action_drop), 488 .type = IBV_FLOW_SPEC_ACTION_DROP, 489 }, 490 }; 491 struct ibv_flow *flow; 492 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 493 uint16_t vprio[] = { 8, 16 }; 494 int i; 495 int priority = 0; 496 497 if (!drop) { 498 rte_errno = ENOTSUP; 499 return -rte_errno; 500 } 501 for (i = 0; i != RTE_DIM(vprio); i++) { 502 flow_attr.attr.priority = vprio[i] - 1; 503 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 504 if (!flow) 505 break; 506 claim_zero(mlx5_glue->destroy_flow(flow)); 507 priority = vprio[i]; 508 } 509 mlx5_hrxq_drop_release(dev); 510 switch (priority) { 511 case 8: 512 priority = RTE_DIM(priority_map_3); 513 break; 514 case 16: 515 priority = RTE_DIM(priority_map_5); 516 break; 517 default: 518 rte_errno = ENOTSUP; 519 DRV_LOG(ERR, 520 "port %u verbs maximum priority: %d expected 8/16", 521 dev->data->port_id, priority); 522 return -rte_errno; 523 } 524 DRV_LOG(INFO, "port %u flow maximum priority: %d", 525 dev->data->port_id, priority); 526 return priority; 527 } 528 529 /** 530 * Adjust flow priority based on the highest layer and the request priority. 531 * 532 * @param[in] dev 533 * Pointer to the Ethernet device structure. 534 * @param[in] priority 535 * The rule base priority. 536 * @param[in] subpriority 537 * The priority based on the items. 538 * 539 * @return 540 * The new priority. 541 */ 542 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 543 uint32_t subpriority) 544 { 545 uint32_t res = 0; 546 struct mlx5_priv *priv = dev->data->dev_private; 547 548 switch (priv->config.flow_prio) { 549 case RTE_DIM(priority_map_3): 550 res = priority_map_3[priority][subpriority]; 551 break; 552 case RTE_DIM(priority_map_5): 553 res = priority_map_5[priority][subpriority]; 554 break; 555 } 556 return res; 557 } 558 559 /** 560 * Verify the @p item specifications (spec, last, mask) are compatible with the 561 * NIC capabilities. 562 * 563 * @param[in] item 564 * Item specification. 565 * @param[in] mask 566 * @p item->mask or flow default bit-masks. 567 * @param[in] nic_mask 568 * Bit-masks covering supported fields by the NIC to compare with user mask. 569 * @param[in] size 570 * Bit-masks size in bytes. 571 * @param[out] error 572 * Pointer to error structure. 573 * 574 * @return 575 * 0 on success, a negative errno value otherwise and rte_errno is set. 576 */ 577 int 578 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 579 const uint8_t *mask, 580 const uint8_t *nic_mask, 581 unsigned int size, 582 struct rte_flow_error *error) 583 { 584 unsigned int i; 585 586 assert(nic_mask); 587 for (i = 0; i < size; ++i) 588 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 589 return rte_flow_error_set(error, ENOTSUP, 590 RTE_FLOW_ERROR_TYPE_ITEM, 591 item, 592 "mask enables non supported" 593 " bits"); 594 if (!item->spec && (item->mask || item->last)) 595 return rte_flow_error_set(error, EINVAL, 596 RTE_FLOW_ERROR_TYPE_ITEM, item, 597 "mask/last without a spec is not" 598 " supported"); 599 if (item->spec && item->last) { 600 uint8_t spec[size]; 601 uint8_t last[size]; 602 unsigned int i; 603 int ret; 604 605 for (i = 0; i < size; ++i) { 606 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 607 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 608 } 609 ret = memcmp(spec, last, size); 610 if (ret != 0) 611 return rte_flow_error_set(error, EINVAL, 612 RTE_FLOW_ERROR_TYPE_ITEM, 613 item, 614 "range is not valid"); 615 } 616 return 0; 617 } 618 619 /** 620 * Adjust the hash fields according to the @p flow information. 621 * 622 * @param[in] dev_flow. 623 * Pointer to the mlx5_flow. 624 * @param[in] tunnel 625 * 1 when the hash field is for a tunnel item. 626 * @param[in] layer_types 627 * ETH_RSS_* types. 628 * @param[in] hash_fields 629 * Item hash fields. 630 * 631 * @return 632 * The hash fields that should be used. 633 */ 634 uint64_t 635 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, 636 int tunnel __rte_unused, uint64_t layer_types, 637 uint64_t hash_fields) 638 { 639 struct rte_flow *flow = dev_flow->flow; 640 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 641 int rss_request_inner = flow->rss.level >= 2; 642 643 /* Check RSS hash level for tunnel. */ 644 if (tunnel && rss_request_inner) 645 hash_fields |= IBV_RX_HASH_INNER; 646 else if (tunnel || rss_request_inner) 647 return 0; 648 #endif 649 /* Check if requested layer matches RSS hash fields. */ 650 if (!(flow->rss.types & layer_types)) 651 return 0; 652 return hash_fields; 653 } 654 655 /** 656 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 657 * if several tunnel rules are used on this queue, the tunnel ptype will be 658 * cleared. 659 * 660 * @param rxq_ctrl 661 * Rx queue to update. 662 */ 663 static void 664 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 665 { 666 unsigned int i; 667 uint32_t tunnel_ptype = 0; 668 669 /* Look up for the ptype to use. */ 670 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 671 if (!rxq_ctrl->flow_tunnels_n[i]) 672 continue; 673 if (!tunnel_ptype) { 674 tunnel_ptype = tunnels_info[i].ptype; 675 } else { 676 tunnel_ptype = 0; 677 break; 678 } 679 } 680 rxq_ctrl->rxq.tunnel = tunnel_ptype; 681 } 682 683 /** 684 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 685 * flow. 686 * 687 * @param[in] dev 688 * Pointer to the Ethernet device structure. 689 * @param[in] dev_flow 690 * Pointer to device flow structure. 691 */ 692 static void 693 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 694 { 695 struct mlx5_priv *priv = dev->data->dev_private; 696 struct rte_flow *flow = dev_flow->flow; 697 const int mark = !!(dev_flow->actions & 698 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 699 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 700 unsigned int i; 701 702 for (i = 0; i != flow->rss.queue_num; ++i) { 703 int idx = (*flow->rss.queue)[i]; 704 struct mlx5_rxq_ctrl *rxq_ctrl = 705 container_of((*priv->rxqs)[idx], 706 struct mlx5_rxq_ctrl, rxq); 707 708 /* 709 * To support metadata register copy on Tx loopback, 710 * this must be always enabled (metadata may arive 711 * from other port - not from local flows only. 712 */ 713 if (priv->config.dv_flow_en && 714 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 715 mlx5_flow_ext_mreg_supported(dev)) { 716 rxq_ctrl->rxq.mark = 1; 717 rxq_ctrl->flow_mark_n = 1; 718 } else if (mark) { 719 rxq_ctrl->rxq.mark = 1; 720 rxq_ctrl->flow_mark_n++; 721 } 722 if (tunnel) { 723 unsigned int j; 724 725 /* Increase the counter matching the flow. */ 726 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 727 if ((tunnels_info[j].tunnel & 728 dev_flow->layers) == 729 tunnels_info[j].tunnel) { 730 rxq_ctrl->flow_tunnels_n[j]++; 731 break; 732 } 733 } 734 flow_rxq_tunnel_ptype_update(rxq_ctrl); 735 } 736 } 737 } 738 739 /** 740 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 741 * 742 * @param[in] dev 743 * Pointer to the Ethernet device structure. 744 * @param[in] flow 745 * Pointer to flow structure. 746 */ 747 static void 748 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 749 { 750 struct mlx5_flow *dev_flow; 751 752 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 753 flow_drv_rxq_flags_set(dev, dev_flow); 754 } 755 756 /** 757 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 758 * device flow if no other flow uses it with the same kind of request. 759 * 760 * @param dev 761 * Pointer to Ethernet device. 762 * @param[in] dev_flow 763 * Pointer to the device flow. 764 */ 765 static void 766 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 767 { 768 struct mlx5_priv *priv = dev->data->dev_private; 769 struct rte_flow *flow = dev_flow->flow; 770 const int mark = !!(dev_flow->actions & 771 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 772 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 773 unsigned int i; 774 775 assert(dev->data->dev_started); 776 for (i = 0; i != flow->rss.queue_num; ++i) { 777 int idx = (*flow->rss.queue)[i]; 778 struct mlx5_rxq_ctrl *rxq_ctrl = 779 container_of((*priv->rxqs)[idx], 780 struct mlx5_rxq_ctrl, rxq); 781 782 if (priv->config.dv_flow_en && 783 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 784 mlx5_flow_ext_mreg_supported(dev)) { 785 rxq_ctrl->rxq.mark = 1; 786 rxq_ctrl->flow_mark_n = 1; 787 } else if (mark) { 788 rxq_ctrl->flow_mark_n--; 789 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 790 } 791 if (tunnel) { 792 unsigned int j; 793 794 /* Decrease the counter matching the flow. */ 795 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 796 if ((tunnels_info[j].tunnel & 797 dev_flow->layers) == 798 tunnels_info[j].tunnel) { 799 rxq_ctrl->flow_tunnels_n[j]--; 800 break; 801 } 802 } 803 flow_rxq_tunnel_ptype_update(rxq_ctrl); 804 } 805 } 806 } 807 808 /** 809 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 810 * @p flow if no other flow uses it with the same kind of request. 811 * 812 * @param dev 813 * Pointer to Ethernet device. 814 * @param[in] flow 815 * Pointer to the flow. 816 */ 817 static void 818 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 819 { 820 struct mlx5_flow *dev_flow; 821 822 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 823 flow_drv_rxq_flags_trim(dev, dev_flow); 824 } 825 826 /** 827 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 828 * 829 * @param dev 830 * Pointer to Ethernet device. 831 */ 832 static void 833 flow_rxq_flags_clear(struct rte_eth_dev *dev) 834 { 835 struct mlx5_priv *priv = dev->data->dev_private; 836 unsigned int i; 837 838 for (i = 0; i != priv->rxqs_n; ++i) { 839 struct mlx5_rxq_ctrl *rxq_ctrl; 840 unsigned int j; 841 842 if (!(*priv->rxqs)[i]) 843 continue; 844 rxq_ctrl = container_of((*priv->rxqs)[i], 845 struct mlx5_rxq_ctrl, rxq); 846 rxq_ctrl->flow_mark_n = 0; 847 rxq_ctrl->rxq.mark = 0; 848 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 849 rxq_ctrl->flow_tunnels_n[j] = 0; 850 rxq_ctrl->rxq.tunnel = 0; 851 } 852 } 853 854 /* 855 * return a pointer to the desired action in the list of actions. 856 * 857 * @param[in] actions 858 * The list of actions to search the action in. 859 * @param[in] action 860 * The action to find. 861 * 862 * @return 863 * Pointer to the action in the list, if found. NULL otherwise. 864 */ 865 const struct rte_flow_action * 866 mlx5_flow_find_action(const struct rte_flow_action *actions, 867 enum rte_flow_action_type action) 868 { 869 if (actions == NULL) 870 return NULL; 871 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 872 if (actions->type == action) 873 return actions; 874 return NULL; 875 } 876 877 /* 878 * Validate the flag action. 879 * 880 * @param[in] action_flags 881 * Bit-fields that holds the actions detected until now. 882 * @param[in] attr 883 * Attributes of flow that includes this action. 884 * @param[out] error 885 * Pointer to error structure. 886 * 887 * @return 888 * 0 on success, a negative errno value otherwise and rte_errno is set. 889 */ 890 int 891 mlx5_flow_validate_action_flag(uint64_t action_flags, 892 const struct rte_flow_attr *attr, 893 struct rte_flow_error *error) 894 { 895 896 if (action_flags & MLX5_FLOW_ACTION_DROP) 897 return rte_flow_error_set(error, EINVAL, 898 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 899 "can't drop and flag in same flow"); 900 if (action_flags & MLX5_FLOW_ACTION_MARK) 901 return rte_flow_error_set(error, EINVAL, 902 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 903 "can't mark and flag in same flow"); 904 if (action_flags & MLX5_FLOW_ACTION_FLAG) 905 return rte_flow_error_set(error, EINVAL, 906 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 907 "can't have 2 flag" 908 " actions in same flow"); 909 if (attr->egress) 910 return rte_flow_error_set(error, ENOTSUP, 911 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 912 "flag action not supported for " 913 "egress"); 914 return 0; 915 } 916 917 /* 918 * Validate the mark action. 919 * 920 * @param[in] action 921 * Pointer to the queue action. 922 * @param[in] action_flags 923 * Bit-fields that holds the actions detected until now. 924 * @param[in] attr 925 * Attributes of flow that includes this action. 926 * @param[out] error 927 * Pointer to error structure. 928 * 929 * @return 930 * 0 on success, a negative errno value otherwise and rte_errno is set. 931 */ 932 int 933 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 934 uint64_t action_flags, 935 const struct rte_flow_attr *attr, 936 struct rte_flow_error *error) 937 { 938 const struct rte_flow_action_mark *mark = action->conf; 939 940 if (!mark) 941 return rte_flow_error_set(error, EINVAL, 942 RTE_FLOW_ERROR_TYPE_ACTION, 943 action, 944 "configuration cannot be null"); 945 if (mark->id >= MLX5_FLOW_MARK_MAX) 946 return rte_flow_error_set(error, EINVAL, 947 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 948 &mark->id, 949 "mark id must in 0 <= id < " 950 RTE_STR(MLX5_FLOW_MARK_MAX)); 951 if (action_flags & MLX5_FLOW_ACTION_DROP) 952 return rte_flow_error_set(error, EINVAL, 953 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 954 "can't drop and mark in same flow"); 955 if (action_flags & MLX5_FLOW_ACTION_FLAG) 956 return rte_flow_error_set(error, EINVAL, 957 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 958 "can't flag and mark in same flow"); 959 if (action_flags & MLX5_FLOW_ACTION_MARK) 960 return rte_flow_error_set(error, EINVAL, 961 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 962 "can't have 2 mark actions in same" 963 " flow"); 964 if (attr->egress) 965 return rte_flow_error_set(error, ENOTSUP, 966 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 967 "mark action not supported for " 968 "egress"); 969 return 0; 970 } 971 972 /* 973 * Validate the drop action. 974 * 975 * @param[in] action_flags 976 * Bit-fields that holds the actions detected until now. 977 * @param[in] attr 978 * Attributes of flow that includes this action. 979 * @param[out] error 980 * Pointer to error structure. 981 * 982 * @return 983 * 0 on success, a negative errno value otherwise and rte_errno is set. 984 */ 985 int 986 mlx5_flow_validate_action_drop(uint64_t action_flags, 987 const struct rte_flow_attr *attr, 988 struct rte_flow_error *error) 989 { 990 if (action_flags & MLX5_FLOW_ACTION_FLAG) 991 return rte_flow_error_set(error, EINVAL, 992 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 993 "can't drop and flag in same flow"); 994 if (action_flags & MLX5_FLOW_ACTION_MARK) 995 return rte_flow_error_set(error, EINVAL, 996 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 997 "can't drop and mark in same flow"); 998 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 999 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 1000 return rte_flow_error_set(error, EINVAL, 1001 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1002 "can't have 2 fate actions in" 1003 " same flow"); 1004 if (attr->egress) 1005 return rte_flow_error_set(error, ENOTSUP, 1006 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1007 "drop action not supported for " 1008 "egress"); 1009 return 0; 1010 } 1011 1012 /* 1013 * Validate the queue action. 1014 * 1015 * @param[in] action 1016 * Pointer to the queue action. 1017 * @param[in] action_flags 1018 * Bit-fields that holds the actions detected until now. 1019 * @param[in] dev 1020 * Pointer to the Ethernet device structure. 1021 * @param[in] attr 1022 * Attributes of flow that includes this action. 1023 * @param[out] error 1024 * Pointer to error structure. 1025 * 1026 * @return 1027 * 0 on success, a negative errno value otherwise and rte_errno is set. 1028 */ 1029 int 1030 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1031 uint64_t action_flags, 1032 struct rte_eth_dev *dev, 1033 const struct rte_flow_attr *attr, 1034 struct rte_flow_error *error) 1035 { 1036 struct mlx5_priv *priv = dev->data->dev_private; 1037 const struct rte_flow_action_queue *queue = action->conf; 1038 1039 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1040 return rte_flow_error_set(error, EINVAL, 1041 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1042 "can't have 2 fate actions in" 1043 " same flow"); 1044 if (!priv->rxqs_n) 1045 return rte_flow_error_set(error, EINVAL, 1046 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1047 NULL, "No Rx queues configured"); 1048 if (queue->index >= priv->rxqs_n) 1049 return rte_flow_error_set(error, EINVAL, 1050 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1051 &queue->index, 1052 "queue index out of range"); 1053 if (!(*priv->rxqs)[queue->index]) 1054 return rte_flow_error_set(error, EINVAL, 1055 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1056 &queue->index, 1057 "queue is not configured"); 1058 if (attr->egress) 1059 return rte_flow_error_set(error, ENOTSUP, 1060 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1061 "queue action not supported for " 1062 "egress"); 1063 return 0; 1064 } 1065 1066 /* 1067 * Validate the rss action. 1068 * 1069 * @param[in] action 1070 * Pointer to the queue action. 1071 * @param[in] action_flags 1072 * Bit-fields that holds the actions detected until now. 1073 * @param[in] dev 1074 * Pointer to the Ethernet device structure. 1075 * @param[in] attr 1076 * Attributes of flow that includes this action. 1077 * @param[in] item_flags 1078 * Items that were detected. 1079 * @param[out] error 1080 * Pointer to error structure. 1081 * 1082 * @return 1083 * 0 on success, a negative errno value otherwise and rte_errno is set. 1084 */ 1085 int 1086 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1087 uint64_t action_flags, 1088 struct rte_eth_dev *dev, 1089 const struct rte_flow_attr *attr, 1090 uint64_t item_flags, 1091 struct rte_flow_error *error) 1092 { 1093 struct mlx5_priv *priv = dev->data->dev_private; 1094 const struct rte_flow_action_rss *rss = action->conf; 1095 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1096 unsigned int i; 1097 1098 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1099 return rte_flow_error_set(error, EINVAL, 1100 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1101 "can't have 2 fate actions" 1102 " in same flow"); 1103 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1104 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1105 return rte_flow_error_set(error, ENOTSUP, 1106 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1107 &rss->func, 1108 "RSS hash function not supported"); 1109 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1110 if (rss->level > 2) 1111 #else 1112 if (rss->level > 1) 1113 #endif 1114 return rte_flow_error_set(error, ENOTSUP, 1115 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1116 &rss->level, 1117 "tunnel RSS is not supported"); 1118 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1119 if (rss->key_len == 0 && rss->key != NULL) 1120 return rte_flow_error_set(error, ENOTSUP, 1121 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1122 &rss->key_len, 1123 "RSS hash key length 0"); 1124 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1125 return rte_flow_error_set(error, ENOTSUP, 1126 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1127 &rss->key_len, 1128 "RSS hash key too small"); 1129 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1130 return rte_flow_error_set(error, ENOTSUP, 1131 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1132 &rss->key_len, 1133 "RSS hash key too large"); 1134 if (rss->queue_num > priv->config.ind_table_max_size) 1135 return rte_flow_error_set(error, ENOTSUP, 1136 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1137 &rss->queue_num, 1138 "number of queues too large"); 1139 if (rss->types & MLX5_RSS_HF_MASK) 1140 return rte_flow_error_set(error, ENOTSUP, 1141 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1142 &rss->types, 1143 "some RSS protocols are not" 1144 " supported"); 1145 if (!priv->rxqs_n) 1146 return rte_flow_error_set(error, EINVAL, 1147 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1148 NULL, "No Rx queues configured"); 1149 if (!rss->queue_num) 1150 return rte_flow_error_set(error, EINVAL, 1151 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1152 NULL, "No queues configured"); 1153 for (i = 0; i != rss->queue_num; ++i) { 1154 if (!(*priv->rxqs)[rss->queue[i]]) 1155 return rte_flow_error_set 1156 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1157 &rss->queue[i], "queue is not configured"); 1158 } 1159 if (attr->egress) 1160 return rte_flow_error_set(error, ENOTSUP, 1161 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1162 "rss action not supported for " 1163 "egress"); 1164 if (rss->level > 1 && !tunnel) 1165 return rte_flow_error_set(error, EINVAL, 1166 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1167 "inner RSS is not supported for " 1168 "non-tunnel flows"); 1169 return 0; 1170 } 1171 1172 /* 1173 * Validate the count action. 1174 * 1175 * @param[in] dev 1176 * Pointer to the Ethernet device structure. 1177 * @param[in] attr 1178 * Attributes of flow that includes this action. 1179 * @param[out] error 1180 * Pointer to error structure. 1181 * 1182 * @return 1183 * 0 on success, a negative errno value otherwise and rte_errno is set. 1184 */ 1185 int 1186 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1187 const struct rte_flow_attr *attr, 1188 struct rte_flow_error *error) 1189 { 1190 if (attr->egress) 1191 return rte_flow_error_set(error, ENOTSUP, 1192 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1193 "count action not supported for " 1194 "egress"); 1195 return 0; 1196 } 1197 1198 /** 1199 * Verify the @p attributes will be correctly understood by the NIC and store 1200 * them in the @p flow if everything is correct. 1201 * 1202 * @param[in] dev 1203 * Pointer to the Ethernet device structure. 1204 * @param[in] attributes 1205 * Pointer to flow attributes 1206 * @param[out] error 1207 * Pointer to error structure. 1208 * 1209 * @return 1210 * 0 on success, a negative errno value otherwise and rte_errno is set. 1211 */ 1212 int 1213 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1214 const struct rte_flow_attr *attributes, 1215 struct rte_flow_error *error) 1216 { 1217 struct mlx5_priv *priv = dev->data->dev_private; 1218 uint32_t priority_max = priv->config.flow_prio - 1; 1219 1220 if (attributes->group) 1221 return rte_flow_error_set(error, ENOTSUP, 1222 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1223 NULL, "groups is not supported"); 1224 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1225 attributes->priority >= priority_max) 1226 return rte_flow_error_set(error, ENOTSUP, 1227 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1228 NULL, "priority out of range"); 1229 if (attributes->egress) 1230 return rte_flow_error_set(error, ENOTSUP, 1231 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1232 "egress is not supported"); 1233 if (attributes->transfer && !priv->config.dv_esw_en) 1234 return rte_flow_error_set(error, ENOTSUP, 1235 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1236 NULL, "transfer is not supported"); 1237 if (!attributes->ingress) 1238 return rte_flow_error_set(error, EINVAL, 1239 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1240 NULL, 1241 "ingress attribute is mandatory"); 1242 return 0; 1243 } 1244 1245 /** 1246 * Validate ICMP6 item. 1247 * 1248 * @param[in] item 1249 * Item specification. 1250 * @param[in] item_flags 1251 * Bit-fields that holds the items detected until now. 1252 * @param[out] error 1253 * Pointer to error structure. 1254 * 1255 * @return 1256 * 0 on success, a negative errno value otherwise and rte_errno is set. 1257 */ 1258 int 1259 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1260 uint64_t item_flags, 1261 uint8_t target_protocol, 1262 struct rte_flow_error *error) 1263 { 1264 const struct rte_flow_item_icmp6 *mask = item->mask; 1265 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1266 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1267 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1268 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1269 MLX5_FLOW_LAYER_OUTER_L4; 1270 int ret; 1271 1272 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1273 return rte_flow_error_set(error, EINVAL, 1274 RTE_FLOW_ERROR_TYPE_ITEM, item, 1275 "protocol filtering not compatible" 1276 " with ICMP6 layer"); 1277 if (!(item_flags & l3m)) 1278 return rte_flow_error_set(error, EINVAL, 1279 RTE_FLOW_ERROR_TYPE_ITEM, item, 1280 "IPv6 is mandatory to filter on" 1281 " ICMP6"); 1282 if (item_flags & l4m) 1283 return rte_flow_error_set(error, EINVAL, 1284 RTE_FLOW_ERROR_TYPE_ITEM, item, 1285 "multiple L4 layers not supported"); 1286 if (!mask) 1287 mask = &rte_flow_item_icmp6_mask; 1288 ret = mlx5_flow_item_acceptable 1289 (item, (const uint8_t *)mask, 1290 (const uint8_t *)&rte_flow_item_icmp6_mask, 1291 sizeof(struct rte_flow_item_icmp6), error); 1292 if (ret < 0) 1293 return ret; 1294 return 0; 1295 } 1296 1297 /** 1298 * Validate ICMP item. 1299 * 1300 * @param[in] item 1301 * Item specification. 1302 * @param[in] item_flags 1303 * Bit-fields that holds the items detected until now. 1304 * @param[out] error 1305 * Pointer to error structure. 1306 * 1307 * @return 1308 * 0 on success, a negative errno value otherwise and rte_errno is set. 1309 */ 1310 int 1311 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1312 uint64_t item_flags, 1313 uint8_t target_protocol, 1314 struct rte_flow_error *error) 1315 { 1316 const struct rte_flow_item_icmp *mask = item->mask; 1317 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1318 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1319 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1320 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1321 MLX5_FLOW_LAYER_OUTER_L4; 1322 int ret; 1323 1324 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1325 return rte_flow_error_set(error, EINVAL, 1326 RTE_FLOW_ERROR_TYPE_ITEM, item, 1327 "protocol filtering not compatible" 1328 " with ICMP layer"); 1329 if (!(item_flags & l3m)) 1330 return rte_flow_error_set(error, EINVAL, 1331 RTE_FLOW_ERROR_TYPE_ITEM, item, 1332 "IPv4 is mandatory to filter" 1333 " on ICMP"); 1334 if (item_flags & l4m) 1335 return rte_flow_error_set(error, EINVAL, 1336 RTE_FLOW_ERROR_TYPE_ITEM, item, 1337 "multiple L4 layers not supported"); 1338 if (!mask) 1339 mask = &rte_flow_item_icmp_mask; 1340 ret = mlx5_flow_item_acceptable 1341 (item, (const uint8_t *)mask, 1342 (const uint8_t *)&rte_flow_item_icmp_mask, 1343 sizeof(struct rte_flow_item_icmp), error); 1344 if (ret < 0) 1345 return ret; 1346 return 0; 1347 } 1348 1349 /** 1350 * Validate Ethernet item. 1351 * 1352 * @param[in] item 1353 * Item specification. 1354 * @param[in] item_flags 1355 * Bit-fields that holds the items detected until now. 1356 * @param[out] error 1357 * Pointer to error structure. 1358 * 1359 * @return 1360 * 0 on success, a negative errno value otherwise and rte_errno is set. 1361 */ 1362 int 1363 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1364 uint64_t item_flags, 1365 struct rte_flow_error *error) 1366 { 1367 const struct rte_flow_item_eth *mask = item->mask; 1368 const struct rte_flow_item_eth nic_mask = { 1369 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1370 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1371 .type = RTE_BE16(0xffff), 1372 }; 1373 int ret; 1374 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1375 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1376 MLX5_FLOW_LAYER_OUTER_L2; 1377 1378 if (item_flags & ethm) 1379 return rte_flow_error_set(error, ENOTSUP, 1380 RTE_FLOW_ERROR_TYPE_ITEM, item, 1381 "multiple L2 layers not supported"); 1382 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1383 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1384 return rte_flow_error_set(error, EINVAL, 1385 RTE_FLOW_ERROR_TYPE_ITEM, item, 1386 "L2 layer should not follow " 1387 "L3 layers"); 1388 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1389 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1390 return rte_flow_error_set(error, EINVAL, 1391 RTE_FLOW_ERROR_TYPE_ITEM, item, 1392 "L2 layer should not follow VLAN"); 1393 if (!mask) 1394 mask = &rte_flow_item_eth_mask; 1395 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1396 (const uint8_t *)&nic_mask, 1397 sizeof(struct rte_flow_item_eth), 1398 error); 1399 return ret; 1400 } 1401 1402 /** 1403 * Validate VLAN item. 1404 * 1405 * @param[in] item 1406 * Item specification. 1407 * @param[in] item_flags 1408 * Bit-fields that holds the items detected until now. 1409 * @param[in] dev 1410 * Ethernet device flow is being created on. 1411 * @param[out] error 1412 * Pointer to error structure. 1413 * 1414 * @return 1415 * 0 on success, a negative errno value otherwise and rte_errno is set. 1416 */ 1417 int 1418 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1419 uint64_t item_flags, 1420 struct rte_eth_dev *dev, 1421 struct rte_flow_error *error) 1422 { 1423 const struct rte_flow_item_vlan *spec = item->spec; 1424 const struct rte_flow_item_vlan *mask = item->mask; 1425 const struct rte_flow_item_vlan nic_mask = { 1426 .tci = RTE_BE16(UINT16_MAX), 1427 .inner_type = RTE_BE16(UINT16_MAX), 1428 }; 1429 uint16_t vlan_tag = 0; 1430 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1431 int ret; 1432 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1433 MLX5_FLOW_LAYER_INNER_L4) : 1434 (MLX5_FLOW_LAYER_OUTER_L3 | 1435 MLX5_FLOW_LAYER_OUTER_L4); 1436 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1437 MLX5_FLOW_LAYER_OUTER_VLAN; 1438 1439 if (item_flags & vlanm) 1440 return rte_flow_error_set(error, EINVAL, 1441 RTE_FLOW_ERROR_TYPE_ITEM, item, 1442 "multiple VLAN layers not supported"); 1443 else if ((item_flags & l34m) != 0) 1444 return rte_flow_error_set(error, EINVAL, 1445 RTE_FLOW_ERROR_TYPE_ITEM, item, 1446 "VLAN cannot follow L3/L4 layer"); 1447 if (!mask) 1448 mask = &rte_flow_item_vlan_mask; 1449 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1450 (const uint8_t *)&nic_mask, 1451 sizeof(struct rte_flow_item_vlan), 1452 error); 1453 if (ret) 1454 return ret; 1455 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1456 struct mlx5_priv *priv = dev->data->dev_private; 1457 1458 if (priv->vmwa_context) { 1459 /* 1460 * Non-NULL context means we have a virtual machine 1461 * and SR-IOV enabled, we have to create VLAN interface 1462 * to make hypervisor to setup E-Switch vport 1463 * context correctly. We avoid creating the multiple 1464 * VLAN interfaces, so we cannot support VLAN tag mask. 1465 */ 1466 return rte_flow_error_set(error, EINVAL, 1467 RTE_FLOW_ERROR_TYPE_ITEM, 1468 item, 1469 "VLAN tag mask is not" 1470 " supported in virtual" 1471 " environment"); 1472 } 1473 } 1474 if (spec) { 1475 vlan_tag = spec->tci; 1476 vlan_tag &= mask->tci; 1477 } 1478 /* 1479 * From verbs perspective an empty VLAN is equivalent 1480 * to a packet without VLAN layer. 1481 */ 1482 if (!vlan_tag) 1483 return rte_flow_error_set(error, EINVAL, 1484 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1485 item->spec, 1486 "VLAN cannot be empty"); 1487 return 0; 1488 } 1489 1490 /** 1491 * Validate IPV4 item. 1492 * 1493 * @param[in] item 1494 * Item specification. 1495 * @param[in] item_flags 1496 * Bit-fields that holds the items detected until now. 1497 * @param[in] acc_mask 1498 * Acceptable mask, if NULL default internal default mask 1499 * will be used to check whether item fields are supported. 1500 * @param[out] error 1501 * Pointer to error structure. 1502 * 1503 * @return 1504 * 0 on success, a negative errno value otherwise and rte_errno is set. 1505 */ 1506 int 1507 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1508 uint64_t item_flags, 1509 uint64_t last_item, 1510 uint16_t ether_type, 1511 const struct rte_flow_item_ipv4 *acc_mask, 1512 struct rte_flow_error *error) 1513 { 1514 const struct rte_flow_item_ipv4 *mask = item->mask; 1515 const struct rte_flow_item_ipv4 *spec = item->spec; 1516 const struct rte_flow_item_ipv4 nic_mask = { 1517 .hdr = { 1518 .src_addr = RTE_BE32(0xffffffff), 1519 .dst_addr = RTE_BE32(0xffffffff), 1520 .type_of_service = 0xff, 1521 .next_proto_id = 0xff, 1522 }, 1523 }; 1524 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1525 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1526 MLX5_FLOW_LAYER_OUTER_L3; 1527 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1528 MLX5_FLOW_LAYER_OUTER_L4; 1529 int ret; 1530 uint8_t next_proto = 0xFF; 1531 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1532 MLX5_FLOW_LAYER_OUTER_VLAN | 1533 MLX5_FLOW_LAYER_INNER_VLAN); 1534 1535 if ((last_item & l2_vlan) && ether_type && 1536 ether_type != RTE_ETHER_TYPE_IPV4) 1537 return rte_flow_error_set(error, EINVAL, 1538 RTE_FLOW_ERROR_TYPE_ITEM, item, 1539 "IPv4 cannot follow L2/VLAN layer " 1540 "which ether type is not IPv4"); 1541 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1542 if (mask && spec) 1543 next_proto = mask->hdr.next_proto_id & 1544 spec->hdr.next_proto_id; 1545 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1546 return rte_flow_error_set(error, EINVAL, 1547 RTE_FLOW_ERROR_TYPE_ITEM, 1548 item, 1549 "multiple tunnel " 1550 "not supported"); 1551 } 1552 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1553 return rte_flow_error_set(error, EINVAL, 1554 RTE_FLOW_ERROR_TYPE_ITEM, item, 1555 "wrong tunnel type - IPv6 specified " 1556 "but IPv4 item provided"); 1557 if (item_flags & l3m) 1558 return rte_flow_error_set(error, ENOTSUP, 1559 RTE_FLOW_ERROR_TYPE_ITEM, item, 1560 "multiple L3 layers not supported"); 1561 else if (item_flags & l4m) 1562 return rte_flow_error_set(error, EINVAL, 1563 RTE_FLOW_ERROR_TYPE_ITEM, item, 1564 "L3 cannot follow an L4 layer."); 1565 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1566 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1567 return rte_flow_error_set(error, EINVAL, 1568 RTE_FLOW_ERROR_TYPE_ITEM, item, 1569 "L3 cannot follow an NVGRE layer."); 1570 if (!mask) 1571 mask = &rte_flow_item_ipv4_mask; 1572 else if (mask->hdr.next_proto_id != 0 && 1573 mask->hdr.next_proto_id != 0xff) 1574 return rte_flow_error_set(error, EINVAL, 1575 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1576 "partial mask is not supported" 1577 " for protocol"); 1578 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1579 acc_mask ? (const uint8_t *)acc_mask 1580 : (const uint8_t *)&nic_mask, 1581 sizeof(struct rte_flow_item_ipv4), 1582 error); 1583 if (ret < 0) 1584 return ret; 1585 return 0; 1586 } 1587 1588 /** 1589 * Validate IPV6 item. 1590 * 1591 * @param[in] item 1592 * Item specification. 1593 * @param[in] item_flags 1594 * Bit-fields that holds the items detected until now. 1595 * @param[in] acc_mask 1596 * Acceptable mask, if NULL default internal default mask 1597 * will be used to check whether item fields are supported. 1598 * @param[out] error 1599 * Pointer to error structure. 1600 * 1601 * @return 1602 * 0 on success, a negative errno value otherwise and rte_errno is set. 1603 */ 1604 int 1605 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1606 uint64_t item_flags, 1607 uint64_t last_item, 1608 uint16_t ether_type, 1609 const struct rte_flow_item_ipv6 *acc_mask, 1610 struct rte_flow_error *error) 1611 { 1612 const struct rte_flow_item_ipv6 *mask = item->mask; 1613 const struct rte_flow_item_ipv6 *spec = item->spec; 1614 const struct rte_flow_item_ipv6 nic_mask = { 1615 .hdr = { 1616 .src_addr = 1617 "\xff\xff\xff\xff\xff\xff\xff\xff" 1618 "\xff\xff\xff\xff\xff\xff\xff\xff", 1619 .dst_addr = 1620 "\xff\xff\xff\xff\xff\xff\xff\xff" 1621 "\xff\xff\xff\xff\xff\xff\xff\xff", 1622 .vtc_flow = RTE_BE32(0xffffffff), 1623 .proto = 0xff, 1624 .hop_limits = 0xff, 1625 }, 1626 }; 1627 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1628 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1629 MLX5_FLOW_LAYER_OUTER_L3; 1630 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1631 MLX5_FLOW_LAYER_OUTER_L4; 1632 int ret; 1633 uint8_t next_proto = 0xFF; 1634 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1635 MLX5_FLOW_LAYER_OUTER_VLAN | 1636 MLX5_FLOW_LAYER_INNER_VLAN); 1637 1638 if ((last_item & l2_vlan) && ether_type && 1639 ether_type != RTE_ETHER_TYPE_IPV6) 1640 return rte_flow_error_set(error, EINVAL, 1641 RTE_FLOW_ERROR_TYPE_ITEM, item, 1642 "IPv6 cannot follow L2/VLAN layer " 1643 "which ether type is not IPv6"); 1644 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1645 if (mask && spec) 1646 next_proto = mask->hdr.proto & spec->hdr.proto; 1647 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1648 return rte_flow_error_set(error, EINVAL, 1649 RTE_FLOW_ERROR_TYPE_ITEM, 1650 item, 1651 "multiple tunnel " 1652 "not supported"); 1653 } 1654 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1655 return rte_flow_error_set(error, EINVAL, 1656 RTE_FLOW_ERROR_TYPE_ITEM, item, 1657 "wrong tunnel type - IPv4 specified " 1658 "but IPv6 item provided"); 1659 if (item_flags & l3m) 1660 return rte_flow_error_set(error, ENOTSUP, 1661 RTE_FLOW_ERROR_TYPE_ITEM, item, 1662 "multiple L3 layers not supported"); 1663 else if (item_flags & l4m) 1664 return rte_flow_error_set(error, EINVAL, 1665 RTE_FLOW_ERROR_TYPE_ITEM, item, 1666 "L3 cannot follow an L4 layer."); 1667 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1668 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1669 return rte_flow_error_set(error, EINVAL, 1670 RTE_FLOW_ERROR_TYPE_ITEM, item, 1671 "L3 cannot follow an NVGRE layer."); 1672 if (!mask) 1673 mask = &rte_flow_item_ipv6_mask; 1674 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1675 acc_mask ? (const uint8_t *)acc_mask 1676 : (const uint8_t *)&nic_mask, 1677 sizeof(struct rte_flow_item_ipv6), 1678 error); 1679 if (ret < 0) 1680 return ret; 1681 return 0; 1682 } 1683 1684 /** 1685 * Validate UDP item. 1686 * 1687 * @param[in] item 1688 * Item specification. 1689 * @param[in] item_flags 1690 * Bit-fields that holds the items detected until now. 1691 * @param[in] target_protocol 1692 * The next protocol in the previous item. 1693 * @param[in] flow_mask 1694 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1695 * @param[out] error 1696 * Pointer to error structure. 1697 * 1698 * @return 1699 * 0 on success, a negative errno value otherwise and rte_errno is set. 1700 */ 1701 int 1702 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1703 uint64_t item_flags, 1704 uint8_t target_protocol, 1705 struct rte_flow_error *error) 1706 { 1707 const struct rte_flow_item_udp *mask = item->mask; 1708 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1709 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1710 MLX5_FLOW_LAYER_OUTER_L3; 1711 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1712 MLX5_FLOW_LAYER_OUTER_L4; 1713 int ret; 1714 1715 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1716 return rte_flow_error_set(error, EINVAL, 1717 RTE_FLOW_ERROR_TYPE_ITEM, item, 1718 "protocol filtering not compatible" 1719 " with UDP layer"); 1720 if (!(item_flags & l3m)) 1721 return rte_flow_error_set(error, EINVAL, 1722 RTE_FLOW_ERROR_TYPE_ITEM, item, 1723 "L3 is mandatory to filter on L4"); 1724 if (item_flags & l4m) 1725 return rte_flow_error_set(error, EINVAL, 1726 RTE_FLOW_ERROR_TYPE_ITEM, item, 1727 "multiple L4 layers not supported"); 1728 if (!mask) 1729 mask = &rte_flow_item_udp_mask; 1730 ret = mlx5_flow_item_acceptable 1731 (item, (const uint8_t *)mask, 1732 (const uint8_t *)&rte_flow_item_udp_mask, 1733 sizeof(struct rte_flow_item_udp), error); 1734 if (ret < 0) 1735 return ret; 1736 return 0; 1737 } 1738 1739 /** 1740 * Validate TCP item. 1741 * 1742 * @param[in] item 1743 * Item specification. 1744 * @param[in] item_flags 1745 * Bit-fields that holds the items detected until now. 1746 * @param[in] target_protocol 1747 * The next protocol in the previous item. 1748 * @param[out] error 1749 * Pointer to error structure. 1750 * 1751 * @return 1752 * 0 on success, a negative errno value otherwise and rte_errno is set. 1753 */ 1754 int 1755 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1756 uint64_t item_flags, 1757 uint8_t target_protocol, 1758 const struct rte_flow_item_tcp *flow_mask, 1759 struct rte_flow_error *error) 1760 { 1761 const struct rte_flow_item_tcp *mask = item->mask; 1762 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1763 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1764 MLX5_FLOW_LAYER_OUTER_L3; 1765 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1766 MLX5_FLOW_LAYER_OUTER_L4; 1767 int ret; 1768 1769 assert(flow_mask); 1770 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1771 return rte_flow_error_set(error, EINVAL, 1772 RTE_FLOW_ERROR_TYPE_ITEM, item, 1773 "protocol filtering not compatible" 1774 " with TCP layer"); 1775 if (!(item_flags & l3m)) 1776 return rte_flow_error_set(error, EINVAL, 1777 RTE_FLOW_ERROR_TYPE_ITEM, item, 1778 "L3 is mandatory to filter on L4"); 1779 if (item_flags & l4m) 1780 return rte_flow_error_set(error, EINVAL, 1781 RTE_FLOW_ERROR_TYPE_ITEM, item, 1782 "multiple L4 layers not supported"); 1783 if (!mask) 1784 mask = &rte_flow_item_tcp_mask; 1785 ret = mlx5_flow_item_acceptable 1786 (item, (const uint8_t *)mask, 1787 (const uint8_t *)flow_mask, 1788 sizeof(struct rte_flow_item_tcp), error); 1789 if (ret < 0) 1790 return ret; 1791 return 0; 1792 } 1793 1794 /** 1795 * Validate VXLAN item. 1796 * 1797 * @param[in] item 1798 * Item specification. 1799 * @param[in] item_flags 1800 * Bit-fields that holds the items detected until now. 1801 * @param[in] target_protocol 1802 * The next protocol in the previous item. 1803 * @param[out] error 1804 * Pointer to error structure. 1805 * 1806 * @return 1807 * 0 on success, a negative errno value otherwise and rte_errno is set. 1808 */ 1809 int 1810 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1811 uint64_t item_flags, 1812 struct rte_flow_error *error) 1813 { 1814 const struct rte_flow_item_vxlan *spec = item->spec; 1815 const struct rte_flow_item_vxlan *mask = item->mask; 1816 int ret; 1817 union vni { 1818 uint32_t vlan_id; 1819 uint8_t vni[4]; 1820 } id = { .vlan_id = 0, }; 1821 uint32_t vlan_id = 0; 1822 1823 1824 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1825 return rte_flow_error_set(error, ENOTSUP, 1826 RTE_FLOW_ERROR_TYPE_ITEM, item, 1827 "multiple tunnel layers not" 1828 " supported"); 1829 /* 1830 * Verify only UDPv4 is present as defined in 1831 * https://tools.ietf.org/html/rfc7348 1832 */ 1833 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1834 return rte_flow_error_set(error, EINVAL, 1835 RTE_FLOW_ERROR_TYPE_ITEM, item, 1836 "no outer UDP layer found"); 1837 if (!mask) 1838 mask = &rte_flow_item_vxlan_mask; 1839 ret = mlx5_flow_item_acceptable 1840 (item, (const uint8_t *)mask, 1841 (const uint8_t *)&rte_flow_item_vxlan_mask, 1842 sizeof(struct rte_flow_item_vxlan), 1843 error); 1844 if (ret < 0) 1845 return ret; 1846 if (spec) { 1847 memcpy(&id.vni[1], spec->vni, 3); 1848 vlan_id = id.vlan_id; 1849 memcpy(&id.vni[1], mask->vni, 3); 1850 vlan_id &= id.vlan_id; 1851 } 1852 /* 1853 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if 1854 * only this layer is defined in the Verbs specification it is 1855 * interpreted as wildcard and all packets will match this 1856 * rule, if it follows a full stack layer (ex: eth / ipv4 / 1857 * udp), all packets matching the layers before will also 1858 * match this rule. To avoid such situation, VNI 0 is 1859 * currently refused. 1860 */ 1861 if (!vlan_id) 1862 return rte_flow_error_set(error, ENOTSUP, 1863 RTE_FLOW_ERROR_TYPE_ITEM, item, 1864 "VXLAN vni cannot be 0"); 1865 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1866 return rte_flow_error_set(error, ENOTSUP, 1867 RTE_FLOW_ERROR_TYPE_ITEM, item, 1868 "VXLAN tunnel must be fully defined"); 1869 return 0; 1870 } 1871 1872 /** 1873 * Validate VXLAN_GPE item. 1874 * 1875 * @param[in] item 1876 * Item specification. 1877 * @param[in] item_flags 1878 * Bit-fields that holds the items detected until now. 1879 * @param[in] priv 1880 * Pointer to the private data structure. 1881 * @param[in] target_protocol 1882 * The next protocol in the previous item. 1883 * @param[out] error 1884 * Pointer to error structure. 1885 * 1886 * @return 1887 * 0 on success, a negative errno value otherwise and rte_errno is set. 1888 */ 1889 int 1890 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1891 uint64_t item_flags, 1892 struct rte_eth_dev *dev, 1893 struct rte_flow_error *error) 1894 { 1895 struct mlx5_priv *priv = dev->data->dev_private; 1896 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1897 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1898 int ret; 1899 union vni { 1900 uint32_t vlan_id; 1901 uint8_t vni[4]; 1902 } id = { .vlan_id = 0, }; 1903 uint32_t vlan_id = 0; 1904 1905 if (!priv->config.l3_vxlan_en) 1906 return rte_flow_error_set(error, ENOTSUP, 1907 RTE_FLOW_ERROR_TYPE_ITEM, item, 1908 "L3 VXLAN is not enabled by device" 1909 " parameter and/or not configured in" 1910 " firmware"); 1911 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1912 return rte_flow_error_set(error, ENOTSUP, 1913 RTE_FLOW_ERROR_TYPE_ITEM, item, 1914 "multiple tunnel layers not" 1915 " supported"); 1916 /* 1917 * Verify only UDPv4 is present as defined in 1918 * https://tools.ietf.org/html/rfc7348 1919 */ 1920 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1921 return rte_flow_error_set(error, EINVAL, 1922 RTE_FLOW_ERROR_TYPE_ITEM, item, 1923 "no outer UDP layer found"); 1924 if (!mask) 1925 mask = &rte_flow_item_vxlan_gpe_mask; 1926 ret = mlx5_flow_item_acceptable 1927 (item, (const uint8_t *)mask, 1928 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1929 sizeof(struct rte_flow_item_vxlan_gpe), 1930 error); 1931 if (ret < 0) 1932 return ret; 1933 if (spec) { 1934 if (spec->protocol) 1935 return rte_flow_error_set(error, ENOTSUP, 1936 RTE_FLOW_ERROR_TYPE_ITEM, 1937 item, 1938 "VxLAN-GPE protocol" 1939 " not supported"); 1940 memcpy(&id.vni[1], spec->vni, 3); 1941 vlan_id = id.vlan_id; 1942 memcpy(&id.vni[1], mask->vni, 3); 1943 vlan_id &= id.vlan_id; 1944 } 1945 /* 1946 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1947 * layer is defined in the Verbs specification it is interpreted as 1948 * wildcard and all packets will match this rule, if it follows a full 1949 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1950 * before will also match this rule. To avoid such situation, VNI 0 1951 * is currently refused. 1952 */ 1953 if (!vlan_id) 1954 return rte_flow_error_set(error, ENOTSUP, 1955 RTE_FLOW_ERROR_TYPE_ITEM, item, 1956 "VXLAN-GPE vni cannot be 0"); 1957 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1958 return rte_flow_error_set(error, ENOTSUP, 1959 RTE_FLOW_ERROR_TYPE_ITEM, item, 1960 "VXLAN-GPE tunnel must be fully" 1961 " defined"); 1962 return 0; 1963 } 1964 /** 1965 * Validate GRE Key item. 1966 * 1967 * @param[in] item 1968 * Item specification. 1969 * @param[in] item_flags 1970 * Bit flags to mark detected items. 1971 * @param[in] gre_item 1972 * Pointer to gre_item 1973 * @param[out] error 1974 * Pointer to error structure. 1975 * 1976 * @return 1977 * 0 on success, a negative errno value otherwise and rte_errno is set. 1978 */ 1979 int 1980 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 1981 uint64_t item_flags, 1982 const struct rte_flow_item *gre_item, 1983 struct rte_flow_error *error) 1984 { 1985 const rte_be32_t *mask = item->mask; 1986 int ret = 0; 1987 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 1988 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 1989 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 1990 1991 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 1992 return rte_flow_error_set(error, ENOTSUP, 1993 RTE_FLOW_ERROR_TYPE_ITEM, item, 1994 "Multiple GRE key not support"); 1995 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 1996 return rte_flow_error_set(error, ENOTSUP, 1997 RTE_FLOW_ERROR_TYPE_ITEM, item, 1998 "No preceding GRE header"); 1999 if (item_flags & MLX5_FLOW_LAYER_INNER) 2000 return rte_flow_error_set(error, ENOTSUP, 2001 RTE_FLOW_ERROR_TYPE_ITEM, item, 2002 "GRE key following a wrong item"); 2003 if (!gre_mask) 2004 gre_mask = &rte_flow_item_gre_mask; 2005 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2006 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2007 return rte_flow_error_set(error, EINVAL, 2008 RTE_FLOW_ERROR_TYPE_ITEM, item, 2009 "Key bit must be on"); 2010 2011 if (!mask) 2012 mask = &gre_key_default_mask; 2013 ret = mlx5_flow_item_acceptable 2014 (item, (const uint8_t *)mask, 2015 (const uint8_t *)&gre_key_default_mask, 2016 sizeof(rte_be32_t), error); 2017 return ret; 2018 } 2019 2020 /** 2021 * Validate GRE item. 2022 * 2023 * @param[in] item 2024 * Item specification. 2025 * @param[in] item_flags 2026 * Bit flags to mark detected items. 2027 * @param[in] target_protocol 2028 * The next protocol in the previous item. 2029 * @param[out] error 2030 * Pointer to error structure. 2031 * 2032 * @return 2033 * 0 on success, a negative errno value otherwise and rte_errno is set. 2034 */ 2035 int 2036 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2037 uint64_t item_flags, 2038 uint8_t target_protocol, 2039 struct rte_flow_error *error) 2040 { 2041 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2042 const struct rte_flow_item_gre *mask = item->mask; 2043 int ret; 2044 const struct rte_flow_item_gre nic_mask = { 2045 .c_rsvd0_ver = RTE_BE16(0xB000), 2046 .protocol = RTE_BE16(UINT16_MAX), 2047 }; 2048 2049 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2050 return rte_flow_error_set(error, EINVAL, 2051 RTE_FLOW_ERROR_TYPE_ITEM, item, 2052 "protocol filtering not compatible" 2053 " with this GRE layer"); 2054 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2055 return rte_flow_error_set(error, ENOTSUP, 2056 RTE_FLOW_ERROR_TYPE_ITEM, item, 2057 "multiple tunnel layers not" 2058 " supported"); 2059 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2060 return rte_flow_error_set(error, ENOTSUP, 2061 RTE_FLOW_ERROR_TYPE_ITEM, item, 2062 "L3 Layer is missing"); 2063 if (!mask) 2064 mask = &rte_flow_item_gre_mask; 2065 ret = mlx5_flow_item_acceptable 2066 (item, (const uint8_t *)mask, 2067 (const uint8_t *)&nic_mask, 2068 sizeof(struct rte_flow_item_gre), error); 2069 if (ret < 0) 2070 return ret; 2071 #ifndef HAVE_MLX5DV_DR 2072 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2073 if (spec && (spec->protocol & mask->protocol)) 2074 return rte_flow_error_set(error, ENOTSUP, 2075 RTE_FLOW_ERROR_TYPE_ITEM, item, 2076 "without MPLS support the" 2077 " specification cannot be used for" 2078 " filtering"); 2079 #endif 2080 #endif 2081 return 0; 2082 } 2083 2084 /** 2085 * Validate Geneve item. 2086 * 2087 * @param[in] item 2088 * Item specification. 2089 * @param[in] itemFlags 2090 * Bit-fields that holds the items detected until now. 2091 * @param[in] enPriv 2092 * Pointer to the private data structure. 2093 * @param[out] error 2094 * Pointer to error structure. 2095 * 2096 * @return 2097 * 0 on success, a negative errno value otherwise and rte_errno is set. 2098 */ 2099 2100 int 2101 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2102 uint64_t item_flags, 2103 struct rte_eth_dev *dev, 2104 struct rte_flow_error *error) 2105 { 2106 struct mlx5_priv *priv = dev->data->dev_private; 2107 const struct rte_flow_item_geneve *spec = item->spec; 2108 const struct rte_flow_item_geneve *mask = item->mask; 2109 int ret; 2110 uint16_t gbhdr; 2111 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2112 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2113 const struct rte_flow_item_geneve nic_mask = { 2114 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2115 .vni = "\xff\xff\xff", 2116 .protocol = RTE_BE16(UINT16_MAX), 2117 }; 2118 2119 if (!(priv->config.hca_attr.flex_parser_protocols & 2120 MLX5_HCA_FLEX_GENEVE_ENABLED) || 2121 !priv->config.hca_attr.tunnel_stateless_geneve_rx) 2122 return rte_flow_error_set(error, ENOTSUP, 2123 RTE_FLOW_ERROR_TYPE_ITEM, item, 2124 "L3 Geneve is not enabled by device" 2125 " parameter and/or not configured in" 2126 " firmware"); 2127 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2128 return rte_flow_error_set(error, ENOTSUP, 2129 RTE_FLOW_ERROR_TYPE_ITEM, item, 2130 "multiple tunnel layers not" 2131 " supported"); 2132 /* 2133 * Verify only UDPv4 is present as defined in 2134 * https://tools.ietf.org/html/rfc7348 2135 */ 2136 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2137 return rte_flow_error_set(error, EINVAL, 2138 RTE_FLOW_ERROR_TYPE_ITEM, item, 2139 "no outer UDP layer found"); 2140 if (!mask) 2141 mask = &rte_flow_item_geneve_mask; 2142 ret = mlx5_flow_item_acceptable 2143 (item, (const uint8_t *)mask, 2144 (const uint8_t *)&nic_mask, 2145 sizeof(struct rte_flow_item_geneve), error); 2146 if (ret) 2147 return ret; 2148 if (spec) { 2149 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2150 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2151 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2152 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2153 return rte_flow_error_set(error, ENOTSUP, 2154 RTE_FLOW_ERROR_TYPE_ITEM, 2155 item, 2156 "Geneve protocol unsupported" 2157 " fields are being used"); 2158 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2159 return rte_flow_error_set 2160 (error, ENOTSUP, 2161 RTE_FLOW_ERROR_TYPE_ITEM, 2162 item, 2163 "Unsupported Geneve options length"); 2164 } 2165 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2166 return rte_flow_error_set 2167 (error, ENOTSUP, 2168 RTE_FLOW_ERROR_TYPE_ITEM, item, 2169 "Geneve tunnel must be fully defined"); 2170 return 0; 2171 } 2172 2173 /** 2174 * Validate MPLS item. 2175 * 2176 * @param[in] dev 2177 * Pointer to the rte_eth_dev structure. 2178 * @param[in] item 2179 * Item specification. 2180 * @param[in] item_flags 2181 * Bit-fields that holds the items detected until now. 2182 * @param[in] prev_layer 2183 * The protocol layer indicated in previous item. 2184 * @param[out] error 2185 * Pointer to error structure. 2186 * 2187 * @return 2188 * 0 on success, a negative errno value otherwise and rte_errno is set. 2189 */ 2190 int 2191 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2192 const struct rte_flow_item *item __rte_unused, 2193 uint64_t item_flags __rte_unused, 2194 uint64_t prev_layer __rte_unused, 2195 struct rte_flow_error *error) 2196 { 2197 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2198 const struct rte_flow_item_mpls *mask = item->mask; 2199 struct mlx5_priv *priv = dev->data->dev_private; 2200 int ret; 2201 2202 if (!priv->config.mpls_en) 2203 return rte_flow_error_set(error, ENOTSUP, 2204 RTE_FLOW_ERROR_TYPE_ITEM, item, 2205 "MPLS not supported or" 2206 " disabled in firmware" 2207 " configuration."); 2208 /* MPLS over IP, UDP, GRE is allowed */ 2209 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2210 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2211 MLX5_FLOW_LAYER_GRE))) 2212 return rte_flow_error_set(error, EINVAL, 2213 RTE_FLOW_ERROR_TYPE_ITEM, item, 2214 "protocol filtering not compatible" 2215 " with MPLS layer"); 2216 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2217 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2218 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2219 return rte_flow_error_set(error, ENOTSUP, 2220 RTE_FLOW_ERROR_TYPE_ITEM, item, 2221 "multiple tunnel layers not" 2222 " supported"); 2223 if (!mask) 2224 mask = &rte_flow_item_mpls_mask; 2225 ret = mlx5_flow_item_acceptable 2226 (item, (const uint8_t *)mask, 2227 (const uint8_t *)&rte_flow_item_mpls_mask, 2228 sizeof(struct rte_flow_item_mpls), error); 2229 if (ret < 0) 2230 return ret; 2231 return 0; 2232 #endif 2233 return rte_flow_error_set(error, ENOTSUP, 2234 RTE_FLOW_ERROR_TYPE_ITEM, item, 2235 "MPLS is not supported by Verbs, please" 2236 " update."); 2237 } 2238 2239 /** 2240 * Validate NVGRE item. 2241 * 2242 * @param[in] item 2243 * Item specification. 2244 * @param[in] item_flags 2245 * Bit flags to mark detected items. 2246 * @param[in] target_protocol 2247 * The next protocol in the previous item. 2248 * @param[out] error 2249 * Pointer to error structure. 2250 * 2251 * @return 2252 * 0 on success, a negative errno value otherwise and rte_errno is set. 2253 */ 2254 int 2255 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2256 uint64_t item_flags, 2257 uint8_t target_protocol, 2258 struct rte_flow_error *error) 2259 { 2260 const struct rte_flow_item_nvgre *mask = item->mask; 2261 int ret; 2262 2263 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2264 return rte_flow_error_set(error, EINVAL, 2265 RTE_FLOW_ERROR_TYPE_ITEM, item, 2266 "protocol filtering not compatible" 2267 " with this GRE layer"); 2268 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2269 return rte_flow_error_set(error, ENOTSUP, 2270 RTE_FLOW_ERROR_TYPE_ITEM, item, 2271 "multiple tunnel layers not" 2272 " supported"); 2273 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2274 return rte_flow_error_set(error, ENOTSUP, 2275 RTE_FLOW_ERROR_TYPE_ITEM, item, 2276 "L3 Layer is missing"); 2277 if (!mask) 2278 mask = &rte_flow_item_nvgre_mask; 2279 ret = mlx5_flow_item_acceptable 2280 (item, (const uint8_t *)mask, 2281 (const uint8_t *)&rte_flow_item_nvgre_mask, 2282 sizeof(struct rte_flow_item_nvgre), error); 2283 if (ret < 0) 2284 return ret; 2285 return 0; 2286 } 2287 2288 /* Allocate unique ID for the split Q/RSS subflows. */ 2289 static uint32_t 2290 flow_qrss_get_id(struct rte_eth_dev *dev) 2291 { 2292 struct mlx5_priv *priv = dev->data->dev_private; 2293 uint32_t qrss_id, ret; 2294 2295 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2296 if (ret) 2297 return 0; 2298 assert(qrss_id); 2299 return qrss_id; 2300 } 2301 2302 /* Free unique ID for the split Q/RSS subflows. */ 2303 static void 2304 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2305 { 2306 struct mlx5_priv *priv = dev->data->dev_private; 2307 2308 if (qrss_id) 2309 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2310 } 2311 2312 /** 2313 * Release resource related QUEUE/RSS action split. 2314 * 2315 * @param dev 2316 * Pointer to Ethernet device. 2317 * @param flow 2318 * Flow to release id's from. 2319 */ 2320 static void 2321 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2322 struct rte_flow *flow) 2323 { 2324 struct mlx5_flow *dev_flow; 2325 2326 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 2327 if (dev_flow->qrss_id) 2328 flow_qrss_free_id(dev, dev_flow->qrss_id); 2329 } 2330 2331 static int 2332 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2333 const struct rte_flow_attr *attr __rte_unused, 2334 const struct rte_flow_item items[] __rte_unused, 2335 const struct rte_flow_action actions[] __rte_unused, 2336 bool external __rte_unused, 2337 struct rte_flow_error *error) 2338 { 2339 return rte_flow_error_set(error, ENOTSUP, 2340 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2341 } 2342 2343 static struct mlx5_flow * 2344 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, 2345 const struct rte_flow_item items[] __rte_unused, 2346 const struct rte_flow_action actions[] __rte_unused, 2347 struct rte_flow_error *error) 2348 { 2349 rte_flow_error_set(error, ENOTSUP, 2350 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2351 return NULL; 2352 } 2353 2354 static int 2355 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2356 struct mlx5_flow *dev_flow __rte_unused, 2357 const struct rte_flow_attr *attr __rte_unused, 2358 const struct rte_flow_item items[] __rte_unused, 2359 const struct rte_flow_action actions[] __rte_unused, 2360 struct rte_flow_error *error) 2361 { 2362 return rte_flow_error_set(error, ENOTSUP, 2363 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2364 } 2365 2366 static int 2367 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2368 struct rte_flow *flow __rte_unused, 2369 struct rte_flow_error *error) 2370 { 2371 return rte_flow_error_set(error, ENOTSUP, 2372 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2373 } 2374 2375 static void 2376 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2377 struct rte_flow *flow __rte_unused) 2378 { 2379 } 2380 2381 static void 2382 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2383 struct rte_flow *flow __rte_unused) 2384 { 2385 } 2386 2387 static int 2388 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2389 struct rte_flow *flow __rte_unused, 2390 const struct rte_flow_action *actions __rte_unused, 2391 void *data __rte_unused, 2392 struct rte_flow_error *error) 2393 { 2394 return rte_flow_error_set(error, ENOTSUP, 2395 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2396 } 2397 2398 /* Void driver to protect from null pointer reference. */ 2399 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2400 .validate = flow_null_validate, 2401 .prepare = flow_null_prepare, 2402 .translate = flow_null_translate, 2403 .apply = flow_null_apply, 2404 .remove = flow_null_remove, 2405 .destroy = flow_null_destroy, 2406 .query = flow_null_query, 2407 }; 2408 2409 /** 2410 * Select flow driver type according to flow attributes and device 2411 * configuration. 2412 * 2413 * @param[in] dev 2414 * Pointer to the dev structure. 2415 * @param[in] attr 2416 * Pointer to the flow attributes. 2417 * 2418 * @return 2419 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2420 */ 2421 static enum mlx5_flow_drv_type 2422 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2423 { 2424 struct mlx5_priv *priv = dev->data->dev_private; 2425 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2426 2427 if (attr->transfer && priv->config.dv_esw_en) 2428 type = MLX5_FLOW_TYPE_DV; 2429 if (!attr->transfer) 2430 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2431 MLX5_FLOW_TYPE_VERBS; 2432 return type; 2433 } 2434 2435 #define flow_get_drv_ops(type) flow_drv_ops[type] 2436 2437 /** 2438 * Flow driver validation API. This abstracts calling driver specific functions. 2439 * The type of flow driver is determined according to flow attributes. 2440 * 2441 * @param[in] dev 2442 * Pointer to the dev structure. 2443 * @param[in] attr 2444 * Pointer to the flow attributes. 2445 * @param[in] items 2446 * Pointer to the list of items. 2447 * @param[in] actions 2448 * Pointer to the list of actions. 2449 * @param[in] external 2450 * This flow rule is created by request external to PMD. 2451 * @param[out] error 2452 * Pointer to the error structure. 2453 * 2454 * @return 2455 * 0 on success, a negative errno value otherwise and rte_errno is set. 2456 */ 2457 static inline int 2458 flow_drv_validate(struct rte_eth_dev *dev, 2459 const struct rte_flow_attr *attr, 2460 const struct rte_flow_item items[], 2461 const struct rte_flow_action actions[], 2462 bool external, struct rte_flow_error *error) 2463 { 2464 const struct mlx5_flow_driver_ops *fops; 2465 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2466 2467 fops = flow_get_drv_ops(type); 2468 return fops->validate(dev, attr, items, actions, external, error); 2469 } 2470 2471 /** 2472 * Flow driver preparation API. This abstracts calling driver specific 2473 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2474 * calculates the size of memory required for device flow, allocates the memory, 2475 * initializes the device flow and returns the pointer. 2476 * 2477 * @note 2478 * This function initializes device flow structure such as dv or verbs in 2479 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2480 * rest. For example, adding returning device flow to flow->dev_flow list and 2481 * setting backward reference to the flow should be done out of this function. 2482 * layers field is not filled either. 2483 * 2484 * @param[in] attr 2485 * Pointer to the flow attributes. 2486 * @param[in] items 2487 * Pointer to the list of items. 2488 * @param[in] actions 2489 * Pointer to the list of actions. 2490 * @param[out] error 2491 * Pointer to the error structure. 2492 * 2493 * @return 2494 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2495 */ 2496 static inline struct mlx5_flow * 2497 flow_drv_prepare(const struct rte_flow *flow, 2498 const struct rte_flow_attr *attr, 2499 const struct rte_flow_item items[], 2500 const struct rte_flow_action actions[], 2501 struct rte_flow_error *error) 2502 { 2503 const struct mlx5_flow_driver_ops *fops; 2504 enum mlx5_flow_drv_type type = flow->drv_type; 2505 2506 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2507 fops = flow_get_drv_ops(type); 2508 return fops->prepare(attr, items, actions, error); 2509 } 2510 2511 /** 2512 * Flow driver translation API. This abstracts calling driver specific 2513 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2514 * translates a generic flow into a driver flow. flow_drv_prepare() must 2515 * precede. 2516 * 2517 * @note 2518 * dev_flow->layers could be filled as a result of parsing during translation 2519 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2520 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2521 * flow->actions could be overwritten even though all the expanded dev_flows 2522 * have the same actions. 2523 * 2524 * @param[in] dev 2525 * Pointer to the rte dev structure. 2526 * @param[in, out] dev_flow 2527 * Pointer to the mlx5 flow. 2528 * @param[in] attr 2529 * Pointer to the flow attributes. 2530 * @param[in] items 2531 * Pointer to the list of items. 2532 * @param[in] actions 2533 * Pointer to the list of actions. 2534 * @param[out] error 2535 * Pointer to the error structure. 2536 * 2537 * @return 2538 * 0 on success, a negative errno value otherwise and rte_errno is set. 2539 */ 2540 static inline int 2541 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2542 const struct rte_flow_attr *attr, 2543 const struct rte_flow_item items[], 2544 const struct rte_flow_action actions[], 2545 struct rte_flow_error *error) 2546 { 2547 const struct mlx5_flow_driver_ops *fops; 2548 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2549 2550 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2551 fops = flow_get_drv_ops(type); 2552 return fops->translate(dev, dev_flow, attr, items, actions, error); 2553 } 2554 2555 /** 2556 * Flow driver apply API. This abstracts calling driver specific functions. 2557 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2558 * translated driver flows on to device. flow_drv_translate() must precede. 2559 * 2560 * @param[in] dev 2561 * Pointer to Ethernet device structure. 2562 * @param[in, out] flow 2563 * Pointer to flow structure. 2564 * @param[out] error 2565 * Pointer to error structure. 2566 * 2567 * @return 2568 * 0 on success, a negative errno value otherwise and rte_errno is set. 2569 */ 2570 static inline int 2571 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2572 struct rte_flow_error *error) 2573 { 2574 const struct mlx5_flow_driver_ops *fops; 2575 enum mlx5_flow_drv_type type = flow->drv_type; 2576 2577 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2578 fops = flow_get_drv_ops(type); 2579 return fops->apply(dev, flow, error); 2580 } 2581 2582 /** 2583 * Flow driver remove API. This abstracts calling driver specific functions. 2584 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2585 * on device. All the resources of the flow should be freed by calling 2586 * flow_drv_destroy(). 2587 * 2588 * @param[in] dev 2589 * Pointer to Ethernet device. 2590 * @param[in, out] flow 2591 * Pointer to flow structure. 2592 */ 2593 static inline void 2594 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2595 { 2596 const struct mlx5_flow_driver_ops *fops; 2597 enum mlx5_flow_drv_type type = flow->drv_type; 2598 2599 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2600 fops = flow_get_drv_ops(type); 2601 fops->remove(dev, flow); 2602 } 2603 2604 /** 2605 * Flow driver destroy API. This abstracts calling driver specific functions. 2606 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2607 * on device and releases resources of the flow. 2608 * 2609 * @param[in] dev 2610 * Pointer to Ethernet device. 2611 * @param[in, out] flow 2612 * Pointer to flow structure. 2613 */ 2614 static inline void 2615 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2616 { 2617 const struct mlx5_flow_driver_ops *fops; 2618 enum mlx5_flow_drv_type type = flow->drv_type; 2619 2620 flow_mreg_split_qrss_release(dev, flow); 2621 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2622 fops = flow_get_drv_ops(type); 2623 fops->destroy(dev, flow); 2624 } 2625 2626 /** 2627 * Validate a flow supported by the NIC. 2628 * 2629 * @see rte_flow_validate() 2630 * @see rte_flow_ops 2631 */ 2632 int 2633 mlx5_flow_validate(struct rte_eth_dev *dev, 2634 const struct rte_flow_attr *attr, 2635 const struct rte_flow_item items[], 2636 const struct rte_flow_action actions[], 2637 struct rte_flow_error *error) 2638 { 2639 int ret; 2640 2641 ret = flow_drv_validate(dev, attr, items, actions, true, error); 2642 if (ret < 0) 2643 return ret; 2644 return 0; 2645 } 2646 2647 /** 2648 * Get RSS action from the action list. 2649 * 2650 * @param[in] actions 2651 * Pointer to the list of actions. 2652 * 2653 * @return 2654 * Pointer to the RSS action if exist, else return NULL. 2655 */ 2656 static const struct rte_flow_action_rss* 2657 flow_get_rss_action(const struct rte_flow_action actions[]) 2658 { 2659 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2660 switch (actions->type) { 2661 case RTE_FLOW_ACTION_TYPE_RSS: 2662 return (const struct rte_flow_action_rss *) 2663 actions->conf; 2664 default: 2665 break; 2666 } 2667 } 2668 return NULL; 2669 } 2670 2671 static unsigned int 2672 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2673 { 2674 const struct rte_flow_item *item; 2675 unsigned int has_vlan = 0; 2676 2677 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2678 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2679 has_vlan = 1; 2680 break; 2681 } 2682 } 2683 if (has_vlan) 2684 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2685 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2686 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2687 MLX5_EXPANSION_ROOT_OUTER; 2688 } 2689 2690 /** 2691 * Get QUEUE/RSS action from the action list. 2692 * 2693 * @param[in] actions 2694 * Pointer to the list of actions. 2695 * @param[out] qrss 2696 * Pointer to the return pointer. 2697 * @param[out] qrss_type 2698 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2699 * if no QUEUE/RSS is found. 2700 * 2701 * @return 2702 * Total number of actions. 2703 */ 2704 static int 2705 flow_parse_qrss_action(const struct rte_flow_action actions[], 2706 const struct rte_flow_action **qrss) 2707 { 2708 int actions_n = 0; 2709 2710 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2711 switch (actions->type) { 2712 case RTE_FLOW_ACTION_TYPE_QUEUE: 2713 case RTE_FLOW_ACTION_TYPE_RSS: 2714 *qrss = actions; 2715 break; 2716 default: 2717 break; 2718 } 2719 actions_n++; 2720 } 2721 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2722 return actions_n + 1; 2723 } 2724 2725 /** 2726 * Check if the flow should be splited due to hairpin. 2727 * The reason for the split is that in current HW we can't 2728 * support encap on Rx, so if a flow have encap we move it 2729 * to Tx. 2730 * 2731 * @param dev 2732 * Pointer to Ethernet device. 2733 * @param[in] attr 2734 * Flow rule attributes. 2735 * @param[in] actions 2736 * Associated actions (list terminated by the END action). 2737 * 2738 * @return 2739 * > 0 the number of actions and the flow should be split, 2740 * 0 when no split required. 2741 */ 2742 static int 2743 flow_check_hairpin_split(struct rte_eth_dev *dev, 2744 const struct rte_flow_attr *attr, 2745 const struct rte_flow_action actions[]) 2746 { 2747 int queue_action = 0; 2748 int action_n = 0; 2749 int encap = 0; 2750 const struct rte_flow_action_queue *queue; 2751 const struct rte_flow_action_rss *rss; 2752 const struct rte_flow_action_raw_encap *raw_encap; 2753 2754 if (!attr->ingress) 2755 return 0; 2756 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2757 switch (actions->type) { 2758 case RTE_FLOW_ACTION_TYPE_QUEUE: 2759 queue = actions->conf; 2760 if (mlx5_rxq_get_type(dev, queue->index) != 2761 MLX5_RXQ_TYPE_HAIRPIN) 2762 return 0; 2763 queue_action = 1; 2764 action_n++; 2765 break; 2766 case RTE_FLOW_ACTION_TYPE_RSS: 2767 rss = actions->conf; 2768 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2769 MLX5_RXQ_TYPE_HAIRPIN) 2770 return 0; 2771 queue_action = 1; 2772 action_n++; 2773 break; 2774 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2775 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2776 encap = 1; 2777 action_n++; 2778 break; 2779 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2780 raw_encap = actions->conf; 2781 if (raw_encap->size > 2782 (sizeof(struct rte_flow_item_eth) + 2783 sizeof(struct rte_flow_item_ipv4))) 2784 encap = 1; 2785 action_n++; 2786 break; 2787 default: 2788 action_n++; 2789 break; 2790 } 2791 } 2792 if (encap == 1 && queue_action) 2793 return action_n; 2794 return 0; 2795 } 2796 2797 /* Declare flow create/destroy prototype in advance. */ 2798 static struct rte_flow * 2799 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 2800 const struct rte_flow_attr *attr, 2801 const struct rte_flow_item items[], 2802 const struct rte_flow_action actions[], 2803 bool external, struct rte_flow_error *error); 2804 2805 static void 2806 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2807 struct rte_flow *flow); 2808 2809 /** 2810 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2811 * 2812 * As mark_id is unique, if there's already a registered flow for the mark_id, 2813 * return by increasing the reference counter of the resource. Otherwise, create 2814 * the resource (mcp_res) and flow. 2815 * 2816 * Flow looks like, 2817 * - If ingress port is ANY and reg_c[1] is mark_id, 2818 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2819 * 2820 * For default flow (zero mark_id), flow is like, 2821 * - If ingress port is ANY, 2822 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2823 * 2824 * @param dev 2825 * Pointer to Ethernet device. 2826 * @param mark_id 2827 * ID of MARK action, zero means default flow for META. 2828 * @param[out] error 2829 * Perform verbose error reporting if not NULL. 2830 * 2831 * @return 2832 * Associated resource on success, NULL otherwise and rte_errno is set. 2833 */ 2834 static struct mlx5_flow_mreg_copy_resource * 2835 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2836 struct rte_flow_error *error) 2837 { 2838 struct mlx5_priv *priv = dev->data->dev_private; 2839 struct rte_flow_attr attr = { 2840 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2841 .ingress = 1, 2842 }; 2843 struct mlx5_rte_flow_item_tag tag_spec = { 2844 .data = mark_id, 2845 }; 2846 struct rte_flow_item items[] = { 2847 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2848 }; 2849 struct rte_flow_action_mark ftag = { 2850 .id = mark_id, 2851 }; 2852 struct mlx5_flow_action_copy_mreg cp_mreg = { 2853 .dst = REG_B, 2854 .src = 0, 2855 }; 2856 struct rte_flow_action_jump jump = { 2857 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2858 }; 2859 struct rte_flow_action actions[] = { 2860 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2861 }; 2862 struct mlx5_flow_mreg_copy_resource *mcp_res; 2863 int ret; 2864 2865 /* Fill the register fileds in the flow. */ 2866 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2867 if (ret < 0) 2868 return NULL; 2869 tag_spec.id = ret; 2870 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 2871 if (ret < 0) 2872 return NULL; 2873 cp_mreg.src = ret; 2874 /* Check if already registered. */ 2875 assert(priv->mreg_cp_tbl); 2876 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 2877 if (mcp_res) { 2878 /* For non-default rule. */ 2879 if (mark_id) 2880 mcp_res->refcnt++; 2881 assert(mark_id || mcp_res->refcnt == 1); 2882 return mcp_res; 2883 } 2884 /* Provide the full width of FLAG specific value. */ 2885 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 2886 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 2887 /* Build a new flow. */ 2888 if (mark_id) { 2889 items[0] = (struct rte_flow_item){ 2890 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 2891 .spec = &tag_spec, 2892 }; 2893 items[1] = (struct rte_flow_item){ 2894 .type = RTE_FLOW_ITEM_TYPE_END, 2895 }; 2896 actions[0] = (struct rte_flow_action){ 2897 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK, 2898 .conf = &ftag, 2899 }; 2900 actions[1] = (struct rte_flow_action){ 2901 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2902 .conf = &cp_mreg, 2903 }; 2904 actions[2] = (struct rte_flow_action){ 2905 .type = RTE_FLOW_ACTION_TYPE_JUMP, 2906 .conf = &jump, 2907 }; 2908 actions[3] = (struct rte_flow_action){ 2909 .type = RTE_FLOW_ACTION_TYPE_END, 2910 }; 2911 } else { 2912 /* Default rule, wildcard match. */ 2913 attr.priority = MLX5_FLOW_PRIO_RSVD; 2914 items[0] = (struct rte_flow_item){ 2915 .type = RTE_FLOW_ITEM_TYPE_END, 2916 }; 2917 actions[0] = (struct rte_flow_action){ 2918 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2919 .conf = &cp_mreg, 2920 }; 2921 actions[1] = (struct rte_flow_action){ 2922 .type = RTE_FLOW_ACTION_TYPE_JUMP, 2923 .conf = &jump, 2924 }; 2925 actions[2] = (struct rte_flow_action){ 2926 .type = RTE_FLOW_ACTION_TYPE_END, 2927 }; 2928 } 2929 /* Build a new entry. */ 2930 mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0); 2931 if (!mcp_res) { 2932 rte_errno = ENOMEM; 2933 return NULL; 2934 } 2935 /* 2936 * The copy Flows are not included in any list. There 2937 * ones are referenced from other Flows and can not 2938 * be applied, removed, deleted in ardbitrary order 2939 * by list traversing. 2940 */ 2941 mcp_res->flow = flow_list_create(dev, NULL, &attr, items, 2942 actions, false, error); 2943 if (!mcp_res->flow) 2944 goto error; 2945 mcp_res->refcnt++; 2946 mcp_res->hlist_ent.key = mark_id; 2947 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 2948 &mcp_res->hlist_ent); 2949 assert(!ret); 2950 if (ret) 2951 goto error; 2952 return mcp_res; 2953 error: 2954 if (mcp_res->flow) 2955 flow_list_destroy(dev, NULL, mcp_res->flow); 2956 rte_free(mcp_res); 2957 return NULL; 2958 } 2959 2960 /** 2961 * Release flow in RX_CP_TBL. 2962 * 2963 * @param dev 2964 * Pointer to Ethernet device. 2965 * @flow 2966 * Parent flow for wich copying is provided. 2967 */ 2968 static void 2969 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 2970 struct rte_flow *flow) 2971 { 2972 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 2973 struct mlx5_priv *priv = dev->data->dev_private; 2974 2975 if (!mcp_res || !priv->mreg_cp_tbl) 2976 return; 2977 if (flow->copy_applied) { 2978 assert(mcp_res->appcnt); 2979 flow->copy_applied = 0; 2980 --mcp_res->appcnt; 2981 if (!mcp_res->appcnt) 2982 flow_drv_remove(dev, mcp_res->flow); 2983 } 2984 /* 2985 * We do not check availability of metadata registers here, 2986 * because copy resources are allocated in this case. 2987 */ 2988 if (--mcp_res->refcnt) 2989 return; 2990 assert(mcp_res->flow); 2991 flow_list_destroy(dev, NULL, mcp_res->flow); 2992 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 2993 rte_free(mcp_res); 2994 flow->mreg_copy = NULL; 2995 } 2996 2997 /** 2998 * Start flow in RX_CP_TBL. 2999 * 3000 * @param dev 3001 * Pointer to Ethernet device. 3002 * @flow 3003 * Parent flow for wich copying is provided. 3004 * 3005 * @return 3006 * 0 on success, a negative errno value otherwise and rte_errno is set. 3007 */ 3008 static int 3009 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3010 struct rte_flow *flow) 3011 { 3012 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3013 int ret; 3014 3015 if (!mcp_res || flow->copy_applied) 3016 return 0; 3017 if (!mcp_res->appcnt) { 3018 ret = flow_drv_apply(dev, mcp_res->flow, NULL); 3019 if (ret) 3020 return ret; 3021 } 3022 ++mcp_res->appcnt; 3023 flow->copy_applied = 1; 3024 return 0; 3025 } 3026 3027 /** 3028 * Stop flow in RX_CP_TBL. 3029 * 3030 * @param dev 3031 * Pointer to Ethernet device. 3032 * @flow 3033 * Parent flow for wich copying is provided. 3034 */ 3035 static void 3036 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3037 struct rte_flow *flow) 3038 { 3039 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3040 3041 if (!mcp_res || !flow->copy_applied) 3042 return; 3043 assert(mcp_res->appcnt); 3044 --mcp_res->appcnt; 3045 flow->copy_applied = 0; 3046 if (!mcp_res->appcnt) 3047 flow_drv_remove(dev, mcp_res->flow); 3048 } 3049 3050 /** 3051 * Remove the default copy action from RX_CP_TBL. 3052 * 3053 * @param dev 3054 * Pointer to Ethernet device. 3055 */ 3056 static void 3057 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3058 { 3059 struct mlx5_flow_mreg_copy_resource *mcp_res; 3060 struct mlx5_priv *priv = dev->data->dev_private; 3061 3062 /* Check if default flow is registered. */ 3063 if (!priv->mreg_cp_tbl) 3064 return; 3065 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 0ULL); 3066 if (!mcp_res) 3067 return; 3068 assert(mcp_res->flow); 3069 flow_list_destroy(dev, NULL, mcp_res->flow); 3070 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3071 rte_free(mcp_res); 3072 } 3073 3074 /** 3075 * Add the default copy action in in RX_CP_TBL. 3076 * 3077 * @param dev 3078 * Pointer to Ethernet device. 3079 * @param[out] error 3080 * Perform verbose error reporting if not NULL. 3081 * 3082 * @return 3083 * 0 for success, negative value otherwise and rte_errno is set. 3084 */ 3085 static int 3086 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3087 struct rte_flow_error *error) 3088 { 3089 struct mlx5_priv *priv = dev->data->dev_private; 3090 struct mlx5_flow_mreg_copy_resource *mcp_res; 3091 3092 /* Check whether extensive metadata feature is engaged. */ 3093 if (!priv->config.dv_flow_en || 3094 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3095 !mlx5_flow_ext_mreg_supported(dev) || 3096 !priv->sh->dv_regc0_mask) 3097 return 0; 3098 mcp_res = flow_mreg_add_copy_action(dev, 0, error); 3099 if (!mcp_res) 3100 return -rte_errno; 3101 return 0; 3102 } 3103 3104 /** 3105 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3106 * 3107 * All the flow having Q/RSS action should be split by 3108 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3109 * performs the following, 3110 * - CQE->flow_tag := reg_c[1] (MARK) 3111 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3112 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3113 * but there should be a flow per each MARK ID set by MARK action. 3114 * 3115 * For the aforementioned reason, if there's a MARK action in flow's action 3116 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3117 * the MARK ID to CQE's flow_tag like, 3118 * - If reg_c[1] is mark_id, 3119 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3120 * 3121 * For SET_META action which stores value in reg_c[0], as the destination is 3122 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3123 * MARK ID means the default flow. The default flow looks like, 3124 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3125 * 3126 * @param dev 3127 * Pointer to Ethernet device. 3128 * @param flow 3129 * Pointer to flow structure. 3130 * @param[in] actions 3131 * Pointer to the list of actions. 3132 * @param[out] error 3133 * Perform verbose error reporting if not NULL. 3134 * 3135 * @return 3136 * 0 on success, negative value otherwise and rte_errno is set. 3137 */ 3138 static int 3139 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3140 struct rte_flow *flow, 3141 const struct rte_flow_action *actions, 3142 struct rte_flow_error *error) 3143 { 3144 struct mlx5_priv *priv = dev->data->dev_private; 3145 struct mlx5_dev_config *config = &priv->config; 3146 struct mlx5_flow_mreg_copy_resource *mcp_res; 3147 const struct rte_flow_action_mark *mark; 3148 3149 /* Check whether extensive metadata feature is engaged. */ 3150 if (!config->dv_flow_en || 3151 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3152 !mlx5_flow_ext_mreg_supported(dev) || 3153 !priv->sh->dv_regc0_mask) 3154 return 0; 3155 /* Find MARK action. */ 3156 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3157 switch (actions->type) { 3158 case RTE_FLOW_ACTION_TYPE_FLAG: 3159 mcp_res = flow_mreg_add_copy_action 3160 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3161 if (!mcp_res) 3162 return -rte_errno; 3163 flow->mreg_copy = mcp_res; 3164 if (dev->data->dev_started) { 3165 mcp_res->appcnt++; 3166 flow->copy_applied = 1; 3167 } 3168 return 0; 3169 case RTE_FLOW_ACTION_TYPE_MARK: 3170 mark = (const struct rte_flow_action_mark *) 3171 actions->conf; 3172 mcp_res = 3173 flow_mreg_add_copy_action(dev, mark->id, error); 3174 if (!mcp_res) 3175 return -rte_errno; 3176 flow->mreg_copy = mcp_res; 3177 if (dev->data->dev_started) { 3178 mcp_res->appcnt++; 3179 flow->copy_applied = 1; 3180 } 3181 return 0; 3182 default: 3183 break; 3184 } 3185 } 3186 return 0; 3187 } 3188 3189 #define MLX5_MAX_SPLIT_ACTIONS 24 3190 #define MLX5_MAX_SPLIT_ITEMS 24 3191 3192 /** 3193 * Split the hairpin flow. 3194 * Since HW can't support encap on Rx we move the encap to Tx. 3195 * If the count action is after the encap then we also 3196 * move the count action. in this case the count will also measure 3197 * the outer bytes. 3198 * 3199 * @param dev 3200 * Pointer to Ethernet device. 3201 * @param[in] actions 3202 * Associated actions (list terminated by the END action). 3203 * @param[out] actions_rx 3204 * Rx flow actions. 3205 * @param[out] actions_tx 3206 * Tx flow actions.. 3207 * @param[out] pattern_tx 3208 * The pattern items for the Tx flow. 3209 * @param[out] flow_id 3210 * The flow ID connected to this flow. 3211 * 3212 * @return 3213 * 0 on success. 3214 */ 3215 static int 3216 flow_hairpin_split(struct rte_eth_dev *dev, 3217 const struct rte_flow_action actions[], 3218 struct rte_flow_action actions_rx[], 3219 struct rte_flow_action actions_tx[], 3220 struct rte_flow_item pattern_tx[], 3221 uint32_t *flow_id) 3222 { 3223 struct mlx5_priv *priv = dev->data->dev_private; 3224 const struct rte_flow_action_raw_encap *raw_encap; 3225 const struct rte_flow_action_raw_decap *raw_decap; 3226 struct mlx5_rte_flow_action_set_tag *set_tag; 3227 struct rte_flow_action *tag_action; 3228 struct mlx5_rte_flow_item_tag *tag_item; 3229 struct rte_flow_item *item; 3230 char *addr; 3231 int encap = 0; 3232 3233 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3234 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3235 switch (actions->type) { 3236 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3237 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3238 rte_memcpy(actions_tx, actions, 3239 sizeof(struct rte_flow_action)); 3240 actions_tx++; 3241 break; 3242 case RTE_FLOW_ACTION_TYPE_COUNT: 3243 if (encap) { 3244 rte_memcpy(actions_tx, actions, 3245 sizeof(struct rte_flow_action)); 3246 actions_tx++; 3247 } else { 3248 rte_memcpy(actions_rx, actions, 3249 sizeof(struct rte_flow_action)); 3250 actions_rx++; 3251 } 3252 break; 3253 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3254 raw_encap = actions->conf; 3255 if (raw_encap->size > 3256 (sizeof(struct rte_flow_item_eth) + 3257 sizeof(struct rte_flow_item_ipv4))) { 3258 memcpy(actions_tx, actions, 3259 sizeof(struct rte_flow_action)); 3260 actions_tx++; 3261 encap = 1; 3262 } else { 3263 rte_memcpy(actions_rx, actions, 3264 sizeof(struct rte_flow_action)); 3265 actions_rx++; 3266 } 3267 break; 3268 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3269 raw_decap = actions->conf; 3270 if (raw_decap->size < 3271 (sizeof(struct rte_flow_item_eth) + 3272 sizeof(struct rte_flow_item_ipv4))) { 3273 memcpy(actions_tx, actions, 3274 sizeof(struct rte_flow_action)); 3275 actions_tx++; 3276 } else { 3277 rte_memcpy(actions_rx, actions, 3278 sizeof(struct rte_flow_action)); 3279 actions_rx++; 3280 } 3281 break; 3282 default: 3283 rte_memcpy(actions_rx, actions, 3284 sizeof(struct rte_flow_action)); 3285 actions_rx++; 3286 break; 3287 } 3288 } 3289 /* Add set meta action and end action for the Rx flow. */ 3290 tag_action = actions_rx; 3291 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3292 actions_rx++; 3293 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3294 actions_rx++; 3295 set_tag = (void *)actions_rx; 3296 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3297 assert(set_tag->id > REG_NONE); 3298 set_tag->data = *flow_id; 3299 tag_action->conf = set_tag; 3300 /* Create Tx item list. */ 3301 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3302 addr = (void *)&pattern_tx[2]; 3303 item = pattern_tx; 3304 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3305 tag_item = (void *)addr; 3306 tag_item->data = *flow_id; 3307 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3308 assert(set_tag->id > REG_NONE); 3309 item->spec = tag_item; 3310 addr += sizeof(struct mlx5_rte_flow_item_tag); 3311 tag_item = (void *)addr; 3312 tag_item->data = UINT32_MAX; 3313 tag_item->id = UINT16_MAX; 3314 item->mask = tag_item; 3315 addr += sizeof(struct mlx5_rte_flow_item_tag); 3316 item->last = NULL; 3317 item++; 3318 item->type = RTE_FLOW_ITEM_TYPE_END; 3319 return 0; 3320 } 3321 3322 /** 3323 * The last stage of splitting chain, just creates the subflow 3324 * without any modification. 3325 * 3326 * @param dev 3327 * Pointer to Ethernet device. 3328 * @param[in] flow 3329 * Parent flow structure pointer. 3330 * @param[in, out] sub_flow 3331 * Pointer to return the created subflow, may be NULL. 3332 * @param[in] attr 3333 * Flow rule attributes. 3334 * @param[in] items 3335 * Pattern specification (list terminated by the END pattern item). 3336 * @param[in] actions 3337 * Associated actions (list terminated by the END action). 3338 * @param[in] external 3339 * This flow rule is created by request external to PMD. 3340 * @param[out] error 3341 * Perform verbose error reporting if not NULL. 3342 * @return 3343 * 0 on success, negative value otherwise 3344 */ 3345 static int 3346 flow_create_split_inner(struct rte_eth_dev *dev, 3347 struct rte_flow *flow, 3348 struct mlx5_flow **sub_flow, 3349 const struct rte_flow_attr *attr, 3350 const struct rte_flow_item items[], 3351 const struct rte_flow_action actions[], 3352 bool external, struct rte_flow_error *error) 3353 { 3354 struct mlx5_flow *dev_flow; 3355 3356 dev_flow = flow_drv_prepare(flow, attr, items, actions, error); 3357 if (!dev_flow) 3358 return -rte_errno; 3359 dev_flow->flow = flow; 3360 dev_flow->external = external; 3361 /* Subflow object was created, we must include one in the list. */ 3362 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 3363 if (sub_flow) 3364 *sub_flow = dev_flow; 3365 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3366 } 3367 3368 /** 3369 * Split action list having QUEUE/RSS for metadata register copy. 3370 * 3371 * Once Q/RSS action is detected in user's action list, the flow action 3372 * should be split in order to copy metadata registers, which will happen in 3373 * RX_CP_TBL like, 3374 * - CQE->flow_tag := reg_c[1] (MARK) 3375 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3376 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3377 * This is because the last action of each flow must be a terminal action 3378 * (QUEUE, RSS or DROP). 3379 * 3380 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3381 * stored and kept in the mlx5_flow structure per each sub_flow. 3382 * 3383 * The Q/RSS action is replaced with, 3384 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3385 * And the following JUMP action is added at the end, 3386 * - JUMP, to RX_CP_TBL. 3387 * 3388 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3389 * flow_create_split_metadata() routine. The flow will look like, 3390 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3391 * 3392 * @param dev 3393 * Pointer to Ethernet device. 3394 * @param[out] split_actions 3395 * Pointer to store split actions to jump to CP_TBL. 3396 * @param[in] actions 3397 * Pointer to the list of original flow actions. 3398 * @param[in] qrss 3399 * Pointer to the Q/RSS action. 3400 * @param[in] actions_n 3401 * Number of original actions. 3402 * @param[out] error 3403 * Perform verbose error reporting if not NULL. 3404 * 3405 * @return 3406 * non-zero unique flow_id on success, otherwise 0 and 3407 * error/rte_error are set. 3408 */ 3409 static uint32_t 3410 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3411 struct rte_flow_action *split_actions, 3412 const struct rte_flow_action *actions, 3413 const struct rte_flow_action *qrss, 3414 int actions_n, struct rte_flow_error *error) 3415 { 3416 struct mlx5_rte_flow_action_set_tag *set_tag; 3417 struct rte_flow_action_jump *jump; 3418 const int qrss_idx = qrss - actions; 3419 uint32_t flow_id; 3420 int ret = 0; 3421 3422 /* 3423 * Given actions will be split 3424 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3425 * - Add jump to mreg CP_TBL. 3426 * As a result, there will be one more action. 3427 */ 3428 ++actions_n; 3429 /* 3430 * Allocate the new subflow ID. This one is unique within 3431 * device and not shared with representors. Otherwise, 3432 * we would have to resolve multi-thread access synch 3433 * issue. Each flow on the shared device is appended 3434 * with source vport identifier, so the resulting 3435 * flows will be unique in the shared (by master and 3436 * representors) domain even if they have coinciding 3437 * IDs. 3438 */ 3439 flow_id = flow_qrss_get_id(dev); 3440 if (!flow_id) 3441 return rte_flow_error_set(error, ENOMEM, 3442 RTE_FLOW_ERROR_TYPE_ACTION, 3443 NULL, "can't allocate id " 3444 "for split Q/RSS subflow"); 3445 /* Internal SET_TAG action to set flow ID. */ 3446 set_tag = (void *)(split_actions + actions_n); 3447 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3448 .data = flow_id, 3449 }; 3450 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3451 if (ret < 0) 3452 return ret; 3453 set_tag->id = ret; 3454 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3455 jump = (void *)(set_tag + 1); 3456 *jump = (struct rte_flow_action_jump){ 3457 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3458 }; 3459 /* Construct new actions array. */ 3460 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3461 /* Replace QUEUE/RSS action. */ 3462 split_actions[qrss_idx] = (struct rte_flow_action){ 3463 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3464 .conf = set_tag, 3465 }; 3466 split_actions[actions_n - 2] = (struct rte_flow_action){ 3467 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3468 .conf = jump, 3469 }; 3470 split_actions[actions_n - 1] = (struct rte_flow_action){ 3471 .type = RTE_FLOW_ACTION_TYPE_END, 3472 }; 3473 return flow_id; 3474 } 3475 3476 /** 3477 * Extend the given action list for Tx metadata copy. 3478 * 3479 * Copy the given action list to the ext_actions and add flow metadata register 3480 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3481 * 3482 * @param[out] ext_actions 3483 * Pointer to the extended action list. 3484 * @param[in] actions 3485 * Pointer to the list of actions. 3486 * @param[in] actions_n 3487 * Number of actions in the list. 3488 * @param[out] error 3489 * Perform verbose error reporting if not NULL. 3490 * 3491 * @return 3492 * 0 on success, negative value otherwise 3493 */ 3494 static int 3495 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3496 struct rte_flow_action *ext_actions, 3497 const struct rte_flow_action *actions, 3498 int actions_n, struct rte_flow_error *error) 3499 { 3500 struct mlx5_flow_action_copy_mreg *cp_mreg = 3501 (struct mlx5_flow_action_copy_mreg *) 3502 (ext_actions + actions_n + 1); 3503 int ret; 3504 3505 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3506 if (ret < 0) 3507 return ret; 3508 cp_mreg->dst = ret; 3509 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3510 if (ret < 0) 3511 return ret; 3512 cp_mreg->src = ret; 3513 memcpy(ext_actions, actions, 3514 sizeof(*ext_actions) * actions_n); 3515 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3516 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3517 .conf = cp_mreg, 3518 }; 3519 ext_actions[actions_n] = (struct rte_flow_action){ 3520 .type = RTE_FLOW_ACTION_TYPE_END, 3521 }; 3522 return 0; 3523 } 3524 3525 /** 3526 * The splitting for metadata feature. 3527 * 3528 * - Q/RSS action on NIC Rx should be split in order to pass by 3529 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3530 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3531 * 3532 * - All the actions on NIC Tx should have a mreg copy action to 3533 * copy reg_a from WQE to reg_c[0]. 3534 * 3535 * @param dev 3536 * Pointer to Ethernet device. 3537 * @param[in] flow 3538 * Parent flow structure pointer. 3539 * @param[in] attr 3540 * Flow rule attributes. 3541 * @param[in] items 3542 * Pattern specification (list terminated by the END pattern item). 3543 * @param[in] actions 3544 * Associated actions (list terminated by the END action). 3545 * @param[in] external 3546 * This flow rule is created by request external to PMD. 3547 * @param[out] error 3548 * Perform verbose error reporting if not NULL. 3549 * @return 3550 * 0 on success, negative value otherwise 3551 */ 3552 static int 3553 flow_create_split_metadata(struct rte_eth_dev *dev, 3554 struct rte_flow *flow, 3555 const struct rte_flow_attr *attr, 3556 const struct rte_flow_item items[], 3557 const struct rte_flow_action actions[], 3558 bool external, struct rte_flow_error *error) 3559 { 3560 struct mlx5_priv *priv = dev->data->dev_private; 3561 struct mlx5_dev_config *config = &priv->config; 3562 const struct rte_flow_action *qrss = NULL; 3563 struct rte_flow_action *ext_actions = NULL; 3564 struct mlx5_flow *dev_flow = NULL; 3565 uint32_t qrss_id = 0; 3566 size_t act_size; 3567 int actions_n; 3568 int ret; 3569 3570 /* Check whether extensive metadata feature is engaged. */ 3571 if (!config->dv_flow_en || 3572 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3573 !mlx5_flow_ext_mreg_supported(dev)) 3574 return flow_create_split_inner(dev, flow, NULL, attr, items, 3575 actions, external, error); 3576 actions_n = flow_parse_qrss_action(actions, &qrss); 3577 if (qrss) { 3578 /* Exclude hairpin flows from splitting. */ 3579 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3580 const struct rte_flow_action_queue *queue; 3581 3582 queue = qrss->conf; 3583 if (mlx5_rxq_get_type(dev, queue->index) == 3584 MLX5_RXQ_TYPE_HAIRPIN) 3585 qrss = NULL; 3586 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3587 const struct rte_flow_action_rss *rss; 3588 3589 rss = qrss->conf; 3590 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3591 MLX5_RXQ_TYPE_HAIRPIN) 3592 qrss = NULL; 3593 } 3594 } 3595 if (qrss) { 3596 /* 3597 * Q/RSS action on NIC Rx should be split in order to pass by 3598 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3599 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3600 */ 3601 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3602 sizeof(struct rte_flow_action_set_tag) + 3603 sizeof(struct rte_flow_action_jump); 3604 ext_actions = rte_zmalloc(__func__, act_size, 0); 3605 if (!ext_actions) 3606 return rte_flow_error_set(error, ENOMEM, 3607 RTE_FLOW_ERROR_TYPE_ACTION, 3608 NULL, "no memory to split " 3609 "metadata flow"); 3610 /* 3611 * Create the new actions list with removed Q/RSS action 3612 * and appended set tag and jump to register copy table 3613 * (RX_CP_TBL). We should preallocate unique tag ID here 3614 * in advance, because it is needed for set tag action. 3615 */ 3616 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3617 qrss, actions_n, error); 3618 if (!qrss_id) { 3619 ret = -rte_errno; 3620 goto exit; 3621 } 3622 } else if (attr->egress && !attr->transfer) { 3623 /* 3624 * All the actions on NIC Tx should have a metadata register 3625 * copy action to copy reg_a from WQE to reg_c[meta] 3626 */ 3627 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3628 sizeof(struct mlx5_flow_action_copy_mreg); 3629 ext_actions = rte_zmalloc(__func__, act_size, 0); 3630 if (!ext_actions) 3631 return rte_flow_error_set(error, ENOMEM, 3632 RTE_FLOW_ERROR_TYPE_ACTION, 3633 NULL, "no memory to split " 3634 "metadata flow"); 3635 /* Create the action list appended with copy register. */ 3636 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 3637 actions_n, error); 3638 if (ret < 0) 3639 goto exit; 3640 } 3641 /* Add the unmodified original or prefix subflow. */ 3642 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, 3643 ext_actions ? ext_actions : actions, 3644 external, error); 3645 if (ret < 0) 3646 goto exit; 3647 assert(dev_flow); 3648 if (qrss_id) { 3649 const struct rte_flow_attr q_attr = { 3650 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3651 .ingress = 1, 3652 }; 3653 /* Internal PMD action to set register. */ 3654 struct mlx5_rte_flow_item_tag q_tag_spec = { 3655 .data = qrss_id, 3656 .id = 0, 3657 }; 3658 struct rte_flow_item q_items[] = { 3659 { 3660 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3661 .spec = &q_tag_spec, 3662 .last = NULL, 3663 .mask = NULL, 3664 }, 3665 { 3666 .type = RTE_FLOW_ITEM_TYPE_END, 3667 }, 3668 }; 3669 struct rte_flow_action q_actions[] = { 3670 { 3671 .type = qrss->type, 3672 .conf = qrss->conf, 3673 }, 3674 { 3675 .type = RTE_FLOW_ACTION_TYPE_END, 3676 }, 3677 }; 3678 uint64_t hash_fields = dev_flow->hash_fields; 3679 /* 3680 * Put unique id in prefix flow due to it is destroyed after 3681 * prefix flow and id will be freed after there is no actual 3682 * flows with this id and identifier reallocation becomes 3683 * possible (for example, for other flows in other threads). 3684 */ 3685 dev_flow->qrss_id = qrss_id; 3686 qrss_id = 0; 3687 dev_flow = NULL; 3688 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3689 if (ret < 0) 3690 goto exit; 3691 q_tag_spec.id = ret; 3692 /* Add suffix subflow to execute Q/RSS. */ 3693 ret = flow_create_split_inner(dev, flow, &dev_flow, 3694 &q_attr, q_items, q_actions, 3695 external, error); 3696 if (ret < 0) 3697 goto exit; 3698 assert(dev_flow); 3699 dev_flow->hash_fields = hash_fields; 3700 } 3701 3702 exit: 3703 /* 3704 * We do not destroy the partially created sub_flows in case of error. 3705 * These ones are included into parent flow list and will be destroyed 3706 * by flow_drv_destroy. 3707 */ 3708 flow_qrss_free_id(dev, qrss_id); 3709 rte_free(ext_actions); 3710 return ret; 3711 } 3712 3713 /** 3714 * Split the flow to subflow set. The splitters might be linked 3715 * in the chain, like this: 3716 * flow_create_split_outer() calls: 3717 * flow_create_split_meter() calls: 3718 * flow_create_split_metadata(meter_subflow_0) calls: 3719 * flow_create_split_inner(metadata_subflow_0) 3720 * flow_create_split_inner(metadata_subflow_1) 3721 * flow_create_split_inner(metadata_subflow_2) 3722 * flow_create_split_metadata(meter_subflow_1) calls: 3723 * flow_create_split_inner(metadata_subflow_0) 3724 * flow_create_split_inner(metadata_subflow_1) 3725 * flow_create_split_inner(metadata_subflow_2) 3726 * 3727 * This provide flexible way to add new levels of flow splitting. 3728 * The all of successfully created subflows are included to the 3729 * parent flow dev_flow list. 3730 * 3731 * @param dev 3732 * Pointer to Ethernet device. 3733 * @param[in] flow 3734 * Parent flow structure pointer. 3735 * @param[in] attr 3736 * Flow rule attributes. 3737 * @param[in] items 3738 * Pattern specification (list terminated by the END pattern item). 3739 * @param[in] actions 3740 * Associated actions (list terminated by the END action). 3741 * @param[in] external 3742 * This flow rule is created by request external to PMD. 3743 * @param[out] error 3744 * Perform verbose error reporting if not NULL. 3745 * @return 3746 * 0 on success, negative value otherwise 3747 */ 3748 static int 3749 flow_create_split_outer(struct rte_eth_dev *dev, 3750 struct rte_flow *flow, 3751 const struct rte_flow_attr *attr, 3752 const struct rte_flow_item items[], 3753 const struct rte_flow_action actions[], 3754 bool external, struct rte_flow_error *error) 3755 { 3756 int ret; 3757 3758 ret = flow_create_split_metadata(dev, flow, attr, items, 3759 actions, external, error); 3760 assert(ret <= 0); 3761 return ret; 3762 } 3763 3764 /** 3765 * Create a flow and add it to @p list. 3766 * 3767 * @param dev 3768 * Pointer to Ethernet device. 3769 * @param list 3770 * Pointer to a TAILQ flow list. If this parameter NULL, 3771 * no list insertion occurred, flow is just created, 3772 * this is caller's responsibility to track the 3773 * created flow. 3774 * @param[in] attr 3775 * Flow rule attributes. 3776 * @param[in] items 3777 * Pattern specification (list terminated by the END pattern item). 3778 * @param[in] actions 3779 * Associated actions (list terminated by the END action). 3780 * @param[in] external 3781 * This flow rule is created by request external to PMD. 3782 * @param[out] error 3783 * Perform verbose error reporting if not NULL. 3784 * 3785 * @return 3786 * A flow on success, NULL otherwise and rte_errno is set. 3787 */ 3788 static struct rte_flow * 3789 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 3790 const struct rte_flow_attr *attr, 3791 const struct rte_flow_item items[], 3792 const struct rte_flow_action actions[], 3793 bool external, struct rte_flow_error *error) 3794 { 3795 struct mlx5_priv *priv = dev->data->dev_private; 3796 struct rte_flow *flow = NULL; 3797 struct mlx5_flow *dev_flow; 3798 const struct rte_flow_action_rss *rss; 3799 union { 3800 struct rte_flow_expand_rss buf; 3801 uint8_t buffer[2048]; 3802 } expand_buffer; 3803 union { 3804 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 3805 uint8_t buffer[2048]; 3806 } actions_rx; 3807 union { 3808 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 3809 uint8_t buffer[2048]; 3810 } actions_hairpin_tx; 3811 union { 3812 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 3813 uint8_t buffer[2048]; 3814 } items_tx; 3815 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 3816 const struct rte_flow_action *p_actions_rx = actions; 3817 int ret; 3818 uint32_t i; 3819 uint32_t flow_size; 3820 int hairpin_flow = 0; 3821 uint32_t hairpin_id = 0; 3822 struct rte_flow_attr attr_tx = { .priority = 0 }; 3823 3824 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 3825 if (hairpin_flow > 0) { 3826 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 3827 rte_errno = EINVAL; 3828 return NULL; 3829 } 3830 flow_hairpin_split(dev, actions, actions_rx.actions, 3831 actions_hairpin_tx.actions, items_tx.items, 3832 &hairpin_id); 3833 p_actions_rx = actions_rx.actions; 3834 } 3835 ret = flow_drv_validate(dev, attr, items, p_actions_rx, external, 3836 error); 3837 if (ret < 0) 3838 goto error_before_flow; 3839 flow_size = sizeof(struct rte_flow); 3840 rss = flow_get_rss_action(p_actions_rx); 3841 if (rss) 3842 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t), 3843 sizeof(void *)); 3844 else 3845 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *)); 3846 flow = rte_calloc(__func__, 1, flow_size, 0); 3847 if (!flow) { 3848 rte_errno = ENOMEM; 3849 goto error_before_flow; 3850 } 3851 flow->drv_type = flow_get_drv_type(dev, attr); 3852 if (hairpin_id != 0) 3853 flow->hairpin_flow_id = hairpin_id; 3854 assert(flow->drv_type > MLX5_FLOW_TYPE_MIN && 3855 flow->drv_type < MLX5_FLOW_TYPE_MAX); 3856 flow->rss.queue = (void *)(flow + 1); 3857 if (rss) { 3858 /* 3859 * The following information is required by 3860 * mlx5_flow_hashfields_adjust() in advance. 3861 */ 3862 flow->rss.level = rss->level; 3863 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 3864 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; 3865 } 3866 LIST_INIT(&flow->dev_flows); 3867 if (rss && rss->types) { 3868 unsigned int graph_root; 3869 3870 graph_root = find_graph_root(items, rss->level); 3871 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 3872 items, rss->types, 3873 mlx5_support_expansion, 3874 graph_root); 3875 assert(ret > 0 && 3876 (unsigned int)ret < sizeof(expand_buffer.buffer)); 3877 } else { 3878 buf->entries = 1; 3879 buf->entry[0].pattern = (void *)(uintptr_t)items; 3880 } 3881 for (i = 0; i < buf->entries; ++i) { 3882 /* 3883 * The splitter may create multiple dev_flows, 3884 * depending on configuration. In the simplest 3885 * case it just creates unmodified original flow. 3886 */ 3887 ret = flow_create_split_outer(dev, flow, attr, 3888 buf->entry[i].pattern, 3889 p_actions_rx, external, 3890 error); 3891 if (ret < 0) 3892 goto error; 3893 } 3894 /* Create the tx flow. */ 3895 if (hairpin_flow) { 3896 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 3897 attr_tx.ingress = 0; 3898 attr_tx.egress = 1; 3899 dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items, 3900 actions_hairpin_tx.actions, error); 3901 if (!dev_flow) 3902 goto error; 3903 dev_flow->flow = flow; 3904 dev_flow->external = 0; 3905 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 3906 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 3907 items_tx.items, 3908 actions_hairpin_tx.actions, error); 3909 if (ret < 0) 3910 goto error; 3911 } 3912 /* 3913 * Update the metadata register copy table. If extensive 3914 * metadata feature is enabled and registers are supported 3915 * we might create the extra rte_flow for each unique 3916 * MARK/FLAG action ID. 3917 * 3918 * The table is updated for ingress Flows only, because 3919 * the egress Flows belong to the different device and 3920 * copy table should be updated in peer NIC Rx domain. 3921 */ 3922 if (attr->ingress && 3923 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 3924 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 3925 if (ret) 3926 goto error; 3927 } 3928 if (dev->data->dev_started) { 3929 ret = flow_drv_apply(dev, flow, error); 3930 if (ret < 0) 3931 goto error; 3932 } 3933 if (list) 3934 TAILQ_INSERT_TAIL(list, flow, next); 3935 flow_rxq_flags_set(dev, flow); 3936 return flow; 3937 error_before_flow: 3938 if (hairpin_id) 3939 mlx5_flow_id_release(priv->sh->flow_id_pool, 3940 hairpin_id); 3941 return NULL; 3942 error: 3943 assert(flow); 3944 flow_mreg_del_copy_action(dev, flow); 3945 ret = rte_errno; /* Save rte_errno before cleanup. */ 3946 if (flow->hairpin_flow_id) 3947 mlx5_flow_id_release(priv->sh->flow_id_pool, 3948 flow->hairpin_flow_id); 3949 assert(flow); 3950 flow_drv_destroy(dev, flow); 3951 rte_free(flow); 3952 rte_errno = ret; /* Restore rte_errno. */ 3953 return NULL; 3954 } 3955 3956 /** 3957 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 3958 * incoming packets to table 1. 3959 * 3960 * Other flow rules, requested for group n, will be created in 3961 * e-switch table n+1. 3962 * Jump action to e-switch group n will be created to group n+1. 3963 * 3964 * Used when working in switchdev mode, to utilise advantages of table 1 3965 * and above. 3966 * 3967 * @param dev 3968 * Pointer to Ethernet device. 3969 * 3970 * @return 3971 * Pointer to flow on success, NULL otherwise and rte_errno is set. 3972 */ 3973 struct rte_flow * 3974 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 3975 { 3976 const struct rte_flow_attr attr = { 3977 .group = 0, 3978 .priority = 0, 3979 .ingress = 1, 3980 .egress = 0, 3981 .transfer = 1, 3982 }; 3983 const struct rte_flow_item pattern = { 3984 .type = RTE_FLOW_ITEM_TYPE_END, 3985 }; 3986 struct rte_flow_action_jump jump = { 3987 .group = 1, 3988 }; 3989 const struct rte_flow_action actions[] = { 3990 { 3991 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3992 .conf = &jump, 3993 }, 3994 { 3995 .type = RTE_FLOW_ACTION_TYPE_END, 3996 }, 3997 }; 3998 struct mlx5_priv *priv = dev->data->dev_private; 3999 struct rte_flow_error error; 4000 4001 return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern, 4002 actions, false, &error); 4003 } 4004 4005 /** 4006 * Create a flow. 4007 * 4008 * @see rte_flow_create() 4009 * @see rte_flow_ops 4010 */ 4011 struct rte_flow * 4012 mlx5_flow_create(struct rte_eth_dev *dev, 4013 const struct rte_flow_attr *attr, 4014 const struct rte_flow_item items[], 4015 const struct rte_flow_action actions[], 4016 struct rte_flow_error *error) 4017 { 4018 struct mlx5_priv *priv = dev->data->dev_private; 4019 4020 return flow_list_create(dev, &priv->flows, 4021 attr, items, actions, true, error); 4022 } 4023 4024 /** 4025 * Destroy a flow in a list. 4026 * 4027 * @param dev 4028 * Pointer to Ethernet device. 4029 * @param list 4030 * Pointer to a TAILQ flow list. If this parameter NULL, 4031 * there is no flow removal from the list. 4032 * @param[in] flow 4033 * Flow to destroy. 4034 */ 4035 static void 4036 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 4037 struct rte_flow *flow) 4038 { 4039 struct mlx5_priv *priv = dev->data->dev_private; 4040 4041 /* 4042 * Update RX queue flags only if port is started, otherwise it is 4043 * already clean. 4044 */ 4045 if (dev->data->dev_started) 4046 flow_rxq_flags_trim(dev, flow); 4047 if (flow->hairpin_flow_id) 4048 mlx5_flow_id_release(priv->sh->flow_id_pool, 4049 flow->hairpin_flow_id); 4050 flow_drv_destroy(dev, flow); 4051 if (list) 4052 TAILQ_REMOVE(list, flow, next); 4053 flow_mreg_del_copy_action(dev, flow); 4054 rte_free(flow->fdir); 4055 rte_free(flow); 4056 } 4057 4058 /** 4059 * Destroy all flows. 4060 * 4061 * @param dev 4062 * Pointer to Ethernet device. 4063 * @param list 4064 * Pointer to a TAILQ flow list. 4065 */ 4066 void 4067 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 4068 { 4069 while (!TAILQ_EMPTY(list)) { 4070 struct rte_flow *flow; 4071 4072 flow = TAILQ_FIRST(list); 4073 flow_list_destroy(dev, list, flow); 4074 } 4075 } 4076 4077 /** 4078 * Remove all flows. 4079 * 4080 * @param dev 4081 * Pointer to Ethernet device. 4082 * @param list 4083 * Pointer to a TAILQ flow list. 4084 */ 4085 void 4086 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 4087 { 4088 struct rte_flow *flow; 4089 4090 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 4091 flow_drv_remove(dev, flow); 4092 flow_mreg_stop_copy_action(dev, flow); 4093 } 4094 flow_mreg_del_default_copy_action(dev); 4095 flow_rxq_flags_clear(dev); 4096 } 4097 4098 /** 4099 * Add all flows. 4100 * 4101 * @param dev 4102 * Pointer to Ethernet device. 4103 * @param list 4104 * Pointer to a TAILQ flow list. 4105 * 4106 * @return 4107 * 0 on success, a negative errno value otherwise and rte_errno is set. 4108 */ 4109 int 4110 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 4111 { 4112 struct rte_flow *flow; 4113 struct rte_flow_error error; 4114 int ret = 0; 4115 4116 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4117 ret = flow_mreg_add_default_copy_action(dev, &error); 4118 if (ret < 0) 4119 return -rte_errno; 4120 /* Apply Flows created by application. */ 4121 TAILQ_FOREACH(flow, list, next) { 4122 ret = flow_mreg_start_copy_action(dev, flow); 4123 if (ret < 0) 4124 goto error; 4125 ret = flow_drv_apply(dev, flow, &error); 4126 if (ret < 0) 4127 goto error; 4128 flow_rxq_flags_set(dev, flow); 4129 } 4130 return 0; 4131 error: 4132 ret = rte_errno; /* Save rte_errno before cleanup. */ 4133 mlx5_flow_stop(dev, list); 4134 rte_errno = ret; /* Restore rte_errno. */ 4135 return -rte_errno; 4136 } 4137 4138 /** 4139 * Verify the flow list is empty 4140 * 4141 * @param dev 4142 * Pointer to Ethernet device. 4143 * 4144 * @return the number of flows not released. 4145 */ 4146 int 4147 mlx5_flow_verify(struct rte_eth_dev *dev) 4148 { 4149 struct mlx5_priv *priv = dev->data->dev_private; 4150 struct rte_flow *flow; 4151 int ret = 0; 4152 4153 TAILQ_FOREACH(flow, &priv->flows, next) { 4154 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4155 dev->data->port_id, (void *)flow); 4156 ++ret; 4157 } 4158 return ret; 4159 } 4160 4161 /** 4162 * Enable default hairpin egress flow. 4163 * 4164 * @param dev 4165 * Pointer to Ethernet device. 4166 * @param queue 4167 * The queue index. 4168 * 4169 * @return 4170 * 0 on success, a negative errno value otherwise and rte_errno is set. 4171 */ 4172 int 4173 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4174 uint32_t queue) 4175 { 4176 struct mlx5_priv *priv = dev->data->dev_private; 4177 const struct rte_flow_attr attr = { 4178 .egress = 1, 4179 .priority = 0, 4180 }; 4181 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4182 .queue = queue, 4183 }; 4184 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4185 .queue = UINT32_MAX, 4186 }; 4187 struct rte_flow_item items[] = { 4188 { 4189 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4190 .spec = &queue_spec, 4191 .last = NULL, 4192 .mask = &queue_mask, 4193 }, 4194 { 4195 .type = RTE_FLOW_ITEM_TYPE_END, 4196 }, 4197 }; 4198 struct rte_flow_action_jump jump = { 4199 .group = MLX5_HAIRPIN_TX_TABLE, 4200 }; 4201 struct rte_flow_action actions[2]; 4202 struct rte_flow *flow; 4203 struct rte_flow_error error; 4204 4205 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4206 actions[0].conf = &jump; 4207 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4208 flow = flow_list_create(dev, &priv->ctrl_flows, 4209 &attr, items, actions, false, &error); 4210 if (!flow) { 4211 DRV_LOG(DEBUG, 4212 "Failed to create ctrl flow: rte_errno(%d)," 4213 " type(%d), message(%s)", 4214 rte_errno, error.type, 4215 error.message ? error.message : " (no stated reason)"); 4216 return -rte_errno; 4217 } 4218 return 0; 4219 } 4220 4221 /** 4222 * Enable a control flow configured from the control plane. 4223 * 4224 * @param dev 4225 * Pointer to Ethernet device. 4226 * @param eth_spec 4227 * An Ethernet flow spec to apply. 4228 * @param eth_mask 4229 * An Ethernet flow mask to apply. 4230 * @param vlan_spec 4231 * A VLAN flow spec to apply. 4232 * @param vlan_mask 4233 * A VLAN flow mask to apply. 4234 * 4235 * @return 4236 * 0 on success, a negative errno value otherwise and rte_errno is set. 4237 */ 4238 int 4239 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4240 struct rte_flow_item_eth *eth_spec, 4241 struct rte_flow_item_eth *eth_mask, 4242 struct rte_flow_item_vlan *vlan_spec, 4243 struct rte_flow_item_vlan *vlan_mask) 4244 { 4245 struct mlx5_priv *priv = dev->data->dev_private; 4246 const struct rte_flow_attr attr = { 4247 .ingress = 1, 4248 .priority = MLX5_FLOW_PRIO_RSVD, 4249 }; 4250 struct rte_flow_item items[] = { 4251 { 4252 .type = RTE_FLOW_ITEM_TYPE_ETH, 4253 .spec = eth_spec, 4254 .last = NULL, 4255 .mask = eth_mask, 4256 }, 4257 { 4258 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4259 RTE_FLOW_ITEM_TYPE_END, 4260 .spec = vlan_spec, 4261 .last = NULL, 4262 .mask = vlan_mask, 4263 }, 4264 { 4265 .type = RTE_FLOW_ITEM_TYPE_END, 4266 }, 4267 }; 4268 uint16_t queue[priv->reta_idx_n]; 4269 struct rte_flow_action_rss action_rss = { 4270 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4271 .level = 0, 4272 .types = priv->rss_conf.rss_hf, 4273 .key_len = priv->rss_conf.rss_key_len, 4274 .queue_num = priv->reta_idx_n, 4275 .key = priv->rss_conf.rss_key, 4276 .queue = queue, 4277 }; 4278 struct rte_flow_action actions[] = { 4279 { 4280 .type = RTE_FLOW_ACTION_TYPE_RSS, 4281 .conf = &action_rss, 4282 }, 4283 { 4284 .type = RTE_FLOW_ACTION_TYPE_END, 4285 }, 4286 }; 4287 struct rte_flow *flow; 4288 struct rte_flow_error error; 4289 unsigned int i; 4290 4291 if (!priv->reta_idx_n || !priv->rxqs_n) { 4292 return 0; 4293 } 4294 for (i = 0; i != priv->reta_idx_n; ++i) 4295 queue[i] = (*priv->reta_idx)[i]; 4296 flow = flow_list_create(dev, &priv->ctrl_flows, 4297 &attr, items, actions, false, &error); 4298 if (!flow) 4299 return -rte_errno; 4300 return 0; 4301 } 4302 4303 /** 4304 * Enable a flow control configured from the control plane. 4305 * 4306 * @param dev 4307 * Pointer to Ethernet device. 4308 * @param eth_spec 4309 * An Ethernet flow spec to apply. 4310 * @param eth_mask 4311 * An Ethernet flow mask to apply. 4312 * 4313 * @return 4314 * 0 on success, a negative errno value otherwise and rte_errno is set. 4315 */ 4316 int 4317 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4318 struct rte_flow_item_eth *eth_spec, 4319 struct rte_flow_item_eth *eth_mask) 4320 { 4321 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4322 } 4323 4324 /** 4325 * Destroy a flow. 4326 * 4327 * @see rte_flow_destroy() 4328 * @see rte_flow_ops 4329 */ 4330 int 4331 mlx5_flow_destroy(struct rte_eth_dev *dev, 4332 struct rte_flow *flow, 4333 struct rte_flow_error *error __rte_unused) 4334 { 4335 struct mlx5_priv *priv = dev->data->dev_private; 4336 4337 flow_list_destroy(dev, &priv->flows, flow); 4338 return 0; 4339 } 4340 4341 /** 4342 * Destroy all flows. 4343 * 4344 * @see rte_flow_flush() 4345 * @see rte_flow_ops 4346 */ 4347 int 4348 mlx5_flow_flush(struct rte_eth_dev *dev, 4349 struct rte_flow_error *error __rte_unused) 4350 { 4351 struct mlx5_priv *priv = dev->data->dev_private; 4352 4353 mlx5_flow_list_flush(dev, &priv->flows); 4354 return 0; 4355 } 4356 4357 /** 4358 * Isolated mode. 4359 * 4360 * @see rte_flow_isolate() 4361 * @see rte_flow_ops 4362 */ 4363 int 4364 mlx5_flow_isolate(struct rte_eth_dev *dev, 4365 int enable, 4366 struct rte_flow_error *error) 4367 { 4368 struct mlx5_priv *priv = dev->data->dev_private; 4369 4370 if (dev->data->dev_started) { 4371 rte_flow_error_set(error, EBUSY, 4372 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4373 NULL, 4374 "port must be stopped first"); 4375 return -rte_errno; 4376 } 4377 priv->isolated = !!enable; 4378 if (enable) 4379 dev->dev_ops = &mlx5_dev_ops_isolate; 4380 else 4381 dev->dev_ops = &mlx5_dev_ops; 4382 return 0; 4383 } 4384 4385 /** 4386 * Query a flow. 4387 * 4388 * @see rte_flow_query() 4389 * @see rte_flow_ops 4390 */ 4391 static int 4392 flow_drv_query(struct rte_eth_dev *dev, 4393 struct rte_flow *flow, 4394 const struct rte_flow_action *actions, 4395 void *data, 4396 struct rte_flow_error *error) 4397 { 4398 const struct mlx5_flow_driver_ops *fops; 4399 enum mlx5_flow_drv_type ftype = flow->drv_type; 4400 4401 assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 4402 fops = flow_get_drv_ops(ftype); 4403 4404 return fops->query(dev, flow, actions, data, error); 4405 } 4406 4407 /** 4408 * Query a flow. 4409 * 4410 * @see rte_flow_query() 4411 * @see rte_flow_ops 4412 */ 4413 int 4414 mlx5_flow_query(struct rte_eth_dev *dev, 4415 struct rte_flow *flow, 4416 const struct rte_flow_action *actions, 4417 void *data, 4418 struct rte_flow_error *error) 4419 { 4420 int ret; 4421 4422 ret = flow_drv_query(dev, flow, actions, data, error); 4423 if (ret < 0) 4424 return ret; 4425 return 0; 4426 } 4427 4428 /** 4429 * Convert a flow director filter to a generic flow. 4430 * 4431 * @param dev 4432 * Pointer to Ethernet device. 4433 * @param fdir_filter 4434 * Flow director filter to add. 4435 * @param attributes 4436 * Generic flow parameters structure. 4437 * 4438 * @return 4439 * 0 on success, a negative errno value otherwise and rte_errno is set. 4440 */ 4441 static int 4442 flow_fdir_filter_convert(struct rte_eth_dev *dev, 4443 const struct rte_eth_fdir_filter *fdir_filter, 4444 struct mlx5_fdir *attributes) 4445 { 4446 struct mlx5_priv *priv = dev->data->dev_private; 4447 const struct rte_eth_fdir_input *input = &fdir_filter->input; 4448 const struct rte_eth_fdir_masks *mask = 4449 &dev->data->dev_conf.fdir_conf.mask; 4450 4451 /* Validate queue number. */ 4452 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 4453 DRV_LOG(ERR, "port %u invalid queue number %d", 4454 dev->data->port_id, fdir_filter->action.rx_queue); 4455 rte_errno = EINVAL; 4456 return -rte_errno; 4457 } 4458 attributes->attr.ingress = 1; 4459 attributes->items[0] = (struct rte_flow_item) { 4460 .type = RTE_FLOW_ITEM_TYPE_ETH, 4461 .spec = &attributes->l2, 4462 .mask = &attributes->l2_mask, 4463 }; 4464 switch (fdir_filter->action.behavior) { 4465 case RTE_ETH_FDIR_ACCEPT: 4466 attributes->actions[0] = (struct rte_flow_action){ 4467 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 4468 .conf = &attributes->queue, 4469 }; 4470 break; 4471 case RTE_ETH_FDIR_REJECT: 4472 attributes->actions[0] = (struct rte_flow_action){ 4473 .type = RTE_FLOW_ACTION_TYPE_DROP, 4474 }; 4475 break; 4476 default: 4477 DRV_LOG(ERR, "port %u invalid behavior %d", 4478 dev->data->port_id, 4479 fdir_filter->action.behavior); 4480 rte_errno = ENOTSUP; 4481 return -rte_errno; 4482 } 4483 attributes->queue.index = fdir_filter->action.rx_queue; 4484 /* Handle L3. */ 4485 switch (fdir_filter->input.flow_type) { 4486 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4487 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4488 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4489 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 4490 .src_addr = input->flow.ip4_flow.src_ip, 4491 .dst_addr = input->flow.ip4_flow.dst_ip, 4492 .time_to_live = input->flow.ip4_flow.ttl, 4493 .type_of_service = input->flow.ip4_flow.tos, 4494 }; 4495 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 4496 .src_addr = mask->ipv4_mask.src_ip, 4497 .dst_addr = mask->ipv4_mask.dst_ip, 4498 .time_to_live = mask->ipv4_mask.ttl, 4499 .type_of_service = mask->ipv4_mask.tos, 4500 .next_proto_id = mask->ipv4_mask.proto, 4501 }; 4502 attributes->items[1] = (struct rte_flow_item){ 4503 .type = RTE_FLOW_ITEM_TYPE_IPV4, 4504 .spec = &attributes->l3, 4505 .mask = &attributes->l3_mask, 4506 }; 4507 break; 4508 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4509 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4510 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4511 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 4512 .hop_limits = input->flow.ipv6_flow.hop_limits, 4513 .proto = input->flow.ipv6_flow.proto, 4514 }; 4515 4516 memcpy(attributes->l3.ipv6.hdr.src_addr, 4517 input->flow.ipv6_flow.src_ip, 4518 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4519 memcpy(attributes->l3.ipv6.hdr.dst_addr, 4520 input->flow.ipv6_flow.dst_ip, 4521 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4522 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 4523 mask->ipv6_mask.src_ip, 4524 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4525 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 4526 mask->ipv6_mask.dst_ip, 4527 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4528 attributes->items[1] = (struct rte_flow_item){ 4529 .type = RTE_FLOW_ITEM_TYPE_IPV6, 4530 .spec = &attributes->l3, 4531 .mask = &attributes->l3_mask, 4532 }; 4533 break; 4534 default: 4535 DRV_LOG(ERR, "port %u invalid flow type%d", 4536 dev->data->port_id, fdir_filter->input.flow_type); 4537 rte_errno = ENOTSUP; 4538 return -rte_errno; 4539 } 4540 /* Handle L4. */ 4541 switch (fdir_filter->input.flow_type) { 4542 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4543 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4544 .src_port = input->flow.udp4_flow.src_port, 4545 .dst_port = input->flow.udp4_flow.dst_port, 4546 }; 4547 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4548 .src_port = mask->src_port_mask, 4549 .dst_port = mask->dst_port_mask, 4550 }; 4551 attributes->items[2] = (struct rte_flow_item){ 4552 .type = RTE_FLOW_ITEM_TYPE_UDP, 4553 .spec = &attributes->l4, 4554 .mask = &attributes->l4_mask, 4555 }; 4556 break; 4557 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4558 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4559 .src_port = input->flow.tcp4_flow.src_port, 4560 .dst_port = input->flow.tcp4_flow.dst_port, 4561 }; 4562 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4563 .src_port = mask->src_port_mask, 4564 .dst_port = mask->dst_port_mask, 4565 }; 4566 attributes->items[2] = (struct rte_flow_item){ 4567 .type = RTE_FLOW_ITEM_TYPE_TCP, 4568 .spec = &attributes->l4, 4569 .mask = &attributes->l4_mask, 4570 }; 4571 break; 4572 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4573 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4574 .src_port = input->flow.udp6_flow.src_port, 4575 .dst_port = input->flow.udp6_flow.dst_port, 4576 }; 4577 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4578 .src_port = mask->src_port_mask, 4579 .dst_port = mask->dst_port_mask, 4580 }; 4581 attributes->items[2] = (struct rte_flow_item){ 4582 .type = RTE_FLOW_ITEM_TYPE_UDP, 4583 .spec = &attributes->l4, 4584 .mask = &attributes->l4_mask, 4585 }; 4586 break; 4587 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4588 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4589 .src_port = input->flow.tcp6_flow.src_port, 4590 .dst_port = input->flow.tcp6_flow.dst_port, 4591 }; 4592 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4593 .src_port = mask->src_port_mask, 4594 .dst_port = mask->dst_port_mask, 4595 }; 4596 attributes->items[2] = (struct rte_flow_item){ 4597 .type = RTE_FLOW_ITEM_TYPE_TCP, 4598 .spec = &attributes->l4, 4599 .mask = &attributes->l4_mask, 4600 }; 4601 break; 4602 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4603 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4604 break; 4605 default: 4606 DRV_LOG(ERR, "port %u invalid flow type%d", 4607 dev->data->port_id, fdir_filter->input.flow_type); 4608 rte_errno = ENOTSUP; 4609 return -rte_errno; 4610 } 4611 return 0; 4612 } 4613 4614 #define FLOW_FDIR_CMP(f1, f2, fld) \ 4615 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 4616 4617 /** 4618 * Compare two FDIR flows. If items and actions are identical, the two flows are 4619 * regarded as same. 4620 * 4621 * @param dev 4622 * Pointer to Ethernet device. 4623 * @param f1 4624 * FDIR flow to compare. 4625 * @param f2 4626 * FDIR flow to compare. 4627 * 4628 * @return 4629 * Zero on match, 1 otherwise. 4630 */ 4631 static int 4632 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 4633 { 4634 if (FLOW_FDIR_CMP(f1, f2, attr) || 4635 FLOW_FDIR_CMP(f1, f2, l2) || 4636 FLOW_FDIR_CMP(f1, f2, l2_mask) || 4637 FLOW_FDIR_CMP(f1, f2, l3) || 4638 FLOW_FDIR_CMP(f1, f2, l3_mask) || 4639 FLOW_FDIR_CMP(f1, f2, l4) || 4640 FLOW_FDIR_CMP(f1, f2, l4_mask) || 4641 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 4642 return 1; 4643 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 4644 FLOW_FDIR_CMP(f1, f2, queue)) 4645 return 1; 4646 return 0; 4647 } 4648 4649 /** 4650 * Search device flow list to find out a matched FDIR flow. 4651 * 4652 * @param dev 4653 * Pointer to Ethernet device. 4654 * @param fdir_flow 4655 * FDIR flow to lookup. 4656 * 4657 * @return 4658 * Pointer of flow if found, NULL otherwise. 4659 */ 4660 static struct rte_flow * 4661 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 4662 { 4663 struct mlx5_priv *priv = dev->data->dev_private; 4664 struct rte_flow *flow = NULL; 4665 4666 assert(fdir_flow); 4667 TAILQ_FOREACH(flow, &priv->flows, next) { 4668 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) { 4669 DRV_LOG(DEBUG, "port %u found FDIR flow %p", 4670 dev->data->port_id, (void *)flow); 4671 break; 4672 } 4673 } 4674 return flow; 4675 } 4676 4677 /** 4678 * Add new flow director filter and store it in list. 4679 * 4680 * @param dev 4681 * Pointer to Ethernet device. 4682 * @param fdir_filter 4683 * Flow director filter to add. 4684 * 4685 * @return 4686 * 0 on success, a negative errno value otherwise and rte_errno is set. 4687 */ 4688 static int 4689 flow_fdir_filter_add(struct rte_eth_dev *dev, 4690 const struct rte_eth_fdir_filter *fdir_filter) 4691 { 4692 struct mlx5_priv *priv = dev->data->dev_private; 4693 struct mlx5_fdir *fdir_flow; 4694 struct rte_flow *flow; 4695 int ret; 4696 4697 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 4698 if (!fdir_flow) { 4699 rte_errno = ENOMEM; 4700 return -rte_errno; 4701 } 4702 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 4703 if (ret) 4704 goto error; 4705 flow = flow_fdir_filter_lookup(dev, fdir_flow); 4706 if (flow) { 4707 rte_errno = EEXIST; 4708 goto error; 4709 } 4710 flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 4711 fdir_flow->items, fdir_flow->actions, true, 4712 NULL); 4713 if (!flow) 4714 goto error; 4715 assert(!flow->fdir); 4716 flow->fdir = fdir_flow; 4717 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 4718 dev->data->port_id, (void *)flow); 4719 return 0; 4720 error: 4721 rte_free(fdir_flow); 4722 return -rte_errno; 4723 } 4724 4725 /** 4726 * Delete specific filter. 4727 * 4728 * @param dev 4729 * Pointer to Ethernet device. 4730 * @param fdir_filter 4731 * Filter to be deleted. 4732 * 4733 * @return 4734 * 0 on success, a negative errno value otherwise and rte_errno is set. 4735 */ 4736 static int 4737 flow_fdir_filter_delete(struct rte_eth_dev *dev, 4738 const struct rte_eth_fdir_filter *fdir_filter) 4739 { 4740 struct mlx5_priv *priv = dev->data->dev_private; 4741 struct rte_flow *flow; 4742 struct mlx5_fdir fdir_flow = { 4743 .attr.group = 0, 4744 }; 4745 int ret; 4746 4747 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 4748 if (ret) 4749 return -rte_errno; 4750 flow = flow_fdir_filter_lookup(dev, &fdir_flow); 4751 if (!flow) { 4752 rte_errno = ENOENT; 4753 return -rte_errno; 4754 } 4755 flow_list_destroy(dev, &priv->flows, flow); 4756 DRV_LOG(DEBUG, "port %u deleted FDIR flow %p", 4757 dev->data->port_id, (void *)flow); 4758 return 0; 4759 } 4760 4761 /** 4762 * Update queue for specific filter. 4763 * 4764 * @param dev 4765 * Pointer to Ethernet device. 4766 * @param fdir_filter 4767 * Filter to be updated. 4768 * 4769 * @return 4770 * 0 on success, a negative errno value otherwise and rte_errno is set. 4771 */ 4772 static int 4773 flow_fdir_filter_update(struct rte_eth_dev *dev, 4774 const struct rte_eth_fdir_filter *fdir_filter) 4775 { 4776 int ret; 4777 4778 ret = flow_fdir_filter_delete(dev, fdir_filter); 4779 if (ret) 4780 return ret; 4781 return flow_fdir_filter_add(dev, fdir_filter); 4782 } 4783 4784 /** 4785 * Flush all filters. 4786 * 4787 * @param dev 4788 * Pointer to Ethernet device. 4789 */ 4790 static void 4791 flow_fdir_filter_flush(struct rte_eth_dev *dev) 4792 { 4793 struct mlx5_priv *priv = dev->data->dev_private; 4794 4795 mlx5_flow_list_flush(dev, &priv->flows); 4796 } 4797 4798 /** 4799 * Get flow director information. 4800 * 4801 * @param dev 4802 * Pointer to Ethernet device. 4803 * @param[out] fdir_info 4804 * Resulting flow director information. 4805 */ 4806 static void 4807 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 4808 { 4809 struct rte_eth_fdir_masks *mask = 4810 &dev->data->dev_conf.fdir_conf.mask; 4811 4812 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 4813 fdir_info->guarant_spc = 0; 4814 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 4815 fdir_info->max_flexpayload = 0; 4816 fdir_info->flow_types_mask[0] = 0; 4817 fdir_info->flex_payload_unit = 0; 4818 fdir_info->max_flex_payload_segment_num = 0; 4819 fdir_info->flex_payload_limit = 0; 4820 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 4821 } 4822 4823 /** 4824 * Deal with flow director operations. 4825 * 4826 * @param dev 4827 * Pointer to Ethernet device. 4828 * @param filter_op 4829 * Operation to perform. 4830 * @param arg 4831 * Pointer to operation-specific structure. 4832 * 4833 * @return 4834 * 0 on success, a negative errno value otherwise and rte_errno is set. 4835 */ 4836 static int 4837 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 4838 void *arg) 4839 { 4840 enum rte_fdir_mode fdir_mode = 4841 dev->data->dev_conf.fdir_conf.mode; 4842 4843 if (filter_op == RTE_ETH_FILTER_NOP) 4844 return 0; 4845 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 4846 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 4847 DRV_LOG(ERR, "port %u flow director mode %d not supported", 4848 dev->data->port_id, fdir_mode); 4849 rte_errno = EINVAL; 4850 return -rte_errno; 4851 } 4852 switch (filter_op) { 4853 case RTE_ETH_FILTER_ADD: 4854 return flow_fdir_filter_add(dev, arg); 4855 case RTE_ETH_FILTER_UPDATE: 4856 return flow_fdir_filter_update(dev, arg); 4857 case RTE_ETH_FILTER_DELETE: 4858 return flow_fdir_filter_delete(dev, arg); 4859 case RTE_ETH_FILTER_FLUSH: 4860 flow_fdir_filter_flush(dev); 4861 break; 4862 case RTE_ETH_FILTER_INFO: 4863 flow_fdir_info_get(dev, arg); 4864 break; 4865 default: 4866 DRV_LOG(DEBUG, "port %u unknown operation %u", 4867 dev->data->port_id, filter_op); 4868 rte_errno = EINVAL; 4869 return -rte_errno; 4870 } 4871 return 0; 4872 } 4873 4874 /** 4875 * Manage filter operations. 4876 * 4877 * @param dev 4878 * Pointer to Ethernet device structure. 4879 * @param filter_type 4880 * Filter type. 4881 * @param filter_op 4882 * Operation to perform. 4883 * @param arg 4884 * Pointer to operation-specific structure. 4885 * 4886 * @return 4887 * 0 on success, a negative errno value otherwise and rte_errno is set. 4888 */ 4889 int 4890 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 4891 enum rte_filter_type filter_type, 4892 enum rte_filter_op filter_op, 4893 void *arg) 4894 { 4895 switch (filter_type) { 4896 case RTE_ETH_FILTER_GENERIC: 4897 if (filter_op != RTE_ETH_FILTER_GET) { 4898 rte_errno = EINVAL; 4899 return -rte_errno; 4900 } 4901 *(const void **)arg = &mlx5_flow_ops; 4902 return 0; 4903 case RTE_ETH_FILTER_FDIR: 4904 return flow_fdir_ctrl_func(dev, filter_op, arg); 4905 default: 4906 DRV_LOG(ERR, "port %u filter type (%d) not supported", 4907 dev->data->port_id, filter_type); 4908 rte_errno = ENOTSUP; 4909 return -rte_errno; 4910 } 4911 return 0; 4912 } 4913 4914 /** 4915 * Create the needed meter and suffix tables. 4916 * 4917 * @param[in] dev 4918 * Pointer to Ethernet device. 4919 * 4920 * @return 4921 * Pointer to table set on success, NULL otherwise. 4922 */ 4923 struct mlx5_meter_domains_infos * 4924 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) 4925 { 4926 const struct mlx5_flow_driver_ops *fops; 4927 4928 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 4929 return fops->create_mtr_tbls(dev); 4930 } 4931 4932 /** 4933 * Destroy the meter table set. 4934 * 4935 * @param[in] dev 4936 * Pointer to Ethernet device. 4937 * @param[in] tbl 4938 * Pointer to the meter table set. 4939 * 4940 * @return 4941 * 0 on success. 4942 */ 4943 int 4944 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 4945 struct mlx5_meter_domains_infos *tbls) 4946 { 4947 const struct mlx5_flow_driver_ops *fops; 4948 4949 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 4950 return fops->destroy_mtr_tbls(dev, tbls); 4951 } 4952 4953 /** 4954 * Create policer rules. 4955 * 4956 * @param[in] dev 4957 * Pointer to Ethernet device. 4958 * @param[in] fm 4959 * Pointer to flow meter structure. 4960 * @param[in] attr 4961 * Pointer to flow attributes. 4962 * 4963 * @return 4964 * 0 on success, -1 otherwise. 4965 */ 4966 int 4967 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 4968 struct mlx5_flow_meter *fm, 4969 const struct rte_flow_attr *attr) 4970 { 4971 const struct mlx5_flow_driver_ops *fops; 4972 4973 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 4974 return fops->create_policer_rules(dev, fm, attr); 4975 } 4976 4977 /** 4978 * Destroy policer rules. 4979 * 4980 * @param[in] fm 4981 * Pointer to flow meter structure. 4982 * @param[in] attr 4983 * Pointer to flow attributes. 4984 * 4985 * @return 4986 * 0 on success, -1 otherwise. 4987 */ 4988 int 4989 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 4990 struct mlx5_flow_meter *fm, 4991 const struct rte_flow_attr *attr) 4992 { 4993 const struct mlx5_flow_driver_ops *fops; 4994 4995 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 4996 return fops->destroy_policer_rules(dev, fm, attr); 4997 } 4998 4999 /** 5000 * Allocate a counter. 5001 * 5002 * @param[in] dev 5003 * Pointer to Ethernet device structure. 5004 * 5005 * @return 5006 * Pointer to allocated counter on success, NULL otherwise. 5007 */ 5008 struct mlx5_flow_counter * 5009 mlx5_counter_alloc(struct rte_eth_dev *dev) 5010 { 5011 const struct mlx5_flow_driver_ops *fops; 5012 struct rte_flow_attr attr = { .transfer = 0 }; 5013 5014 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5015 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5016 return fops->counter_alloc(dev); 5017 } 5018 DRV_LOG(ERR, 5019 "port %u counter allocate is not supported.", 5020 dev->data->port_id); 5021 return NULL; 5022 } 5023 5024 /** 5025 * Free a counter. 5026 * 5027 * @param[in] dev 5028 * Pointer to Ethernet device structure. 5029 * @param[in] cnt 5030 * Pointer to counter to be free. 5031 */ 5032 void 5033 mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) 5034 { 5035 const struct mlx5_flow_driver_ops *fops; 5036 struct rte_flow_attr attr = { .transfer = 0 }; 5037 5038 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5039 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5040 fops->counter_free(dev, cnt); 5041 return; 5042 } 5043 DRV_LOG(ERR, 5044 "port %u counter free is not supported.", 5045 dev->data->port_id); 5046 } 5047 5048 /** 5049 * Query counter statistics. 5050 * 5051 * @param[in] dev 5052 * Pointer to Ethernet device structure. 5053 * @param[in] cnt 5054 * Pointer to counter to query. 5055 * @param[in] clear 5056 * Set to clear counter statistics. 5057 * @param[out] pkts 5058 * The counter hits packets number to save. 5059 * @param[out] bytes 5060 * The counter hits bytes number to save. 5061 * 5062 * @return 5063 * 0 on success, a negative errno value otherwise. 5064 */ 5065 int 5066 mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt, 5067 bool clear, uint64_t *pkts, uint64_t *bytes) 5068 { 5069 const struct mlx5_flow_driver_ops *fops; 5070 struct rte_flow_attr attr = { .transfer = 0 }; 5071 5072 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5073 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5074 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5075 } 5076 DRV_LOG(ERR, 5077 "port %u counter query is not supported.", 5078 dev->data->port_id); 5079 return -ENOTSUP; 5080 } 5081 5082 #define MLX5_POOL_QUERY_FREQ_US 1000000 5083 5084 /** 5085 * Set the periodic procedure for triggering asynchronous batch queries for all 5086 * the counter pools. 5087 * 5088 * @param[in] sh 5089 * Pointer to mlx5_ibv_shared object. 5090 */ 5091 void 5092 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5093 { 5094 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0); 5095 uint32_t pools_n = rte_atomic16_read(&cont->n_valid); 5096 uint32_t us; 5097 5098 cont = MLX5_CNT_CONTAINER(sh, 1, 0); 5099 pools_n += rte_atomic16_read(&cont->n_valid); 5100 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5101 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5102 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5103 sh->cmng.query_thread_on = 0; 5104 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5105 } else { 5106 sh->cmng.query_thread_on = 1; 5107 } 5108 } 5109 5110 /** 5111 * The periodic procedure for triggering asynchronous batch queries for all the 5112 * counter pools. This function is probably called by the host thread. 5113 * 5114 * @param[in] arg 5115 * The parameter for the alarm process. 5116 */ 5117 void 5118 mlx5_flow_query_alarm(void *arg) 5119 { 5120 struct mlx5_ibv_shared *sh = arg; 5121 struct mlx5_devx_obj *dcs; 5122 uint16_t offset; 5123 int ret; 5124 uint8_t batch = sh->cmng.batch; 5125 uint16_t pool_index = sh->cmng.pool_index; 5126 struct mlx5_pools_container *cont; 5127 struct mlx5_pools_container *mcont; 5128 struct mlx5_flow_counter_pool *pool; 5129 5130 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5131 goto set_alarm; 5132 next_container: 5133 cont = MLX5_CNT_CONTAINER(sh, batch, 1); 5134 mcont = MLX5_CNT_CONTAINER(sh, batch, 0); 5135 /* Check if resize was done and need to flip a container. */ 5136 if (cont != mcont) { 5137 if (cont->pools) { 5138 /* Clean the old container. */ 5139 rte_free(cont->pools); 5140 memset(cont, 0, sizeof(*cont)); 5141 } 5142 rte_cio_wmb(); 5143 /* Flip the host container. */ 5144 sh->cmng.mhi[batch] ^= (uint8_t)2; 5145 cont = mcont; 5146 } 5147 if (!cont->pools) { 5148 /* 2 empty containers case is unexpected. */ 5149 if (unlikely(batch != sh->cmng.batch)) 5150 goto set_alarm; 5151 batch ^= 0x1; 5152 pool_index = 0; 5153 goto next_container; 5154 } 5155 pool = cont->pools[pool_index]; 5156 if (pool->raw_hw) 5157 /* There is a pool query in progress. */ 5158 goto set_alarm; 5159 pool->raw_hw = 5160 LIST_FIRST(&sh->cmng.free_stat_raws); 5161 if (!pool->raw_hw) 5162 /* No free counter statistics raw memory. */ 5163 goto set_alarm; 5164 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5165 (&pool->a64_dcs); 5166 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5167 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5168 offset, NULL, NULL, 5169 pool->raw_hw->mem_mng->dm->id, 5170 (void *)(uintptr_t) 5171 (pool->raw_hw->data + offset), 5172 sh->devx_comp, 5173 (uint64_t)(uintptr_t)pool); 5174 if (ret) { 5175 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5176 " %d", pool->min_dcs->id); 5177 pool->raw_hw = NULL; 5178 goto set_alarm; 5179 } 5180 pool->raw_hw->min_dcs_id = dcs->id; 5181 LIST_REMOVE(pool->raw_hw, next); 5182 sh->cmng.pending_queries++; 5183 pool_index++; 5184 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5185 batch ^= 0x1; 5186 pool_index = 0; 5187 } 5188 set_alarm: 5189 sh->cmng.batch = batch; 5190 sh->cmng.pool_index = pool_index; 5191 mlx5_set_query_alarm(sh); 5192 } 5193 5194 /** 5195 * Handler for the HW respond about ready values from an asynchronous batch 5196 * query. This function is probably called by the host thread. 5197 * 5198 * @param[in] sh 5199 * The pointer to the shared IB device context. 5200 * @param[in] async_id 5201 * The Devx async ID. 5202 * @param[in] status 5203 * The status of the completion. 5204 */ 5205 void 5206 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 5207 uint64_t async_id, int status) 5208 { 5209 struct mlx5_flow_counter_pool *pool = 5210 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 5211 struct mlx5_counter_stats_raw *raw_to_free; 5212 5213 if (unlikely(status)) { 5214 raw_to_free = pool->raw_hw; 5215 } else { 5216 raw_to_free = pool->raw; 5217 rte_spinlock_lock(&pool->sl); 5218 pool->raw = pool->raw_hw; 5219 rte_spinlock_unlock(&pool->sl); 5220 rte_atomic64_add(&pool->query_gen, 1); 5221 /* Be sure the new raw counters data is updated in memory. */ 5222 rte_cio_wmb(); 5223 } 5224 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 5225 pool->raw_hw = NULL; 5226 sh->cmng.pending_queries--; 5227 } 5228 5229 /** 5230 * Translate the rte_flow group index to HW table value. 5231 * 5232 * @param[in] attributes 5233 * Pointer to flow attributes 5234 * @param[in] external 5235 * Value is part of flow rule created by request external to PMD. 5236 * @param[in] group 5237 * rte_flow group index value. 5238 * @param[out] table 5239 * HW table value. 5240 * @param[out] error 5241 * Pointer to error structure. 5242 * 5243 * @return 5244 * 0 on success, a negative errno value otherwise and rte_errno is set. 5245 */ 5246 int 5247 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 5248 uint32_t group, uint32_t *table, 5249 struct rte_flow_error *error) 5250 { 5251 if (attributes->transfer && external) { 5252 if (group == UINT32_MAX) 5253 return rte_flow_error_set 5254 (error, EINVAL, 5255 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 5256 NULL, 5257 "group index not supported"); 5258 *table = group + 1; 5259 } else { 5260 *table = group; 5261 } 5262 return 0; 5263 } 5264 5265 /** 5266 * Discover availability of metadata reg_c's. 5267 * 5268 * Iteratively use test flows to check availability. 5269 * 5270 * @param[in] dev 5271 * Pointer to the Ethernet device structure. 5272 * 5273 * @return 5274 * 0 on success, a negative errno value otherwise and rte_errno is set. 5275 */ 5276 int 5277 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 5278 { 5279 struct mlx5_priv *priv = dev->data->dev_private; 5280 struct mlx5_dev_config *config = &priv->config; 5281 enum modify_reg idx; 5282 int n = 0; 5283 5284 /* reg_c[0] and reg_c[1] are reserved. */ 5285 config->flow_mreg_c[n++] = REG_C_0; 5286 config->flow_mreg_c[n++] = REG_C_1; 5287 /* Discover availability of other reg_c's. */ 5288 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 5289 struct rte_flow_attr attr = { 5290 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5291 .priority = MLX5_FLOW_PRIO_RSVD, 5292 .ingress = 1, 5293 }; 5294 struct rte_flow_item items[] = { 5295 [0] = { 5296 .type = RTE_FLOW_ITEM_TYPE_END, 5297 }, 5298 }; 5299 struct rte_flow_action actions[] = { 5300 [0] = { 5301 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5302 .conf = &(struct mlx5_flow_action_copy_mreg){ 5303 .src = REG_C_1, 5304 .dst = idx, 5305 }, 5306 }, 5307 [1] = { 5308 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5309 .conf = &(struct rte_flow_action_jump){ 5310 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5311 }, 5312 }, 5313 [2] = { 5314 .type = RTE_FLOW_ACTION_TYPE_END, 5315 }, 5316 }; 5317 struct rte_flow *flow; 5318 struct rte_flow_error error; 5319 5320 if (!config->dv_flow_en) 5321 break; 5322 /* Create internal flow, validation skips copy action. */ 5323 flow = flow_list_create(dev, NULL, &attr, items, 5324 actions, false, &error); 5325 if (!flow) 5326 continue; 5327 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 5328 config->flow_mreg_c[n++] = idx; 5329 flow_list_destroy(dev, NULL, flow); 5330 } 5331 for (; n < MLX5_MREG_C_NUM; ++n) 5332 config->flow_mreg_c[n] = REG_NONE; 5333 return 0; 5334 } 5335