1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 12 /* Verbs header. */ 13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 14 #ifdef PEDANTIC 15 #pragma GCC diagnostic ignored "-Wpedantic" 16 #endif 17 #include <infiniband/verbs.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_common.h> 23 #include <rte_ether.h> 24 #include <rte_ethdev_driver.h> 25 #include <rte_flow.h> 26 #include <rte_flow_driver.h> 27 #include <rte_malloc.h> 28 #include <rte_ip.h> 29 30 #include "mlx5.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_flow.h" 33 #include "mlx5_glue.h" 34 #include "mlx5_prm.h" 35 #include "mlx5_rxtx.h" 36 37 /* Dev ops structure defined in mlx5.c */ 38 extern const struct eth_dev_ops mlx5_dev_ops; 39 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 40 41 /** Device flow drivers. */ 42 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 43 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 44 #endif 45 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 46 47 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 48 49 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 50 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 51 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 52 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 53 #endif 54 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 55 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 56 }; 57 58 enum mlx5_expansion { 59 MLX5_EXPANSION_ROOT, 60 MLX5_EXPANSION_ROOT_OUTER, 61 MLX5_EXPANSION_ROOT_ETH_VLAN, 62 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 63 MLX5_EXPANSION_OUTER_ETH, 64 MLX5_EXPANSION_OUTER_ETH_VLAN, 65 MLX5_EXPANSION_OUTER_VLAN, 66 MLX5_EXPANSION_OUTER_IPV4, 67 MLX5_EXPANSION_OUTER_IPV4_UDP, 68 MLX5_EXPANSION_OUTER_IPV4_TCP, 69 MLX5_EXPANSION_OUTER_IPV6, 70 MLX5_EXPANSION_OUTER_IPV6_UDP, 71 MLX5_EXPANSION_OUTER_IPV6_TCP, 72 MLX5_EXPANSION_VXLAN, 73 MLX5_EXPANSION_VXLAN_GPE, 74 MLX5_EXPANSION_GRE, 75 MLX5_EXPANSION_MPLS, 76 MLX5_EXPANSION_ETH, 77 MLX5_EXPANSION_ETH_VLAN, 78 MLX5_EXPANSION_VLAN, 79 MLX5_EXPANSION_IPV4, 80 MLX5_EXPANSION_IPV4_UDP, 81 MLX5_EXPANSION_IPV4_TCP, 82 MLX5_EXPANSION_IPV6, 83 MLX5_EXPANSION_IPV6_UDP, 84 MLX5_EXPANSION_IPV6_TCP, 85 }; 86 87 /** Supported expansion of items. */ 88 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 89 [MLX5_EXPANSION_ROOT] = { 90 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 91 MLX5_EXPANSION_IPV4, 92 MLX5_EXPANSION_IPV6), 93 .type = RTE_FLOW_ITEM_TYPE_END, 94 }, 95 [MLX5_EXPANSION_ROOT_OUTER] = { 96 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 97 MLX5_EXPANSION_OUTER_IPV4, 98 MLX5_EXPANSION_OUTER_IPV6), 99 .type = RTE_FLOW_ITEM_TYPE_END, 100 }, 101 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 102 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 103 .type = RTE_FLOW_ITEM_TYPE_END, 104 }, 105 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 106 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 107 .type = RTE_FLOW_ITEM_TYPE_END, 108 }, 109 [MLX5_EXPANSION_OUTER_ETH] = { 110 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 111 MLX5_EXPANSION_OUTER_IPV6, 112 MLX5_EXPANSION_MPLS), 113 .type = RTE_FLOW_ITEM_TYPE_ETH, 114 .rss_types = 0, 115 }, 116 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 117 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 118 .type = RTE_FLOW_ITEM_TYPE_ETH, 119 .rss_types = 0, 120 }, 121 [MLX5_EXPANSION_OUTER_VLAN] = { 122 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 123 MLX5_EXPANSION_OUTER_IPV6), 124 .type = RTE_FLOW_ITEM_TYPE_VLAN, 125 }, 126 [MLX5_EXPANSION_OUTER_IPV4] = { 127 .next = RTE_FLOW_EXPAND_RSS_NEXT 128 (MLX5_EXPANSION_OUTER_IPV4_UDP, 129 MLX5_EXPANSION_OUTER_IPV4_TCP, 130 MLX5_EXPANSION_GRE, 131 MLX5_EXPANSION_IPV4, 132 MLX5_EXPANSION_IPV6), 133 .type = RTE_FLOW_ITEM_TYPE_IPV4, 134 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 135 ETH_RSS_NONFRAG_IPV4_OTHER, 136 }, 137 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 138 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 139 MLX5_EXPANSION_VXLAN_GPE), 140 .type = RTE_FLOW_ITEM_TYPE_UDP, 141 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 142 }, 143 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 144 .type = RTE_FLOW_ITEM_TYPE_TCP, 145 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 146 }, 147 [MLX5_EXPANSION_OUTER_IPV6] = { 148 .next = RTE_FLOW_EXPAND_RSS_NEXT 149 (MLX5_EXPANSION_OUTER_IPV6_UDP, 150 MLX5_EXPANSION_OUTER_IPV6_TCP, 151 MLX5_EXPANSION_IPV4, 152 MLX5_EXPANSION_IPV6), 153 .type = RTE_FLOW_ITEM_TYPE_IPV6, 154 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 155 ETH_RSS_NONFRAG_IPV6_OTHER, 156 }, 157 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 158 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 159 MLX5_EXPANSION_VXLAN_GPE), 160 .type = RTE_FLOW_ITEM_TYPE_UDP, 161 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 162 }, 163 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 164 .type = RTE_FLOW_ITEM_TYPE_TCP, 165 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 166 }, 167 [MLX5_EXPANSION_VXLAN] = { 168 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 169 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 170 }, 171 [MLX5_EXPANSION_VXLAN_GPE] = { 172 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 173 MLX5_EXPANSION_IPV4, 174 MLX5_EXPANSION_IPV6), 175 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 176 }, 177 [MLX5_EXPANSION_GRE] = { 178 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 179 .type = RTE_FLOW_ITEM_TYPE_GRE, 180 }, 181 [MLX5_EXPANSION_MPLS] = { 182 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 183 MLX5_EXPANSION_IPV6), 184 .type = RTE_FLOW_ITEM_TYPE_MPLS, 185 }, 186 [MLX5_EXPANSION_ETH] = { 187 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 188 MLX5_EXPANSION_IPV6), 189 .type = RTE_FLOW_ITEM_TYPE_ETH, 190 }, 191 [MLX5_EXPANSION_ETH_VLAN] = { 192 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 193 .type = RTE_FLOW_ITEM_TYPE_ETH, 194 }, 195 [MLX5_EXPANSION_VLAN] = { 196 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 197 MLX5_EXPANSION_IPV6), 198 .type = RTE_FLOW_ITEM_TYPE_VLAN, 199 }, 200 [MLX5_EXPANSION_IPV4] = { 201 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 202 MLX5_EXPANSION_IPV4_TCP), 203 .type = RTE_FLOW_ITEM_TYPE_IPV4, 204 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 205 ETH_RSS_NONFRAG_IPV4_OTHER, 206 }, 207 [MLX5_EXPANSION_IPV4_UDP] = { 208 .type = RTE_FLOW_ITEM_TYPE_UDP, 209 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 210 }, 211 [MLX5_EXPANSION_IPV4_TCP] = { 212 .type = RTE_FLOW_ITEM_TYPE_TCP, 213 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 214 }, 215 [MLX5_EXPANSION_IPV6] = { 216 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 217 MLX5_EXPANSION_IPV6_TCP), 218 .type = RTE_FLOW_ITEM_TYPE_IPV6, 219 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 220 ETH_RSS_NONFRAG_IPV6_OTHER, 221 }, 222 [MLX5_EXPANSION_IPV6_UDP] = { 223 .type = RTE_FLOW_ITEM_TYPE_UDP, 224 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 225 }, 226 [MLX5_EXPANSION_IPV6_TCP] = { 227 .type = RTE_FLOW_ITEM_TYPE_TCP, 228 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 229 }, 230 }; 231 232 static const struct rte_flow_ops mlx5_flow_ops = { 233 .validate = mlx5_flow_validate, 234 .create = mlx5_flow_create, 235 .destroy = mlx5_flow_destroy, 236 .flush = mlx5_flow_flush, 237 .isolate = mlx5_flow_isolate, 238 .query = mlx5_flow_query, 239 .dev_dump = mlx5_flow_dev_dump, 240 }; 241 242 /* Convert FDIR request to Generic flow. */ 243 struct mlx5_fdir { 244 struct rte_flow_attr attr; 245 struct rte_flow_item items[4]; 246 struct rte_flow_item_eth l2; 247 struct rte_flow_item_eth l2_mask; 248 union { 249 struct rte_flow_item_ipv4 ipv4; 250 struct rte_flow_item_ipv6 ipv6; 251 } l3; 252 union { 253 struct rte_flow_item_ipv4 ipv4; 254 struct rte_flow_item_ipv6 ipv6; 255 } l3_mask; 256 union { 257 struct rte_flow_item_udp udp; 258 struct rte_flow_item_tcp tcp; 259 } l4; 260 union { 261 struct rte_flow_item_udp udp; 262 struct rte_flow_item_tcp tcp; 263 } l4_mask; 264 struct rte_flow_action actions[2]; 265 struct rte_flow_action_queue queue; 266 }; 267 268 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 269 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 270 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 271 }; 272 273 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 274 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 275 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 276 { 9, 10, 11 }, { 12, 13, 14 }, 277 }; 278 279 /* Tunnel information. */ 280 struct mlx5_flow_tunnel_info { 281 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 282 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 283 }; 284 285 static struct mlx5_flow_tunnel_info tunnels_info[] = { 286 { 287 .tunnel = MLX5_FLOW_LAYER_VXLAN, 288 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 289 }, 290 { 291 .tunnel = MLX5_FLOW_LAYER_GENEVE, 292 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 293 }, 294 { 295 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 296 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 297 }, 298 { 299 .tunnel = MLX5_FLOW_LAYER_GRE, 300 .ptype = RTE_PTYPE_TUNNEL_GRE, 301 }, 302 { 303 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 304 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 305 }, 306 { 307 .tunnel = MLX5_FLOW_LAYER_MPLS, 308 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 309 }, 310 { 311 .tunnel = MLX5_FLOW_LAYER_NVGRE, 312 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 313 }, 314 { 315 .tunnel = MLX5_FLOW_LAYER_IPIP, 316 .ptype = RTE_PTYPE_TUNNEL_IP, 317 }, 318 { 319 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 320 .ptype = RTE_PTYPE_TUNNEL_IP, 321 }, 322 { 323 .tunnel = MLX5_FLOW_LAYER_GTP, 324 .ptype = RTE_PTYPE_TUNNEL_GTPU, 325 }, 326 }; 327 328 /** 329 * Translate tag ID to register. 330 * 331 * @param[in] dev 332 * Pointer to the Ethernet device structure. 333 * @param[in] feature 334 * The feature that request the register. 335 * @param[in] id 336 * The request register ID. 337 * @param[out] error 338 * Error description in case of any. 339 * 340 * @return 341 * The request register on success, a negative errno 342 * value otherwise and rte_errno is set. 343 */ 344 enum modify_reg 345 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 346 enum mlx5_feature_name feature, 347 uint32_t id, 348 struct rte_flow_error *error) 349 { 350 struct mlx5_priv *priv = dev->data->dev_private; 351 struct mlx5_dev_config *config = &priv->config; 352 enum modify_reg start_reg; 353 354 switch (feature) { 355 case MLX5_HAIRPIN_RX: 356 return REG_B; 357 case MLX5_HAIRPIN_TX: 358 return REG_A; 359 case MLX5_METADATA_RX: 360 switch (config->dv_xmeta_en) { 361 case MLX5_XMETA_MODE_LEGACY: 362 return REG_B; 363 case MLX5_XMETA_MODE_META16: 364 return REG_C_0; 365 case MLX5_XMETA_MODE_META32: 366 return REG_C_1; 367 } 368 break; 369 case MLX5_METADATA_TX: 370 return REG_A; 371 case MLX5_METADATA_FDB: 372 switch (config->dv_xmeta_en) { 373 case MLX5_XMETA_MODE_LEGACY: 374 return REG_NONE; 375 case MLX5_XMETA_MODE_META16: 376 return REG_C_0; 377 case MLX5_XMETA_MODE_META32: 378 return REG_C_1; 379 } 380 break; 381 case MLX5_FLOW_MARK: 382 switch (config->dv_xmeta_en) { 383 case MLX5_XMETA_MODE_LEGACY: 384 return REG_NONE; 385 case MLX5_XMETA_MODE_META16: 386 return REG_C_1; 387 case MLX5_XMETA_MODE_META32: 388 return REG_C_0; 389 } 390 break; 391 case MLX5_COPY_MARK: 392 case MLX5_MTR_SFX: 393 /* 394 * Metadata COPY_MARK register using is in meter suffix sub 395 * flow while with meter. It's safe to share the same register. 396 */ 397 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 398 case MLX5_MTR_COLOR: 399 RTE_ASSERT(priv->mtr_color_reg != REG_NONE); 400 return priv->mtr_color_reg; 401 case MLX5_APP_TAG: 402 /* 403 * If meter is enable, it will engage two registers for color 404 * match and flow match. If meter color match is not using the 405 * REG_C_2, need to skip the REG_C_x be used by meter color 406 * match. 407 * If meter is disable, free to use all available registers. 408 */ 409 if (priv->mtr_color_reg != REG_NONE) 410 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_3 : 411 REG_C_4; 412 else 413 start_reg = REG_C_2; 414 if (id > (REG_C_7 - start_reg)) 415 return rte_flow_error_set(error, EINVAL, 416 RTE_FLOW_ERROR_TYPE_ITEM, 417 NULL, "invalid tag id"); 418 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 419 return rte_flow_error_set(error, ENOTSUP, 420 RTE_FLOW_ERROR_TYPE_ITEM, 421 NULL, "unsupported tag id"); 422 /* 423 * This case means meter is using the REG_C_x great than 2. 424 * Take care not to conflict with meter color REG_C_x. 425 * If the available index REG_C_y >= REG_C_x, skip the 426 * color register. 427 */ 428 if (start_reg == REG_C_3 && config->flow_mreg_c 429 [id + REG_C_3 - REG_C_0] >= priv->mtr_color_reg) { 430 if (config->flow_mreg_c[id + 1 + REG_C_3 - REG_C_0] != 431 REG_NONE) 432 return config->flow_mreg_c 433 [id + 1 + REG_C_3 - REG_C_0]; 434 return rte_flow_error_set(error, ENOTSUP, 435 RTE_FLOW_ERROR_TYPE_ITEM, 436 NULL, "unsupported tag id"); 437 } 438 return config->flow_mreg_c[id + start_reg - REG_C_0]; 439 } 440 assert(false); 441 return rte_flow_error_set(error, EINVAL, 442 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 443 NULL, "invalid feature name"); 444 } 445 446 /** 447 * Check extensive flow metadata register support. 448 * 449 * @param dev 450 * Pointer to rte_eth_dev structure. 451 * 452 * @return 453 * True if device supports extensive flow metadata register, otherwise false. 454 */ 455 bool 456 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 457 { 458 struct mlx5_priv *priv = dev->data->dev_private; 459 struct mlx5_dev_config *config = &priv->config; 460 461 /* 462 * Having available reg_c can be regarded inclusively as supporting 463 * extensive flow metadata register, which could mean, 464 * - metadata register copy action by modify header. 465 * - 16 modify header actions is supported. 466 * - reg_c's are preserved across different domain (FDB and NIC) on 467 * packet loopback by flow lookup miss. 468 */ 469 return config->flow_mreg_c[2] != REG_NONE; 470 } 471 472 /** 473 * Discover the maximum number of priority available. 474 * 475 * @param[in] dev 476 * Pointer to the Ethernet device structure. 477 * 478 * @return 479 * number of supported flow priority on success, a negative errno 480 * value otherwise and rte_errno is set. 481 */ 482 int 483 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 484 { 485 struct mlx5_priv *priv = dev->data->dev_private; 486 struct { 487 struct ibv_flow_attr attr; 488 struct ibv_flow_spec_eth eth; 489 struct ibv_flow_spec_action_drop drop; 490 } flow_attr = { 491 .attr = { 492 .num_of_specs = 2, 493 .port = (uint8_t)priv->ibv_port, 494 }, 495 .eth = { 496 .type = IBV_FLOW_SPEC_ETH, 497 .size = sizeof(struct ibv_flow_spec_eth), 498 }, 499 .drop = { 500 .size = sizeof(struct ibv_flow_spec_action_drop), 501 .type = IBV_FLOW_SPEC_ACTION_DROP, 502 }, 503 }; 504 struct ibv_flow *flow; 505 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 506 uint16_t vprio[] = { 8, 16 }; 507 int i; 508 int priority = 0; 509 510 if (!drop) { 511 rte_errno = ENOTSUP; 512 return -rte_errno; 513 } 514 for (i = 0; i != RTE_DIM(vprio); i++) { 515 flow_attr.attr.priority = vprio[i] - 1; 516 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 517 if (!flow) 518 break; 519 claim_zero(mlx5_glue->destroy_flow(flow)); 520 priority = vprio[i]; 521 } 522 mlx5_hrxq_drop_release(dev); 523 switch (priority) { 524 case 8: 525 priority = RTE_DIM(priority_map_3); 526 break; 527 case 16: 528 priority = RTE_DIM(priority_map_5); 529 break; 530 default: 531 rte_errno = ENOTSUP; 532 DRV_LOG(ERR, 533 "port %u verbs maximum priority: %d expected 8/16", 534 dev->data->port_id, priority); 535 return -rte_errno; 536 } 537 DRV_LOG(INFO, "port %u flow maximum priority: %d", 538 dev->data->port_id, priority); 539 return priority; 540 } 541 542 /** 543 * Adjust flow priority based on the highest layer and the request priority. 544 * 545 * @param[in] dev 546 * Pointer to the Ethernet device structure. 547 * @param[in] priority 548 * The rule base priority. 549 * @param[in] subpriority 550 * The priority based on the items. 551 * 552 * @return 553 * The new priority. 554 */ 555 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 556 uint32_t subpriority) 557 { 558 uint32_t res = 0; 559 struct mlx5_priv *priv = dev->data->dev_private; 560 561 switch (priv->config.flow_prio) { 562 case RTE_DIM(priority_map_3): 563 res = priority_map_3[priority][subpriority]; 564 break; 565 case RTE_DIM(priority_map_5): 566 res = priority_map_5[priority][subpriority]; 567 break; 568 } 569 return res; 570 } 571 572 /** 573 * Verify the @p item specifications (spec, last, mask) are compatible with the 574 * NIC capabilities. 575 * 576 * @param[in] item 577 * Item specification. 578 * @param[in] mask 579 * @p item->mask or flow default bit-masks. 580 * @param[in] nic_mask 581 * Bit-masks covering supported fields by the NIC to compare with user mask. 582 * @param[in] size 583 * Bit-masks size in bytes. 584 * @param[out] error 585 * Pointer to error structure. 586 * 587 * @return 588 * 0 on success, a negative errno value otherwise and rte_errno is set. 589 */ 590 int 591 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 592 const uint8_t *mask, 593 const uint8_t *nic_mask, 594 unsigned int size, 595 struct rte_flow_error *error) 596 { 597 unsigned int i; 598 599 assert(nic_mask); 600 for (i = 0; i < size; ++i) 601 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 602 return rte_flow_error_set(error, ENOTSUP, 603 RTE_FLOW_ERROR_TYPE_ITEM, 604 item, 605 "mask enables non supported" 606 " bits"); 607 if (!item->spec && (item->mask || item->last)) 608 return rte_flow_error_set(error, EINVAL, 609 RTE_FLOW_ERROR_TYPE_ITEM, item, 610 "mask/last without a spec is not" 611 " supported"); 612 if (item->spec && item->last) { 613 uint8_t spec[size]; 614 uint8_t last[size]; 615 unsigned int i; 616 int ret; 617 618 for (i = 0; i < size; ++i) { 619 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 620 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 621 } 622 ret = memcmp(spec, last, size); 623 if (ret != 0) 624 return rte_flow_error_set(error, EINVAL, 625 RTE_FLOW_ERROR_TYPE_ITEM, 626 item, 627 "range is not valid"); 628 } 629 return 0; 630 } 631 632 /** 633 * Adjust the hash fields according to the @p flow information. 634 * 635 * @param[in] dev_flow. 636 * Pointer to the mlx5_flow. 637 * @param[in] tunnel 638 * 1 when the hash field is for a tunnel item. 639 * @param[in] layer_types 640 * ETH_RSS_* types. 641 * @param[in] hash_fields 642 * Item hash fields. 643 * 644 * @return 645 * The hash fields that should be used. 646 */ 647 uint64_t 648 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, 649 int tunnel __rte_unused, uint64_t layer_types, 650 uint64_t hash_fields) 651 { 652 struct rte_flow *flow = dev_flow->flow; 653 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 654 int rss_request_inner = flow->rss.level >= 2; 655 656 /* Check RSS hash level for tunnel. */ 657 if (tunnel && rss_request_inner) 658 hash_fields |= IBV_RX_HASH_INNER; 659 else if (tunnel || rss_request_inner) 660 return 0; 661 #endif 662 /* Check if requested layer matches RSS hash fields. */ 663 if (!(flow->rss.types & layer_types)) 664 return 0; 665 return hash_fields; 666 } 667 668 /** 669 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 670 * if several tunnel rules are used on this queue, the tunnel ptype will be 671 * cleared. 672 * 673 * @param rxq_ctrl 674 * Rx queue to update. 675 */ 676 static void 677 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 678 { 679 unsigned int i; 680 uint32_t tunnel_ptype = 0; 681 682 /* Look up for the ptype to use. */ 683 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 684 if (!rxq_ctrl->flow_tunnels_n[i]) 685 continue; 686 if (!tunnel_ptype) { 687 tunnel_ptype = tunnels_info[i].ptype; 688 } else { 689 tunnel_ptype = 0; 690 break; 691 } 692 } 693 rxq_ctrl->rxq.tunnel = tunnel_ptype; 694 } 695 696 /** 697 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 698 * flow. 699 * 700 * @param[in] dev 701 * Pointer to the Ethernet device structure. 702 * @param[in] dev_flow 703 * Pointer to device flow structure. 704 */ 705 static void 706 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 707 { 708 struct mlx5_priv *priv = dev->data->dev_private; 709 struct rte_flow *flow = dev_flow->flow; 710 const int mark = !!(dev_flow->actions & 711 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 712 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 713 unsigned int i; 714 715 for (i = 0; i != flow->rss.queue_num; ++i) { 716 int idx = (*flow->rss.queue)[i]; 717 struct mlx5_rxq_ctrl *rxq_ctrl = 718 container_of((*priv->rxqs)[idx], 719 struct mlx5_rxq_ctrl, rxq); 720 721 /* 722 * To support metadata register copy on Tx loopback, 723 * this must be always enabled (metadata may arive 724 * from other port - not from local flows only. 725 */ 726 if (priv->config.dv_flow_en && 727 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 728 mlx5_flow_ext_mreg_supported(dev)) { 729 rxq_ctrl->rxq.mark = 1; 730 rxq_ctrl->flow_mark_n = 1; 731 } else if (mark) { 732 rxq_ctrl->rxq.mark = 1; 733 rxq_ctrl->flow_mark_n++; 734 } 735 if (tunnel) { 736 unsigned int j; 737 738 /* Increase the counter matching the flow. */ 739 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 740 if ((tunnels_info[j].tunnel & 741 dev_flow->layers) == 742 tunnels_info[j].tunnel) { 743 rxq_ctrl->flow_tunnels_n[j]++; 744 break; 745 } 746 } 747 flow_rxq_tunnel_ptype_update(rxq_ctrl); 748 } 749 } 750 } 751 752 /** 753 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 754 * 755 * @param[in] dev 756 * Pointer to the Ethernet device structure. 757 * @param[in] flow 758 * Pointer to flow structure. 759 */ 760 static void 761 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 762 { 763 struct mlx5_flow *dev_flow; 764 765 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 766 flow_drv_rxq_flags_set(dev, dev_flow); 767 } 768 769 /** 770 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 771 * device flow if no other flow uses it with the same kind of request. 772 * 773 * @param dev 774 * Pointer to Ethernet device. 775 * @param[in] dev_flow 776 * Pointer to the device flow. 777 */ 778 static void 779 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 780 { 781 struct mlx5_priv *priv = dev->data->dev_private; 782 struct rte_flow *flow = dev_flow->flow; 783 const int mark = !!(dev_flow->actions & 784 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 785 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 786 unsigned int i; 787 788 assert(dev->data->dev_started); 789 for (i = 0; i != flow->rss.queue_num; ++i) { 790 int idx = (*flow->rss.queue)[i]; 791 struct mlx5_rxq_ctrl *rxq_ctrl = 792 container_of((*priv->rxqs)[idx], 793 struct mlx5_rxq_ctrl, rxq); 794 795 if (priv->config.dv_flow_en && 796 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 797 mlx5_flow_ext_mreg_supported(dev)) { 798 rxq_ctrl->rxq.mark = 1; 799 rxq_ctrl->flow_mark_n = 1; 800 } else if (mark) { 801 rxq_ctrl->flow_mark_n--; 802 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 803 } 804 if (tunnel) { 805 unsigned int j; 806 807 /* Decrease the counter matching the flow. */ 808 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 809 if ((tunnels_info[j].tunnel & 810 dev_flow->layers) == 811 tunnels_info[j].tunnel) { 812 rxq_ctrl->flow_tunnels_n[j]--; 813 break; 814 } 815 } 816 flow_rxq_tunnel_ptype_update(rxq_ctrl); 817 } 818 } 819 } 820 821 /** 822 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 823 * @p flow if no other flow uses it with the same kind of request. 824 * 825 * @param dev 826 * Pointer to Ethernet device. 827 * @param[in] flow 828 * Pointer to the flow. 829 */ 830 static void 831 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 832 { 833 struct mlx5_flow *dev_flow; 834 835 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 836 flow_drv_rxq_flags_trim(dev, dev_flow); 837 } 838 839 /** 840 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 841 * 842 * @param dev 843 * Pointer to Ethernet device. 844 */ 845 static void 846 flow_rxq_flags_clear(struct rte_eth_dev *dev) 847 { 848 struct mlx5_priv *priv = dev->data->dev_private; 849 unsigned int i; 850 851 for (i = 0; i != priv->rxqs_n; ++i) { 852 struct mlx5_rxq_ctrl *rxq_ctrl; 853 unsigned int j; 854 855 if (!(*priv->rxqs)[i]) 856 continue; 857 rxq_ctrl = container_of((*priv->rxqs)[i], 858 struct mlx5_rxq_ctrl, rxq); 859 rxq_ctrl->flow_mark_n = 0; 860 rxq_ctrl->rxq.mark = 0; 861 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 862 rxq_ctrl->flow_tunnels_n[j] = 0; 863 rxq_ctrl->rxq.tunnel = 0; 864 } 865 } 866 867 /* 868 * return a pointer to the desired action in the list of actions. 869 * 870 * @param[in] actions 871 * The list of actions to search the action in. 872 * @param[in] action 873 * The action to find. 874 * 875 * @return 876 * Pointer to the action in the list, if found. NULL otherwise. 877 */ 878 const struct rte_flow_action * 879 mlx5_flow_find_action(const struct rte_flow_action *actions, 880 enum rte_flow_action_type action) 881 { 882 if (actions == NULL) 883 return NULL; 884 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 885 if (actions->type == action) 886 return actions; 887 return NULL; 888 } 889 890 /* 891 * Validate the flag action. 892 * 893 * @param[in] action_flags 894 * Bit-fields that holds the actions detected until now. 895 * @param[in] attr 896 * Attributes of flow that includes this action. 897 * @param[out] error 898 * Pointer to error structure. 899 * 900 * @return 901 * 0 on success, a negative errno value otherwise and rte_errno is set. 902 */ 903 int 904 mlx5_flow_validate_action_flag(uint64_t action_flags, 905 const struct rte_flow_attr *attr, 906 struct rte_flow_error *error) 907 { 908 909 if (action_flags & MLX5_FLOW_ACTION_DROP) 910 return rte_flow_error_set(error, EINVAL, 911 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 912 "can't drop and flag in same flow"); 913 if (action_flags & MLX5_FLOW_ACTION_MARK) 914 return rte_flow_error_set(error, EINVAL, 915 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 916 "can't mark and flag in same flow"); 917 if (action_flags & MLX5_FLOW_ACTION_FLAG) 918 return rte_flow_error_set(error, EINVAL, 919 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 920 "can't have 2 flag" 921 " actions in same flow"); 922 if (attr->egress) 923 return rte_flow_error_set(error, ENOTSUP, 924 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 925 "flag action not supported for " 926 "egress"); 927 return 0; 928 } 929 930 /* 931 * Validate the mark action. 932 * 933 * @param[in] action 934 * Pointer to the queue action. 935 * @param[in] action_flags 936 * Bit-fields that holds the actions detected until now. 937 * @param[in] attr 938 * Attributes of flow that includes this action. 939 * @param[out] error 940 * Pointer to error structure. 941 * 942 * @return 943 * 0 on success, a negative errno value otherwise and rte_errno is set. 944 */ 945 int 946 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 947 uint64_t action_flags, 948 const struct rte_flow_attr *attr, 949 struct rte_flow_error *error) 950 { 951 const struct rte_flow_action_mark *mark = action->conf; 952 953 if (!mark) 954 return rte_flow_error_set(error, EINVAL, 955 RTE_FLOW_ERROR_TYPE_ACTION, 956 action, 957 "configuration cannot be null"); 958 if (mark->id >= MLX5_FLOW_MARK_MAX) 959 return rte_flow_error_set(error, EINVAL, 960 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 961 &mark->id, 962 "mark id must in 0 <= id < " 963 RTE_STR(MLX5_FLOW_MARK_MAX)); 964 if (action_flags & MLX5_FLOW_ACTION_DROP) 965 return rte_flow_error_set(error, EINVAL, 966 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 967 "can't drop and mark in same flow"); 968 if (action_flags & MLX5_FLOW_ACTION_FLAG) 969 return rte_flow_error_set(error, EINVAL, 970 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 971 "can't flag and mark in same flow"); 972 if (action_flags & MLX5_FLOW_ACTION_MARK) 973 return rte_flow_error_set(error, EINVAL, 974 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 975 "can't have 2 mark actions in same" 976 " flow"); 977 if (attr->egress) 978 return rte_flow_error_set(error, ENOTSUP, 979 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 980 "mark action not supported for " 981 "egress"); 982 return 0; 983 } 984 985 /* 986 * Validate the drop action. 987 * 988 * @param[in] action_flags 989 * Bit-fields that holds the actions detected until now. 990 * @param[in] attr 991 * Attributes of flow that includes this action. 992 * @param[out] error 993 * Pointer to error structure. 994 * 995 * @return 996 * 0 on success, a negative errno value otherwise and rte_errno is set. 997 */ 998 int 999 mlx5_flow_validate_action_drop(uint64_t action_flags, 1000 const struct rte_flow_attr *attr, 1001 struct rte_flow_error *error) 1002 { 1003 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1004 return rte_flow_error_set(error, EINVAL, 1005 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1006 "can't drop and flag in same flow"); 1007 if (action_flags & MLX5_FLOW_ACTION_MARK) 1008 return rte_flow_error_set(error, EINVAL, 1009 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1010 "can't drop and mark in same flow"); 1011 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 1012 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 1013 return rte_flow_error_set(error, EINVAL, 1014 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1015 "can't have 2 fate actions in" 1016 " same flow"); 1017 if (attr->egress) 1018 return rte_flow_error_set(error, ENOTSUP, 1019 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1020 "drop action not supported for " 1021 "egress"); 1022 return 0; 1023 } 1024 1025 /* 1026 * Validate the queue action. 1027 * 1028 * @param[in] action 1029 * Pointer to the queue action. 1030 * @param[in] action_flags 1031 * Bit-fields that holds the actions detected until now. 1032 * @param[in] dev 1033 * Pointer to the Ethernet device structure. 1034 * @param[in] attr 1035 * Attributes of flow that includes this action. 1036 * @param[out] error 1037 * Pointer to error structure. 1038 * 1039 * @return 1040 * 0 on success, a negative errno value otherwise and rte_errno is set. 1041 */ 1042 int 1043 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1044 uint64_t action_flags, 1045 struct rte_eth_dev *dev, 1046 const struct rte_flow_attr *attr, 1047 struct rte_flow_error *error) 1048 { 1049 struct mlx5_priv *priv = dev->data->dev_private; 1050 const struct rte_flow_action_queue *queue = action->conf; 1051 1052 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1053 return rte_flow_error_set(error, EINVAL, 1054 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1055 "can't have 2 fate actions in" 1056 " same flow"); 1057 if (!priv->rxqs_n) 1058 return rte_flow_error_set(error, EINVAL, 1059 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1060 NULL, "No Rx queues configured"); 1061 if (queue->index >= priv->rxqs_n) 1062 return rte_flow_error_set(error, EINVAL, 1063 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1064 &queue->index, 1065 "queue index out of range"); 1066 if (!(*priv->rxqs)[queue->index]) 1067 return rte_flow_error_set(error, EINVAL, 1068 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1069 &queue->index, 1070 "queue is not configured"); 1071 if (attr->egress) 1072 return rte_flow_error_set(error, ENOTSUP, 1073 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1074 "queue action not supported for " 1075 "egress"); 1076 return 0; 1077 } 1078 1079 /* 1080 * Validate the rss action. 1081 * 1082 * @param[in] action 1083 * Pointer to the queue action. 1084 * @param[in] action_flags 1085 * Bit-fields that holds the actions detected until now. 1086 * @param[in] dev 1087 * Pointer to the Ethernet device structure. 1088 * @param[in] attr 1089 * Attributes of flow that includes this action. 1090 * @param[in] item_flags 1091 * Items that were detected. 1092 * @param[out] error 1093 * Pointer to error structure. 1094 * 1095 * @return 1096 * 0 on success, a negative errno value otherwise and rte_errno is set. 1097 */ 1098 int 1099 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1100 uint64_t action_flags, 1101 struct rte_eth_dev *dev, 1102 const struct rte_flow_attr *attr, 1103 uint64_t item_flags, 1104 struct rte_flow_error *error) 1105 { 1106 struct mlx5_priv *priv = dev->data->dev_private; 1107 const struct rte_flow_action_rss *rss = action->conf; 1108 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1109 unsigned int i; 1110 1111 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1112 return rte_flow_error_set(error, EINVAL, 1113 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1114 "can't have 2 fate actions" 1115 " in same flow"); 1116 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1117 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1118 return rte_flow_error_set(error, ENOTSUP, 1119 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1120 &rss->func, 1121 "RSS hash function not supported"); 1122 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1123 if (rss->level > 2) 1124 #else 1125 if (rss->level > 1) 1126 #endif 1127 return rte_flow_error_set(error, ENOTSUP, 1128 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1129 &rss->level, 1130 "tunnel RSS is not supported"); 1131 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1132 if (rss->key_len == 0 && rss->key != NULL) 1133 return rte_flow_error_set(error, ENOTSUP, 1134 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1135 &rss->key_len, 1136 "RSS hash key length 0"); 1137 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1138 return rte_flow_error_set(error, ENOTSUP, 1139 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1140 &rss->key_len, 1141 "RSS hash key too small"); 1142 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1143 return rte_flow_error_set(error, ENOTSUP, 1144 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1145 &rss->key_len, 1146 "RSS hash key too large"); 1147 if (rss->queue_num > priv->config.ind_table_max_size) 1148 return rte_flow_error_set(error, ENOTSUP, 1149 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1150 &rss->queue_num, 1151 "number of queues too large"); 1152 if (rss->types & MLX5_RSS_HF_MASK) 1153 return rte_flow_error_set(error, ENOTSUP, 1154 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1155 &rss->types, 1156 "some RSS protocols are not" 1157 " supported"); 1158 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1159 !(rss->types & ETH_RSS_IP)) 1160 return rte_flow_error_set(error, EINVAL, 1161 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1162 "L3 partial RSS requested but L3 RSS" 1163 " type not specified"); 1164 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1165 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1166 return rte_flow_error_set(error, EINVAL, 1167 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1168 "L4 partial RSS requested but L4 RSS" 1169 " type not specified"); 1170 if (!priv->rxqs_n) 1171 return rte_flow_error_set(error, EINVAL, 1172 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1173 NULL, "No Rx queues configured"); 1174 if (!rss->queue_num) 1175 return rte_flow_error_set(error, EINVAL, 1176 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1177 NULL, "No queues configured"); 1178 for (i = 0; i != rss->queue_num; ++i) { 1179 if (rss->queue[i] >= priv->rxqs_n) 1180 return rte_flow_error_set 1181 (error, EINVAL, 1182 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1183 &rss->queue[i], "queue index out of range"); 1184 if (!(*priv->rxqs)[rss->queue[i]]) 1185 return rte_flow_error_set 1186 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1187 &rss->queue[i], "queue is not configured"); 1188 } 1189 if (attr->egress) 1190 return rte_flow_error_set(error, ENOTSUP, 1191 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1192 "rss action not supported for " 1193 "egress"); 1194 if (rss->level > 1 && !tunnel) 1195 return rte_flow_error_set(error, EINVAL, 1196 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1197 "inner RSS is not supported for " 1198 "non-tunnel flows"); 1199 return 0; 1200 } 1201 1202 /* 1203 * Validate the count action. 1204 * 1205 * @param[in] dev 1206 * Pointer to the Ethernet device structure. 1207 * @param[in] attr 1208 * Attributes of flow that includes this action. 1209 * @param[out] error 1210 * Pointer to error structure. 1211 * 1212 * @return 1213 * 0 on success, a negative errno value otherwise and rte_errno is set. 1214 */ 1215 int 1216 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1217 const struct rte_flow_attr *attr, 1218 struct rte_flow_error *error) 1219 { 1220 if (attr->egress) 1221 return rte_flow_error_set(error, ENOTSUP, 1222 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1223 "count action not supported for " 1224 "egress"); 1225 return 0; 1226 } 1227 1228 /** 1229 * Verify the @p attributes will be correctly understood by the NIC and store 1230 * them in the @p flow if everything is correct. 1231 * 1232 * @param[in] dev 1233 * Pointer to the Ethernet device structure. 1234 * @param[in] attributes 1235 * Pointer to flow attributes 1236 * @param[out] error 1237 * Pointer to error structure. 1238 * 1239 * @return 1240 * 0 on success, a negative errno value otherwise and rte_errno is set. 1241 */ 1242 int 1243 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1244 const struct rte_flow_attr *attributes, 1245 struct rte_flow_error *error) 1246 { 1247 struct mlx5_priv *priv = dev->data->dev_private; 1248 uint32_t priority_max = priv->config.flow_prio - 1; 1249 1250 if (attributes->group) 1251 return rte_flow_error_set(error, ENOTSUP, 1252 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1253 NULL, "groups is not supported"); 1254 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1255 attributes->priority >= priority_max) 1256 return rte_flow_error_set(error, ENOTSUP, 1257 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1258 NULL, "priority out of range"); 1259 if (attributes->egress) 1260 return rte_flow_error_set(error, ENOTSUP, 1261 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1262 "egress is not supported"); 1263 if (attributes->transfer && !priv->config.dv_esw_en) 1264 return rte_flow_error_set(error, ENOTSUP, 1265 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1266 NULL, "transfer is not supported"); 1267 if (!attributes->ingress) 1268 return rte_flow_error_set(error, EINVAL, 1269 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1270 NULL, 1271 "ingress attribute is mandatory"); 1272 return 0; 1273 } 1274 1275 /** 1276 * Validate ICMP6 item. 1277 * 1278 * @param[in] item 1279 * Item specification. 1280 * @param[in] item_flags 1281 * Bit-fields that holds the items detected until now. 1282 * @param[out] error 1283 * Pointer to error structure. 1284 * 1285 * @return 1286 * 0 on success, a negative errno value otherwise and rte_errno is set. 1287 */ 1288 int 1289 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1290 uint64_t item_flags, 1291 uint8_t target_protocol, 1292 struct rte_flow_error *error) 1293 { 1294 const struct rte_flow_item_icmp6 *mask = item->mask; 1295 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1296 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1297 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1298 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1299 MLX5_FLOW_LAYER_OUTER_L4; 1300 int ret; 1301 1302 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1303 return rte_flow_error_set(error, EINVAL, 1304 RTE_FLOW_ERROR_TYPE_ITEM, item, 1305 "protocol filtering not compatible" 1306 " with ICMP6 layer"); 1307 if (!(item_flags & l3m)) 1308 return rte_flow_error_set(error, EINVAL, 1309 RTE_FLOW_ERROR_TYPE_ITEM, item, 1310 "IPv6 is mandatory to filter on" 1311 " ICMP6"); 1312 if (item_flags & l4m) 1313 return rte_flow_error_set(error, EINVAL, 1314 RTE_FLOW_ERROR_TYPE_ITEM, item, 1315 "multiple L4 layers not supported"); 1316 if (!mask) 1317 mask = &rte_flow_item_icmp6_mask; 1318 ret = mlx5_flow_item_acceptable 1319 (item, (const uint8_t *)mask, 1320 (const uint8_t *)&rte_flow_item_icmp6_mask, 1321 sizeof(struct rte_flow_item_icmp6), error); 1322 if (ret < 0) 1323 return ret; 1324 return 0; 1325 } 1326 1327 /** 1328 * Validate ICMP item. 1329 * 1330 * @param[in] item 1331 * Item specification. 1332 * @param[in] item_flags 1333 * Bit-fields that holds the items detected until now. 1334 * @param[out] error 1335 * Pointer to error structure. 1336 * 1337 * @return 1338 * 0 on success, a negative errno value otherwise and rte_errno is set. 1339 */ 1340 int 1341 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1342 uint64_t item_flags, 1343 uint8_t target_protocol, 1344 struct rte_flow_error *error) 1345 { 1346 const struct rte_flow_item_icmp *mask = item->mask; 1347 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1348 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1349 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1350 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1351 MLX5_FLOW_LAYER_OUTER_L4; 1352 int ret; 1353 1354 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1355 return rte_flow_error_set(error, EINVAL, 1356 RTE_FLOW_ERROR_TYPE_ITEM, item, 1357 "protocol filtering not compatible" 1358 " with ICMP layer"); 1359 if (!(item_flags & l3m)) 1360 return rte_flow_error_set(error, EINVAL, 1361 RTE_FLOW_ERROR_TYPE_ITEM, item, 1362 "IPv4 is mandatory to filter" 1363 " on ICMP"); 1364 if (item_flags & l4m) 1365 return rte_flow_error_set(error, EINVAL, 1366 RTE_FLOW_ERROR_TYPE_ITEM, item, 1367 "multiple L4 layers not supported"); 1368 if (!mask) 1369 mask = &rte_flow_item_icmp_mask; 1370 ret = mlx5_flow_item_acceptable 1371 (item, (const uint8_t *)mask, 1372 (const uint8_t *)&rte_flow_item_icmp_mask, 1373 sizeof(struct rte_flow_item_icmp), error); 1374 if (ret < 0) 1375 return ret; 1376 return 0; 1377 } 1378 1379 /** 1380 * Validate Ethernet item. 1381 * 1382 * @param[in] item 1383 * Item specification. 1384 * @param[in] item_flags 1385 * Bit-fields that holds the items detected until now. 1386 * @param[out] error 1387 * Pointer to error structure. 1388 * 1389 * @return 1390 * 0 on success, a negative errno value otherwise and rte_errno is set. 1391 */ 1392 int 1393 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1394 uint64_t item_flags, 1395 struct rte_flow_error *error) 1396 { 1397 const struct rte_flow_item_eth *mask = item->mask; 1398 const struct rte_flow_item_eth nic_mask = { 1399 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1400 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1401 .type = RTE_BE16(0xffff), 1402 }; 1403 int ret; 1404 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1405 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1406 MLX5_FLOW_LAYER_OUTER_L2; 1407 1408 if (item_flags & ethm) 1409 return rte_flow_error_set(error, ENOTSUP, 1410 RTE_FLOW_ERROR_TYPE_ITEM, item, 1411 "multiple L2 layers not supported"); 1412 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1413 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1414 return rte_flow_error_set(error, EINVAL, 1415 RTE_FLOW_ERROR_TYPE_ITEM, item, 1416 "L2 layer should not follow " 1417 "L3 layers"); 1418 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1419 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1420 return rte_flow_error_set(error, EINVAL, 1421 RTE_FLOW_ERROR_TYPE_ITEM, item, 1422 "L2 layer should not follow VLAN"); 1423 if (!mask) 1424 mask = &rte_flow_item_eth_mask; 1425 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1426 (const uint8_t *)&nic_mask, 1427 sizeof(struct rte_flow_item_eth), 1428 error); 1429 return ret; 1430 } 1431 1432 /** 1433 * Validate VLAN item. 1434 * 1435 * @param[in] item 1436 * Item specification. 1437 * @param[in] item_flags 1438 * Bit-fields that holds the items detected until now. 1439 * @param[in] dev 1440 * Ethernet device flow is being created on. 1441 * @param[out] error 1442 * Pointer to error structure. 1443 * 1444 * @return 1445 * 0 on success, a negative errno value otherwise and rte_errno is set. 1446 */ 1447 int 1448 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1449 uint64_t item_flags, 1450 struct rte_eth_dev *dev, 1451 struct rte_flow_error *error) 1452 { 1453 const struct rte_flow_item_vlan *spec = item->spec; 1454 const struct rte_flow_item_vlan *mask = item->mask; 1455 const struct rte_flow_item_vlan nic_mask = { 1456 .tci = RTE_BE16(UINT16_MAX), 1457 .inner_type = RTE_BE16(UINT16_MAX), 1458 }; 1459 uint16_t vlan_tag = 0; 1460 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1461 int ret; 1462 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1463 MLX5_FLOW_LAYER_INNER_L4) : 1464 (MLX5_FLOW_LAYER_OUTER_L3 | 1465 MLX5_FLOW_LAYER_OUTER_L4); 1466 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1467 MLX5_FLOW_LAYER_OUTER_VLAN; 1468 1469 if (item_flags & vlanm) 1470 return rte_flow_error_set(error, EINVAL, 1471 RTE_FLOW_ERROR_TYPE_ITEM, item, 1472 "multiple VLAN layers not supported"); 1473 else if ((item_flags & l34m) != 0) 1474 return rte_flow_error_set(error, EINVAL, 1475 RTE_FLOW_ERROR_TYPE_ITEM, item, 1476 "VLAN cannot follow L3/L4 layer"); 1477 if (!mask) 1478 mask = &rte_flow_item_vlan_mask; 1479 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1480 (const uint8_t *)&nic_mask, 1481 sizeof(struct rte_flow_item_vlan), 1482 error); 1483 if (ret) 1484 return ret; 1485 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1486 struct mlx5_priv *priv = dev->data->dev_private; 1487 1488 if (priv->vmwa_context) { 1489 /* 1490 * Non-NULL context means we have a virtual machine 1491 * and SR-IOV enabled, we have to create VLAN interface 1492 * to make hypervisor to setup E-Switch vport 1493 * context correctly. We avoid creating the multiple 1494 * VLAN interfaces, so we cannot support VLAN tag mask. 1495 */ 1496 return rte_flow_error_set(error, EINVAL, 1497 RTE_FLOW_ERROR_TYPE_ITEM, 1498 item, 1499 "VLAN tag mask is not" 1500 " supported in virtual" 1501 " environment"); 1502 } 1503 } 1504 if (spec) { 1505 vlan_tag = spec->tci; 1506 vlan_tag &= mask->tci; 1507 } 1508 /* 1509 * From verbs perspective an empty VLAN is equivalent 1510 * to a packet without VLAN layer. 1511 */ 1512 if (!vlan_tag) 1513 return rte_flow_error_set(error, EINVAL, 1514 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1515 item->spec, 1516 "VLAN cannot be empty"); 1517 return 0; 1518 } 1519 1520 /** 1521 * Validate IPV4 item. 1522 * 1523 * @param[in] item 1524 * Item specification. 1525 * @param[in] item_flags 1526 * Bit-fields that holds the items detected until now. 1527 * @param[in] acc_mask 1528 * Acceptable mask, if NULL default internal default mask 1529 * will be used to check whether item fields are supported. 1530 * @param[out] error 1531 * Pointer to error structure. 1532 * 1533 * @return 1534 * 0 on success, a negative errno value otherwise and rte_errno is set. 1535 */ 1536 int 1537 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1538 uint64_t item_flags, 1539 uint64_t last_item, 1540 uint16_t ether_type, 1541 const struct rte_flow_item_ipv4 *acc_mask, 1542 struct rte_flow_error *error) 1543 { 1544 const struct rte_flow_item_ipv4 *mask = item->mask; 1545 const struct rte_flow_item_ipv4 *spec = item->spec; 1546 const struct rte_flow_item_ipv4 nic_mask = { 1547 .hdr = { 1548 .src_addr = RTE_BE32(0xffffffff), 1549 .dst_addr = RTE_BE32(0xffffffff), 1550 .type_of_service = 0xff, 1551 .next_proto_id = 0xff, 1552 }, 1553 }; 1554 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1555 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1556 MLX5_FLOW_LAYER_OUTER_L3; 1557 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1558 MLX5_FLOW_LAYER_OUTER_L4; 1559 int ret; 1560 uint8_t next_proto = 0xFF; 1561 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1562 MLX5_FLOW_LAYER_OUTER_VLAN | 1563 MLX5_FLOW_LAYER_INNER_VLAN); 1564 1565 if ((last_item & l2_vlan) && ether_type && 1566 ether_type != RTE_ETHER_TYPE_IPV4) 1567 return rte_flow_error_set(error, EINVAL, 1568 RTE_FLOW_ERROR_TYPE_ITEM, item, 1569 "IPv4 cannot follow L2/VLAN layer " 1570 "which ether type is not IPv4"); 1571 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1572 if (mask && spec) 1573 next_proto = mask->hdr.next_proto_id & 1574 spec->hdr.next_proto_id; 1575 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1576 return rte_flow_error_set(error, EINVAL, 1577 RTE_FLOW_ERROR_TYPE_ITEM, 1578 item, 1579 "multiple tunnel " 1580 "not supported"); 1581 } 1582 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1583 return rte_flow_error_set(error, EINVAL, 1584 RTE_FLOW_ERROR_TYPE_ITEM, item, 1585 "wrong tunnel type - IPv6 specified " 1586 "but IPv4 item provided"); 1587 if (item_flags & l3m) 1588 return rte_flow_error_set(error, ENOTSUP, 1589 RTE_FLOW_ERROR_TYPE_ITEM, item, 1590 "multiple L3 layers not supported"); 1591 else if (item_flags & l4m) 1592 return rte_flow_error_set(error, EINVAL, 1593 RTE_FLOW_ERROR_TYPE_ITEM, item, 1594 "L3 cannot follow an L4 layer."); 1595 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1596 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1597 return rte_flow_error_set(error, EINVAL, 1598 RTE_FLOW_ERROR_TYPE_ITEM, item, 1599 "L3 cannot follow an NVGRE layer."); 1600 if (!mask) 1601 mask = &rte_flow_item_ipv4_mask; 1602 else if (mask->hdr.next_proto_id != 0 && 1603 mask->hdr.next_proto_id != 0xff) 1604 return rte_flow_error_set(error, EINVAL, 1605 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1606 "partial mask is not supported" 1607 " for protocol"); 1608 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1609 acc_mask ? (const uint8_t *)acc_mask 1610 : (const uint8_t *)&nic_mask, 1611 sizeof(struct rte_flow_item_ipv4), 1612 error); 1613 if (ret < 0) 1614 return ret; 1615 return 0; 1616 } 1617 1618 /** 1619 * Validate IPV6 item. 1620 * 1621 * @param[in] item 1622 * Item specification. 1623 * @param[in] item_flags 1624 * Bit-fields that holds the items detected until now. 1625 * @param[in] acc_mask 1626 * Acceptable mask, if NULL default internal default mask 1627 * will be used to check whether item fields are supported. 1628 * @param[out] error 1629 * Pointer to error structure. 1630 * 1631 * @return 1632 * 0 on success, a negative errno value otherwise and rte_errno is set. 1633 */ 1634 int 1635 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1636 uint64_t item_flags, 1637 uint64_t last_item, 1638 uint16_t ether_type, 1639 const struct rte_flow_item_ipv6 *acc_mask, 1640 struct rte_flow_error *error) 1641 { 1642 const struct rte_flow_item_ipv6 *mask = item->mask; 1643 const struct rte_flow_item_ipv6 *spec = item->spec; 1644 const struct rte_flow_item_ipv6 nic_mask = { 1645 .hdr = { 1646 .src_addr = 1647 "\xff\xff\xff\xff\xff\xff\xff\xff" 1648 "\xff\xff\xff\xff\xff\xff\xff\xff", 1649 .dst_addr = 1650 "\xff\xff\xff\xff\xff\xff\xff\xff" 1651 "\xff\xff\xff\xff\xff\xff\xff\xff", 1652 .vtc_flow = RTE_BE32(0xffffffff), 1653 .proto = 0xff, 1654 .hop_limits = 0xff, 1655 }, 1656 }; 1657 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1658 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1659 MLX5_FLOW_LAYER_OUTER_L3; 1660 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1661 MLX5_FLOW_LAYER_OUTER_L4; 1662 int ret; 1663 uint8_t next_proto = 0xFF; 1664 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1665 MLX5_FLOW_LAYER_OUTER_VLAN | 1666 MLX5_FLOW_LAYER_INNER_VLAN); 1667 1668 if ((last_item & l2_vlan) && ether_type && 1669 ether_type != RTE_ETHER_TYPE_IPV6) 1670 return rte_flow_error_set(error, EINVAL, 1671 RTE_FLOW_ERROR_TYPE_ITEM, item, 1672 "IPv6 cannot follow L2/VLAN layer " 1673 "which ether type is not IPv6"); 1674 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1675 if (mask && spec) 1676 next_proto = mask->hdr.proto & spec->hdr.proto; 1677 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1678 return rte_flow_error_set(error, EINVAL, 1679 RTE_FLOW_ERROR_TYPE_ITEM, 1680 item, 1681 "multiple tunnel " 1682 "not supported"); 1683 } 1684 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1685 return rte_flow_error_set(error, EINVAL, 1686 RTE_FLOW_ERROR_TYPE_ITEM, item, 1687 "wrong tunnel type - IPv4 specified " 1688 "but IPv6 item provided"); 1689 if (item_flags & l3m) 1690 return rte_flow_error_set(error, ENOTSUP, 1691 RTE_FLOW_ERROR_TYPE_ITEM, item, 1692 "multiple L3 layers not supported"); 1693 else if (item_flags & l4m) 1694 return rte_flow_error_set(error, EINVAL, 1695 RTE_FLOW_ERROR_TYPE_ITEM, item, 1696 "L3 cannot follow an L4 layer."); 1697 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1698 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1699 return rte_flow_error_set(error, EINVAL, 1700 RTE_FLOW_ERROR_TYPE_ITEM, item, 1701 "L3 cannot follow an NVGRE layer."); 1702 if (!mask) 1703 mask = &rte_flow_item_ipv6_mask; 1704 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1705 acc_mask ? (const uint8_t *)acc_mask 1706 : (const uint8_t *)&nic_mask, 1707 sizeof(struct rte_flow_item_ipv6), 1708 error); 1709 if (ret < 0) 1710 return ret; 1711 return 0; 1712 } 1713 1714 /** 1715 * Validate UDP item. 1716 * 1717 * @param[in] item 1718 * Item specification. 1719 * @param[in] item_flags 1720 * Bit-fields that holds the items detected until now. 1721 * @param[in] target_protocol 1722 * The next protocol in the previous item. 1723 * @param[in] flow_mask 1724 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1725 * @param[out] error 1726 * Pointer to error structure. 1727 * 1728 * @return 1729 * 0 on success, a negative errno value otherwise and rte_errno is set. 1730 */ 1731 int 1732 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1733 uint64_t item_flags, 1734 uint8_t target_protocol, 1735 struct rte_flow_error *error) 1736 { 1737 const struct rte_flow_item_udp *mask = item->mask; 1738 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1739 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1740 MLX5_FLOW_LAYER_OUTER_L3; 1741 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1742 MLX5_FLOW_LAYER_OUTER_L4; 1743 int ret; 1744 1745 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1746 return rte_flow_error_set(error, EINVAL, 1747 RTE_FLOW_ERROR_TYPE_ITEM, item, 1748 "protocol filtering not compatible" 1749 " with UDP layer"); 1750 if (!(item_flags & l3m)) 1751 return rte_flow_error_set(error, EINVAL, 1752 RTE_FLOW_ERROR_TYPE_ITEM, item, 1753 "L3 is mandatory to filter on L4"); 1754 if (item_flags & l4m) 1755 return rte_flow_error_set(error, EINVAL, 1756 RTE_FLOW_ERROR_TYPE_ITEM, item, 1757 "multiple L4 layers not supported"); 1758 if (!mask) 1759 mask = &rte_flow_item_udp_mask; 1760 ret = mlx5_flow_item_acceptable 1761 (item, (const uint8_t *)mask, 1762 (const uint8_t *)&rte_flow_item_udp_mask, 1763 sizeof(struct rte_flow_item_udp), error); 1764 if (ret < 0) 1765 return ret; 1766 return 0; 1767 } 1768 1769 /** 1770 * Validate TCP item. 1771 * 1772 * @param[in] item 1773 * Item specification. 1774 * @param[in] item_flags 1775 * Bit-fields that holds the items detected until now. 1776 * @param[in] target_protocol 1777 * The next protocol in the previous item. 1778 * @param[out] error 1779 * Pointer to error structure. 1780 * 1781 * @return 1782 * 0 on success, a negative errno value otherwise and rte_errno is set. 1783 */ 1784 int 1785 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1786 uint64_t item_flags, 1787 uint8_t target_protocol, 1788 const struct rte_flow_item_tcp *flow_mask, 1789 struct rte_flow_error *error) 1790 { 1791 const struct rte_flow_item_tcp *mask = item->mask; 1792 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1793 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1794 MLX5_FLOW_LAYER_OUTER_L3; 1795 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1796 MLX5_FLOW_LAYER_OUTER_L4; 1797 int ret; 1798 1799 assert(flow_mask); 1800 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1801 return rte_flow_error_set(error, EINVAL, 1802 RTE_FLOW_ERROR_TYPE_ITEM, item, 1803 "protocol filtering not compatible" 1804 " with TCP layer"); 1805 if (!(item_flags & l3m)) 1806 return rte_flow_error_set(error, EINVAL, 1807 RTE_FLOW_ERROR_TYPE_ITEM, item, 1808 "L3 is mandatory to filter on L4"); 1809 if (item_flags & l4m) 1810 return rte_flow_error_set(error, EINVAL, 1811 RTE_FLOW_ERROR_TYPE_ITEM, item, 1812 "multiple L4 layers not supported"); 1813 if (!mask) 1814 mask = &rte_flow_item_tcp_mask; 1815 ret = mlx5_flow_item_acceptable 1816 (item, (const uint8_t *)mask, 1817 (const uint8_t *)flow_mask, 1818 sizeof(struct rte_flow_item_tcp), error); 1819 if (ret < 0) 1820 return ret; 1821 return 0; 1822 } 1823 1824 /** 1825 * Validate VXLAN item. 1826 * 1827 * @param[in] item 1828 * Item specification. 1829 * @param[in] item_flags 1830 * Bit-fields that holds the items detected until now. 1831 * @param[in] target_protocol 1832 * The next protocol in the previous item. 1833 * @param[out] error 1834 * Pointer to error structure. 1835 * 1836 * @return 1837 * 0 on success, a negative errno value otherwise and rte_errno is set. 1838 */ 1839 int 1840 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1841 uint64_t item_flags, 1842 struct rte_flow_error *error) 1843 { 1844 const struct rte_flow_item_vxlan *spec = item->spec; 1845 const struct rte_flow_item_vxlan *mask = item->mask; 1846 int ret; 1847 union vni { 1848 uint32_t vlan_id; 1849 uint8_t vni[4]; 1850 } id = { .vlan_id = 0, }; 1851 uint32_t vlan_id = 0; 1852 1853 1854 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1855 return rte_flow_error_set(error, ENOTSUP, 1856 RTE_FLOW_ERROR_TYPE_ITEM, item, 1857 "multiple tunnel layers not" 1858 " supported"); 1859 /* 1860 * Verify only UDPv4 is present as defined in 1861 * https://tools.ietf.org/html/rfc7348 1862 */ 1863 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1864 return rte_flow_error_set(error, EINVAL, 1865 RTE_FLOW_ERROR_TYPE_ITEM, item, 1866 "no outer UDP layer found"); 1867 if (!mask) 1868 mask = &rte_flow_item_vxlan_mask; 1869 ret = mlx5_flow_item_acceptable 1870 (item, (const uint8_t *)mask, 1871 (const uint8_t *)&rte_flow_item_vxlan_mask, 1872 sizeof(struct rte_flow_item_vxlan), 1873 error); 1874 if (ret < 0) 1875 return ret; 1876 if (spec) { 1877 memcpy(&id.vni[1], spec->vni, 3); 1878 vlan_id = id.vlan_id; 1879 memcpy(&id.vni[1], mask->vni, 3); 1880 vlan_id &= id.vlan_id; 1881 } 1882 /* 1883 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if 1884 * only this layer is defined in the Verbs specification it is 1885 * interpreted as wildcard and all packets will match this 1886 * rule, if it follows a full stack layer (ex: eth / ipv4 / 1887 * udp), all packets matching the layers before will also 1888 * match this rule. To avoid such situation, VNI 0 is 1889 * currently refused. 1890 */ 1891 if (!vlan_id) 1892 return rte_flow_error_set(error, ENOTSUP, 1893 RTE_FLOW_ERROR_TYPE_ITEM, item, 1894 "VXLAN vni cannot be 0"); 1895 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1896 return rte_flow_error_set(error, ENOTSUP, 1897 RTE_FLOW_ERROR_TYPE_ITEM, item, 1898 "VXLAN tunnel must be fully defined"); 1899 return 0; 1900 } 1901 1902 /** 1903 * Validate VXLAN_GPE item. 1904 * 1905 * @param[in] item 1906 * Item specification. 1907 * @param[in] item_flags 1908 * Bit-fields that holds the items detected until now. 1909 * @param[in] priv 1910 * Pointer to the private data structure. 1911 * @param[in] target_protocol 1912 * The next protocol in the previous item. 1913 * @param[out] error 1914 * Pointer to error structure. 1915 * 1916 * @return 1917 * 0 on success, a negative errno value otherwise and rte_errno is set. 1918 */ 1919 int 1920 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1921 uint64_t item_flags, 1922 struct rte_eth_dev *dev, 1923 struct rte_flow_error *error) 1924 { 1925 struct mlx5_priv *priv = dev->data->dev_private; 1926 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1927 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1928 int ret; 1929 union vni { 1930 uint32_t vlan_id; 1931 uint8_t vni[4]; 1932 } id = { .vlan_id = 0, }; 1933 uint32_t vlan_id = 0; 1934 1935 if (!priv->config.l3_vxlan_en) 1936 return rte_flow_error_set(error, ENOTSUP, 1937 RTE_FLOW_ERROR_TYPE_ITEM, item, 1938 "L3 VXLAN is not enabled by device" 1939 " parameter and/or not configured in" 1940 " firmware"); 1941 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1942 return rte_flow_error_set(error, ENOTSUP, 1943 RTE_FLOW_ERROR_TYPE_ITEM, item, 1944 "multiple tunnel layers not" 1945 " supported"); 1946 /* 1947 * Verify only UDPv4 is present as defined in 1948 * https://tools.ietf.org/html/rfc7348 1949 */ 1950 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1951 return rte_flow_error_set(error, EINVAL, 1952 RTE_FLOW_ERROR_TYPE_ITEM, item, 1953 "no outer UDP layer found"); 1954 if (!mask) 1955 mask = &rte_flow_item_vxlan_gpe_mask; 1956 ret = mlx5_flow_item_acceptable 1957 (item, (const uint8_t *)mask, 1958 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1959 sizeof(struct rte_flow_item_vxlan_gpe), 1960 error); 1961 if (ret < 0) 1962 return ret; 1963 if (spec) { 1964 if (spec->protocol) 1965 return rte_flow_error_set(error, ENOTSUP, 1966 RTE_FLOW_ERROR_TYPE_ITEM, 1967 item, 1968 "VxLAN-GPE protocol" 1969 " not supported"); 1970 memcpy(&id.vni[1], spec->vni, 3); 1971 vlan_id = id.vlan_id; 1972 memcpy(&id.vni[1], mask->vni, 3); 1973 vlan_id &= id.vlan_id; 1974 } 1975 /* 1976 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1977 * layer is defined in the Verbs specification it is interpreted as 1978 * wildcard and all packets will match this rule, if it follows a full 1979 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1980 * before will also match this rule. To avoid such situation, VNI 0 1981 * is currently refused. 1982 */ 1983 if (!vlan_id) 1984 return rte_flow_error_set(error, ENOTSUP, 1985 RTE_FLOW_ERROR_TYPE_ITEM, item, 1986 "VXLAN-GPE vni cannot be 0"); 1987 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1988 return rte_flow_error_set(error, ENOTSUP, 1989 RTE_FLOW_ERROR_TYPE_ITEM, item, 1990 "VXLAN-GPE tunnel must be fully" 1991 " defined"); 1992 return 0; 1993 } 1994 /** 1995 * Validate GRE Key item. 1996 * 1997 * @param[in] item 1998 * Item specification. 1999 * @param[in] item_flags 2000 * Bit flags to mark detected items. 2001 * @param[in] gre_item 2002 * Pointer to gre_item 2003 * @param[out] error 2004 * Pointer to error structure. 2005 * 2006 * @return 2007 * 0 on success, a negative errno value otherwise and rte_errno is set. 2008 */ 2009 int 2010 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2011 uint64_t item_flags, 2012 const struct rte_flow_item *gre_item, 2013 struct rte_flow_error *error) 2014 { 2015 const rte_be32_t *mask = item->mask; 2016 int ret = 0; 2017 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2018 const struct rte_flow_item_gre *gre_spec; 2019 const struct rte_flow_item_gre *gre_mask; 2020 2021 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2022 return rte_flow_error_set(error, ENOTSUP, 2023 RTE_FLOW_ERROR_TYPE_ITEM, item, 2024 "Multiple GRE key not support"); 2025 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2026 return rte_flow_error_set(error, ENOTSUP, 2027 RTE_FLOW_ERROR_TYPE_ITEM, item, 2028 "No preceding GRE header"); 2029 if (item_flags & MLX5_FLOW_LAYER_INNER) 2030 return rte_flow_error_set(error, ENOTSUP, 2031 RTE_FLOW_ERROR_TYPE_ITEM, item, 2032 "GRE key following a wrong item"); 2033 gre_mask = gre_item->mask; 2034 if (!gre_mask) 2035 gre_mask = &rte_flow_item_gre_mask; 2036 gre_spec = gre_item->spec; 2037 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2038 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2039 return rte_flow_error_set(error, EINVAL, 2040 RTE_FLOW_ERROR_TYPE_ITEM, item, 2041 "Key bit must be on"); 2042 2043 if (!mask) 2044 mask = &gre_key_default_mask; 2045 ret = mlx5_flow_item_acceptable 2046 (item, (const uint8_t *)mask, 2047 (const uint8_t *)&gre_key_default_mask, 2048 sizeof(rte_be32_t), error); 2049 return ret; 2050 } 2051 2052 /** 2053 * Validate GRE item. 2054 * 2055 * @param[in] item 2056 * Item specification. 2057 * @param[in] item_flags 2058 * Bit flags to mark detected items. 2059 * @param[in] target_protocol 2060 * The next protocol in the previous item. 2061 * @param[out] error 2062 * Pointer to error structure. 2063 * 2064 * @return 2065 * 0 on success, a negative errno value otherwise and rte_errno is set. 2066 */ 2067 int 2068 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2069 uint64_t item_flags, 2070 uint8_t target_protocol, 2071 struct rte_flow_error *error) 2072 { 2073 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2074 const struct rte_flow_item_gre *mask = item->mask; 2075 int ret; 2076 const struct rte_flow_item_gre nic_mask = { 2077 .c_rsvd0_ver = RTE_BE16(0xB000), 2078 .protocol = RTE_BE16(UINT16_MAX), 2079 }; 2080 2081 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2082 return rte_flow_error_set(error, EINVAL, 2083 RTE_FLOW_ERROR_TYPE_ITEM, item, 2084 "protocol filtering not compatible" 2085 " with this GRE layer"); 2086 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2087 return rte_flow_error_set(error, ENOTSUP, 2088 RTE_FLOW_ERROR_TYPE_ITEM, item, 2089 "multiple tunnel layers not" 2090 " supported"); 2091 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2092 return rte_flow_error_set(error, ENOTSUP, 2093 RTE_FLOW_ERROR_TYPE_ITEM, item, 2094 "L3 Layer is missing"); 2095 if (!mask) 2096 mask = &rte_flow_item_gre_mask; 2097 ret = mlx5_flow_item_acceptable 2098 (item, (const uint8_t *)mask, 2099 (const uint8_t *)&nic_mask, 2100 sizeof(struct rte_flow_item_gre), error); 2101 if (ret < 0) 2102 return ret; 2103 #ifndef HAVE_MLX5DV_DR 2104 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2105 if (spec && (spec->protocol & mask->protocol)) 2106 return rte_flow_error_set(error, ENOTSUP, 2107 RTE_FLOW_ERROR_TYPE_ITEM, item, 2108 "without MPLS support the" 2109 " specification cannot be used for" 2110 " filtering"); 2111 #endif 2112 #endif 2113 return 0; 2114 } 2115 2116 /** 2117 * Validate Geneve item. 2118 * 2119 * @param[in] item 2120 * Item specification. 2121 * @param[in] itemFlags 2122 * Bit-fields that holds the items detected until now. 2123 * @param[in] enPriv 2124 * Pointer to the private data structure. 2125 * @param[out] error 2126 * Pointer to error structure. 2127 * 2128 * @return 2129 * 0 on success, a negative errno value otherwise and rte_errno is set. 2130 */ 2131 2132 int 2133 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2134 uint64_t item_flags, 2135 struct rte_eth_dev *dev, 2136 struct rte_flow_error *error) 2137 { 2138 struct mlx5_priv *priv = dev->data->dev_private; 2139 const struct rte_flow_item_geneve *spec = item->spec; 2140 const struct rte_flow_item_geneve *mask = item->mask; 2141 int ret; 2142 uint16_t gbhdr; 2143 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2144 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2145 const struct rte_flow_item_geneve nic_mask = { 2146 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2147 .vni = "\xff\xff\xff", 2148 .protocol = RTE_BE16(UINT16_MAX), 2149 }; 2150 2151 if (!(priv->config.hca_attr.flex_parser_protocols & 2152 MLX5_HCA_FLEX_GENEVE_ENABLED) || 2153 !priv->config.hca_attr.tunnel_stateless_geneve_rx) 2154 return rte_flow_error_set(error, ENOTSUP, 2155 RTE_FLOW_ERROR_TYPE_ITEM, item, 2156 "L3 Geneve is not enabled by device" 2157 " parameter and/or not configured in" 2158 " firmware"); 2159 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2160 return rte_flow_error_set(error, ENOTSUP, 2161 RTE_FLOW_ERROR_TYPE_ITEM, item, 2162 "multiple tunnel layers not" 2163 " supported"); 2164 /* 2165 * Verify only UDPv4 is present as defined in 2166 * https://tools.ietf.org/html/rfc7348 2167 */ 2168 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2169 return rte_flow_error_set(error, EINVAL, 2170 RTE_FLOW_ERROR_TYPE_ITEM, item, 2171 "no outer UDP layer found"); 2172 if (!mask) 2173 mask = &rte_flow_item_geneve_mask; 2174 ret = mlx5_flow_item_acceptable 2175 (item, (const uint8_t *)mask, 2176 (const uint8_t *)&nic_mask, 2177 sizeof(struct rte_flow_item_geneve), error); 2178 if (ret) 2179 return ret; 2180 if (spec) { 2181 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2182 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2183 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2184 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2185 return rte_flow_error_set(error, ENOTSUP, 2186 RTE_FLOW_ERROR_TYPE_ITEM, 2187 item, 2188 "Geneve protocol unsupported" 2189 " fields are being used"); 2190 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2191 return rte_flow_error_set 2192 (error, ENOTSUP, 2193 RTE_FLOW_ERROR_TYPE_ITEM, 2194 item, 2195 "Unsupported Geneve options length"); 2196 } 2197 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2198 return rte_flow_error_set 2199 (error, ENOTSUP, 2200 RTE_FLOW_ERROR_TYPE_ITEM, item, 2201 "Geneve tunnel must be fully defined"); 2202 return 0; 2203 } 2204 2205 /** 2206 * Validate MPLS item. 2207 * 2208 * @param[in] dev 2209 * Pointer to the rte_eth_dev structure. 2210 * @param[in] item 2211 * Item specification. 2212 * @param[in] item_flags 2213 * Bit-fields that holds the items detected until now. 2214 * @param[in] prev_layer 2215 * The protocol layer indicated in previous item. 2216 * @param[out] error 2217 * Pointer to error structure. 2218 * 2219 * @return 2220 * 0 on success, a negative errno value otherwise and rte_errno is set. 2221 */ 2222 int 2223 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2224 const struct rte_flow_item *item __rte_unused, 2225 uint64_t item_flags __rte_unused, 2226 uint64_t prev_layer __rte_unused, 2227 struct rte_flow_error *error) 2228 { 2229 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2230 const struct rte_flow_item_mpls *mask = item->mask; 2231 struct mlx5_priv *priv = dev->data->dev_private; 2232 int ret; 2233 2234 if (!priv->config.mpls_en) 2235 return rte_flow_error_set(error, ENOTSUP, 2236 RTE_FLOW_ERROR_TYPE_ITEM, item, 2237 "MPLS not supported or" 2238 " disabled in firmware" 2239 " configuration."); 2240 /* MPLS over IP, UDP, GRE is allowed */ 2241 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2242 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2243 MLX5_FLOW_LAYER_GRE))) 2244 return rte_flow_error_set(error, EINVAL, 2245 RTE_FLOW_ERROR_TYPE_ITEM, item, 2246 "protocol filtering not compatible" 2247 " with MPLS layer"); 2248 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2249 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2250 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2251 return rte_flow_error_set(error, ENOTSUP, 2252 RTE_FLOW_ERROR_TYPE_ITEM, item, 2253 "multiple tunnel layers not" 2254 " supported"); 2255 if (!mask) 2256 mask = &rte_flow_item_mpls_mask; 2257 ret = mlx5_flow_item_acceptable 2258 (item, (const uint8_t *)mask, 2259 (const uint8_t *)&rte_flow_item_mpls_mask, 2260 sizeof(struct rte_flow_item_mpls), error); 2261 if (ret < 0) 2262 return ret; 2263 return 0; 2264 #endif 2265 return rte_flow_error_set(error, ENOTSUP, 2266 RTE_FLOW_ERROR_TYPE_ITEM, item, 2267 "MPLS is not supported by Verbs, please" 2268 " update."); 2269 } 2270 2271 /** 2272 * Validate NVGRE item. 2273 * 2274 * @param[in] item 2275 * Item specification. 2276 * @param[in] item_flags 2277 * Bit flags to mark detected items. 2278 * @param[in] target_protocol 2279 * The next protocol in the previous item. 2280 * @param[out] error 2281 * Pointer to error structure. 2282 * 2283 * @return 2284 * 0 on success, a negative errno value otherwise and rte_errno is set. 2285 */ 2286 int 2287 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2288 uint64_t item_flags, 2289 uint8_t target_protocol, 2290 struct rte_flow_error *error) 2291 { 2292 const struct rte_flow_item_nvgre *mask = item->mask; 2293 int ret; 2294 2295 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2296 return rte_flow_error_set(error, EINVAL, 2297 RTE_FLOW_ERROR_TYPE_ITEM, item, 2298 "protocol filtering not compatible" 2299 " with this GRE layer"); 2300 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2301 return rte_flow_error_set(error, ENOTSUP, 2302 RTE_FLOW_ERROR_TYPE_ITEM, item, 2303 "multiple tunnel layers not" 2304 " supported"); 2305 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2306 return rte_flow_error_set(error, ENOTSUP, 2307 RTE_FLOW_ERROR_TYPE_ITEM, item, 2308 "L3 Layer is missing"); 2309 if (!mask) 2310 mask = &rte_flow_item_nvgre_mask; 2311 ret = mlx5_flow_item_acceptable 2312 (item, (const uint8_t *)mask, 2313 (const uint8_t *)&rte_flow_item_nvgre_mask, 2314 sizeof(struct rte_flow_item_nvgre), error); 2315 if (ret < 0) 2316 return ret; 2317 return 0; 2318 } 2319 2320 /* Allocate unique ID for the split Q/RSS subflows. */ 2321 static uint32_t 2322 flow_qrss_get_id(struct rte_eth_dev *dev) 2323 { 2324 struct mlx5_priv *priv = dev->data->dev_private; 2325 uint32_t qrss_id, ret; 2326 2327 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2328 if (ret) 2329 return 0; 2330 assert(qrss_id); 2331 return qrss_id; 2332 } 2333 2334 /* Free unique ID for the split Q/RSS subflows. */ 2335 static void 2336 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2337 { 2338 struct mlx5_priv *priv = dev->data->dev_private; 2339 2340 if (qrss_id) 2341 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2342 } 2343 2344 /** 2345 * Release resource related QUEUE/RSS action split. 2346 * 2347 * @param dev 2348 * Pointer to Ethernet device. 2349 * @param flow 2350 * Flow to release id's from. 2351 */ 2352 static void 2353 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2354 struct rte_flow *flow) 2355 { 2356 struct mlx5_flow *dev_flow; 2357 2358 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 2359 if (dev_flow->qrss_id) 2360 flow_qrss_free_id(dev, dev_flow->qrss_id); 2361 } 2362 2363 static int 2364 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2365 const struct rte_flow_attr *attr __rte_unused, 2366 const struct rte_flow_item items[] __rte_unused, 2367 const struct rte_flow_action actions[] __rte_unused, 2368 bool external __rte_unused, 2369 struct rte_flow_error *error) 2370 { 2371 return rte_flow_error_set(error, ENOTSUP, 2372 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2373 } 2374 2375 static struct mlx5_flow * 2376 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, 2377 const struct rte_flow_item items[] __rte_unused, 2378 const struct rte_flow_action actions[] __rte_unused, 2379 struct rte_flow_error *error) 2380 { 2381 rte_flow_error_set(error, ENOTSUP, 2382 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2383 return NULL; 2384 } 2385 2386 static int 2387 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2388 struct mlx5_flow *dev_flow __rte_unused, 2389 const struct rte_flow_attr *attr __rte_unused, 2390 const struct rte_flow_item items[] __rte_unused, 2391 const struct rte_flow_action actions[] __rte_unused, 2392 struct rte_flow_error *error) 2393 { 2394 return rte_flow_error_set(error, ENOTSUP, 2395 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2396 } 2397 2398 static int 2399 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2400 struct rte_flow *flow __rte_unused, 2401 struct rte_flow_error *error) 2402 { 2403 return rte_flow_error_set(error, ENOTSUP, 2404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2405 } 2406 2407 static void 2408 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2409 struct rte_flow *flow __rte_unused) 2410 { 2411 } 2412 2413 static void 2414 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2415 struct rte_flow *flow __rte_unused) 2416 { 2417 } 2418 2419 static int 2420 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2421 struct rte_flow *flow __rte_unused, 2422 const struct rte_flow_action *actions __rte_unused, 2423 void *data __rte_unused, 2424 struct rte_flow_error *error) 2425 { 2426 return rte_flow_error_set(error, ENOTSUP, 2427 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2428 } 2429 2430 /* Void driver to protect from null pointer reference. */ 2431 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2432 .validate = flow_null_validate, 2433 .prepare = flow_null_prepare, 2434 .translate = flow_null_translate, 2435 .apply = flow_null_apply, 2436 .remove = flow_null_remove, 2437 .destroy = flow_null_destroy, 2438 .query = flow_null_query, 2439 }; 2440 2441 /** 2442 * Select flow driver type according to flow attributes and device 2443 * configuration. 2444 * 2445 * @param[in] dev 2446 * Pointer to the dev structure. 2447 * @param[in] attr 2448 * Pointer to the flow attributes. 2449 * 2450 * @return 2451 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2452 */ 2453 static enum mlx5_flow_drv_type 2454 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2455 { 2456 struct mlx5_priv *priv = dev->data->dev_private; 2457 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2458 2459 if (attr->transfer && priv->config.dv_esw_en) 2460 type = MLX5_FLOW_TYPE_DV; 2461 if (!attr->transfer) 2462 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2463 MLX5_FLOW_TYPE_VERBS; 2464 return type; 2465 } 2466 2467 #define flow_get_drv_ops(type) flow_drv_ops[type] 2468 2469 /** 2470 * Flow driver validation API. This abstracts calling driver specific functions. 2471 * The type of flow driver is determined according to flow attributes. 2472 * 2473 * @param[in] dev 2474 * Pointer to the dev structure. 2475 * @param[in] attr 2476 * Pointer to the flow attributes. 2477 * @param[in] items 2478 * Pointer to the list of items. 2479 * @param[in] actions 2480 * Pointer to the list of actions. 2481 * @param[in] external 2482 * This flow rule is created by request external to PMD. 2483 * @param[out] error 2484 * Pointer to the error structure. 2485 * 2486 * @return 2487 * 0 on success, a negative errno value otherwise and rte_errno is set. 2488 */ 2489 static inline int 2490 flow_drv_validate(struct rte_eth_dev *dev, 2491 const struct rte_flow_attr *attr, 2492 const struct rte_flow_item items[], 2493 const struct rte_flow_action actions[], 2494 bool external, struct rte_flow_error *error) 2495 { 2496 const struct mlx5_flow_driver_ops *fops; 2497 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2498 2499 fops = flow_get_drv_ops(type); 2500 return fops->validate(dev, attr, items, actions, external, error); 2501 } 2502 2503 /** 2504 * Flow driver preparation API. This abstracts calling driver specific 2505 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2506 * calculates the size of memory required for device flow, allocates the memory, 2507 * initializes the device flow and returns the pointer. 2508 * 2509 * @note 2510 * This function initializes device flow structure such as dv or verbs in 2511 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2512 * rest. For example, adding returning device flow to flow->dev_flow list and 2513 * setting backward reference to the flow should be done out of this function. 2514 * layers field is not filled either. 2515 * 2516 * @param[in] attr 2517 * Pointer to the flow attributes. 2518 * @param[in] items 2519 * Pointer to the list of items. 2520 * @param[in] actions 2521 * Pointer to the list of actions. 2522 * @param[out] error 2523 * Pointer to the error structure. 2524 * 2525 * @return 2526 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2527 */ 2528 static inline struct mlx5_flow * 2529 flow_drv_prepare(const struct rte_flow *flow, 2530 const struct rte_flow_attr *attr, 2531 const struct rte_flow_item items[], 2532 const struct rte_flow_action actions[], 2533 struct rte_flow_error *error) 2534 { 2535 const struct mlx5_flow_driver_ops *fops; 2536 enum mlx5_flow_drv_type type = flow->drv_type; 2537 2538 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2539 fops = flow_get_drv_ops(type); 2540 return fops->prepare(attr, items, actions, error); 2541 } 2542 2543 /** 2544 * Flow driver translation API. This abstracts calling driver specific 2545 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2546 * translates a generic flow into a driver flow. flow_drv_prepare() must 2547 * precede. 2548 * 2549 * @note 2550 * dev_flow->layers could be filled as a result of parsing during translation 2551 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2552 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2553 * flow->actions could be overwritten even though all the expanded dev_flows 2554 * have the same actions. 2555 * 2556 * @param[in] dev 2557 * Pointer to the rte dev structure. 2558 * @param[in, out] dev_flow 2559 * Pointer to the mlx5 flow. 2560 * @param[in] attr 2561 * Pointer to the flow attributes. 2562 * @param[in] items 2563 * Pointer to the list of items. 2564 * @param[in] actions 2565 * Pointer to the list of actions. 2566 * @param[out] error 2567 * Pointer to the error structure. 2568 * 2569 * @return 2570 * 0 on success, a negative errno value otherwise and rte_errno is set. 2571 */ 2572 static inline int 2573 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2574 const struct rte_flow_attr *attr, 2575 const struct rte_flow_item items[], 2576 const struct rte_flow_action actions[], 2577 struct rte_flow_error *error) 2578 { 2579 const struct mlx5_flow_driver_ops *fops; 2580 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2581 2582 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2583 fops = flow_get_drv_ops(type); 2584 return fops->translate(dev, dev_flow, attr, items, actions, error); 2585 } 2586 2587 /** 2588 * Flow driver apply API. This abstracts calling driver specific functions. 2589 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2590 * translated driver flows on to device. flow_drv_translate() must precede. 2591 * 2592 * @param[in] dev 2593 * Pointer to Ethernet device structure. 2594 * @param[in, out] flow 2595 * Pointer to flow structure. 2596 * @param[out] error 2597 * Pointer to error structure. 2598 * 2599 * @return 2600 * 0 on success, a negative errno value otherwise and rte_errno is set. 2601 */ 2602 static inline int 2603 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2604 struct rte_flow_error *error) 2605 { 2606 const struct mlx5_flow_driver_ops *fops; 2607 enum mlx5_flow_drv_type type = flow->drv_type; 2608 2609 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2610 fops = flow_get_drv_ops(type); 2611 return fops->apply(dev, flow, error); 2612 } 2613 2614 /** 2615 * Flow driver remove API. This abstracts calling driver specific functions. 2616 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2617 * on device. All the resources of the flow should be freed by calling 2618 * flow_drv_destroy(). 2619 * 2620 * @param[in] dev 2621 * Pointer to Ethernet device. 2622 * @param[in, out] flow 2623 * Pointer to flow structure. 2624 */ 2625 static inline void 2626 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2627 { 2628 const struct mlx5_flow_driver_ops *fops; 2629 enum mlx5_flow_drv_type type = flow->drv_type; 2630 2631 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2632 fops = flow_get_drv_ops(type); 2633 fops->remove(dev, flow); 2634 } 2635 2636 /** 2637 * Flow driver destroy API. This abstracts calling driver specific functions. 2638 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2639 * on device and releases resources of the flow. 2640 * 2641 * @param[in] dev 2642 * Pointer to Ethernet device. 2643 * @param[in, out] flow 2644 * Pointer to flow structure. 2645 */ 2646 static inline void 2647 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2648 { 2649 const struct mlx5_flow_driver_ops *fops; 2650 enum mlx5_flow_drv_type type = flow->drv_type; 2651 2652 flow_mreg_split_qrss_release(dev, flow); 2653 assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2654 fops = flow_get_drv_ops(type); 2655 fops->destroy(dev, flow); 2656 } 2657 2658 /** 2659 * Validate a flow supported by the NIC. 2660 * 2661 * @see rte_flow_validate() 2662 * @see rte_flow_ops 2663 */ 2664 int 2665 mlx5_flow_validate(struct rte_eth_dev *dev, 2666 const struct rte_flow_attr *attr, 2667 const struct rte_flow_item items[], 2668 const struct rte_flow_action actions[], 2669 struct rte_flow_error *error) 2670 { 2671 int ret; 2672 2673 ret = flow_drv_validate(dev, attr, items, actions, true, error); 2674 if (ret < 0) 2675 return ret; 2676 return 0; 2677 } 2678 2679 /** 2680 * Get port id item from the item list. 2681 * 2682 * @param[in] item 2683 * Pointer to the list of items. 2684 * 2685 * @return 2686 * Pointer to the port id item if exist, else return NULL. 2687 */ 2688 static const struct rte_flow_item * 2689 find_port_id_item(const struct rte_flow_item *item) 2690 { 2691 assert(item); 2692 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2693 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) 2694 return item; 2695 } 2696 return NULL; 2697 } 2698 2699 /** 2700 * Get RSS action from the action list. 2701 * 2702 * @param[in] actions 2703 * Pointer to the list of actions. 2704 * 2705 * @return 2706 * Pointer to the RSS action if exist, else return NULL. 2707 */ 2708 static const struct rte_flow_action_rss* 2709 flow_get_rss_action(const struct rte_flow_action actions[]) 2710 { 2711 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2712 switch (actions->type) { 2713 case RTE_FLOW_ACTION_TYPE_RSS: 2714 return (const struct rte_flow_action_rss *) 2715 actions->conf; 2716 default: 2717 break; 2718 } 2719 } 2720 return NULL; 2721 } 2722 2723 static unsigned int 2724 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2725 { 2726 const struct rte_flow_item *item; 2727 unsigned int has_vlan = 0; 2728 2729 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2730 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2731 has_vlan = 1; 2732 break; 2733 } 2734 } 2735 if (has_vlan) 2736 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2737 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2738 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2739 MLX5_EXPANSION_ROOT_OUTER; 2740 } 2741 2742 /** 2743 * Get QUEUE/RSS action from the action list. 2744 * 2745 * @param[in] actions 2746 * Pointer to the list of actions. 2747 * @param[out] qrss 2748 * Pointer to the return pointer. 2749 * @param[out] qrss_type 2750 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2751 * if no QUEUE/RSS is found. 2752 * 2753 * @return 2754 * Total number of actions. 2755 */ 2756 static int 2757 flow_parse_qrss_action(const struct rte_flow_action actions[], 2758 const struct rte_flow_action **qrss) 2759 { 2760 int actions_n = 0; 2761 2762 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2763 switch (actions->type) { 2764 case RTE_FLOW_ACTION_TYPE_QUEUE: 2765 case RTE_FLOW_ACTION_TYPE_RSS: 2766 *qrss = actions; 2767 break; 2768 default: 2769 break; 2770 } 2771 actions_n++; 2772 } 2773 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2774 return actions_n + 1; 2775 } 2776 2777 /** 2778 * Check meter action from the action list. 2779 * 2780 * @param[in] actions 2781 * Pointer to the list of actions. 2782 * @param[out] mtr 2783 * Pointer to the meter exist flag. 2784 * 2785 * @return 2786 * Total number of actions. 2787 */ 2788 static int 2789 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2790 { 2791 int actions_n = 0; 2792 2793 assert(mtr); 2794 *mtr = 0; 2795 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2796 switch (actions->type) { 2797 case RTE_FLOW_ACTION_TYPE_METER: 2798 *mtr = 1; 2799 break; 2800 default: 2801 break; 2802 } 2803 actions_n++; 2804 } 2805 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2806 return actions_n + 1; 2807 } 2808 2809 /** 2810 * Check if the flow should be splited due to hairpin. 2811 * The reason for the split is that in current HW we can't 2812 * support encap on Rx, so if a flow have encap we move it 2813 * to Tx. 2814 * 2815 * @param dev 2816 * Pointer to Ethernet device. 2817 * @param[in] attr 2818 * Flow rule attributes. 2819 * @param[in] actions 2820 * Associated actions (list terminated by the END action). 2821 * 2822 * @return 2823 * > 0 the number of actions and the flow should be split, 2824 * 0 when no split required. 2825 */ 2826 static int 2827 flow_check_hairpin_split(struct rte_eth_dev *dev, 2828 const struct rte_flow_attr *attr, 2829 const struct rte_flow_action actions[]) 2830 { 2831 int queue_action = 0; 2832 int action_n = 0; 2833 int encap = 0; 2834 const struct rte_flow_action_queue *queue; 2835 const struct rte_flow_action_rss *rss; 2836 const struct rte_flow_action_raw_encap *raw_encap; 2837 2838 if (!attr->ingress) 2839 return 0; 2840 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2841 switch (actions->type) { 2842 case RTE_FLOW_ACTION_TYPE_QUEUE: 2843 queue = actions->conf; 2844 if (queue == NULL) 2845 return 0; 2846 if (mlx5_rxq_get_type(dev, queue->index) != 2847 MLX5_RXQ_TYPE_HAIRPIN) 2848 return 0; 2849 queue_action = 1; 2850 action_n++; 2851 break; 2852 case RTE_FLOW_ACTION_TYPE_RSS: 2853 rss = actions->conf; 2854 if (rss == NULL || rss->queue_num == 0) 2855 return 0; 2856 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2857 MLX5_RXQ_TYPE_HAIRPIN) 2858 return 0; 2859 queue_action = 1; 2860 action_n++; 2861 break; 2862 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2863 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2864 encap = 1; 2865 action_n++; 2866 break; 2867 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2868 raw_encap = actions->conf; 2869 if (raw_encap->size > 2870 (sizeof(struct rte_flow_item_eth) + 2871 sizeof(struct rte_flow_item_ipv4))) 2872 encap = 1; 2873 action_n++; 2874 break; 2875 default: 2876 action_n++; 2877 break; 2878 } 2879 } 2880 if (encap == 1 && queue_action) 2881 return action_n; 2882 return 0; 2883 } 2884 2885 /* Declare flow create/destroy prototype in advance. */ 2886 static struct rte_flow * 2887 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 2888 const struct rte_flow_attr *attr, 2889 const struct rte_flow_item items[], 2890 const struct rte_flow_action actions[], 2891 bool external, struct rte_flow_error *error); 2892 2893 static void 2894 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2895 struct rte_flow *flow); 2896 2897 /** 2898 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2899 * 2900 * As mark_id is unique, if there's already a registered flow for the mark_id, 2901 * return by increasing the reference counter of the resource. Otherwise, create 2902 * the resource (mcp_res) and flow. 2903 * 2904 * Flow looks like, 2905 * - If ingress port is ANY and reg_c[1] is mark_id, 2906 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2907 * 2908 * For default flow (zero mark_id), flow is like, 2909 * - If ingress port is ANY, 2910 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2911 * 2912 * @param dev 2913 * Pointer to Ethernet device. 2914 * @param mark_id 2915 * ID of MARK action, zero means default flow for META. 2916 * @param[out] error 2917 * Perform verbose error reporting if not NULL. 2918 * 2919 * @return 2920 * Associated resource on success, NULL otherwise and rte_errno is set. 2921 */ 2922 static struct mlx5_flow_mreg_copy_resource * 2923 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2924 struct rte_flow_error *error) 2925 { 2926 struct mlx5_priv *priv = dev->data->dev_private; 2927 struct rte_flow_attr attr = { 2928 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2929 .ingress = 1, 2930 }; 2931 struct mlx5_rte_flow_item_tag tag_spec = { 2932 .data = mark_id, 2933 }; 2934 struct rte_flow_item items[] = { 2935 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2936 }; 2937 struct rte_flow_action_mark ftag = { 2938 .id = mark_id, 2939 }; 2940 struct mlx5_flow_action_copy_mreg cp_mreg = { 2941 .dst = REG_B, 2942 .src = 0, 2943 }; 2944 struct rte_flow_action_jump jump = { 2945 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2946 }; 2947 struct rte_flow_action actions[] = { 2948 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2949 }; 2950 struct mlx5_flow_mreg_copy_resource *mcp_res; 2951 int ret; 2952 2953 /* Fill the register fileds in the flow. */ 2954 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2955 if (ret < 0) 2956 return NULL; 2957 tag_spec.id = ret; 2958 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 2959 if (ret < 0) 2960 return NULL; 2961 cp_mreg.src = ret; 2962 /* Check if already registered. */ 2963 assert(priv->mreg_cp_tbl); 2964 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 2965 if (mcp_res) { 2966 /* For non-default rule. */ 2967 if (mark_id != MLX5_DEFAULT_COPY_ID) 2968 mcp_res->refcnt++; 2969 assert(mark_id != MLX5_DEFAULT_COPY_ID || mcp_res->refcnt == 1); 2970 return mcp_res; 2971 } 2972 /* Provide the full width of FLAG specific value. */ 2973 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 2974 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 2975 /* Build a new flow. */ 2976 if (mark_id != MLX5_DEFAULT_COPY_ID) { 2977 items[0] = (struct rte_flow_item){ 2978 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 2979 .spec = &tag_spec, 2980 }; 2981 items[1] = (struct rte_flow_item){ 2982 .type = RTE_FLOW_ITEM_TYPE_END, 2983 }; 2984 actions[0] = (struct rte_flow_action){ 2985 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK, 2986 .conf = &ftag, 2987 }; 2988 actions[1] = (struct rte_flow_action){ 2989 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 2990 .conf = &cp_mreg, 2991 }; 2992 actions[2] = (struct rte_flow_action){ 2993 .type = RTE_FLOW_ACTION_TYPE_JUMP, 2994 .conf = &jump, 2995 }; 2996 actions[3] = (struct rte_flow_action){ 2997 .type = RTE_FLOW_ACTION_TYPE_END, 2998 }; 2999 } else { 3000 /* Default rule, wildcard match. */ 3001 attr.priority = MLX5_FLOW_PRIO_RSVD; 3002 items[0] = (struct rte_flow_item){ 3003 .type = RTE_FLOW_ITEM_TYPE_END, 3004 }; 3005 actions[0] = (struct rte_flow_action){ 3006 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3007 .conf = &cp_mreg, 3008 }; 3009 actions[1] = (struct rte_flow_action){ 3010 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3011 .conf = &jump, 3012 }; 3013 actions[2] = (struct rte_flow_action){ 3014 .type = RTE_FLOW_ACTION_TYPE_END, 3015 }; 3016 } 3017 /* Build a new entry. */ 3018 mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0); 3019 if (!mcp_res) { 3020 rte_errno = ENOMEM; 3021 return NULL; 3022 } 3023 /* 3024 * The copy Flows are not included in any list. There 3025 * ones are referenced from other Flows and can not 3026 * be applied, removed, deleted in ardbitrary order 3027 * by list traversing. 3028 */ 3029 mcp_res->flow = flow_list_create(dev, NULL, &attr, items, 3030 actions, false, error); 3031 if (!mcp_res->flow) 3032 goto error; 3033 mcp_res->refcnt++; 3034 mcp_res->hlist_ent.key = mark_id; 3035 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3036 &mcp_res->hlist_ent); 3037 assert(!ret); 3038 if (ret) 3039 goto error; 3040 return mcp_res; 3041 error: 3042 if (mcp_res->flow) 3043 flow_list_destroy(dev, NULL, mcp_res->flow); 3044 rte_free(mcp_res); 3045 return NULL; 3046 } 3047 3048 /** 3049 * Release flow in RX_CP_TBL. 3050 * 3051 * @param dev 3052 * Pointer to Ethernet device. 3053 * @flow 3054 * Parent flow for wich copying is provided. 3055 */ 3056 static void 3057 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3058 struct rte_flow *flow) 3059 { 3060 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3061 struct mlx5_priv *priv = dev->data->dev_private; 3062 3063 if (!mcp_res || !priv->mreg_cp_tbl) 3064 return; 3065 if (flow->copy_applied) { 3066 assert(mcp_res->appcnt); 3067 flow->copy_applied = 0; 3068 --mcp_res->appcnt; 3069 if (!mcp_res->appcnt) 3070 flow_drv_remove(dev, mcp_res->flow); 3071 } 3072 /* 3073 * We do not check availability of metadata registers here, 3074 * because copy resources are not allocated in this case. 3075 */ 3076 if (--mcp_res->refcnt) 3077 return; 3078 assert(mcp_res->flow); 3079 flow_list_destroy(dev, NULL, mcp_res->flow); 3080 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3081 rte_free(mcp_res); 3082 flow->mreg_copy = NULL; 3083 } 3084 3085 /** 3086 * Start flow in RX_CP_TBL. 3087 * 3088 * @param dev 3089 * Pointer to Ethernet device. 3090 * @flow 3091 * Parent flow for wich copying is provided. 3092 * 3093 * @return 3094 * 0 on success, a negative errno value otherwise and rte_errno is set. 3095 */ 3096 static int 3097 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3098 struct rte_flow *flow) 3099 { 3100 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3101 int ret; 3102 3103 if (!mcp_res || flow->copy_applied) 3104 return 0; 3105 if (!mcp_res->appcnt) { 3106 ret = flow_drv_apply(dev, mcp_res->flow, NULL); 3107 if (ret) 3108 return ret; 3109 } 3110 ++mcp_res->appcnt; 3111 flow->copy_applied = 1; 3112 return 0; 3113 } 3114 3115 /** 3116 * Stop flow in RX_CP_TBL. 3117 * 3118 * @param dev 3119 * Pointer to Ethernet device. 3120 * @flow 3121 * Parent flow for wich copying is provided. 3122 */ 3123 static void 3124 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3125 struct rte_flow *flow) 3126 { 3127 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3128 3129 if (!mcp_res || !flow->copy_applied) 3130 return; 3131 assert(mcp_res->appcnt); 3132 --mcp_res->appcnt; 3133 flow->copy_applied = 0; 3134 if (!mcp_res->appcnt) 3135 flow_drv_remove(dev, mcp_res->flow); 3136 } 3137 3138 /** 3139 * Remove the default copy action from RX_CP_TBL. 3140 * 3141 * @param dev 3142 * Pointer to Ethernet device. 3143 */ 3144 static void 3145 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3146 { 3147 struct mlx5_flow_mreg_copy_resource *mcp_res; 3148 struct mlx5_priv *priv = dev->data->dev_private; 3149 3150 /* Check if default flow is registered. */ 3151 if (!priv->mreg_cp_tbl) 3152 return; 3153 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3154 MLX5_DEFAULT_COPY_ID); 3155 if (!mcp_res) 3156 return; 3157 assert(mcp_res->flow); 3158 flow_list_destroy(dev, NULL, mcp_res->flow); 3159 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3160 rte_free(mcp_res); 3161 } 3162 3163 /** 3164 * Add the default copy action in in RX_CP_TBL. 3165 * 3166 * @param dev 3167 * Pointer to Ethernet device. 3168 * @param[out] error 3169 * Perform verbose error reporting if not NULL. 3170 * 3171 * @return 3172 * 0 for success, negative value otherwise and rte_errno is set. 3173 */ 3174 static int 3175 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3176 struct rte_flow_error *error) 3177 { 3178 struct mlx5_priv *priv = dev->data->dev_private; 3179 struct mlx5_flow_mreg_copy_resource *mcp_res; 3180 3181 /* Check whether extensive metadata feature is engaged. */ 3182 if (!priv->config.dv_flow_en || 3183 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3184 !mlx5_flow_ext_mreg_supported(dev) || 3185 !priv->sh->dv_regc0_mask) 3186 return 0; 3187 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3188 if (!mcp_res) 3189 return -rte_errno; 3190 return 0; 3191 } 3192 3193 /** 3194 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3195 * 3196 * All the flow having Q/RSS action should be split by 3197 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3198 * performs the following, 3199 * - CQE->flow_tag := reg_c[1] (MARK) 3200 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3201 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3202 * but there should be a flow per each MARK ID set by MARK action. 3203 * 3204 * For the aforementioned reason, if there's a MARK action in flow's action 3205 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3206 * the MARK ID to CQE's flow_tag like, 3207 * - If reg_c[1] is mark_id, 3208 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3209 * 3210 * For SET_META action which stores value in reg_c[0], as the destination is 3211 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3212 * MARK ID means the default flow. The default flow looks like, 3213 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3214 * 3215 * @param dev 3216 * Pointer to Ethernet device. 3217 * @param flow 3218 * Pointer to flow structure. 3219 * @param[in] actions 3220 * Pointer to the list of actions. 3221 * @param[out] error 3222 * Perform verbose error reporting if not NULL. 3223 * 3224 * @return 3225 * 0 on success, negative value otherwise and rte_errno is set. 3226 */ 3227 static int 3228 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3229 struct rte_flow *flow, 3230 const struct rte_flow_action *actions, 3231 struct rte_flow_error *error) 3232 { 3233 struct mlx5_priv *priv = dev->data->dev_private; 3234 struct mlx5_dev_config *config = &priv->config; 3235 struct mlx5_flow_mreg_copy_resource *mcp_res; 3236 const struct rte_flow_action_mark *mark; 3237 3238 /* Check whether extensive metadata feature is engaged. */ 3239 if (!config->dv_flow_en || 3240 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3241 !mlx5_flow_ext_mreg_supported(dev) || 3242 !priv->sh->dv_regc0_mask) 3243 return 0; 3244 /* Find MARK action. */ 3245 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3246 switch (actions->type) { 3247 case RTE_FLOW_ACTION_TYPE_FLAG: 3248 mcp_res = flow_mreg_add_copy_action 3249 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3250 if (!mcp_res) 3251 return -rte_errno; 3252 flow->mreg_copy = mcp_res; 3253 if (dev->data->dev_started) { 3254 mcp_res->appcnt++; 3255 flow->copy_applied = 1; 3256 } 3257 return 0; 3258 case RTE_FLOW_ACTION_TYPE_MARK: 3259 mark = (const struct rte_flow_action_mark *) 3260 actions->conf; 3261 mcp_res = 3262 flow_mreg_add_copy_action(dev, mark->id, error); 3263 if (!mcp_res) 3264 return -rte_errno; 3265 flow->mreg_copy = mcp_res; 3266 if (dev->data->dev_started) { 3267 mcp_res->appcnt++; 3268 flow->copy_applied = 1; 3269 } 3270 return 0; 3271 default: 3272 break; 3273 } 3274 } 3275 return 0; 3276 } 3277 3278 #define MLX5_MAX_SPLIT_ACTIONS 24 3279 #define MLX5_MAX_SPLIT_ITEMS 24 3280 3281 /** 3282 * Split the hairpin flow. 3283 * Since HW can't support encap on Rx we move the encap to Tx. 3284 * If the count action is after the encap then we also 3285 * move the count action. in this case the count will also measure 3286 * the outer bytes. 3287 * 3288 * @param dev 3289 * Pointer to Ethernet device. 3290 * @param[in] actions 3291 * Associated actions (list terminated by the END action). 3292 * @param[out] actions_rx 3293 * Rx flow actions. 3294 * @param[out] actions_tx 3295 * Tx flow actions.. 3296 * @param[out] pattern_tx 3297 * The pattern items for the Tx flow. 3298 * @param[out] flow_id 3299 * The flow ID connected to this flow. 3300 * 3301 * @return 3302 * 0 on success. 3303 */ 3304 static int 3305 flow_hairpin_split(struct rte_eth_dev *dev, 3306 const struct rte_flow_action actions[], 3307 struct rte_flow_action actions_rx[], 3308 struct rte_flow_action actions_tx[], 3309 struct rte_flow_item pattern_tx[], 3310 uint32_t *flow_id) 3311 { 3312 struct mlx5_priv *priv = dev->data->dev_private; 3313 const struct rte_flow_action_raw_encap *raw_encap; 3314 const struct rte_flow_action_raw_decap *raw_decap; 3315 struct mlx5_rte_flow_action_set_tag *set_tag; 3316 struct rte_flow_action *tag_action; 3317 struct mlx5_rte_flow_item_tag *tag_item; 3318 struct rte_flow_item *item; 3319 char *addr; 3320 int encap = 0; 3321 3322 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3323 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3324 switch (actions->type) { 3325 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3326 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3327 rte_memcpy(actions_tx, actions, 3328 sizeof(struct rte_flow_action)); 3329 actions_tx++; 3330 break; 3331 case RTE_FLOW_ACTION_TYPE_COUNT: 3332 if (encap) { 3333 rte_memcpy(actions_tx, actions, 3334 sizeof(struct rte_flow_action)); 3335 actions_tx++; 3336 } else { 3337 rte_memcpy(actions_rx, actions, 3338 sizeof(struct rte_flow_action)); 3339 actions_rx++; 3340 } 3341 break; 3342 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3343 raw_encap = actions->conf; 3344 if (raw_encap->size > 3345 (sizeof(struct rte_flow_item_eth) + 3346 sizeof(struct rte_flow_item_ipv4))) { 3347 memcpy(actions_tx, actions, 3348 sizeof(struct rte_flow_action)); 3349 actions_tx++; 3350 encap = 1; 3351 } else { 3352 rte_memcpy(actions_rx, actions, 3353 sizeof(struct rte_flow_action)); 3354 actions_rx++; 3355 } 3356 break; 3357 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3358 raw_decap = actions->conf; 3359 if (raw_decap->size < 3360 (sizeof(struct rte_flow_item_eth) + 3361 sizeof(struct rte_flow_item_ipv4))) { 3362 memcpy(actions_tx, actions, 3363 sizeof(struct rte_flow_action)); 3364 actions_tx++; 3365 } else { 3366 rte_memcpy(actions_rx, actions, 3367 sizeof(struct rte_flow_action)); 3368 actions_rx++; 3369 } 3370 break; 3371 default: 3372 rte_memcpy(actions_rx, actions, 3373 sizeof(struct rte_flow_action)); 3374 actions_rx++; 3375 break; 3376 } 3377 } 3378 /* Add set meta action and end action for the Rx flow. */ 3379 tag_action = actions_rx; 3380 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3381 actions_rx++; 3382 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3383 actions_rx++; 3384 set_tag = (void *)actions_rx; 3385 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3386 assert(set_tag->id > REG_NONE); 3387 set_tag->data = *flow_id; 3388 tag_action->conf = set_tag; 3389 /* Create Tx item list. */ 3390 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3391 addr = (void *)&pattern_tx[2]; 3392 item = pattern_tx; 3393 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3394 tag_item = (void *)addr; 3395 tag_item->data = *flow_id; 3396 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3397 assert(set_tag->id > REG_NONE); 3398 item->spec = tag_item; 3399 addr += sizeof(struct mlx5_rte_flow_item_tag); 3400 tag_item = (void *)addr; 3401 tag_item->data = UINT32_MAX; 3402 tag_item->id = UINT16_MAX; 3403 item->mask = tag_item; 3404 addr += sizeof(struct mlx5_rte_flow_item_tag); 3405 item->last = NULL; 3406 item++; 3407 item->type = RTE_FLOW_ITEM_TYPE_END; 3408 return 0; 3409 } 3410 3411 /** 3412 * The last stage of splitting chain, just creates the subflow 3413 * without any modification. 3414 * 3415 * @param dev 3416 * Pointer to Ethernet device. 3417 * @param[in] flow 3418 * Parent flow structure pointer. 3419 * @param[in, out] sub_flow 3420 * Pointer to return the created subflow, may be NULL. 3421 * @param[in] attr 3422 * Flow rule attributes. 3423 * @param[in] items 3424 * Pattern specification (list terminated by the END pattern item). 3425 * @param[in] actions 3426 * Associated actions (list terminated by the END action). 3427 * @param[in] external 3428 * This flow rule is created by request external to PMD. 3429 * @param[out] error 3430 * Perform verbose error reporting if not NULL. 3431 * @return 3432 * 0 on success, negative value otherwise 3433 */ 3434 static int 3435 flow_create_split_inner(struct rte_eth_dev *dev, 3436 struct rte_flow *flow, 3437 struct mlx5_flow **sub_flow, 3438 const struct rte_flow_attr *attr, 3439 const struct rte_flow_item items[], 3440 const struct rte_flow_action actions[], 3441 bool external, struct rte_flow_error *error) 3442 { 3443 struct mlx5_flow *dev_flow; 3444 3445 dev_flow = flow_drv_prepare(flow, attr, items, actions, error); 3446 if (!dev_flow) 3447 return -rte_errno; 3448 dev_flow->flow = flow; 3449 dev_flow->external = external; 3450 /* Subflow object was created, we must include one in the list. */ 3451 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 3452 if (sub_flow) 3453 *sub_flow = dev_flow; 3454 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3455 } 3456 3457 /** 3458 * Split the meter flow. 3459 * 3460 * As meter flow will split to three sub flow, other than meter 3461 * action, the other actions make sense to only meter accepts 3462 * the packet. If it need to be dropped, no other additional 3463 * actions should be take. 3464 * 3465 * One kind of special action which decapsulates the L3 tunnel 3466 * header will be in the prefix sub flow, as not to take the 3467 * L3 tunnel header into account. 3468 * 3469 * @param dev 3470 * Pointer to Ethernet device. 3471 * @param[in] actions 3472 * Associated actions (list terminated by the END action). 3473 * @param[out] actions_sfx 3474 * Suffix flow actions. 3475 * @param[out] actions_pre 3476 * Prefix flow actions. 3477 * @param[out] pattern_sfx 3478 * The pattern items for the suffix flow. 3479 * @param[out] tag_sfx 3480 * Pointer to suffix flow tag. 3481 * 3482 * @return 3483 * 0 on success. 3484 */ 3485 static int 3486 flow_meter_split_prep(struct rte_eth_dev *dev, 3487 const struct rte_flow_action actions[], 3488 struct rte_flow_action actions_sfx[], 3489 struct rte_flow_action actions_pre[]) 3490 { 3491 struct rte_flow_action *tag_action; 3492 struct mlx5_rte_flow_action_set_tag *set_tag; 3493 struct rte_flow_error error; 3494 const struct rte_flow_action_raw_encap *raw_encap; 3495 const struct rte_flow_action_raw_decap *raw_decap; 3496 uint32_t tag_id; 3497 3498 /* Add the extra tag action first. */ 3499 tag_action = actions_pre; 3500 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3501 actions_pre++; 3502 /* Prepare the actions for prefix and suffix flow. */ 3503 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3504 switch (actions->type) { 3505 case RTE_FLOW_ACTION_TYPE_METER: 3506 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3507 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3508 memcpy(actions_pre, actions, 3509 sizeof(struct rte_flow_action)); 3510 actions_pre++; 3511 break; 3512 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3513 raw_encap = actions->conf; 3514 if (raw_encap->size > 3515 (sizeof(struct rte_flow_item_eth) + 3516 sizeof(struct rte_flow_item_ipv4))) { 3517 memcpy(actions_sfx, actions, 3518 sizeof(struct rte_flow_action)); 3519 actions_sfx++; 3520 } else { 3521 rte_memcpy(actions_pre, actions, 3522 sizeof(struct rte_flow_action)); 3523 actions_pre++; 3524 } 3525 break; 3526 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3527 raw_decap = actions->conf; 3528 /* Size 0 decap means 50 bytes as vxlan decap. */ 3529 if (raw_decap->size && (raw_decap->size < 3530 (sizeof(struct rte_flow_item_eth) + 3531 sizeof(struct rte_flow_item_ipv4)))) { 3532 memcpy(actions_sfx, actions, 3533 sizeof(struct rte_flow_action)); 3534 actions_sfx++; 3535 } else { 3536 rte_memcpy(actions_pre, actions, 3537 sizeof(struct rte_flow_action)); 3538 actions_pre++; 3539 } 3540 break; 3541 default: 3542 memcpy(actions_sfx, actions, 3543 sizeof(struct rte_flow_action)); 3544 actions_sfx++; 3545 break; 3546 } 3547 } 3548 /* Add end action to the actions. */ 3549 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3550 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3551 actions_pre++; 3552 /* Set the tag. */ 3553 set_tag = (void *)actions_pre; 3554 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3555 /* 3556 * Get the id from the qrss_pool to make qrss share the id with meter. 3557 */ 3558 tag_id = flow_qrss_get_id(dev); 3559 set_tag->data = rte_cpu_to_be_32(tag_id); 3560 tag_action->conf = set_tag; 3561 return tag_id; 3562 } 3563 3564 /** 3565 * Split action list having QUEUE/RSS for metadata register copy. 3566 * 3567 * Once Q/RSS action is detected in user's action list, the flow action 3568 * should be split in order to copy metadata registers, which will happen in 3569 * RX_CP_TBL like, 3570 * - CQE->flow_tag := reg_c[1] (MARK) 3571 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3572 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3573 * This is because the last action of each flow must be a terminal action 3574 * (QUEUE, RSS or DROP). 3575 * 3576 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3577 * stored and kept in the mlx5_flow structure per each sub_flow. 3578 * 3579 * The Q/RSS action is replaced with, 3580 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3581 * And the following JUMP action is added at the end, 3582 * - JUMP, to RX_CP_TBL. 3583 * 3584 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3585 * flow_create_split_metadata() routine. The flow will look like, 3586 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3587 * 3588 * @param dev 3589 * Pointer to Ethernet device. 3590 * @param[out] split_actions 3591 * Pointer to store split actions to jump to CP_TBL. 3592 * @param[in] actions 3593 * Pointer to the list of original flow actions. 3594 * @param[in] qrss 3595 * Pointer to the Q/RSS action. 3596 * @param[in] actions_n 3597 * Number of original actions. 3598 * @param[out] error 3599 * Perform verbose error reporting if not NULL. 3600 * 3601 * @return 3602 * non-zero unique flow_id on success, otherwise 0 and 3603 * error/rte_error are set. 3604 */ 3605 static uint32_t 3606 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3607 struct rte_flow_action *split_actions, 3608 const struct rte_flow_action *actions, 3609 const struct rte_flow_action *qrss, 3610 int actions_n, struct rte_flow_error *error) 3611 { 3612 struct mlx5_rte_flow_action_set_tag *set_tag; 3613 struct rte_flow_action_jump *jump; 3614 const int qrss_idx = qrss - actions; 3615 uint32_t flow_id = 0; 3616 int ret = 0; 3617 3618 /* 3619 * Given actions will be split 3620 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3621 * - Add jump to mreg CP_TBL. 3622 * As a result, there will be one more action. 3623 */ 3624 ++actions_n; 3625 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3626 set_tag = (void *)(split_actions + actions_n); 3627 /* 3628 * If tag action is not set to void(it means we are not the meter 3629 * suffix flow), add the tag action. Since meter suffix flow already 3630 * has the tag added. 3631 */ 3632 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3633 /* 3634 * Allocate the new subflow ID. This one is unique within 3635 * device and not shared with representors. Otherwise, 3636 * we would have to resolve multi-thread access synch 3637 * issue. Each flow on the shared device is appended 3638 * with source vport identifier, so the resulting 3639 * flows will be unique in the shared (by master and 3640 * representors) domain even if they have coinciding 3641 * IDs. 3642 */ 3643 flow_id = flow_qrss_get_id(dev); 3644 if (!flow_id) 3645 return rte_flow_error_set(error, ENOMEM, 3646 RTE_FLOW_ERROR_TYPE_ACTION, 3647 NULL, "can't allocate id " 3648 "for split Q/RSS subflow"); 3649 /* Internal SET_TAG action to set flow ID. */ 3650 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3651 .data = flow_id, 3652 }; 3653 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3654 if (ret < 0) 3655 return ret; 3656 set_tag->id = ret; 3657 /* Construct new actions array. */ 3658 /* Replace QUEUE/RSS action. */ 3659 split_actions[qrss_idx] = (struct rte_flow_action){ 3660 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3661 .conf = set_tag, 3662 }; 3663 } 3664 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3665 jump = (void *)(set_tag + 1); 3666 *jump = (struct rte_flow_action_jump){ 3667 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3668 }; 3669 split_actions[actions_n - 2] = (struct rte_flow_action){ 3670 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3671 .conf = jump, 3672 }; 3673 split_actions[actions_n - 1] = (struct rte_flow_action){ 3674 .type = RTE_FLOW_ACTION_TYPE_END, 3675 }; 3676 return flow_id; 3677 } 3678 3679 /** 3680 * Extend the given action list for Tx metadata copy. 3681 * 3682 * Copy the given action list to the ext_actions and add flow metadata register 3683 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3684 * 3685 * @param[out] ext_actions 3686 * Pointer to the extended action list. 3687 * @param[in] actions 3688 * Pointer to the list of actions. 3689 * @param[in] actions_n 3690 * Number of actions in the list. 3691 * @param[out] error 3692 * Perform verbose error reporting if not NULL. 3693 * 3694 * @return 3695 * 0 on success, negative value otherwise 3696 */ 3697 static int 3698 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3699 struct rte_flow_action *ext_actions, 3700 const struct rte_flow_action *actions, 3701 int actions_n, struct rte_flow_error *error) 3702 { 3703 struct mlx5_flow_action_copy_mreg *cp_mreg = 3704 (struct mlx5_flow_action_copy_mreg *) 3705 (ext_actions + actions_n + 1); 3706 int ret; 3707 3708 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3709 if (ret < 0) 3710 return ret; 3711 cp_mreg->dst = ret; 3712 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3713 if (ret < 0) 3714 return ret; 3715 cp_mreg->src = ret; 3716 memcpy(ext_actions, actions, 3717 sizeof(*ext_actions) * actions_n); 3718 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3719 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3720 .conf = cp_mreg, 3721 }; 3722 ext_actions[actions_n] = (struct rte_flow_action){ 3723 .type = RTE_FLOW_ACTION_TYPE_END, 3724 }; 3725 return 0; 3726 } 3727 3728 /** 3729 * The splitting for metadata feature. 3730 * 3731 * - Q/RSS action on NIC Rx should be split in order to pass by 3732 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3733 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3734 * 3735 * - All the actions on NIC Tx should have a mreg copy action to 3736 * copy reg_a from WQE to reg_c[0]. 3737 * 3738 * @param dev 3739 * Pointer to Ethernet device. 3740 * @param[in] flow 3741 * Parent flow structure pointer. 3742 * @param[in] attr 3743 * Flow rule attributes. 3744 * @param[in] items 3745 * Pattern specification (list terminated by the END pattern item). 3746 * @param[in] actions 3747 * Associated actions (list terminated by the END action). 3748 * @param[in] external 3749 * This flow rule is created by request external to PMD. 3750 * @param[out] error 3751 * Perform verbose error reporting if not NULL. 3752 * @return 3753 * 0 on success, negative value otherwise 3754 */ 3755 static int 3756 flow_create_split_metadata(struct rte_eth_dev *dev, 3757 struct rte_flow *flow, 3758 const struct rte_flow_attr *attr, 3759 const struct rte_flow_item items[], 3760 const struct rte_flow_action actions[], 3761 bool external, struct rte_flow_error *error) 3762 { 3763 struct mlx5_priv *priv = dev->data->dev_private; 3764 struct mlx5_dev_config *config = &priv->config; 3765 const struct rte_flow_action *qrss = NULL; 3766 struct rte_flow_action *ext_actions = NULL; 3767 struct mlx5_flow *dev_flow = NULL; 3768 uint32_t qrss_id = 0; 3769 int mtr_sfx = 0; 3770 size_t act_size; 3771 int actions_n; 3772 int ret; 3773 3774 /* Check whether extensive metadata feature is engaged. */ 3775 if (!config->dv_flow_en || 3776 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3777 !mlx5_flow_ext_mreg_supported(dev)) 3778 return flow_create_split_inner(dev, flow, NULL, attr, items, 3779 actions, external, error); 3780 actions_n = flow_parse_qrss_action(actions, &qrss); 3781 if (qrss) { 3782 /* Exclude hairpin flows from splitting. */ 3783 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3784 const struct rte_flow_action_queue *queue; 3785 3786 queue = qrss->conf; 3787 if (mlx5_rxq_get_type(dev, queue->index) == 3788 MLX5_RXQ_TYPE_HAIRPIN) 3789 qrss = NULL; 3790 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3791 const struct rte_flow_action_rss *rss; 3792 3793 rss = qrss->conf; 3794 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3795 MLX5_RXQ_TYPE_HAIRPIN) 3796 qrss = NULL; 3797 } 3798 } 3799 if (qrss) { 3800 /* Check if it is in meter suffix table. */ 3801 mtr_sfx = attr->group == (attr->transfer ? 3802 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3803 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3804 /* 3805 * Q/RSS action on NIC Rx should be split in order to pass by 3806 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3807 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3808 */ 3809 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3810 sizeof(struct rte_flow_action_set_tag) + 3811 sizeof(struct rte_flow_action_jump); 3812 ext_actions = rte_zmalloc(__func__, act_size, 0); 3813 if (!ext_actions) 3814 return rte_flow_error_set(error, ENOMEM, 3815 RTE_FLOW_ERROR_TYPE_ACTION, 3816 NULL, "no memory to split " 3817 "metadata flow"); 3818 /* 3819 * If we are the suffix flow of meter, tag already exist. 3820 * Set the tag action to void. 3821 */ 3822 if (mtr_sfx) 3823 ext_actions[qrss - actions].type = 3824 RTE_FLOW_ACTION_TYPE_VOID; 3825 else 3826 ext_actions[qrss - actions].type = 3827 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3828 /* 3829 * Create the new actions list with removed Q/RSS action 3830 * and appended set tag and jump to register copy table 3831 * (RX_CP_TBL). We should preallocate unique tag ID here 3832 * in advance, because it is needed for set tag action. 3833 */ 3834 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3835 qrss, actions_n, error); 3836 if (!mtr_sfx && !qrss_id) { 3837 ret = -rte_errno; 3838 goto exit; 3839 } 3840 } else if (attr->egress && !attr->transfer) { 3841 /* 3842 * All the actions on NIC Tx should have a metadata register 3843 * copy action to copy reg_a from WQE to reg_c[meta] 3844 */ 3845 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3846 sizeof(struct mlx5_flow_action_copy_mreg); 3847 ext_actions = rte_zmalloc(__func__, act_size, 0); 3848 if (!ext_actions) 3849 return rte_flow_error_set(error, ENOMEM, 3850 RTE_FLOW_ERROR_TYPE_ACTION, 3851 NULL, "no memory to split " 3852 "metadata flow"); 3853 /* Create the action list appended with copy register. */ 3854 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 3855 actions_n, error); 3856 if (ret < 0) 3857 goto exit; 3858 } 3859 /* Add the unmodified original or prefix subflow. */ 3860 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, 3861 ext_actions ? ext_actions : actions, 3862 external, error); 3863 if (ret < 0) 3864 goto exit; 3865 assert(dev_flow); 3866 if (qrss) { 3867 const struct rte_flow_attr q_attr = { 3868 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3869 .ingress = 1, 3870 }; 3871 /* Internal PMD action to set register. */ 3872 struct mlx5_rte_flow_item_tag q_tag_spec = { 3873 .data = qrss_id, 3874 .id = 0, 3875 }; 3876 struct rte_flow_item q_items[] = { 3877 { 3878 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3879 .spec = &q_tag_spec, 3880 .last = NULL, 3881 .mask = NULL, 3882 }, 3883 { 3884 .type = RTE_FLOW_ITEM_TYPE_END, 3885 }, 3886 }; 3887 struct rte_flow_action q_actions[] = { 3888 { 3889 .type = qrss->type, 3890 .conf = qrss->conf, 3891 }, 3892 { 3893 .type = RTE_FLOW_ACTION_TYPE_END, 3894 }, 3895 }; 3896 uint64_t hash_fields = dev_flow->hash_fields; 3897 3898 /* 3899 * Configure the tag item only if there is no meter subflow. 3900 * Since tag is already marked in the meter suffix subflow 3901 * we can just use the meter suffix items as is. 3902 */ 3903 if (qrss_id) { 3904 /* Not meter subflow. */ 3905 assert(!mtr_sfx); 3906 /* 3907 * Put unique id in prefix flow due to it is destroyed 3908 * after suffix flow and id will be freed after there 3909 * is no actual flows with this id and identifier 3910 * reallocation becomes possible (for example, for 3911 * other flows in other threads). 3912 */ 3913 dev_flow->qrss_id = qrss_id; 3914 qrss_id = 0; 3915 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 3916 error); 3917 if (ret < 0) 3918 goto exit; 3919 q_tag_spec.id = ret; 3920 } 3921 dev_flow = NULL; 3922 /* Add suffix subflow to execute Q/RSS. */ 3923 ret = flow_create_split_inner(dev, flow, &dev_flow, 3924 &q_attr, mtr_sfx ? items : 3925 q_items, q_actions, 3926 external, error); 3927 if (ret < 0) 3928 goto exit; 3929 assert(dev_flow); 3930 dev_flow->hash_fields = hash_fields; 3931 } 3932 3933 exit: 3934 /* 3935 * We do not destroy the partially created sub_flows in case of error. 3936 * These ones are included into parent flow list and will be destroyed 3937 * by flow_drv_destroy. 3938 */ 3939 flow_qrss_free_id(dev, qrss_id); 3940 rte_free(ext_actions); 3941 return ret; 3942 } 3943 3944 /** 3945 * The splitting for meter feature. 3946 * 3947 * - The meter flow will be split to two flows as prefix and 3948 * suffix flow. The packets make sense only it pass the prefix 3949 * meter action. 3950 * 3951 * - Reg_C_5 is used for the packet to match betweend prefix and 3952 * suffix flow. 3953 * 3954 * @param dev 3955 * Pointer to Ethernet device. 3956 * @param[in] flow 3957 * Parent flow structure pointer. 3958 * @param[in] attr 3959 * Flow rule attributes. 3960 * @param[in] items 3961 * Pattern specification (list terminated by the END pattern item). 3962 * @param[in] actions 3963 * Associated actions (list terminated by the END action). 3964 * @param[in] external 3965 * This flow rule is created by request external to PMD. 3966 * @param[out] error 3967 * Perform verbose error reporting if not NULL. 3968 * @return 3969 * 0 on success, negative value otherwise 3970 */ 3971 static int 3972 flow_create_split_meter(struct rte_eth_dev *dev, 3973 struct rte_flow *flow, 3974 const struct rte_flow_attr *attr, 3975 const struct rte_flow_item items[], 3976 const struct rte_flow_action actions[], 3977 bool external, struct rte_flow_error *error) 3978 { 3979 struct mlx5_priv *priv = dev->data->dev_private; 3980 struct rte_flow_action *sfx_actions = NULL; 3981 struct rte_flow_action *pre_actions = NULL; 3982 struct rte_flow_item *sfx_items = NULL; 3983 const struct rte_flow_item *sfx_port_id_item; 3984 struct mlx5_flow *dev_flow = NULL; 3985 struct rte_flow_attr sfx_attr = *attr; 3986 uint32_t mtr = 0; 3987 uint32_t mtr_tag_id = 0; 3988 size_t act_size; 3989 size_t item_size; 3990 int actions_n = 0; 3991 int ret; 3992 3993 if (priv->mtr_en) 3994 actions_n = flow_check_meter_action(actions, &mtr); 3995 if (mtr) { 3996 struct mlx5_rte_flow_item_tag *tag_spec; 3997 /* The five prefix actions: meter, decap, encap, tag, end. */ 3998 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 3999 sizeof(struct rte_flow_action_set_tag); 4000 /* tag, end. */ 4001 #define METER_SUFFIX_ITEM 3 4002 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4003 sizeof(struct mlx5_rte_flow_item_tag); 4004 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 4005 if (!sfx_actions) 4006 return rte_flow_error_set(error, ENOMEM, 4007 RTE_FLOW_ERROR_TYPE_ACTION, 4008 NULL, "no memory to split " 4009 "meter flow"); 4010 pre_actions = sfx_actions + actions_n; 4011 mtr_tag_id = flow_meter_split_prep(dev, actions, sfx_actions, 4012 pre_actions); 4013 if (!mtr_tag_id) { 4014 ret = -rte_errno; 4015 goto exit; 4016 } 4017 /* Add the prefix subflow. */ 4018 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, 4019 pre_actions, external, error); 4020 if (ret) { 4021 ret = -rte_errno; 4022 goto exit; 4023 } 4024 dev_flow->mtr_flow_id = mtr_tag_id; 4025 /* Prepare the suffix flow match pattern. */ 4026 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4027 act_size); 4028 tag_spec = (struct mlx5_rte_flow_item_tag *)(sfx_items + 4029 METER_SUFFIX_ITEM); 4030 tag_spec->data = rte_cpu_to_be_32(dev_flow->mtr_flow_id); 4031 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, 4032 error); 4033 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4034 sfx_items->spec = tag_spec; 4035 sfx_items->last = NULL; 4036 sfx_items->mask = NULL; 4037 sfx_items++; 4038 sfx_port_id_item = find_port_id_item(items); 4039 if (sfx_port_id_item) { 4040 memcpy(sfx_items, sfx_port_id_item, 4041 sizeof(*sfx_items)); 4042 sfx_items++; 4043 } 4044 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 4045 sfx_items -= sfx_port_id_item ? 2 : 1; 4046 /* Setting the sfx group atrr. */ 4047 sfx_attr.group = sfx_attr.transfer ? 4048 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4049 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4050 } 4051 /* Add the prefix subflow. */ 4052 ret = flow_create_split_metadata(dev, flow, &sfx_attr, 4053 sfx_items ? sfx_items : items, 4054 sfx_actions ? sfx_actions : actions, 4055 external, error); 4056 exit: 4057 if (sfx_actions) 4058 rte_free(sfx_actions); 4059 return ret; 4060 } 4061 4062 /** 4063 * Split the flow to subflow set. The splitters might be linked 4064 * in the chain, like this: 4065 * flow_create_split_outer() calls: 4066 * flow_create_split_meter() calls: 4067 * flow_create_split_metadata(meter_subflow_0) calls: 4068 * flow_create_split_inner(metadata_subflow_0) 4069 * flow_create_split_inner(metadata_subflow_1) 4070 * flow_create_split_inner(metadata_subflow_2) 4071 * flow_create_split_metadata(meter_subflow_1) calls: 4072 * flow_create_split_inner(metadata_subflow_0) 4073 * flow_create_split_inner(metadata_subflow_1) 4074 * flow_create_split_inner(metadata_subflow_2) 4075 * 4076 * This provide flexible way to add new levels of flow splitting. 4077 * The all of successfully created subflows are included to the 4078 * parent flow dev_flow list. 4079 * 4080 * @param dev 4081 * Pointer to Ethernet device. 4082 * @param[in] flow 4083 * Parent flow structure pointer. 4084 * @param[in] attr 4085 * Flow rule attributes. 4086 * @param[in] items 4087 * Pattern specification (list terminated by the END pattern item). 4088 * @param[in] actions 4089 * Associated actions (list terminated by the END action). 4090 * @param[in] external 4091 * This flow rule is created by request external to PMD. 4092 * @param[out] error 4093 * Perform verbose error reporting if not NULL. 4094 * @return 4095 * 0 on success, negative value otherwise 4096 */ 4097 static int 4098 flow_create_split_outer(struct rte_eth_dev *dev, 4099 struct rte_flow *flow, 4100 const struct rte_flow_attr *attr, 4101 const struct rte_flow_item items[], 4102 const struct rte_flow_action actions[], 4103 bool external, struct rte_flow_error *error) 4104 { 4105 int ret; 4106 4107 ret = flow_create_split_meter(dev, flow, attr, items, 4108 actions, external, error); 4109 assert(ret <= 0); 4110 return ret; 4111 } 4112 4113 /** 4114 * Create a flow and add it to @p list. 4115 * 4116 * @param dev 4117 * Pointer to Ethernet device. 4118 * @param list 4119 * Pointer to a TAILQ flow list. If this parameter NULL, 4120 * no list insertion occurred, flow is just created, 4121 * this is caller's responsibility to track the 4122 * created flow. 4123 * @param[in] attr 4124 * Flow rule attributes. 4125 * @param[in] items 4126 * Pattern specification (list terminated by the END pattern item). 4127 * @param[in] actions 4128 * Associated actions (list terminated by the END action). 4129 * @param[in] external 4130 * This flow rule is created by request external to PMD. 4131 * @param[out] error 4132 * Perform verbose error reporting if not NULL. 4133 * 4134 * @return 4135 * A flow on success, NULL otherwise and rte_errno is set. 4136 */ 4137 static struct rte_flow * 4138 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 4139 const struct rte_flow_attr *attr, 4140 const struct rte_flow_item items[], 4141 const struct rte_flow_action actions[], 4142 bool external, struct rte_flow_error *error) 4143 { 4144 struct mlx5_priv *priv = dev->data->dev_private; 4145 struct rte_flow *flow = NULL; 4146 struct mlx5_flow *dev_flow; 4147 const struct rte_flow_action_rss *rss; 4148 union { 4149 struct rte_flow_expand_rss buf; 4150 uint8_t buffer[2048]; 4151 } expand_buffer; 4152 union { 4153 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4154 uint8_t buffer[2048]; 4155 } actions_rx; 4156 union { 4157 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4158 uint8_t buffer[2048]; 4159 } actions_hairpin_tx; 4160 union { 4161 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4162 uint8_t buffer[2048]; 4163 } items_tx; 4164 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4165 const struct rte_flow_action *p_actions_rx = actions; 4166 int ret; 4167 uint32_t i; 4168 uint32_t flow_size; 4169 int hairpin_flow = 0; 4170 uint32_t hairpin_id = 0; 4171 struct rte_flow_attr attr_tx = { .priority = 0 }; 4172 4173 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4174 if (hairpin_flow > 0) { 4175 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4176 rte_errno = EINVAL; 4177 return NULL; 4178 } 4179 flow_hairpin_split(dev, actions, actions_rx.actions, 4180 actions_hairpin_tx.actions, items_tx.items, 4181 &hairpin_id); 4182 p_actions_rx = actions_rx.actions; 4183 } 4184 ret = flow_drv_validate(dev, attr, items, p_actions_rx, external, 4185 error); 4186 if (ret < 0) 4187 goto error_before_flow; 4188 flow_size = sizeof(struct rte_flow); 4189 rss = flow_get_rss_action(p_actions_rx); 4190 if (rss) 4191 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t), 4192 sizeof(void *)); 4193 else 4194 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *)); 4195 flow = rte_calloc(__func__, 1, flow_size, 0); 4196 if (!flow) { 4197 rte_errno = ENOMEM; 4198 goto error_before_flow; 4199 } 4200 flow->drv_type = flow_get_drv_type(dev, attr); 4201 if (hairpin_id != 0) 4202 flow->hairpin_flow_id = hairpin_id; 4203 assert(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4204 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4205 flow->rss.queue = (void *)(flow + 1); 4206 if (rss) { 4207 /* 4208 * The following information is required by 4209 * mlx5_flow_hashfields_adjust() in advance. 4210 */ 4211 flow->rss.level = rss->level; 4212 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4213 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; 4214 } 4215 LIST_INIT(&flow->dev_flows); 4216 if (rss && rss->types) { 4217 unsigned int graph_root; 4218 4219 graph_root = find_graph_root(items, rss->level); 4220 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4221 items, rss->types, 4222 mlx5_support_expansion, 4223 graph_root); 4224 assert(ret > 0 && 4225 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4226 } else { 4227 buf->entries = 1; 4228 buf->entry[0].pattern = (void *)(uintptr_t)items; 4229 } 4230 for (i = 0; i < buf->entries; ++i) { 4231 /* 4232 * The splitter may create multiple dev_flows, 4233 * depending on configuration. In the simplest 4234 * case it just creates unmodified original flow. 4235 */ 4236 ret = flow_create_split_outer(dev, flow, attr, 4237 buf->entry[i].pattern, 4238 p_actions_rx, external, 4239 error); 4240 if (ret < 0) 4241 goto error; 4242 } 4243 /* Create the tx flow. */ 4244 if (hairpin_flow) { 4245 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4246 attr_tx.ingress = 0; 4247 attr_tx.egress = 1; 4248 dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items, 4249 actions_hairpin_tx.actions, error); 4250 if (!dev_flow) 4251 goto error; 4252 dev_flow->flow = flow; 4253 dev_flow->external = 0; 4254 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 4255 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4256 items_tx.items, 4257 actions_hairpin_tx.actions, error); 4258 if (ret < 0) 4259 goto error; 4260 } 4261 /* 4262 * Update the metadata register copy table. If extensive 4263 * metadata feature is enabled and registers are supported 4264 * we might create the extra rte_flow for each unique 4265 * MARK/FLAG action ID. 4266 * 4267 * The table is updated for ingress Flows only, because 4268 * the egress Flows belong to the different device and 4269 * copy table should be updated in peer NIC Rx domain. 4270 */ 4271 if (attr->ingress && 4272 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4273 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4274 if (ret) 4275 goto error; 4276 } 4277 if (dev->data->dev_started) { 4278 ret = flow_drv_apply(dev, flow, error); 4279 if (ret < 0) 4280 goto error; 4281 } 4282 if (list) 4283 TAILQ_INSERT_TAIL(list, flow, next); 4284 flow_rxq_flags_set(dev, flow); 4285 return flow; 4286 error_before_flow: 4287 if (hairpin_id) 4288 mlx5_flow_id_release(priv->sh->flow_id_pool, 4289 hairpin_id); 4290 return NULL; 4291 error: 4292 assert(flow); 4293 flow_mreg_del_copy_action(dev, flow); 4294 ret = rte_errno; /* Save rte_errno before cleanup. */ 4295 if (flow->hairpin_flow_id) 4296 mlx5_flow_id_release(priv->sh->flow_id_pool, 4297 flow->hairpin_flow_id); 4298 assert(flow); 4299 flow_drv_destroy(dev, flow); 4300 rte_free(flow); 4301 rte_errno = ret; /* Restore rte_errno. */ 4302 return NULL; 4303 } 4304 4305 /** 4306 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4307 * incoming packets to table 1. 4308 * 4309 * Other flow rules, requested for group n, will be created in 4310 * e-switch table n+1. 4311 * Jump action to e-switch group n will be created to group n+1. 4312 * 4313 * Used when working in switchdev mode, to utilise advantages of table 1 4314 * and above. 4315 * 4316 * @param dev 4317 * Pointer to Ethernet device. 4318 * 4319 * @return 4320 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4321 */ 4322 struct rte_flow * 4323 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4324 { 4325 const struct rte_flow_attr attr = { 4326 .group = 0, 4327 .priority = 0, 4328 .ingress = 1, 4329 .egress = 0, 4330 .transfer = 1, 4331 }; 4332 const struct rte_flow_item pattern = { 4333 .type = RTE_FLOW_ITEM_TYPE_END, 4334 }; 4335 struct rte_flow_action_jump jump = { 4336 .group = 1, 4337 }; 4338 const struct rte_flow_action actions[] = { 4339 { 4340 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4341 .conf = &jump, 4342 }, 4343 { 4344 .type = RTE_FLOW_ACTION_TYPE_END, 4345 }, 4346 }; 4347 struct mlx5_priv *priv = dev->data->dev_private; 4348 struct rte_flow_error error; 4349 4350 return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern, 4351 actions, false, &error); 4352 } 4353 4354 /** 4355 * Create a flow. 4356 * 4357 * @see rte_flow_create() 4358 * @see rte_flow_ops 4359 */ 4360 struct rte_flow * 4361 mlx5_flow_create(struct rte_eth_dev *dev, 4362 const struct rte_flow_attr *attr, 4363 const struct rte_flow_item items[], 4364 const struct rte_flow_action actions[], 4365 struct rte_flow_error *error) 4366 { 4367 struct mlx5_priv *priv = dev->data->dev_private; 4368 4369 return flow_list_create(dev, &priv->flows, 4370 attr, items, actions, true, error); 4371 } 4372 4373 /** 4374 * Destroy a flow in a list. 4375 * 4376 * @param dev 4377 * Pointer to Ethernet device. 4378 * @param list 4379 * Pointer to a TAILQ flow list. If this parameter NULL, 4380 * there is no flow removal from the list. 4381 * @param[in] flow 4382 * Flow to destroy. 4383 */ 4384 static void 4385 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 4386 struct rte_flow *flow) 4387 { 4388 struct mlx5_priv *priv = dev->data->dev_private; 4389 4390 /* 4391 * Update RX queue flags only if port is started, otherwise it is 4392 * already clean. 4393 */ 4394 if (dev->data->dev_started) 4395 flow_rxq_flags_trim(dev, flow); 4396 if (flow->hairpin_flow_id) 4397 mlx5_flow_id_release(priv->sh->flow_id_pool, 4398 flow->hairpin_flow_id); 4399 flow_drv_destroy(dev, flow); 4400 if (list) 4401 TAILQ_REMOVE(list, flow, next); 4402 flow_mreg_del_copy_action(dev, flow); 4403 rte_free(flow->fdir); 4404 rte_free(flow); 4405 } 4406 4407 /** 4408 * Destroy all flows. 4409 * 4410 * @param dev 4411 * Pointer to Ethernet device. 4412 * @param list 4413 * Pointer to a TAILQ flow list. 4414 */ 4415 void 4416 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 4417 { 4418 while (!TAILQ_EMPTY(list)) { 4419 struct rte_flow *flow; 4420 4421 flow = TAILQ_FIRST(list); 4422 flow_list_destroy(dev, list, flow); 4423 } 4424 } 4425 4426 /** 4427 * Remove all flows. 4428 * 4429 * @param dev 4430 * Pointer to Ethernet device. 4431 * @param list 4432 * Pointer to a TAILQ flow list. 4433 */ 4434 void 4435 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 4436 { 4437 struct rte_flow *flow; 4438 4439 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 4440 flow_drv_remove(dev, flow); 4441 flow_mreg_stop_copy_action(dev, flow); 4442 } 4443 flow_mreg_del_default_copy_action(dev); 4444 flow_rxq_flags_clear(dev); 4445 } 4446 4447 /** 4448 * Add all flows. 4449 * 4450 * @param dev 4451 * Pointer to Ethernet device. 4452 * @param list 4453 * Pointer to a TAILQ flow list. 4454 * 4455 * @return 4456 * 0 on success, a negative errno value otherwise and rte_errno is set. 4457 */ 4458 int 4459 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 4460 { 4461 struct rte_flow *flow; 4462 struct rte_flow_error error; 4463 int ret = 0; 4464 4465 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4466 ret = flow_mreg_add_default_copy_action(dev, &error); 4467 if (ret < 0) 4468 return -rte_errno; 4469 /* Apply Flows created by application. */ 4470 TAILQ_FOREACH(flow, list, next) { 4471 ret = flow_mreg_start_copy_action(dev, flow); 4472 if (ret < 0) 4473 goto error; 4474 ret = flow_drv_apply(dev, flow, &error); 4475 if (ret < 0) 4476 goto error; 4477 flow_rxq_flags_set(dev, flow); 4478 } 4479 return 0; 4480 error: 4481 ret = rte_errno; /* Save rte_errno before cleanup. */ 4482 mlx5_flow_stop(dev, list); 4483 rte_errno = ret; /* Restore rte_errno. */ 4484 return -rte_errno; 4485 } 4486 4487 /** 4488 * Verify the flow list is empty 4489 * 4490 * @param dev 4491 * Pointer to Ethernet device. 4492 * 4493 * @return the number of flows not released. 4494 */ 4495 int 4496 mlx5_flow_verify(struct rte_eth_dev *dev) 4497 { 4498 struct mlx5_priv *priv = dev->data->dev_private; 4499 struct rte_flow *flow; 4500 int ret = 0; 4501 4502 TAILQ_FOREACH(flow, &priv->flows, next) { 4503 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4504 dev->data->port_id, (void *)flow); 4505 ++ret; 4506 } 4507 return ret; 4508 } 4509 4510 /** 4511 * Enable default hairpin egress flow. 4512 * 4513 * @param dev 4514 * Pointer to Ethernet device. 4515 * @param queue 4516 * The queue index. 4517 * 4518 * @return 4519 * 0 on success, a negative errno value otherwise and rte_errno is set. 4520 */ 4521 int 4522 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4523 uint32_t queue) 4524 { 4525 struct mlx5_priv *priv = dev->data->dev_private; 4526 const struct rte_flow_attr attr = { 4527 .egress = 1, 4528 .priority = 0, 4529 }; 4530 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4531 .queue = queue, 4532 }; 4533 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4534 .queue = UINT32_MAX, 4535 }; 4536 struct rte_flow_item items[] = { 4537 { 4538 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4539 .spec = &queue_spec, 4540 .last = NULL, 4541 .mask = &queue_mask, 4542 }, 4543 { 4544 .type = RTE_FLOW_ITEM_TYPE_END, 4545 }, 4546 }; 4547 struct rte_flow_action_jump jump = { 4548 .group = MLX5_HAIRPIN_TX_TABLE, 4549 }; 4550 struct rte_flow_action actions[2]; 4551 struct rte_flow *flow; 4552 struct rte_flow_error error; 4553 4554 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4555 actions[0].conf = &jump; 4556 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4557 flow = flow_list_create(dev, &priv->ctrl_flows, 4558 &attr, items, actions, false, &error); 4559 if (!flow) { 4560 DRV_LOG(DEBUG, 4561 "Failed to create ctrl flow: rte_errno(%d)," 4562 " type(%d), message(%s)", 4563 rte_errno, error.type, 4564 error.message ? error.message : " (no stated reason)"); 4565 return -rte_errno; 4566 } 4567 return 0; 4568 } 4569 4570 /** 4571 * Enable a control flow configured from the control plane. 4572 * 4573 * @param dev 4574 * Pointer to Ethernet device. 4575 * @param eth_spec 4576 * An Ethernet flow spec to apply. 4577 * @param eth_mask 4578 * An Ethernet flow mask to apply. 4579 * @param vlan_spec 4580 * A VLAN flow spec to apply. 4581 * @param vlan_mask 4582 * A VLAN flow mask to apply. 4583 * 4584 * @return 4585 * 0 on success, a negative errno value otherwise and rte_errno is set. 4586 */ 4587 int 4588 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4589 struct rte_flow_item_eth *eth_spec, 4590 struct rte_flow_item_eth *eth_mask, 4591 struct rte_flow_item_vlan *vlan_spec, 4592 struct rte_flow_item_vlan *vlan_mask) 4593 { 4594 struct mlx5_priv *priv = dev->data->dev_private; 4595 const struct rte_flow_attr attr = { 4596 .ingress = 1, 4597 .priority = MLX5_FLOW_PRIO_RSVD, 4598 }; 4599 struct rte_flow_item items[] = { 4600 { 4601 .type = RTE_FLOW_ITEM_TYPE_ETH, 4602 .spec = eth_spec, 4603 .last = NULL, 4604 .mask = eth_mask, 4605 }, 4606 { 4607 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4608 RTE_FLOW_ITEM_TYPE_END, 4609 .spec = vlan_spec, 4610 .last = NULL, 4611 .mask = vlan_mask, 4612 }, 4613 { 4614 .type = RTE_FLOW_ITEM_TYPE_END, 4615 }, 4616 }; 4617 uint16_t queue[priv->reta_idx_n]; 4618 struct rte_flow_action_rss action_rss = { 4619 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4620 .level = 0, 4621 .types = priv->rss_conf.rss_hf, 4622 .key_len = priv->rss_conf.rss_key_len, 4623 .queue_num = priv->reta_idx_n, 4624 .key = priv->rss_conf.rss_key, 4625 .queue = queue, 4626 }; 4627 struct rte_flow_action actions[] = { 4628 { 4629 .type = RTE_FLOW_ACTION_TYPE_RSS, 4630 .conf = &action_rss, 4631 }, 4632 { 4633 .type = RTE_FLOW_ACTION_TYPE_END, 4634 }, 4635 }; 4636 struct rte_flow *flow; 4637 struct rte_flow_error error; 4638 unsigned int i; 4639 4640 if (!priv->reta_idx_n || !priv->rxqs_n) { 4641 return 0; 4642 } 4643 for (i = 0; i != priv->reta_idx_n; ++i) 4644 queue[i] = (*priv->reta_idx)[i]; 4645 flow = flow_list_create(dev, &priv->ctrl_flows, 4646 &attr, items, actions, false, &error); 4647 if (!flow) 4648 return -rte_errno; 4649 return 0; 4650 } 4651 4652 /** 4653 * Enable a flow control configured from the control plane. 4654 * 4655 * @param dev 4656 * Pointer to Ethernet device. 4657 * @param eth_spec 4658 * An Ethernet flow spec to apply. 4659 * @param eth_mask 4660 * An Ethernet flow mask to apply. 4661 * 4662 * @return 4663 * 0 on success, a negative errno value otherwise and rte_errno is set. 4664 */ 4665 int 4666 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4667 struct rte_flow_item_eth *eth_spec, 4668 struct rte_flow_item_eth *eth_mask) 4669 { 4670 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4671 } 4672 4673 /** 4674 * Destroy a flow. 4675 * 4676 * @see rte_flow_destroy() 4677 * @see rte_flow_ops 4678 */ 4679 int 4680 mlx5_flow_destroy(struct rte_eth_dev *dev, 4681 struct rte_flow *flow, 4682 struct rte_flow_error *error __rte_unused) 4683 { 4684 struct mlx5_priv *priv = dev->data->dev_private; 4685 4686 flow_list_destroy(dev, &priv->flows, flow); 4687 return 0; 4688 } 4689 4690 /** 4691 * Destroy all flows. 4692 * 4693 * @see rte_flow_flush() 4694 * @see rte_flow_ops 4695 */ 4696 int 4697 mlx5_flow_flush(struct rte_eth_dev *dev, 4698 struct rte_flow_error *error __rte_unused) 4699 { 4700 struct mlx5_priv *priv = dev->data->dev_private; 4701 4702 mlx5_flow_list_flush(dev, &priv->flows); 4703 return 0; 4704 } 4705 4706 /** 4707 * Isolated mode. 4708 * 4709 * @see rte_flow_isolate() 4710 * @see rte_flow_ops 4711 */ 4712 int 4713 mlx5_flow_isolate(struct rte_eth_dev *dev, 4714 int enable, 4715 struct rte_flow_error *error) 4716 { 4717 struct mlx5_priv *priv = dev->data->dev_private; 4718 4719 if (dev->data->dev_started) { 4720 rte_flow_error_set(error, EBUSY, 4721 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4722 NULL, 4723 "port must be stopped first"); 4724 return -rte_errno; 4725 } 4726 priv->isolated = !!enable; 4727 if (enable) 4728 dev->dev_ops = &mlx5_dev_ops_isolate; 4729 else 4730 dev->dev_ops = &mlx5_dev_ops; 4731 return 0; 4732 } 4733 4734 /** 4735 * Query a flow. 4736 * 4737 * @see rte_flow_query() 4738 * @see rte_flow_ops 4739 */ 4740 static int 4741 flow_drv_query(struct rte_eth_dev *dev, 4742 struct rte_flow *flow, 4743 const struct rte_flow_action *actions, 4744 void *data, 4745 struct rte_flow_error *error) 4746 { 4747 const struct mlx5_flow_driver_ops *fops; 4748 enum mlx5_flow_drv_type ftype = flow->drv_type; 4749 4750 assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 4751 fops = flow_get_drv_ops(ftype); 4752 4753 return fops->query(dev, flow, actions, data, error); 4754 } 4755 4756 /** 4757 * Query a flow. 4758 * 4759 * @see rte_flow_query() 4760 * @see rte_flow_ops 4761 */ 4762 int 4763 mlx5_flow_query(struct rte_eth_dev *dev, 4764 struct rte_flow *flow, 4765 const struct rte_flow_action *actions, 4766 void *data, 4767 struct rte_flow_error *error) 4768 { 4769 int ret; 4770 4771 ret = flow_drv_query(dev, flow, actions, data, error); 4772 if (ret < 0) 4773 return ret; 4774 return 0; 4775 } 4776 4777 /** 4778 * Convert a flow director filter to a generic flow. 4779 * 4780 * @param dev 4781 * Pointer to Ethernet device. 4782 * @param fdir_filter 4783 * Flow director filter to add. 4784 * @param attributes 4785 * Generic flow parameters structure. 4786 * 4787 * @return 4788 * 0 on success, a negative errno value otherwise and rte_errno is set. 4789 */ 4790 static int 4791 flow_fdir_filter_convert(struct rte_eth_dev *dev, 4792 const struct rte_eth_fdir_filter *fdir_filter, 4793 struct mlx5_fdir *attributes) 4794 { 4795 struct mlx5_priv *priv = dev->data->dev_private; 4796 const struct rte_eth_fdir_input *input = &fdir_filter->input; 4797 const struct rte_eth_fdir_masks *mask = 4798 &dev->data->dev_conf.fdir_conf.mask; 4799 4800 /* Validate queue number. */ 4801 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 4802 DRV_LOG(ERR, "port %u invalid queue number %d", 4803 dev->data->port_id, fdir_filter->action.rx_queue); 4804 rte_errno = EINVAL; 4805 return -rte_errno; 4806 } 4807 attributes->attr.ingress = 1; 4808 attributes->items[0] = (struct rte_flow_item) { 4809 .type = RTE_FLOW_ITEM_TYPE_ETH, 4810 .spec = &attributes->l2, 4811 .mask = &attributes->l2_mask, 4812 }; 4813 switch (fdir_filter->action.behavior) { 4814 case RTE_ETH_FDIR_ACCEPT: 4815 attributes->actions[0] = (struct rte_flow_action){ 4816 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 4817 .conf = &attributes->queue, 4818 }; 4819 break; 4820 case RTE_ETH_FDIR_REJECT: 4821 attributes->actions[0] = (struct rte_flow_action){ 4822 .type = RTE_FLOW_ACTION_TYPE_DROP, 4823 }; 4824 break; 4825 default: 4826 DRV_LOG(ERR, "port %u invalid behavior %d", 4827 dev->data->port_id, 4828 fdir_filter->action.behavior); 4829 rte_errno = ENOTSUP; 4830 return -rte_errno; 4831 } 4832 attributes->queue.index = fdir_filter->action.rx_queue; 4833 /* Handle L3. */ 4834 switch (fdir_filter->input.flow_type) { 4835 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4836 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4837 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4838 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 4839 .src_addr = input->flow.ip4_flow.src_ip, 4840 .dst_addr = input->flow.ip4_flow.dst_ip, 4841 .time_to_live = input->flow.ip4_flow.ttl, 4842 .type_of_service = input->flow.ip4_flow.tos, 4843 }; 4844 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 4845 .src_addr = mask->ipv4_mask.src_ip, 4846 .dst_addr = mask->ipv4_mask.dst_ip, 4847 .time_to_live = mask->ipv4_mask.ttl, 4848 .type_of_service = mask->ipv4_mask.tos, 4849 .next_proto_id = mask->ipv4_mask.proto, 4850 }; 4851 attributes->items[1] = (struct rte_flow_item){ 4852 .type = RTE_FLOW_ITEM_TYPE_IPV4, 4853 .spec = &attributes->l3, 4854 .mask = &attributes->l3_mask, 4855 }; 4856 break; 4857 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4858 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4859 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4860 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 4861 .hop_limits = input->flow.ipv6_flow.hop_limits, 4862 .proto = input->flow.ipv6_flow.proto, 4863 }; 4864 4865 memcpy(attributes->l3.ipv6.hdr.src_addr, 4866 input->flow.ipv6_flow.src_ip, 4867 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4868 memcpy(attributes->l3.ipv6.hdr.dst_addr, 4869 input->flow.ipv6_flow.dst_ip, 4870 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4871 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 4872 mask->ipv6_mask.src_ip, 4873 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4874 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 4875 mask->ipv6_mask.dst_ip, 4876 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4877 attributes->items[1] = (struct rte_flow_item){ 4878 .type = RTE_FLOW_ITEM_TYPE_IPV6, 4879 .spec = &attributes->l3, 4880 .mask = &attributes->l3_mask, 4881 }; 4882 break; 4883 default: 4884 DRV_LOG(ERR, "port %u invalid flow type%d", 4885 dev->data->port_id, fdir_filter->input.flow_type); 4886 rte_errno = ENOTSUP; 4887 return -rte_errno; 4888 } 4889 /* Handle L4. */ 4890 switch (fdir_filter->input.flow_type) { 4891 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4892 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4893 .src_port = input->flow.udp4_flow.src_port, 4894 .dst_port = input->flow.udp4_flow.dst_port, 4895 }; 4896 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4897 .src_port = mask->src_port_mask, 4898 .dst_port = mask->dst_port_mask, 4899 }; 4900 attributes->items[2] = (struct rte_flow_item){ 4901 .type = RTE_FLOW_ITEM_TYPE_UDP, 4902 .spec = &attributes->l4, 4903 .mask = &attributes->l4_mask, 4904 }; 4905 break; 4906 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4907 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4908 .src_port = input->flow.tcp4_flow.src_port, 4909 .dst_port = input->flow.tcp4_flow.dst_port, 4910 }; 4911 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4912 .src_port = mask->src_port_mask, 4913 .dst_port = mask->dst_port_mask, 4914 }; 4915 attributes->items[2] = (struct rte_flow_item){ 4916 .type = RTE_FLOW_ITEM_TYPE_TCP, 4917 .spec = &attributes->l4, 4918 .mask = &attributes->l4_mask, 4919 }; 4920 break; 4921 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4922 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4923 .src_port = input->flow.udp6_flow.src_port, 4924 .dst_port = input->flow.udp6_flow.dst_port, 4925 }; 4926 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4927 .src_port = mask->src_port_mask, 4928 .dst_port = mask->dst_port_mask, 4929 }; 4930 attributes->items[2] = (struct rte_flow_item){ 4931 .type = RTE_FLOW_ITEM_TYPE_UDP, 4932 .spec = &attributes->l4, 4933 .mask = &attributes->l4_mask, 4934 }; 4935 break; 4936 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4937 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4938 .src_port = input->flow.tcp6_flow.src_port, 4939 .dst_port = input->flow.tcp6_flow.dst_port, 4940 }; 4941 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4942 .src_port = mask->src_port_mask, 4943 .dst_port = mask->dst_port_mask, 4944 }; 4945 attributes->items[2] = (struct rte_flow_item){ 4946 .type = RTE_FLOW_ITEM_TYPE_TCP, 4947 .spec = &attributes->l4, 4948 .mask = &attributes->l4_mask, 4949 }; 4950 break; 4951 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4952 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4953 break; 4954 default: 4955 DRV_LOG(ERR, "port %u invalid flow type%d", 4956 dev->data->port_id, fdir_filter->input.flow_type); 4957 rte_errno = ENOTSUP; 4958 return -rte_errno; 4959 } 4960 return 0; 4961 } 4962 4963 #define FLOW_FDIR_CMP(f1, f2, fld) \ 4964 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 4965 4966 /** 4967 * Compare two FDIR flows. If items and actions are identical, the two flows are 4968 * regarded as same. 4969 * 4970 * @param dev 4971 * Pointer to Ethernet device. 4972 * @param f1 4973 * FDIR flow to compare. 4974 * @param f2 4975 * FDIR flow to compare. 4976 * 4977 * @return 4978 * Zero on match, 1 otherwise. 4979 */ 4980 static int 4981 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 4982 { 4983 if (FLOW_FDIR_CMP(f1, f2, attr) || 4984 FLOW_FDIR_CMP(f1, f2, l2) || 4985 FLOW_FDIR_CMP(f1, f2, l2_mask) || 4986 FLOW_FDIR_CMP(f1, f2, l3) || 4987 FLOW_FDIR_CMP(f1, f2, l3_mask) || 4988 FLOW_FDIR_CMP(f1, f2, l4) || 4989 FLOW_FDIR_CMP(f1, f2, l4_mask) || 4990 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 4991 return 1; 4992 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 4993 FLOW_FDIR_CMP(f1, f2, queue)) 4994 return 1; 4995 return 0; 4996 } 4997 4998 /** 4999 * Search device flow list to find out a matched FDIR flow. 5000 * 5001 * @param dev 5002 * Pointer to Ethernet device. 5003 * @param fdir_flow 5004 * FDIR flow to lookup. 5005 * 5006 * @return 5007 * Pointer of flow if found, NULL otherwise. 5008 */ 5009 static struct rte_flow * 5010 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5011 { 5012 struct mlx5_priv *priv = dev->data->dev_private; 5013 struct rte_flow *flow = NULL; 5014 5015 assert(fdir_flow); 5016 TAILQ_FOREACH(flow, &priv->flows, next) { 5017 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) { 5018 DRV_LOG(DEBUG, "port %u found FDIR flow %p", 5019 dev->data->port_id, (void *)flow); 5020 break; 5021 } 5022 } 5023 return flow; 5024 } 5025 5026 /** 5027 * Add new flow director filter and store it in list. 5028 * 5029 * @param dev 5030 * Pointer to Ethernet device. 5031 * @param fdir_filter 5032 * Flow director filter to add. 5033 * 5034 * @return 5035 * 0 on success, a negative errno value otherwise and rte_errno is set. 5036 */ 5037 static int 5038 flow_fdir_filter_add(struct rte_eth_dev *dev, 5039 const struct rte_eth_fdir_filter *fdir_filter) 5040 { 5041 struct mlx5_priv *priv = dev->data->dev_private; 5042 struct mlx5_fdir *fdir_flow; 5043 struct rte_flow *flow; 5044 int ret; 5045 5046 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5047 if (!fdir_flow) { 5048 rte_errno = ENOMEM; 5049 return -rte_errno; 5050 } 5051 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5052 if (ret) 5053 goto error; 5054 flow = flow_fdir_filter_lookup(dev, fdir_flow); 5055 if (flow) { 5056 rte_errno = EEXIST; 5057 goto error; 5058 } 5059 flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5060 fdir_flow->items, fdir_flow->actions, true, 5061 NULL); 5062 if (!flow) 5063 goto error; 5064 assert(!flow->fdir); 5065 flow->fdir = fdir_flow; 5066 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5067 dev->data->port_id, (void *)flow); 5068 return 0; 5069 error: 5070 rte_free(fdir_flow); 5071 return -rte_errno; 5072 } 5073 5074 /** 5075 * Delete specific filter. 5076 * 5077 * @param dev 5078 * Pointer to Ethernet device. 5079 * @param fdir_filter 5080 * Filter to be deleted. 5081 * 5082 * @return 5083 * 0 on success, a negative errno value otherwise and rte_errno is set. 5084 */ 5085 static int 5086 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5087 const struct rte_eth_fdir_filter *fdir_filter) 5088 { 5089 struct mlx5_priv *priv = dev->data->dev_private; 5090 struct rte_flow *flow; 5091 struct mlx5_fdir fdir_flow = { 5092 .attr.group = 0, 5093 }; 5094 int ret; 5095 5096 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5097 if (ret) 5098 return -rte_errno; 5099 flow = flow_fdir_filter_lookup(dev, &fdir_flow); 5100 if (!flow) { 5101 rte_errno = ENOENT; 5102 return -rte_errno; 5103 } 5104 flow_list_destroy(dev, &priv->flows, flow); 5105 DRV_LOG(DEBUG, "port %u deleted FDIR flow %p", 5106 dev->data->port_id, (void *)flow); 5107 return 0; 5108 } 5109 5110 /** 5111 * Update queue for specific filter. 5112 * 5113 * @param dev 5114 * Pointer to Ethernet device. 5115 * @param fdir_filter 5116 * Filter to be updated. 5117 * 5118 * @return 5119 * 0 on success, a negative errno value otherwise and rte_errno is set. 5120 */ 5121 static int 5122 flow_fdir_filter_update(struct rte_eth_dev *dev, 5123 const struct rte_eth_fdir_filter *fdir_filter) 5124 { 5125 int ret; 5126 5127 ret = flow_fdir_filter_delete(dev, fdir_filter); 5128 if (ret) 5129 return ret; 5130 return flow_fdir_filter_add(dev, fdir_filter); 5131 } 5132 5133 /** 5134 * Flush all filters. 5135 * 5136 * @param dev 5137 * Pointer to Ethernet device. 5138 */ 5139 static void 5140 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5141 { 5142 struct mlx5_priv *priv = dev->data->dev_private; 5143 5144 mlx5_flow_list_flush(dev, &priv->flows); 5145 } 5146 5147 /** 5148 * Get flow director information. 5149 * 5150 * @param dev 5151 * Pointer to Ethernet device. 5152 * @param[out] fdir_info 5153 * Resulting flow director information. 5154 */ 5155 static void 5156 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5157 { 5158 struct rte_eth_fdir_masks *mask = 5159 &dev->data->dev_conf.fdir_conf.mask; 5160 5161 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5162 fdir_info->guarant_spc = 0; 5163 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5164 fdir_info->max_flexpayload = 0; 5165 fdir_info->flow_types_mask[0] = 0; 5166 fdir_info->flex_payload_unit = 0; 5167 fdir_info->max_flex_payload_segment_num = 0; 5168 fdir_info->flex_payload_limit = 0; 5169 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5170 } 5171 5172 /** 5173 * Deal with flow director operations. 5174 * 5175 * @param dev 5176 * Pointer to Ethernet device. 5177 * @param filter_op 5178 * Operation to perform. 5179 * @param arg 5180 * Pointer to operation-specific structure. 5181 * 5182 * @return 5183 * 0 on success, a negative errno value otherwise and rte_errno is set. 5184 */ 5185 static int 5186 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5187 void *arg) 5188 { 5189 enum rte_fdir_mode fdir_mode = 5190 dev->data->dev_conf.fdir_conf.mode; 5191 5192 if (filter_op == RTE_ETH_FILTER_NOP) 5193 return 0; 5194 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5195 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5196 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5197 dev->data->port_id, fdir_mode); 5198 rte_errno = EINVAL; 5199 return -rte_errno; 5200 } 5201 switch (filter_op) { 5202 case RTE_ETH_FILTER_ADD: 5203 return flow_fdir_filter_add(dev, arg); 5204 case RTE_ETH_FILTER_UPDATE: 5205 return flow_fdir_filter_update(dev, arg); 5206 case RTE_ETH_FILTER_DELETE: 5207 return flow_fdir_filter_delete(dev, arg); 5208 case RTE_ETH_FILTER_FLUSH: 5209 flow_fdir_filter_flush(dev); 5210 break; 5211 case RTE_ETH_FILTER_INFO: 5212 flow_fdir_info_get(dev, arg); 5213 break; 5214 default: 5215 DRV_LOG(DEBUG, "port %u unknown operation %u", 5216 dev->data->port_id, filter_op); 5217 rte_errno = EINVAL; 5218 return -rte_errno; 5219 } 5220 return 0; 5221 } 5222 5223 /** 5224 * Manage filter operations. 5225 * 5226 * @param dev 5227 * Pointer to Ethernet device structure. 5228 * @param filter_type 5229 * Filter type. 5230 * @param filter_op 5231 * Operation to perform. 5232 * @param arg 5233 * Pointer to operation-specific structure. 5234 * 5235 * @return 5236 * 0 on success, a negative errno value otherwise and rte_errno is set. 5237 */ 5238 int 5239 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5240 enum rte_filter_type filter_type, 5241 enum rte_filter_op filter_op, 5242 void *arg) 5243 { 5244 switch (filter_type) { 5245 case RTE_ETH_FILTER_GENERIC: 5246 if (filter_op != RTE_ETH_FILTER_GET) { 5247 rte_errno = EINVAL; 5248 return -rte_errno; 5249 } 5250 *(const void **)arg = &mlx5_flow_ops; 5251 return 0; 5252 case RTE_ETH_FILTER_FDIR: 5253 return flow_fdir_ctrl_func(dev, filter_op, arg); 5254 default: 5255 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5256 dev->data->port_id, filter_type); 5257 rte_errno = ENOTSUP; 5258 return -rte_errno; 5259 } 5260 return 0; 5261 } 5262 5263 /** 5264 * Create the needed meter and suffix tables. 5265 * 5266 * @param[in] dev 5267 * Pointer to Ethernet device. 5268 * @param[in] fm 5269 * Pointer to the flow meter. 5270 * 5271 * @return 5272 * Pointer to table set on success, NULL otherwise. 5273 */ 5274 struct mlx5_meter_domains_infos * 5275 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5276 const struct mlx5_flow_meter *fm) 5277 { 5278 const struct mlx5_flow_driver_ops *fops; 5279 5280 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5281 return fops->create_mtr_tbls(dev, fm); 5282 } 5283 5284 /** 5285 * Destroy the meter table set. 5286 * 5287 * @param[in] dev 5288 * Pointer to Ethernet device. 5289 * @param[in] tbl 5290 * Pointer to the meter table set. 5291 * 5292 * @return 5293 * 0 on success. 5294 */ 5295 int 5296 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5297 struct mlx5_meter_domains_infos *tbls) 5298 { 5299 const struct mlx5_flow_driver_ops *fops; 5300 5301 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5302 return fops->destroy_mtr_tbls(dev, tbls); 5303 } 5304 5305 /** 5306 * Create policer rules. 5307 * 5308 * @param[in] dev 5309 * Pointer to Ethernet device. 5310 * @param[in] fm 5311 * Pointer to flow meter structure. 5312 * @param[in] attr 5313 * Pointer to flow attributes. 5314 * 5315 * @return 5316 * 0 on success, -1 otherwise. 5317 */ 5318 int 5319 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5320 struct mlx5_flow_meter *fm, 5321 const struct rte_flow_attr *attr) 5322 { 5323 const struct mlx5_flow_driver_ops *fops; 5324 5325 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5326 return fops->create_policer_rules(dev, fm, attr); 5327 } 5328 5329 /** 5330 * Destroy policer rules. 5331 * 5332 * @param[in] fm 5333 * Pointer to flow meter structure. 5334 * @param[in] attr 5335 * Pointer to flow attributes. 5336 * 5337 * @return 5338 * 0 on success, -1 otherwise. 5339 */ 5340 int 5341 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5342 struct mlx5_flow_meter *fm, 5343 const struct rte_flow_attr *attr) 5344 { 5345 const struct mlx5_flow_driver_ops *fops; 5346 5347 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5348 return fops->destroy_policer_rules(dev, fm, attr); 5349 } 5350 5351 /** 5352 * Allocate a counter. 5353 * 5354 * @param[in] dev 5355 * Pointer to Ethernet device structure. 5356 * 5357 * @return 5358 * Pointer to allocated counter on success, NULL otherwise. 5359 */ 5360 struct mlx5_flow_counter * 5361 mlx5_counter_alloc(struct rte_eth_dev *dev) 5362 { 5363 const struct mlx5_flow_driver_ops *fops; 5364 struct rte_flow_attr attr = { .transfer = 0 }; 5365 5366 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5367 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5368 return fops->counter_alloc(dev); 5369 } 5370 DRV_LOG(ERR, 5371 "port %u counter allocate is not supported.", 5372 dev->data->port_id); 5373 return NULL; 5374 } 5375 5376 /** 5377 * Free a counter. 5378 * 5379 * @param[in] dev 5380 * Pointer to Ethernet device structure. 5381 * @param[in] cnt 5382 * Pointer to counter to be free. 5383 */ 5384 void 5385 mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) 5386 { 5387 const struct mlx5_flow_driver_ops *fops; 5388 struct rte_flow_attr attr = { .transfer = 0 }; 5389 5390 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5391 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5392 fops->counter_free(dev, cnt); 5393 return; 5394 } 5395 DRV_LOG(ERR, 5396 "port %u counter free is not supported.", 5397 dev->data->port_id); 5398 } 5399 5400 /** 5401 * Query counter statistics. 5402 * 5403 * @param[in] dev 5404 * Pointer to Ethernet device structure. 5405 * @param[in] cnt 5406 * Pointer to counter to query. 5407 * @param[in] clear 5408 * Set to clear counter statistics. 5409 * @param[out] pkts 5410 * The counter hits packets number to save. 5411 * @param[out] bytes 5412 * The counter hits bytes number to save. 5413 * 5414 * @return 5415 * 0 on success, a negative errno value otherwise. 5416 */ 5417 int 5418 mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt, 5419 bool clear, uint64_t *pkts, uint64_t *bytes) 5420 { 5421 const struct mlx5_flow_driver_ops *fops; 5422 struct rte_flow_attr attr = { .transfer = 0 }; 5423 5424 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5425 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5426 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5427 } 5428 DRV_LOG(ERR, 5429 "port %u counter query is not supported.", 5430 dev->data->port_id); 5431 return -ENOTSUP; 5432 } 5433 5434 #define MLX5_POOL_QUERY_FREQ_US 1000000 5435 5436 /** 5437 * Set the periodic procedure for triggering asynchronous batch queries for all 5438 * the counter pools. 5439 * 5440 * @param[in] sh 5441 * Pointer to mlx5_ibv_shared object. 5442 */ 5443 void 5444 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5445 { 5446 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0); 5447 uint32_t pools_n = rte_atomic16_read(&cont->n_valid); 5448 uint32_t us; 5449 5450 cont = MLX5_CNT_CONTAINER(sh, 1, 0); 5451 pools_n += rte_atomic16_read(&cont->n_valid); 5452 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5453 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5454 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5455 sh->cmng.query_thread_on = 0; 5456 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5457 } else { 5458 sh->cmng.query_thread_on = 1; 5459 } 5460 } 5461 5462 /** 5463 * The periodic procedure for triggering asynchronous batch queries for all the 5464 * counter pools. This function is probably called by the host thread. 5465 * 5466 * @param[in] arg 5467 * The parameter for the alarm process. 5468 */ 5469 void 5470 mlx5_flow_query_alarm(void *arg) 5471 { 5472 struct mlx5_ibv_shared *sh = arg; 5473 struct mlx5_devx_obj *dcs; 5474 uint16_t offset; 5475 int ret; 5476 uint8_t batch = sh->cmng.batch; 5477 uint16_t pool_index = sh->cmng.pool_index; 5478 struct mlx5_pools_container *cont; 5479 struct mlx5_pools_container *mcont; 5480 struct mlx5_flow_counter_pool *pool; 5481 5482 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5483 goto set_alarm; 5484 next_container: 5485 cont = MLX5_CNT_CONTAINER(sh, batch, 1); 5486 mcont = MLX5_CNT_CONTAINER(sh, batch, 0); 5487 /* Check if resize was done and need to flip a container. */ 5488 if (cont != mcont) { 5489 if (cont->pools) { 5490 /* Clean the old container. */ 5491 rte_free(cont->pools); 5492 memset(cont, 0, sizeof(*cont)); 5493 } 5494 rte_cio_wmb(); 5495 /* Flip the host container. */ 5496 sh->cmng.mhi[batch] ^= (uint8_t)2; 5497 cont = mcont; 5498 } 5499 if (!cont->pools) { 5500 /* 2 empty containers case is unexpected. */ 5501 if (unlikely(batch != sh->cmng.batch)) 5502 goto set_alarm; 5503 batch ^= 0x1; 5504 pool_index = 0; 5505 goto next_container; 5506 } 5507 pool = cont->pools[pool_index]; 5508 if (pool->raw_hw) 5509 /* There is a pool query in progress. */ 5510 goto set_alarm; 5511 pool->raw_hw = 5512 LIST_FIRST(&sh->cmng.free_stat_raws); 5513 if (!pool->raw_hw) 5514 /* No free counter statistics raw memory. */ 5515 goto set_alarm; 5516 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5517 (&pool->a64_dcs); 5518 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5519 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5520 offset, NULL, NULL, 5521 pool->raw_hw->mem_mng->dm->id, 5522 (void *)(uintptr_t) 5523 (pool->raw_hw->data + offset), 5524 sh->devx_comp, 5525 (uint64_t)(uintptr_t)pool); 5526 if (ret) { 5527 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5528 " %d", pool->min_dcs->id); 5529 pool->raw_hw = NULL; 5530 goto set_alarm; 5531 } 5532 pool->raw_hw->min_dcs_id = dcs->id; 5533 LIST_REMOVE(pool->raw_hw, next); 5534 sh->cmng.pending_queries++; 5535 pool_index++; 5536 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5537 batch ^= 0x1; 5538 pool_index = 0; 5539 } 5540 set_alarm: 5541 sh->cmng.batch = batch; 5542 sh->cmng.pool_index = pool_index; 5543 mlx5_set_query_alarm(sh); 5544 } 5545 5546 /** 5547 * Handler for the HW respond about ready values from an asynchronous batch 5548 * query. This function is probably called by the host thread. 5549 * 5550 * @param[in] sh 5551 * The pointer to the shared IB device context. 5552 * @param[in] async_id 5553 * The Devx async ID. 5554 * @param[in] status 5555 * The status of the completion. 5556 */ 5557 void 5558 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 5559 uint64_t async_id, int status) 5560 { 5561 struct mlx5_flow_counter_pool *pool = 5562 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 5563 struct mlx5_counter_stats_raw *raw_to_free; 5564 5565 if (unlikely(status)) { 5566 raw_to_free = pool->raw_hw; 5567 } else { 5568 raw_to_free = pool->raw; 5569 rte_spinlock_lock(&pool->sl); 5570 pool->raw = pool->raw_hw; 5571 rte_spinlock_unlock(&pool->sl); 5572 rte_atomic64_add(&pool->query_gen, 1); 5573 /* Be sure the new raw counters data is updated in memory. */ 5574 rte_cio_wmb(); 5575 } 5576 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 5577 pool->raw_hw = NULL; 5578 sh->cmng.pending_queries--; 5579 } 5580 5581 /** 5582 * Translate the rte_flow group index to HW table value. 5583 * 5584 * @param[in] attributes 5585 * Pointer to flow attributes 5586 * @param[in] external 5587 * Value is part of flow rule created by request external to PMD. 5588 * @param[in] group 5589 * rte_flow group index value. 5590 * @param[out] table 5591 * HW table value. 5592 * @param[out] error 5593 * Pointer to error structure. 5594 * 5595 * @return 5596 * 0 on success, a negative errno value otherwise and rte_errno is set. 5597 */ 5598 int 5599 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 5600 uint32_t group, uint32_t *table, 5601 struct rte_flow_error *error) 5602 { 5603 if (attributes->transfer && external) { 5604 if (group == UINT32_MAX) 5605 return rte_flow_error_set 5606 (error, EINVAL, 5607 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 5608 NULL, 5609 "group index not supported"); 5610 *table = group + 1; 5611 } else { 5612 *table = group; 5613 } 5614 return 0; 5615 } 5616 5617 /** 5618 * Discover availability of metadata reg_c's. 5619 * 5620 * Iteratively use test flows to check availability. 5621 * 5622 * @param[in] dev 5623 * Pointer to the Ethernet device structure. 5624 * 5625 * @return 5626 * 0 on success, a negative errno value otherwise and rte_errno is set. 5627 */ 5628 int 5629 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 5630 { 5631 struct mlx5_priv *priv = dev->data->dev_private; 5632 struct mlx5_dev_config *config = &priv->config; 5633 enum modify_reg idx; 5634 int n = 0; 5635 5636 /* reg_c[0] and reg_c[1] are reserved. */ 5637 config->flow_mreg_c[n++] = REG_C_0; 5638 config->flow_mreg_c[n++] = REG_C_1; 5639 /* Discover availability of other reg_c's. */ 5640 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 5641 struct rte_flow_attr attr = { 5642 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5643 .priority = MLX5_FLOW_PRIO_RSVD, 5644 .ingress = 1, 5645 }; 5646 struct rte_flow_item items[] = { 5647 [0] = { 5648 .type = RTE_FLOW_ITEM_TYPE_END, 5649 }, 5650 }; 5651 struct rte_flow_action actions[] = { 5652 [0] = { 5653 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5654 .conf = &(struct mlx5_flow_action_copy_mreg){ 5655 .src = REG_C_1, 5656 .dst = idx, 5657 }, 5658 }, 5659 [1] = { 5660 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5661 .conf = &(struct rte_flow_action_jump){ 5662 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5663 }, 5664 }, 5665 [2] = { 5666 .type = RTE_FLOW_ACTION_TYPE_END, 5667 }, 5668 }; 5669 struct rte_flow *flow; 5670 struct rte_flow_error error; 5671 5672 if (!config->dv_flow_en) 5673 break; 5674 /* Create internal flow, validation skips copy action. */ 5675 flow = flow_list_create(dev, NULL, &attr, items, 5676 actions, false, &error); 5677 if (!flow) 5678 continue; 5679 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 5680 config->flow_mreg_c[n++] = idx; 5681 flow_list_destroy(dev, NULL, flow); 5682 } 5683 for (; n < MLX5_MREG_C_NUM; ++n) 5684 config->flow_mreg_c[n] = REG_NONE; 5685 return 0; 5686 } 5687 5688 /** 5689 * Dump flow raw hw data to file 5690 * 5691 * @param[in] dev 5692 * The pointer to Ethernet device. 5693 * @param[in] file 5694 * A pointer to a file for output. 5695 * @param[out] error 5696 * Perform verbose error reporting if not NULL. PMDs initialize this 5697 * structure in case of error only. 5698 * @return 5699 * 0 on success, a nagative value otherwise. 5700 */ 5701 int 5702 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 5703 FILE *file, 5704 struct rte_flow_error *error __rte_unused) 5705 { 5706 struct mlx5_priv *priv = dev->data->dev_private; 5707 5708 return mlx5_devx_cmd_flow_dump(priv->sh, file); 5709 } 5710