1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <stdbool.h> 12 13 /* Verbs header. */ 14 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 15 #ifdef PEDANTIC 16 #pragma GCC diagnostic ignored "-Wpedantic" 17 #endif 18 #include <infiniband/verbs.h> 19 #ifdef PEDANTIC 20 #pragma GCC diagnostic error "-Wpedantic" 21 #endif 22 23 #include <rte_common.h> 24 #include <rte_ether.h> 25 #include <rte_ethdev_driver.h> 26 #include <rte_flow.h> 27 #include <rte_flow_driver.h> 28 #include <rte_malloc.h> 29 #include <rte_ip.h> 30 31 #include <mlx5_glue.h> 32 #include <mlx5_devx_cmds.h> 33 #include <mlx5_prm.h> 34 35 #include "mlx5_defs.h" 36 #include "mlx5.h" 37 #include "mlx5_flow.h" 38 #include "mlx5_rxtx.h" 39 40 /* Dev ops structure defined in mlx5.c */ 41 extern const struct eth_dev_ops mlx5_dev_ops; 42 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 43 44 /** Device flow drivers. */ 45 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 46 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 47 #endif 48 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 49 50 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 51 52 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 53 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 54 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 55 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 56 #endif 57 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 58 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 59 }; 60 61 enum mlx5_expansion { 62 MLX5_EXPANSION_ROOT, 63 MLX5_EXPANSION_ROOT_OUTER, 64 MLX5_EXPANSION_ROOT_ETH_VLAN, 65 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 66 MLX5_EXPANSION_OUTER_ETH, 67 MLX5_EXPANSION_OUTER_ETH_VLAN, 68 MLX5_EXPANSION_OUTER_VLAN, 69 MLX5_EXPANSION_OUTER_IPV4, 70 MLX5_EXPANSION_OUTER_IPV4_UDP, 71 MLX5_EXPANSION_OUTER_IPV4_TCP, 72 MLX5_EXPANSION_OUTER_IPV6, 73 MLX5_EXPANSION_OUTER_IPV6_UDP, 74 MLX5_EXPANSION_OUTER_IPV6_TCP, 75 MLX5_EXPANSION_VXLAN, 76 MLX5_EXPANSION_VXLAN_GPE, 77 MLX5_EXPANSION_GRE, 78 MLX5_EXPANSION_MPLS, 79 MLX5_EXPANSION_ETH, 80 MLX5_EXPANSION_ETH_VLAN, 81 MLX5_EXPANSION_VLAN, 82 MLX5_EXPANSION_IPV4, 83 MLX5_EXPANSION_IPV4_UDP, 84 MLX5_EXPANSION_IPV4_TCP, 85 MLX5_EXPANSION_IPV6, 86 MLX5_EXPANSION_IPV6_UDP, 87 MLX5_EXPANSION_IPV6_TCP, 88 }; 89 90 /** Supported expansion of items. */ 91 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 92 [MLX5_EXPANSION_ROOT] = { 93 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 94 MLX5_EXPANSION_IPV4, 95 MLX5_EXPANSION_IPV6), 96 .type = RTE_FLOW_ITEM_TYPE_END, 97 }, 98 [MLX5_EXPANSION_ROOT_OUTER] = { 99 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 100 MLX5_EXPANSION_OUTER_IPV4, 101 MLX5_EXPANSION_OUTER_IPV6), 102 .type = RTE_FLOW_ITEM_TYPE_END, 103 }, 104 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 105 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 106 .type = RTE_FLOW_ITEM_TYPE_END, 107 }, 108 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 109 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 110 .type = RTE_FLOW_ITEM_TYPE_END, 111 }, 112 [MLX5_EXPANSION_OUTER_ETH] = { 113 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 114 MLX5_EXPANSION_OUTER_IPV6, 115 MLX5_EXPANSION_MPLS), 116 .type = RTE_FLOW_ITEM_TYPE_ETH, 117 .rss_types = 0, 118 }, 119 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 120 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 121 .type = RTE_FLOW_ITEM_TYPE_ETH, 122 .rss_types = 0, 123 }, 124 [MLX5_EXPANSION_OUTER_VLAN] = { 125 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 126 MLX5_EXPANSION_OUTER_IPV6), 127 .type = RTE_FLOW_ITEM_TYPE_VLAN, 128 }, 129 [MLX5_EXPANSION_OUTER_IPV4] = { 130 .next = RTE_FLOW_EXPAND_RSS_NEXT 131 (MLX5_EXPANSION_OUTER_IPV4_UDP, 132 MLX5_EXPANSION_OUTER_IPV4_TCP, 133 MLX5_EXPANSION_GRE, 134 MLX5_EXPANSION_IPV4, 135 MLX5_EXPANSION_IPV6), 136 .type = RTE_FLOW_ITEM_TYPE_IPV4, 137 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 138 ETH_RSS_NONFRAG_IPV4_OTHER, 139 }, 140 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 141 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 142 MLX5_EXPANSION_VXLAN_GPE), 143 .type = RTE_FLOW_ITEM_TYPE_UDP, 144 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 145 }, 146 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 147 .type = RTE_FLOW_ITEM_TYPE_TCP, 148 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 149 }, 150 [MLX5_EXPANSION_OUTER_IPV6] = { 151 .next = RTE_FLOW_EXPAND_RSS_NEXT 152 (MLX5_EXPANSION_OUTER_IPV6_UDP, 153 MLX5_EXPANSION_OUTER_IPV6_TCP, 154 MLX5_EXPANSION_IPV4, 155 MLX5_EXPANSION_IPV6), 156 .type = RTE_FLOW_ITEM_TYPE_IPV6, 157 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 158 ETH_RSS_NONFRAG_IPV6_OTHER, 159 }, 160 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 161 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 162 MLX5_EXPANSION_VXLAN_GPE), 163 .type = RTE_FLOW_ITEM_TYPE_UDP, 164 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 165 }, 166 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 167 .type = RTE_FLOW_ITEM_TYPE_TCP, 168 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 169 }, 170 [MLX5_EXPANSION_VXLAN] = { 171 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 172 MLX5_EXPANSION_IPV4, 173 MLX5_EXPANSION_IPV6), 174 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 175 }, 176 [MLX5_EXPANSION_VXLAN_GPE] = { 177 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 178 MLX5_EXPANSION_IPV4, 179 MLX5_EXPANSION_IPV6), 180 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 181 }, 182 [MLX5_EXPANSION_GRE] = { 183 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 184 .type = RTE_FLOW_ITEM_TYPE_GRE, 185 }, 186 [MLX5_EXPANSION_MPLS] = { 187 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 188 MLX5_EXPANSION_IPV6), 189 .type = RTE_FLOW_ITEM_TYPE_MPLS, 190 }, 191 [MLX5_EXPANSION_ETH] = { 192 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 193 MLX5_EXPANSION_IPV6), 194 .type = RTE_FLOW_ITEM_TYPE_ETH, 195 }, 196 [MLX5_EXPANSION_ETH_VLAN] = { 197 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 198 .type = RTE_FLOW_ITEM_TYPE_ETH, 199 }, 200 [MLX5_EXPANSION_VLAN] = { 201 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 202 MLX5_EXPANSION_IPV6), 203 .type = RTE_FLOW_ITEM_TYPE_VLAN, 204 }, 205 [MLX5_EXPANSION_IPV4] = { 206 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 207 MLX5_EXPANSION_IPV4_TCP), 208 .type = RTE_FLOW_ITEM_TYPE_IPV4, 209 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 210 ETH_RSS_NONFRAG_IPV4_OTHER, 211 }, 212 [MLX5_EXPANSION_IPV4_UDP] = { 213 .type = RTE_FLOW_ITEM_TYPE_UDP, 214 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 215 }, 216 [MLX5_EXPANSION_IPV4_TCP] = { 217 .type = RTE_FLOW_ITEM_TYPE_TCP, 218 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 219 }, 220 [MLX5_EXPANSION_IPV6] = { 221 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 222 MLX5_EXPANSION_IPV6_TCP), 223 .type = RTE_FLOW_ITEM_TYPE_IPV6, 224 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 225 ETH_RSS_NONFRAG_IPV6_OTHER, 226 }, 227 [MLX5_EXPANSION_IPV6_UDP] = { 228 .type = RTE_FLOW_ITEM_TYPE_UDP, 229 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 230 }, 231 [MLX5_EXPANSION_IPV6_TCP] = { 232 .type = RTE_FLOW_ITEM_TYPE_TCP, 233 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 234 }, 235 }; 236 237 static const struct rte_flow_ops mlx5_flow_ops = { 238 .validate = mlx5_flow_validate, 239 .create = mlx5_flow_create, 240 .destroy = mlx5_flow_destroy, 241 .flush = mlx5_flow_flush, 242 .isolate = mlx5_flow_isolate, 243 .query = mlx5_flow_query, 244 .dev_dump = mlx5_flow_dev_dump, 245 }; 246 247 /* Convert FDIR request to Generic flow. */ 248 struct mlx5_fdir { 249 struct rte_flow_attr attr; 250 struct rte_flow_item items[4]; 251 struct rte_flow_item_eth l2; 252 struct rte_flow_item_eth l2_mask; 253 union { 254 struct rte_flow_item_ipv4 ipv4; 255 struct rte_flow_item_ipv6 ipv6; 256 } l3; 257 union { 258 struct rte_flow_item_ipv4 ipv4; 259 struct rte_flow_item_ipv6 ipv6; 260 } l3_mask; 261 union { 262 struct rte_flow_item_udp udp; 263 struct rte_flow_item_tcp tcp; 264 } l4; 265 union { 266 struct rte_flow_item_udp udp; 267 struct rte_flow_item_tcp tcp; 268 } l4_mask; 269 struct rte_flow_action actions[2]; 270 struct rte_flow_action_queue queue; 271 }; 272 273 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 274 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 275 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 276 }; 277 278 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 279 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 280 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 281 { 9, 10, 11 }, { 12, 13, 14 }, 282 }; 283 284 /* Tunnel information. */ 285 struct mlx5_flow_tunnel_info { 286 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 287 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 288 }; 289 290 static struct mlx5_flow_tunnel_info tunnels_info[] = { 291 { 292 .tunnel = MLX5_FLOW_LAYER_VXLAN, 293 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 294 }, 295 { 296 .tunnel = MLX5_FLOW_LAYER_GENEVE, 297 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 298 }, 299 { 300 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 301 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 302 }, 303 { 304 .tunnel = MLX5_FLOW_LAYER_GRE, 305 .ptype = RTE_PTYPE_TUNNEL_GRE, 306 }, 307 { 308 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 309 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 310 }, 311 { 312 .tunnel = MLX5_FLOW_LAYER_MPLS, 313 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 314 }, 315 { 316 .tunnel = MLX5_FLOW_LAYER_NVGRE, 317 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 318 }, 319 { 320 .tunnel = MLX5_FLOW_LAYER_IPIP, 321 .ptype = RTE_PTYPE_TUNNEL_IP, 322 }, 323 { 324 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 325 .ptype = RTE_PTYPE_TUNNEL_IP, 326 }, 327 { 328 .tunnel = MLX5_FLOW_LAYER_GTP, 329 .ptype = RTE_PTYPE_TUNNEL_GTPU, 330 }, 331 }; 332 333 /** 334 * Translate tag ID to register. 335 * 336 * @param[in] dev 337 * Pointer to the Ethernet device structure. 338 * @param[in] feature 339 * The feature that request the register. 340 * @param[in] id 341 * The request register ID. 342 * @param[out] error 343 * Error description in case of any. 344 * 345 * @return 346 * The request register on success, a negative errno 347 * value otherwise and rte_errno is set. 348 */ 349 int 350 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 351 enum mlx5_feature_name feature, 352 uint32_t id, 353 struct rte_flow_error *error) 354 { 355 struct mlx5_priv *priv = dev->data->dev_private; 356 struct mlx5_dev_config *config = &priv->config; 357 enum modify_reg start_reg; 358 bool skip_mtr_reg = false; 359 360 switch (feature) { 361 case MLX5_HAIRPIN_RX: 362 return REG_B; 363 case MLX5_HAIRPIN_TX: 364 return REG_A; 365 case MLX5_METADATA_RX: 366 switch (config->dv_xmeta_en) { 367 case MLX5_XMETA_MODE_LEGACY: 368 return REG_B; 369 case MLX5_XMETA_MODE_META16: 370 return REG_C_0; 371 case MLX5_XMETA_MODE_META32: 372 return REG_C_1; 373 } 374 break; 375 case MLX5_METADATA_TX: 376 return REG_A; 377 case MLX5_METADATA_FDB: 378 switch (config->dv_xmeta_en) { 379 case MLX5_XMETA_MODE_LEGACY: 380 return REG_NONE; 381 case MLX5_XMETA_MODE_META16: 382 return REG_C_0; 383 case MLX5_XMETA_MODE_META32: 384 return REG_C_1; 385 } 386 break; 387 case MLX5_FLOW_MARK: 388 switch (config->dv_xmeta_en) { 389 case MLX5_XMETA_MODE_LEGACY: 390 return REG_NONE; 391 case MLX5_XMETA_MODE_META16: 392 return REG_C_1; 393 case MLX5_XMETA_MODE_META32: 394 return REG_C_0; 395 } 396 break; 397 case MLX5_MTR_SFX: 398 /* 399 * If meter color and flow match share one register, flow match 400 * should use the meter color register for match. 401 */ 402 if (priv->mtr_reg_share) 403 return priv->mtr_color_reg; 404 else 405 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 406 REG_C_3; 407 case MLX5_MTR_COLOR: 408 MLX5_ASSERT(priv->mtr_color_reg != REG_NONE); 409 return priv->mtr_color_reg; 410 case MLX5_COPY_MARK: 411 /* 412 * Metadata COPY_MARK register using is in meter suffix sub 413 * flow while with meter. It's safe to share the same register. 414 */ 415 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 416 case MLX5_APP_TAG: 417 /* 418 * If meter is enable, it will engage the register for color 419 * match and flow match. If meter color match is not using the 420 * REG_C_2, need to skip the REG_C_x be used by meter color 421 * match. 422 * If meter is disable, free to use all available registers. 423 */ 424 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 425 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 426 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 427 if (id > (REG_C_7 - start_reg)) 428 return rte_flow_error_set(error, EINVAL, 429 RTE_FLOW_ERROR_TYPE_ITEM, 430 NULL, "invalid tag id"); 431 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 432 return rte_flow_error_set(error, ENOTSUP, 433 RTE_FLOW_ERROR_TYPE_ITEM, 434 NULL, "unsupported tag id"); 435 /* 436 * This case means meter is using the REG_C_x great than 2. 437 * Take care not to conflict with meter color REG_C_x. 438 * If the available index REG_C_y >= REG_C_x, skip the 439 * color register. 440 */ 441 if (skip_mtr_reg && config->flow_mreg_c 442 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 443 if (config->flow_mreg_c 444 [id + 1 + start_reg - REG_C_0] != REG_NONE) 445 return config->flow_mreg_c 446 [id + 1 + start_reg - REG_C_0]; 447 return rte_flow_error_set(error, ENOTSUP, 448 RTE_FLOW_ERROR_TYPE_ITEM, 449 NULL, "unsupported tag id"); 450 } 451 return config->flow_mreg_c[id + start_reg - REG_C_0]; 452 } 453 MLX5_ASSERT(false); 454 return rte_flow_error_set(error, EINVAL, 455 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 456 NULL, "invalid feature name"); 457 } 458 459 /** 460 * Check extensive flow metadata register support. 461 * 462 * @param dev 463 * Pointer to rte_eth_dev structure. 464 * 465 * @return 466 * True if device supports extensive flow metadata register, otherwise false. 467 */ 468 bool 469 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 470 { 471 struct mlx5_priv *priv = dev->data->dev_private; 472 struct mlx5_dev_config *config = &priv->config; 473 474 /* 475 * Having available reg_c can be regarded inclusively as supporting 476 * extensive flow metadata register, which could mean, 477 * - metadata register copy action by modify header. 478 * - 16 modify header actions is supported. 479 * - reg_c's are preserved across different domain (FDB and NIC) on 480 * packet loopback by flow lookup miss. 481 */ 482 return config->flow_mreg_c[2] != REG_NONE; 483 } 484 485 /** 486 * Discover the maximum number of priority available. 487 * 488 * @param[in] dev 489 * Pointer to the Ethernet device structure. 490 * 491 * @return 492 * number of supported flow priority on success, a negative errno 493 * value otherwise and rte_errno is set. 494 */ 495 int 496 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 497 { 498 struct mlx5_priv *priv = dev->data->dev_private; 499 struct { 500 struct ibv_flow_attr attr; 501 struct ibv_flow_spec_eth eth; 502 struct ibv_flow_spec_action_drop drop; 503 } flow_attr = { 504 .attr = { 505 .num_of_specs = 2, 506 .port = (uint8_t)priv->ibv_port, 507 }, 508 .eth = { 509 .type = IBV_FLOW_SPEC_ETH, 510 .size = sizeof(struct ibv_flow_spec_eth), 511 }, 512 .drop = { 513 .size = sizeof(struct ibv_flow_spec_action_drop), 514 .type = IBV_FLOW_SPEC_ACTION_DROP, 515 }, 516 }; 517 struct ibv_flow *flow; 518 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 519 uint16_t vprio[] = { 8, 16 }; 520 int i; 521 int priority = 0; 522 523 if (!drop) { 524 rte_errno = ENOTSUP; 525 return -rte_errno; 526 } 527 for (i = 0; i != RTE_DIM(vprio); i++) { 528 flow_attr.attr.priority = vprio[i] - 1; 529 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 530 if (!flow) 531 break; 532 claim_zero(mlx5_glue->destroy_flow(flow)); 533 priority = vprio[i]; 534 } 535 mlx5_hrxq_drop_release(dev); 536 switch (priority) { 537 case 8: 538 priority = RTE_DIM(priority_map_3); 539 break; 540 case 16: 541 priority = RTE_DIM(priority_map_5); 542 break; 543 default: 544 rte_errno = ENOTSUP; 545 DRV_LOG(ERR, 546 "port %u verbs maximum priority: %d expected 8/16", 547 dev->data->port_id, priority); 548 return -rte_errno; 549 } 550 DRV_LOG(INFO, "port %u flow maximum priority: %d", 551 dev->data->port_id, priority); 552 return priority; 553 } 554 555 /** 556 * Adjust flow priority based on the highest layer and the request priority. 557 * 558 * @param[in] dev 559 * Pointer to the Ethernet device structure. 560 * @param[in] priority 561 * The rule base priority. 562 * @param[in] subpriority 563 * The priority based on the items. 564 * 565 * @return 566 * The new priority. 567 */ 568 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 569 uint32_t subpriority) 570 { 571 uint32_t res = 0; 572 struct mlx5_priv *priv = dev->data->dev_private; 573 574 switch (priv->config.flow_prio) { 575 case RTE_DIM(priority_map_3): 576 res = priority_map_3[priority][subpriority]; 577 break; 578 case RTE_DIM(priority_map_5): 579 res = priority_map_5[priority][subpriority]; 580 break; 581 } 582 return res; 583 } 584 585 /** 586 * Verify the @p item specifications (spec, last, mask) are compatible with the 587 * NIC capabilities. 588 * 589 * @param[in] item 590 * Item specification. 591 * @param[in] mask 592 * @p item->mask or flow default bit-masks. 593 * @param[in] nic_mask 594 * Bit-masks covering supported fields by the NIC to compare with user mask. 595 * @param[in] size 596 * Bit-masks size in bytes. 597 * @param[out] error 598 * Pointer to error structure. 599 * 600 * @return 601 * 0 on success, a negative errno value otherwise and rte_errno is set. 602 */ 603 int 604 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 605 const uint8_t *mask, 606 const uint8_t *nic_mask, 607 unsigned int size, 608 struct rte_flow_error *error) 609 { 610 unsigned int i; 611 612 MLX5_ASSERT(nic_mask); 613 for (i = 0; i < size; ++i) 614 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 615 return rte_flow_error_set(error, ENOTSUP, 616 RTE_FLOW_ERROR_TYPE_ITEM, 617 item, 618 "mask enables non supported" 619 " bits"); 620 if (!item->spec && (item->mask || item->last)) 621 return rte_flow_error_set(error, EINVAL, 622 RTE_FLOW_ERROR_TYPE_ITEM, item, 623 "mask/last without a spec is not" 624 " supported"); 625 if (item->spec && item->last) { 626 uint8_t spec[size]; 627 uint8_t last[size]; 628 unsigned int i; 629 int ret; 630 631 for (i = 0; i < size; ++i) { 632 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 633 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 634 } 635 ret = memcmp(spec, last, size); 636 if (ret != 0) 637 return rte_flow_error_set(error, EINVAL, 638 RTE_FLOW_ERROR_TYPE_ITEM, 639 item, 640 "range is not valid"); 641 } 642 return 0; 643 } 644 645 /** 646 * Adjust the hash fields according to the @p flow information. 647 * 648 * @param[in] dev_flow. 649 * Pointer to the mlx5_flow. 650 * @param[in] tunnel 651 * 1 when the hash field is for a tunnel item. 652 * @param[in] layer_types 653 * ETH_RSS_* types. 654 * @param[in] hash_fields 655 * Item hash fields. 656 * 657 * @return 658 * The hash fields that should be used. 659 */ 660 uint64_t 661 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 662 int tunnel __rte_unused, uint64_t layer_types, 663 uint64_t hash_fields) 664 { 665 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 666 int rss_request_inner = rss_desc->level >= 2; 667 668 /* Check RSS hash level for tunnel. */ 669 if (tunnel && rss_request_inner) 670 hash_fields |= IBV_RX_HASH_INNER; 671 else if (tunnel || rss_request_inner) 672 return 0; 673 #endif 674 /* Check if requested layer matches RSS hash fields. */ 675 if (!(rss_desc->types & layer_types)) 676 return 0; 677 return hash_fields; 678 } 679 680 /** 681 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 682 * if several tunnel rules are used on this queue, the tunnel ptype will be 683 * cleared. 684 * 685 * @param rxq_ctrl 686 * Rx queue to update. 687 */ 688 static void 689 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 690 { 691 unsigned int i; 692 uint32_t tunnel_ptype = 0; 693 694 /* Look up for the ptype to use. */ 695 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 696 if (!rxq_ctrl->flow_tunnels_n[i]) 697 continue; 698 if (!tunnel_ptype) { 699 tunnel_ptype = tunnels_info[i].ptype; 700 } else { 701 tunnel_ptype = 0; 702 break; 703 } 704 } 705 rxq_ctrl->rxq.tunnel = tunnel_ptype; 706 } 707 708 /** 709 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 710 * flow. 711 * 712 * @param[in] dev 713 * Pointer to the Ethernet device structure. 714 * @param[in] dev_handle 715 * Pointer to device flow handle structure. 716 */ 717 static void 718 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 719 struct mlx5_flow_handle *dev_handle) 720 { 721 struct mlx5_priv *priv = dev->data->dev_private; 722 const int mark = dev_handle->mark; 723 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 724 struct mlx5_hrxq *hrxq; 725 unsigned int i; 726 727 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 728 return; 729 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 730 dev_handle->rix_hrxq); 731 if (!hrxq) 732 return; 733 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 734 int idx = hrxq->ind_table->queues[i]; 735 struct mlx5_rxq_ctrl *rxq_ctrl = 736 container_of((*priv->rxqs)[idx], 737 struct mlx5_rxq_ctrl, rxq); 738 739 /* 740 * To support metadata register copy on Tx loopback, 741 * this must be always enabled (metadata may arive 742 * from other port - not from local flows only. 743 */ 744 if (priv->config.dv_flow_en && 745 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 746 mlx5_flow_ext_mreg_supported(dev)) { 747 rxq_ctrl->rxq.mark = 1; 748 rxq_ctrl->flow_mark_n = 1; 749 } else if (mark) { 750 rxq_ctrl->rxq.mark = 1; 751 rxq_ctrl->flow_mark_n++; 752 } 753 if (tunnel) { 754 unsigned int j; 755 756 /* Increase the counter matching the flow. */ 757 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 758 if ((tunnels_info[j].tunnel & 759 dev_handle->layers) == 760 tunnels_info[j].tunnel) { 761 rxq_ctrl->flow_tunnels_n[j]++; 762 break; 763 } 764 } 765 flow_rxq_tunnel_ptype_update(rxq_ctrl); 766 } 767 } 768 } 769 770 /** 771 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 772 * 773 * @param[in] dev 774 * Pointer to the Ethernet device structure. 775 * @param[in] flow 776 * Pointer to flow structure. 777 */ 778 static void 779 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 780 { 781 struct mlx5_priv *priv = dev->data->dev_private; 782 uint32_t handle_idx; 783 struct mlx5_flow_handle *dev_handle; 784 785 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 786 handle_idx, dev_handle, next) 787 flow_drv_rxq_flags_set(dev, dev_handle); 788 } 789 790 /** 791 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 792 * device flow if no other flow uses it with the same kind of request. 793 * 794 * @param dev 795 * Pointer to Ethernet device. 796 * @param[in] dev_handle 797 * Pointer to the device flow handle structure. 798 */ 799 static void 800 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 801 struct mlx5_flow_handle *dev_handle) 802 { 803 struct mlx5_priv *priv = dev->data->dev_private; 804 const int mark = dev_handle->mark; 805 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 806 struct mlx5_hrxq *hrxq; 807 unsigned int i; 808 809 if (dev_handle->fate_action != MLX5_FLOW_FATE_QUEUE) 810 return; 811 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 812 dev_handle->rix_hrxq); 813 if (!hrxq) 814 return; 815 MLX5_ASSERT(dev->data->dev_started); 816 for (i = 0; i != hrxq->ind_table->queues_n; ++i) { 817 int idx = hrxq->ind_table->queues[i]; 818 struct mlx5_rxq_ctrl *rxq_ctrl = 819 container_of((*priv->rxqs)[idx], 820 struct mlx5_rxq_ctrl, rxq); 821 822 if (priv->config.dv_flow_en && 823 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 824 mlx5_flow_ext_mreg_supported(dev)) { 825 rxq_ctrl->rxq.mark = 1; 826 rxq_ctrl->flow_mark_n = 1; 827 } else if (mark) { 828 rxq_ctrl->flow_mark_n--; 829 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 830 } 831 if (tunnel) { 832 unsigned int j; 833 834 /* Decrease the counter matching the flow. */ 835 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 836 if ((tunnels_info[j].tunnel & 837 dev_handle->layers) == 838 tunnels_info[j].tunnel) { 839 rxq_ctrl->flow_tunnels_n[j]--; 840 break; 841 } 842 } 843 flow_rxq_tunnel_ptype_update(rxq_ctrl); 844 } 845 } 846 } 847 848 /** 849 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 850 * @p flow if no other flow uses it with the same kind of request. 851 * 852 * @param dev 853 * Pointer to Ethernet device. 854 * @param[in] flow 855 * Pointer to the flow. 856 */ 857 static void 858 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 859 { 860 struct mlx5_priv *priv = dev->data->dev_private; 861 uint32_t handle_idx; 862 struct mlx5_flow_handle *dev_handle; 863 864 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 865 handle_idx, dev_handle, next) 866 flow_drv_rxq_flags_trim(dev, dev_handle); 867 } 868 869 /** 870 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 871 * 872 * @param dev 873 * Pointer to Ethernet device. 874 */ 875 static void 876 flow_rxq_flags_clear(struct rte_eth_dev *dev) 877 { 878 struct mlx5_priv *priv = dev->data->dev_private; 879 unsigned int i; 880 881 for (i = 0; i != priv->rxqs_n; ++i) { 882 struct mlx5_rxq_ctrl *rxq_ctrl; 883 unsigned int j; 884 885 if (!(*priv->rxqs)[i]) 886 continue; 887 rxq_ctrl = container_of((*priv->rxqs)[i], 888 struct mlx5_rxq_ctrl, rxq); 889 rxq_ctrl->flow_mark_n = 0; 890 rxq_ctrl->rxq.mark = 0; 891 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 892 rxq_ctrl->flow_tunnels_n[j] = 0; 893 rxq_ctrl->rxq.tunnel = 0; 894 } 895 } 896 897 /** 898 * Set the Rx queue dynamic metadata (mask and offset) for a flow 899 * 900 * @param[in] dev 901 * Pointer to the Ethernet device structure. 902 */ 903 void 904 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 905 { 906 struct mlx5_priv *priv = dev->data->dev_private; 907 struct mlx5_rxq_data *data; 908 unsigned int i; 909 910 for (i = 0; i != priv->rxqs_n; ++i) { 911 if (!(*priv->rxqs)[i]) 912 continue; 913 data = (*priv->rxqs)[i]; 914 if (!rte_flow_dynf_metadata_avail()) { 915 data->dynf_meta = 0; 916 data->flow_meta_mask = 0; 917 data->flow_meta_offset = -1; 918 } else { 919 data->dynf_meta = 1; 920 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 921 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 922 } 923 } 924 } 925 926 /* 927 * return a pointer to the desired action in the list of actions. 928 * 929 * @param[in] actions 930 * The list of actions to search the action in. 931 * @param[in] action 932 * The action to find. 933 * 934 * @return 935 * Pointer to the action in the list, if found. NULL otherwise. 936 */ 937 const struct rte_flow_action * 938 mlx5_flow_find_action(const struct rte_flow_action *actions, 939 enum rte_flow_action_type action) 940 { 941 if (actions == NULL) 942 return NULL; 943 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 944 if (actions->type == action) 945 return actions; 946 return NULL; 947 } 948 949 /* 950 * Validate the flag action. 951 * 952 * @param[in] action_flags 953 * Bit-fields that holds the actions detected until now. 954 * @param[in] attr 955 * Attributes of flow that includes this action. 956 * @param[out] error 957 * Pointer to error structure. 958 * 959 * @return 960 * 0 on success, a negative errno value otherwise and rte_errno is set. 961 */ 962 int 963 mlx5_flow_validate_action_flag(uint64_t action_flags, 964 const struct rte_flow_attr *attr, 965 struct rte_flow_error *error) 966 { 967 if (action_flags & MLX5_FLOW_ACTION_MARK) 968 return rte_flow_error_set(error, EINVAL, 969 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 970 "can't mark and flag in same flow"); 971 if (action_flags & MLX5_FLOW_ACTION_FLAG) 972 return rte_flow_error_set(error, EINVAL, 973 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 974 "can't have 2 flag" 975 " actions in same flow"); 976 if (attr->egress) 977 return rte_flow_error_set(error, ENOTSUP, 978 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 979 "flag action not supported for " 980 "egress"); 981 return 0; 982 } 983 984 /* 985 * Validate the mark action. 986 * 987 * @param[in] action 988 * Pointer to the queue action. 989 * @param[in] action_flags 990 * Bit-fields that holds the actions detected until now. 991 * @param[in] attr 992 * Attributes of flow that includes this action. 993 * @param[out] error 994 * Pointer to error structure. 995 * 996 * @return 997 * 0 on success, a negative errno value otherwise and rte_errno is set. 998 */ 999 int 1000 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1001 uint64_t action_flags, 1002 const struct rte_flow_attr *attr, 1003 struct rte_flow_error *error) 1004 { 1005 const struct rte_flow_action_mark *mark = action->conf; 1006 1007 if (!mark) 1008 return rte_flow_error_set(error, EINVAL, 1009 RTE_FLOW_ERROR_TYPE_ACTION, 1010 action, 1011 "configuration cannot be null"); 1012 if (mark->id >= MLX5_FLOW_MARK_MAX) 1013 return rte_flow_error_set(error, EINVAL, 1014 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1015 &mark->id, 1016 "mark id must in 0 <= id < " 1017 RTE_STR(MLX5_FLOW_MARK_MAX)); 1018 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1019 return rte_flow_error_set(error, EINVAL, 1020 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1021 "can't flag and mark in same flow"); 1022 if (action_flags & MLX5_FLOW_ACTION_MARK) 1023 return rte_flow_error_set(error, EINVAL, 1024 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1025 "can't have 2 mark actions in same" 1026 " flow"); 1027 if (attr->egress) 1028 return rte_flow_error_set(error, ENOTSUP, 1029 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1030 "mark action not supported for " 1031 "egress"); 1032 return 0; 1033 } 1034 1035 /* 1036 * Validate the drop action. 1037 * 1038 * @param[in] action_flags 1039 * Bit-fields that holds the actions detected until now. 1040 * @param[in] attr 1041 * Attributes of flow that includes this action. 1042 * @param[out] error 1043 * Pointer to error structure. 1044 * 1045 * @return 1046 * 0 on success, a negative errno value otherwise and rte_errno is set. 1047 */ 1048 int 1049 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1050 const struct rte_flow_attr *attr, 1051 struct rte_flow_error *error) 1052 { 1053 if (attr->egress) 1054 return rte_flow_error_set(error, ENOTSUP, 1055 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1056 "drop action not supported for " 1057 "egress"); 1058 return 0; 1059 } 1060 1061 /* 1062 * Validate the queue action. 1063 * 1064 * @param[in] action 1065 * Pointer to the queue action. 1066 * @param[in] action_flags 1067 * Bit-fields that holds the actions detected until now. 1068 * @param[in] dev 1069 * Pointer to the Ethernet device structure. 1070 * @param[in] attr 1071 * Attributes of flow that includes this action. 1072 * @param[out] error 1073 * Pointer to error structure. 1074 * 1075 * @return 1076 * 0 on success, a negative errno value otherwise and rte_errno is set. 1077 */ 1078 int 1079 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1080 uint64_t action_flags, 1081 struct rte_eth_dev *dev, 1082 const struct rte_flow_attr *attr, 1083 struct rte_flow_error *error) 1084 { 1085 struct mlx5_priv *priv = dev->data->dev_private; 1086 const struct rte_flow_action_queue *queue = action->conf; 1087 1088 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1089 return rte_flow_error_set(error, EINVAL, 1090 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1091 "can't have 2 fate actions in" 1092 " same flow"); 1093 if (!priv->rxqs_n) 1094 return rte_flow_error_set(error, EINVAL, 1095 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1096 NULL, "No Rx queues configured"); 1097 if (queue->index >= priv->rxqs_n) 1098 return rte_flow_error_set(error, EINVAL, 1099 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1100 &queue->index, 1101 "queue index out of range"); 1102 if (!(*priv->rxqs)[queue->index]) 1103 return rte_flow_error_set(error, EINVAL, 1104 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1105 &queue->index, 1106 "queue is not configured"); 1107 if (attr->egress) 1108 return rte_flow_error_set(error, ENOTSUP, 1109 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1110 "queue action not supported for " 1111 "egress"); 1112 return 0; 1113 } 1114 1115 /* 1116 * Validate the rss action. 1117 * 1118 * @param[in] action 1119 * Pointer to the queue action. 1120 * @param[in] action_flags 1121 * Bit-fields that holds the actions detected until now. 1122 * @param[in] dev 1123 * Pointer to the Ethernet device structure. 1124 * @param[in] attr 1125 * Attributes of flow that includes this action. 1126 * @param[in] item_flags 1127 * Items that were detected. 1128 * @param[out] error 1129 * Pointer to error structure. 1130 * 1131 * @return 1132 * 0 on success, a negative errno value otherwise and rte_errno is set. 1133 */ 1134 int 1135 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1136 uint64_t action_flags, 1137 struct rte_eth_dev *dev, 1138 const struct rte_flow_attr *attr, 1139 uint64_t item_flags, 1140 struct rte_flow_error *error) 1141 { 1142 struct mlx5_priv *priv = dev->data->dev_private; 1143 const struct rte_flow_action_rss *rss = action->conf; 1144 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1145 unsigned int i; 1146 1147 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1148 return rte_flow_error_set(error, EINVAL, 1149 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1150 "can't have 2 fate actions" 1151 " in same flow"); 1152 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1153 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1154 return rte_flow_error_set(error, ENOTSUP, 1155 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1156 &rss->func, 1157 "RSS hash function not supported"); 1158 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1159 if (rss->level > 2) 1160 #else 1161 if (rss->level > 1) 1162 #endif 1163 return rte_flow_error_set(error, ENOTSUP, 1164 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1165 &rss->level, 1166 "tunnel RSS is not supported"); 1167 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1168 if (rss->key_len == 0 && rss->key != NULL) 1169 return rte_flow_error_set(error, ENOTSUP, 1170 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1171 &rss->key_len, 1172 "RSS hash key length 0"); 1173 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1174 return rte_flow_error_set(error, ENOTSUP, 1175 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1176 &rss->key_len, 1177 "RSS hash key too small"); 1178 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1179 return rte_flow_error_set(error, ENOTSUP, 1180 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1181 &rss->key_len, 1182 "RSS hash key too large"); 1183 if (rss->queue_num > priv->config.ind_table_max_size) 1184 return rte_flow_error_set(error, ENOTSUP, 1185 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1186 &rss->queue_num, 1187 "number of queues too large"); 1188 if (rss->types & MLX5_RSS_HF_MASK) 1189 return rte_flow_error_set(error, ENOTSUP, 1190 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1191 &rss->types, 1192 "some RSS protocols are not" 1193 " supported"); 1194 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1195 !(rss->types & ETH_RSS_IP)) 1196 return rte_flow_error_set(error, EINVAL, 1197 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1198 "L3 partial RSS requested but L3 RSS" 1199 " type not specified"); 1200 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1201 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1202 return rte_flow_error_set(error, EINVAL, 1203 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1204 "L4 partial RSS requested but L4 RSS" 1205 " type not specified"); 1206 if (!priv->rxqs_n) 1207 return rte_flow_error_set(error, EINVAL, 1208 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1209 NULL, "No Rx queues configured"); 1210 if (!rss->queue_num) 1211 return rte_flow_error_set(error, EINVAL, 1212 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1213 NULL, "No queues configured"); 1214 for (i = 0; i != rss->queue_num; ++i) { 1215 if (rss->queue[i] >= priv->rxqs_n) 1216 return rte_flow_error_set 1217 (error, EINVAL, 1218 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1219 &rss->queue[i], "queue index out of range"); 1220 if (!(*priv->rxqs)[rss->queue[i]]) 1221 return rte_flow_error_set 1222 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1223 &rss->queue[i], "queue is not configured"); 1224 } 1225 if (attr->egress) 1226 return rte_flow_error_set(error, ENOTSUP, 1227 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1228 "rss action not supported for " 1229 "egress"); 1230 if (rss->level > 1 && !tunnel) 1231 return rte_flow_error_set(error, EINVAL, 1232 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1233 "inner RSS is not supported for " 1234 "non-tunnel flows"); 1235 return 0; 1236 } 1237 1238 /* 1239 * Validate the count action. 1240 * 1241 * @param[in] dev 1242 * Pointer to the Ethernet device structure. 1243 * @param[in] attr 1244 * Attributes of flow that includes this action. 1245 * @param[out] error 1246 * Pointer to error structure. 1247 * 1248 * @return 1249 * 0 on success, a negative errno value otherwise and rte_errno is set. 1250 */ 1251 int 1252 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1253 const struct rte_flow_attr *attr, 1254 struct rte_flow_error *error) 1255 { 1256 if (attr->egress) 1257 return rte_flow_error_set(error, ENOTSUP, 1258 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1259 "count action not supported for " 1260 "egress"); 1261 return 0; 1262 } 1263 1264 /** 1265 * Verify the @p attributes will be correctly understood by the NIC and store 1266 * them in the @p flow if everything is correct. 1267 * 1268 * @param[in] dev 1269 * Pointer to the Ethernet device structure. 1270 * @param[in] attributes 1271 * Pointer to flow attributes 1272 * @param[out] error 1273 * Pointer to error structure. 1274 * 1275 * @return 1276 * 0 on success, a negative errno value otherwise and rte_errno is set. 1277 */ 1278 int 1279 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1280 const struct rte_flow_attr *attributes, 1281 struct rte_flow_error *error) 1282 { 1283 struct mlx5_priv *priv = dev->data->dev_private; 1284 uint32_t priority_max = priv->config.flow_prio - 1; 1285 1286 if (attributes->group) 1287 return rte_flow_error_set(error, ENOTSUP, 1288 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1289 NULL, "groups is not supported"); 1290 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1291 attributes->priority >= priority_max) 1292 return rte_flow_error_set(error, ENOTSUP, 1293 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1294 NULL, "priority out of range"); 1295 if (attributes->egress) 1296 return rte_flow_error_set(error, ENOTSUP, 1297 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1298 "egress is not supported"); 1299 if (attributes->transfer && !priv->config.dv_esw_en) 1300 return rte_flow_error_set(error, ENOTSUP, 1301 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1302 NULL, "transfer is not supported"); 1303 if (!attributes->ingress) 1304 return rte_flow_error_set(error, EINVAL, 1305 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1306 NULL, 1307 "ingress attribute is mandatory"); 1308 return 0; 1309 } 1310 1311 /** 1312 * Validate ICMP6 item. 1313 * 1314 * @param[in] item 1315 * Item specification. 1316 * @param[in] item_flags 1317 * Bit-fields that holds the items detected until now. 1318 * @param[out] error 1319 * Pointer to error structure. 1320 * 1321 * @return 1322 * 0 on success, a negative errno value otherwise and rte_errno is set. 1323 */ 1324 int 1325 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1326 uint64_t item_flags, 1327 uint8_t target_protocol, 1328 struct rte_flow_error *error) 1329 { 1330 const struct rte_flow_item_icmp6 *mask = item->mask; 1331 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1332 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1333 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1334 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1335 MLX5_FLOW_LAYER_OUTER_L4; 1336 int ret; 1337 1338 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1339 return rte_flow_error_set(error, EINVAL, 1340 RTE_FLOW_ERROR_TYPE_ITEM, item, 1341 "protocol filtering not compatible" 1342 " with ICMP6 layer"); 1343 if (!(item_flags & l3m)) 1344 return rte_flow_error_set(error, EINVAL, 1345 RTE_FLOW_ERROR_TYPE_ITEM, item, 1346 "IPv6 is mandatory to filter on" 1347 " ICMP6"); 1348 if (item_flags & l4m) 1349 return rte_flow_error_set(error, EINVAL, 1350 RTE_FLOW_ERROR_TYPE_ITEM, item, 1351 "multiple L4 layers not supported"); 1352 if (!mask) 1353 mask = &rte_flow_item_icmp6_mask; 1354 ret = mlx5_flow_item_acceptable 1355 (item, (const uint8_t *)mask, 1356 (const uint8_t *)&rte_flow_item_icmp6_mask, 1357 sizeof(struct rte_flow_item_icmp6), error); 1358 if (ret < 0) 1359 return ret; 1360 return 0; 1361 } 1362 1363 /** 1364 * Validate ICMP item. 1365 * 1366 * @param[in] item 1367 * Item specification. 1368 * @param[in] item_flags 1369 * Bit-fields that holds the items detected until now. 1370 * @param[out] error 1371 * Pointer to error structure. 1372 * 1373 * @return 1374 * 0 on success, a negative errno value otherwise and rte_errno is set. 1375 */ 1376 int 1377 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1378 uint64_t item_flags, 1379 uint8_t target_protocol, 1380 struct rte_flow_error *error) 1381 { 1382 const struct rte_flow_item_icmp *mask = item->mask; 1383 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1384 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1385 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1386 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1387 MLX5_FLOW_LAYER_OUTER_L4; 1388 int ret; 1389 1390 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1391 return rte_flow_error_set(error, EINVAL, 1392 RTE_FLOW_ERROR_TYPE_ITEM, item, 1393 "protocol filtering not compatible" 1394 " with ICMP layer"); 1395 if (!(item_flags & l3m)) 1396 return rte_flow_error_set(error, EINVAL, 1397 RTE_FLOW_ERROR_TYPE_ITEM, item, 1398 "IPv4 is mandatory to filter" 1399 " on ICMP"); 1400 if (item_flags & l4m) 1401 return rte_flow_error_set(error, EINVAL, 1402 RTE_FLOW_ERROR_TYPE_ITEM, item, 1403 "multiple L4 layers not supported"); 1404 if (!mask) 1405 mask = &rte_flow_item_icmp_mask; 1406 ret = mlx5_flow_item_acceptable 1407 (item, (const uint8_t *)mask, 1408 (const uint8_t *)&rte_flow_item_icmp_mask, 1409 sizeof(struct rte_flow_item_icmp), error); 1410 if (ret < 0) 1411 return ret; 1412 return 0; 1413 } 1414 1415 /** 1416 * Validate Ethernet item. 1417 * 1418 * @param[in] item 1419 * Item specification. 1420 * @param[in] item_flags 1421 * Bit-fields that holds the items detected until now. 1422 * @param[out] error 1423 * Pointer to error structure. 1424 * 1425 * @return 1426 * 0 on success, a negative errno value otherwise and rte_errno is set. 1427 */ 1428 int 1429 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1430 uint64_t item_flags, 1431 struct rte_flow_error *error) 1432 { 1433 const struct rte_flow_item_eth *mask = item->mask; 1434 const struct rte_flow_item_eth nic_mask = { 1435 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1436 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1437 .type = RTE_BE16(0xffff), 1438 }; 1439 int ret; 1440 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1441 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1442 MLX5_FLOW_LAYER_OUTER_L2; 1443 1444 if (item_flags & ethm) 1445 return rte_flow_error_set(error, ENOTSUP, 1446 RTE_FLOW_ERROR_TYPE_ITEM, item, 1447 "multiple L2 layers not supported"); 1448 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1449 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1450 return rte_flow_error_set(error, EINVAL, 1451 RTE_FLOW_ERROR_TYPE_ITEM, item, 1452 "L2 layer should not follow " 1453 "L3 layers"); 1454 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1455 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1456 return rte_flow_error_set(error, EINVAL, 1457 RTE_FLOW_ERROR_TYPE_ITEM, item, 1458 "L2 layer should not follow VLAN"); 1459 if (!mask) 1460 mask = &rte_flow_item_eth_mask; 1461 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1462 (const uint8_t *)&nic_mask, 1463 sizeof(struct rte_flow_item_eth), 1464 error); 1465 return ret; 1466 } 1467 1468 /** 1469 * Validate VLAN item. 1470 * 1471 * @param[in] item 1472 * Item specification. 1473 * @param[in] item_flags 1474 * Bit-fields that holds the items detected until now. 1475 * @param[in] dev 1476 * Ethernet device flow is being created on. 1477 * @param[out] error 1478 * Pointer to error structure. 1479 * 1480 * @return 1481 * 0 on success, a negative errno value otherwise and rte_errno is set. 1482 */ 1483 int 1484 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1485 uint64_t item_flags, 1486 struct rte_eth_dev *dev, 1487 struct rte_flow_error *error) 1488 { 1489 const struct rte_flow_item_vlan *spec = item->spec; 1490 const struct rte_flow_item_vlan *mask = item->mask; 1491 const struct rte_flow_item_vlan nic_mask = { 1492 .tci = RTE_BE16(UINT16_MAX), 1493 .inner_type = RTE_BE16(UINT16_MAX), 1494 }; 1495 uint16_t vlan_tag = 0; 1496 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1497 int ret; 1498 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1499 MLX5_FLOW_LAYER_INNER_L4) : 1500 (MLX5_FLOW_LAYER_OUTER_L3 | 1501 MLX5_FLOW_LAYER_OUTER_L4); 1502 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1503 MLX5_FLOW_LAYER_OUTER_VLAN; 1504 1505 if (item_flags & vlanm) 1506 return rte_flow_error_set(error, EINVAL, 1507 RTE_FLOW_ERROR_TYPE_ITEM, item, 1508 "multiple VLAN layers not supported"); 1509 else if ((item_flags & l34m) != 0) 1510 return rte_flow_error_set(error, EINVAL, 1511 RTE_FLOW_ERROR_TYPE_ITEM, item, 1512 "VLAN cannot follow L3/L4 layer"); 1513 if (!mask) 1514 mask = &rte_flow_item_vlan_mask; 1515 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1516 (const uint8_t *)&nic_mask, 1517 sizeof(struct rte_flow_item_vlan), 1518 error); 1519 if (ret) 1520 return ret; 1521 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1522 struct mlx5_priv *priv = dev->data->dev_private; 1523 1524 if (priv->vmwa_context) { 1525 /* 1526 * Non-NULL context means we have a virtual machine 1527 * and SR-IOV enabled, we have to create VLAN interface 1528 * to make hypervisor to setup E-Switch vport 1529 * context correctly. We avoid creating the multiple 1530 * VLAN interfaces, so we cannot support VLAN tag mask. 1531 */ 1532 return rte_flow_error_set(error, EINVAL, 1533 RTE_FLOW_ERROR_TYPE_ITEM, 1534 item, 1535 "VLAN tag mask is not" 1536 " supported in virtual" 1537 " environment"); 1538 } 1539 } 1540 if (spec) { 1541 vlan_tag = spec->tci; 1542 vlan_tag &= mask->tci; 1543 } 1544 /* 1545 * From verbs perspective an empty VLAN is equivalent 1546 * to a packet without VLAN layer. 1547 */ 1548 if (!vlan_tag) 1549 return rte_flow_error_set(error, EINVAL, 1550 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1551 item->spec, 1552 "VLAN cannot be empty"); 1553 return 0; 1554 } 1555 1556 /** 1557 * Validate IPV4 item. 1558 * 1559 * @param[in] item 1560 * Item specification. 1561 * @param[in] item_flags 1562 * Bit-fields that holds the items detected until now. 1563 * @param[in] acc_mask 1564 * Acceptable mask, if NULL default internal default mask 1565 * will be used to check whether item fields are supported. 1566 * @param[out] error 1567 * Pointer to error structure. 1568 * 1569 * @return 1570 * 0 on success, a negative errno value otherwise and rte_errno is set. 1571 */ 1572 int 1573 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1574 uint64_t item_flags, 1575 uint64_t last_item, 1576 uint16_t ether_type, 1577 const struct rte_flow_item_ipv4 *acc_mask, 1578 struct rte_flow_error *error) 1579 { 1580 const struct rte_flow_item_ipv4 *mask = item->mask; 1581 const struct rte_flow_item_ipv4 *spec = item->spec; 1582 const struct rte_flow_item_ipv4 nic_mask = { 1583 .hdr = { 1584 .src_addr = RTE_BE32(0xffffffff), 1585 .dst_addr = RTE_BE32(0xffffffff), 1586 .type_of_service = 0xff, 1587 .next_proto_id = 0xff, 1588 }, 1589 }; 1590 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1591 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1592 MLX5_FLOW_LAYER_OUTER_L3; 1593 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1594 MLX5_FLOW_LAYER_OUTER_L4; 1595 int ret; 1596 uint8_t next_proto = 0xFF; 1597 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1598 MLX5_FLOW_LAYER_OUTER_VLAN | 1599 MLX5_FLOW_LAYER_INNER_VLAN); 1600 1601 if ((last_item & l2_vlan) && ether_type && 1602 ether_type != RTE_ETHER_TYPE_IPV4) 1603 return rte_flow_error_set(error, EINVAL, 1604 RTE_FLOW_ERROR_TYPE_ITEM, item, 1605 "IPv4 cannot follow L2/VLAN layer " 1606 "which ether type is not IPv4"); 1607 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1608 if (mask && spec) 1609 next_proto = mask->hdr.next_proto_id & 1610 spec->hdr.next_proto_id; 1611 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1612 return rte_flow_error_set(error, EINVAL, 1613 RTE_FLOW_ERROR_TYPE_ITEM, 1614 item, 1615 "multiple tunnel " 1616 "not supported"); 1617 } 1618 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1619 return rte_flow_error_set(error, EINVAL, 1620 RTE_FLOW_ERROR_TYPE_ITEM, item, 1621 "wrong tunnel type - IPv6 specified " 1622 "but IPv4 item provided"); 1623 if (item_flags & l3m) 1624 return rte_flow_error_set(error, ENOTSUP, 1625 RTE_FLOW_ERROR_TYPE_ITEM, item, 1626 "multiple L3 layers not supported"); 1627 else if (item_flags & l4m) 1628 return rte_flow_error_set(error, EINVAL, 1629 RTE_FLOW_ERROR_TYPE_ITEM, item, 1630 "L3 cannot follow an L4 layer."); 1631 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1632 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1633 return rte_flow_error_set(error, EINVAL, 1634 RTE_FLOW_ERROR_TYPE_ITEM, item, 1635 "L3 cannot follow an NVGRE layer."); 1636 if (!mask) 1637 mask = &rte_flow_item_ipv4_mask; 1638 else if (mask->hdr.next_proto_id != 0 && 1639 mask->hdr.next_proto_id != 0xff) 1640 return rte_flow_error_set(error, EINVAL, 1641 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1642 "partial mask is not supported" 1643 " for protocol"); 1644 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1645 acc_mask ? (const uint8_t *)acc_mask 1646 : (const uint8_t *)&nic_mask, 1647 sizeof(struct rte_flow_item_ipv4), 1648 error); 1649 if (ret < 0) 1650 return ret; 1651 return 0; 1652 } 1653 1654 /** 1655 * Validate IPV6 item. 1656 * 1657 * @param[in] item 1658 * Item specification. 1659 * @param[in] item_flags 1660 * Bit-fields that holds the items detected until now. 1661 * @param[in] acc_mask 1662 * Acceptable mask, if NULL default internal default mask 1663 * will be used to check whether item fields are supported. 1664 * @param[out] error 1665 * Pointer to error structure. 1666 * 1667 * @return 1668 * 0 on success, a negative errno value otherwise and rte_errno is set. 1669 */ 1670 int 1671 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1672 uint64_t item_flags, 1673 uint64_t last_item, 1674 uint16_t ether_type, 1675 const struct rte_flow_item_ipv6 *acc_mask, 1676 struct rte_flow_error *error) 1677 { 1678 const struct rte_flow_item_ipv6 *mask = item->mask; 1679 const struct rte_flow_item_ipv6 *spec = item->spec; 1680 const struct rte_flow_item_ipv6 nic_mask = { 1681 .hdr = { 1682 .src_addr = 1683 "\xff\xff\xff\xff\xff\xff\xff\xff" 1684 "\xff\xff\xff\xff\xff\xff\xff\xff", 1685 .dst_addr = 1686 "\xff\xff\xff\xff\xff\xff\xff\xff" 1687 "\xff\xff\xff\xff\xff\xff\xff\xff", 1688 .vtc_flow = RTE_BE32(0xffffffff), 1689 .proto = 0xff, 1690 }, 1691 }; 1692 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1693 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1694 MLX5_FLOW_LAYER_OUTER_L3; 1695 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1696 MLX5_FLOW_LAYER_OUTER_L4; 1697 int ret; 1698 uint8_t next_proto = 0xFF; 1699 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1700 MLX5_FLOW_LAYER_OUTER_VLAN | 1701 MLX5_FLOW_LAYER_INNER_VLAN); 1702 1703 if ((last_item & l2_vlan) && ether_type && 1704 ether_type != RTE_ETHER_TYPE_IPV6) 1705 return rte_flow_error_set(error, EINVAL, 1706 RTE_FLOW_ERROR_TYPE_ITEM, item, 1707 "IPv6 cannot follow L2/VLAN layer " 1708 "which ether type is not IPv6"); 1709 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1710 if (mask && spec) 1711 next_proto = mask->hdr.proto & spec->hdr.proto; 1712 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1713 return rte_flow_error_set(error, EINVAL, 1714 RTE_FLOW_ERROR_TYPE_ITEM, 1715 item, 1716 "multiple tunnel " 1717 "not supported"); 1718 } 1719 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1720 return rte_flow_error_set(error, EINVAL, 1721 RTE_FLOW_ERROR_TYPE_ITEM, item, 1722 "wrong tunnel type - IPv4 specified " 1723 "but IPv6 item provided"); 1724 if (item_flags & l3m) 1725 return rte_flow_error_set(error, ENOTSUP, 1726 RTE_FLOW_ERROR_TYPE_ITEM, item, 1727 "multiple L3 layers not supported"); 1728 else if (item_flags & l4m) 1729 return rte_flow_error_set(error, EINVAL, 1730 RTE_FLOW_ERROR_TYPE_ITEM, item, 1731 "L3 cannot follow an L4 layer."); 1732 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1733 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1734 return rte_flow_error_set(error, EINVAL, 1735 RTE_FLOW_ERROR_TYPE_ITEM, item, 1736 "L3 cannot follow an NVGRE layer."); 1737 if (!mask) 1738 mask = &rte_flow_item_ipv6_mask; 1739 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1740 acc_mask ? (const uint8_t *)acc_mask 1741 : (const uint8_t *)&nic_mask, 1742 sizeof(struct rte_flow_item_ipv6), 1743 error); 1744 if (ret < 0) 1745 return ret; 1746 return 0; 1747 } 1748 1749 /** 1750 * Validate UDP item. 1751 * 1752 * @param[in] item 1753 * Item specification. 1754 * @param[in] item_flags 1755 * Bit-fields that holds the items detected until now. 1756 * @param[in] target_protocol 1757 * The next protocol in the previous item. 1758 * @param[in] flow_mask 1759 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1760 * @param[out] error 1761 * Pointer to error structure. 1762 * 1763 * @return 1764 * 0 on success, a negative errno value otherwise and rte_errno is set. 1765 */ 1766 int 1767 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1768 uint64_t item_flags, 1769 uint8_t target_protocol, 1770 struct rte_flow_error *error) 1771 { 1772 const struct rte_flow_item_udp *mask = item->mask; 1773 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1774 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1775 MLX5_FLOW_LAYER_OUTER_L3; 1776 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1777 MLX5_FLOW_LAYER_OUTER_L4; 1778 int ret; 1779 1780 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1781 return rte_flow_error_set(error, EINVAL, 1782 RTE_FLOW_ERROR_TYPE_ITEM, item, 1783 "protocol filtering not compatible" 1784 " with UDP layer"); 1785 if (!(item_flags & l3m)) 1786 return rte_flow_error_set(error, EINVAL, 1787 RTE_FLOW_ERROR_TYPE_ITEM, item, 1788 "L3 is mandatory to filter on L4"); 1789 if (item_flags & l4m) 1790 return rte_flow_error_set(error, EINVAL, 1791 RTE_FLOW_ERROR_TYPE_ITEM, item, 1792 "multiple L4 layers not supported"); 1793 if (!mask) 1794 mask = &rte_flow_item_udp_mask; 1795 ret = mlx5_flow_item_acceptable 1796 (item, (const uint8_t *)mask, 1797 (const uint8_t *)&rte_flow_item_udp_mask, 1798 sizeof(struct rte_flow_item_udp), error); 1799 if (ret < 0) 1800 return ret; 1801 return 0; 1802 } 1803 1804 /** 1805 * Validate TCP item. 1806 * 1807 * @param[in] item 1808 * Item specification. 1809 * @param[in] item_flags 1810 * Bit-fields that holds the items detected until now. 1811 * @param[in] target_protocol 1812 * The next protocol in the previous item. 1813 * @param[out] error 1814 * Pointer to error structure. 1815 * 1816 * @return 1817 * 0 on success, a negative errno value otherwise and rte_errno is set. 1818 */ 1819 int 1820 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1821 uint64_t item_flags, 1822 uint8_t target_protocol, 1823 const struct rte_flow_item_tcp *flow_mask, 1824 struct rte_flow_error *error) 1825 { 1826 const struct rte_flow_item_tcp *mask = item->mask; 1827 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1828 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1829 MLX5_FLOW_LAYER_OUTER_L3; 1830 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1831 MLX5_FLOW_LAYER_OUTER_L4; 1832 int ret; 1833 1834 MLX5_ASSERT(flow_mask); 1835 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1836 return rte_flow_error_set(error, EINVAL, 1837 RTE_FLOW_ERROR_TYPE_ITEM, item, 1838 "protocol filtering not compatible" 1839 " with TCP layer"); 1840 if (!(item_flags & l3m)) 1841 return rte_flow_error_set(error, EINVAL, 1842 RTE_FLOW_ERROR_TYPE_ITEM, item, 1843 "L3 is mandatory to filter on L4"); 1844 if (item_flags & l4m) 1845 return rte_flow_error_set(error, EINVAL, 1846 RTE_FLOW_ERROR_TYPE_ITEM, item, 1847 "multiple L4 layers not supported"); 1848 if (!mask) 1849 mask = &rte_flow_item_tcp_mask; 1850 ret = mlx5_flow_item_acceptable 1851 (item, (const uint8_t *)mask, 1852 (const uint8_t *)flow_mask, 1853 sizeof(struct rte_flow_item_tcp), error); 1854 if (ret < 0) 1855 return ret; 1856 return 0; 1857 } 1858 1859 /** 1860 * Validate VXLAN item. 1861 * 1862 * @param[in] item 1863 * Item specification. 1864 * @param[in] item_flags 1865 * Bit-fields that holds the items detected until now. 1866 * @param[in] target_protocol 1867 * The next protocol in the previous item. 1868 * @param[out] error 1869 * Pointer to error structure. 1870 * 1871 * @return 1872 * 0 on success, a negative errno value otherwise and rte_errno is set. 1873 */ 1874 int 1875 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1876 uint64_t item_flags, 1877 struct rte_flow_error *error) 1878 { 1879 const struct rte_flow_item_vxlan *spec = item->spec; 1880 const struct rte_flow_item_vxlan *mask = item->mask; 1881 int ret; 1882 union vni { 1883 uint32_t vlan_id; 1884 uint8_t vni[4]; 1885 } id = { .vlan_id = 0, }; 1886 1887 1888 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1889 return rte_flow_error_set(error, ENOTSUP, 1890 RTE_FLOW_ERROR_TYPE_ITEM, item, 1891 "multiple tunnel layers not" 1892 " supported"); 1893 /* 1894 * Verify only UDPv4 is present as defined in 1895 * https://tools.ietf.org/html/rfc7348 1896 */ 1897 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1898 return rte_flow_error_set(error, EINVAL, 1899 RTE_FLOW_ERROR_TYPE_ITEM, item, 1900 "no outer UDP layer found"); 1901 if (!mask) 1902 mask = &rte_flow_item_vxlan_mask; 1903 ret = mlx5_flow_item_acceptable 1904 (item, (const uint8_t *)mask, 1905 (const uint8_t *)&rte_flow_item_vxlan_mask, 1906 sizeof(struct rte_flow_item_vxlan), 1907 error); 1908 if (ret < 0) 1909 return ret; 1910 if (spec) { 1911 memcpy(&id.vni[1], spec->vni, 3); 1912 memcpy(&id.vni[1], mask->vni, 3); 1913 } 1914 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1915 return rte_flow_error_set(error, ENOTSUP, 1916 RTE_FLOW_ERROR_TYPE_ITEM, item, 1917 "VXLAN tunnel must be fully defined"); 1918 return 0; 1919 } 1920 1921 /** 1922 * Validate VXLAN_GPE item. 1923 * 1924 * @param[in] item 1925 * Item specification. 1926 * @param[in] item_flags 1927 * Bit-fields that holds the items detected until now. 1928 * @param[in] priv 1929 * Pointer to the private data structure. 1930 * @param[in] target_protocol 1931 * The next protocol in the previous item. 1932 * @param[out] error 1933 * Pointer to error structure. 1934 * 1935 * @return 1936 * 0 on success, a negative errno value otherwise and rte_errno is set. 1937 */ 1938 int 1939 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1940 uint64_t item_flags, 1941 struct rte_eth_dev *dev, 1942 struct rte_flow_error *error) 1943 { 1944 struct mlx5_priv *priv = dev->data->dev_private; 1945 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1946 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1947 int ret; 1948 union vni { 1949 uint32_t vlan_id; 1950 uint8_t vni[4]; 1951 } id = { .vlan_id = 0, }; 1952 1953 if (!priv->config.l3_vxlan_en) 1954 return rte_flow_error_set(error, ENOTSUP, 1955 RTE_FLOW_ERROR_TYPE_ITEM, item, 1956 "L3 VXLAN is not enabled by device" 1957 " parameter and/or not configured in" 1958 " firmware"); 1959 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1960 return rte_flow_error_set(error, ENOTSUP, 1961 RTE_FLOW_ERROR_TYPE_ITEM, item, 1962 "multiple tunnel layers not" 1963 " supported"); 1964 /* 1965 * Verify only UDPv4 is present as defined in 1966 * https://tools.ietf.org/html/rfc7348 1967 */ 1968 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1969 return rte_flow_error_set(error, EINVAL, 1970 RTE_FLOW_ERROR_TYPE_ITEM, item, 1971 "no outer UDP layer found"); 1972 if (!mask) 1973 mask = &rte_flow_item_vxlan_gpe_mask; 1974 ret = mlx5_flow_item_acceptable 1975 (item, (const uint8_t *)mask, 1976 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1977 sizeof(struct rte_flow_item_vxlan_gpe), 1978 error); 1979 if (ret < 0) 1980 return ret; 1981 if (spec) { 1982 if (spec->protocol) 1983 return rte_flow_error_set(error, ENOTSUP, 1984 RTE_FLOW_ERROR_TYPE_ITEM, 1985 item, 1986 "VxLAN-GPE protocol" 1987 " not supported"); 1988 memcpy(&id.vni[1], spec->vni, 3); 1989 memcpy(&id.vni[1], mask->vni, 3); 1990 } 1991 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1992 return rte_flow_error_set(error, ENOTSUP, 1993 RTE_FLOW_ERROR_TYPE_ITEM, item, 1994 "VXLAN-GPE tunnel must be fully" 1995 " defined"); 1996 return 0; 1997 } 1998 /** 1999 * Validate GRE Key item. 2000 * 2001 * @param[in] item 2002 * Item specification. 2003 * @param[in] item_flags 2004 * Bit flags to mark detected items. 2005 * @param[in] gre_item 2006 * Pointer to gre_item 2007 * @param[out] error 2008 * Pointer to error structure. 2009 * 2010 * @return 2011 * 0 on success, a negative errno value otherwise and rte_errno is set. 2012 */ 2013 int 2014 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2015 uint64_t item_flags, 2016 const struct rte_flow_item *gre_item, 2017 struct rte_flow_error *error) 2018 { 2019 const rte_be32_t *mask = item->mask; 2020 int ret = 0; 2021 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2022 const struct rte_flow_item_gre *gre_spec; 2023 const struct rte_flow_item_gre *gre_mask; 2024 2025 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2026 return rte_flow_error_set(error, ENOTSUP, 2027 RTE_FLOW_ERROR_TYPE_ITEM, item, 2028 "Multiple GRE key not support"); 2029 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2030 return rte_flow_error_set(error, ENOTSUP, 2031 RTE_FLOW_ERROR_TYPE_ITEM, item, 2032 "No preceding GRE header"); 2033 if (item_flags & MLX5_FLOW_LAYER_INNER) 2034 return rte_flow_error_set(error, ENOTSUP, 2035 RTE_FLOW_ERROR_TYPE_ITEM, item, 2036 "GRE key following a wrong item"); 2037 gre_mask = gre_item->mask; 2038 if (!gre_mask) 2039 gre_mask = &rte_flow_item_gre_mask; 2040 gre_spec = gre_item->spec; 2041 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2042 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2043 return rte_flow_error_set(error, EINVAL, 2044 RTE_FLOW_ERROR_TYPE_ITEM, item, 2045 "Key bit must be on"); 2046 2047 if (!mask) 2048 mask = &gre_key_default_mask; 2049 ret = mlx5_flow_item_acceptable 2050 (item, (const uint8_t *)mask, 2051 (const uint8_t *)&gre_key_default_mask, 2052 sizeof(rte_be32_t), error); 2053 return ret; 2054 } 2055 2056 /** 2057 * Validate GRE item. 2058 * 2059 * @param[in] item 2060 * Item specification. 2061 * @param[in] item_flags 2062 * Bit flags to mark detected items. 2063 * @param[in] target_protocol 2064 * The next protocol in the previous item. 2065 * @param[out] error 2066 * Pointer to error structure. 2067 * 2068 * @return 2069 * 0 on success, a negative errno value otherwise and rte_errno is set. 2070 */ 2071 int 2072 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2073 uint64_t item_flags, 2074 uint8_t target_protocol, 2075 struct rte_flow_error *error) 2076 { 2077 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2078 const struct rte_flow_item_gre *mask = item->mask; 2079 int ret; 2080 const struct rte_flow_item_gre nic_mask = { 2081 .c_rsvd0_ver = RTE_BE16(0xB000), 2082 .protocol = RTE_BE16(UINT16_MAX), 2083 }; 2084 2085 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2086 return rte_flow_error_set(error, EINVAL, 2087 RTE_FLOW_ERROR_TYPE_ITEM, item, 2088 "protocol filtering not compatible" 2089 " with this GRE layer"); 2090 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2091 return rte_flow_error_set(error, ENOTSUP, 2092 RTE_FLOW_ERROR_TYPE_ITEM, item, 2093 "multiple tunnel layers not" 2094 " supported"); 2095 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2096 return rte_flow_error_set(error, ENOTSUP, 2097 RTE_FLOW_ERROR_TYPE_ITEM, item, 2098 "L3 Layer is missing"); 2099 if (!mask) 2100 mask = &rte_flow_item_gre_mask; 2101 ret = mlx5_flow_item_acceptable 2102 (item, (const uint8_t *)mask, 2103 (const uint8_t *)&nic_mask, 2104 sizeof(struct rte_flow_item_gre), error); 2105 if (ret < 0) 2106 return ret; 2107 #ifndef HAVE_MLX5DV_DR 2108 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2109 if (spec && (spec->protocol & mask->protocol)) 2110 return rte_flow_error_set(error, ENOTSUP, 2111 RTE_FLOW_ERROR_TYPE_ITEM, item, 2112 "without MPLS support the" 2113 " specification cannot be used for" 2114 " filtering"); 2115 #endif 2116 #endif 2117 return 0; 2118 } 2119 2120 /** 2121 * Validate Geneve item. 2122 * 2123 * @param[in] item 2124 * Item specification. 2125 * @param[in] itemFlags 2126 * Bit-fields that holds the items detected until now. 2127 * @param[in] enPriv 2128 * Pointer to the private data structure. 2129 * @param[out] error 2130 * Pointer to error structure. 2131 * 2132 * @return 2133 * 0 on success, a negative errno value otherwise and rte_errno is set. 2134 */ 2135 2136 int 2137 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2138 uint64_t item_flags, 2139 struct rte_eth_dev *dev, 2140 struct rte_flow_error *error) 2141 { 2142 struct mlx5_priv *priv = dev->data->dev_private; 2143 const struct rte_flow_item_geneve *spec = item->spec; 2144 const struct rte_flow_item_geneve *mask = item->mask; 2145 int ret; 2146 uint16_t gbhdr; 2147 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2148 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2149 const struct rte_flow_item_geneve nic_mask = { 2150 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2151 .vni = "\xff\xff\xff", 2152 .protocol = RTE_BE16(UINT16_MAX), 2153 }; 2154 2155 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2156 return rte_flow_error_set(error, ENOTSUP, 2157 RTE_FLOW_ERROR_TYPE_ITEM, item, 2158 "L3 Geneve is not enabled by device" 2159 " parameter and/or not configured in" 2160 " firmware"); 2161 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2162 return rte_flow_error_set(error, ENOTSUP, 2163 RTE_FLOW_ERROR_TYPE_ITEM, item, 2164 "multiple tunnel layers not" 2165 " supported"); 2166 /* 2167 * Verify only UDPv4 is present as defined in 2168 * https://tools.ietf.org/html/rfc7348 2169 */ 2170 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2171 return rte_flow_error_set(error, EINVAL, 2172 RTE_FLOW_ERROR_TYPE_ITEM, item, 2173 "no outer UDP layer found"); 2174 if (!mask) 2175 mask = &rte_flow_item_geneve_mask; 2176 ret = mlx5_flow_item_acceptable 2177 (item, (const uint8_t *)mask, 2178 (const uint8_t *)&nic_mask, 2179 sizeof(struct rte_flow_item_geneve), error); 2180 if (ret) 2181 return ret; 2182 if (spec) { 2183 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2184 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2185 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2186 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2187 return rte_flow_error_set(error, ENOTSUP, 2188 RTE_FLOW_ERROR_TYPE_ITEM, 2189 item, 2190 "Geneve protocol unsupported" 2191 " fields are being used"); 2192 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2193 return rte_flow_error_set 2194 (error, ENOTSUP, 2195 RTE_FLOW_ERROR_TYPE_ITEM, 2196 item, 2197 "Unsupported Geneve options length"); 2198 } 2199 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2200 return rte_flow_error_set 2201 (error, ENOTSUP, 2202 RTE_FLOW_ERROR_TYPE_ITEM, item, 2203 "Geneve tunnel must be fully defined"); 2204 return 0; 2205 } 2206 2207 /** 2208 * Validate MPLS item. 2209 * 2210 * @param[in] dev 2211 * Pointer to the rte_eth_dev structure. 2212 * @param[in] item 2213 * Item specification. 2214 * @param[in] item_flags 2215 * Bit-fields that holds the items detected until now. 2216 * @param[in] prev_layer 2217 * The protocol layer indicated in previous item. 2218 * @param[out] error 2219 * Pointer to error structure. 2220 * 2221 * @return 2222 * 0 on success, a negative errno value otherwise and rte_errno is set. 2223 */ 2224 int 2225 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2226 const struct rte_flow_item *item __rte_unused, 2227 uint64_t item_flags __rte_unused, 2228 uint64_t prev_layer __rte_unused, 2229 struct rte_flow_error *error) 2230 { 2231 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2232 const struct rte_flow_item_mpls *mask = item->mask; 2233 struct mlx5_priv *priv = dev->data->dev_private; 2234 int ret; 2235 2236 if (!priv->config.mpls_en) 2237 return rte_flow_error_set(error, ENOTSUP, 2238 RTE_FLOW_ERROR_TYPE_ITEM, item, 2239 "MPLS not supported or" 2240 " disabled in firmware" 2241 " configuration."); 2242 /* MPLS over IP, UDP, GRE is allowed */ 2243 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2244 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2245 MLX5_FLOW_LAYER_GRE))) 2246 return rte_flow_error_set(error, EINVAL, 2247 RTE_FLOW_ERROR_TYPE_ITEM, item, 2248 "protocol filtering not compatible" 2249 " with MPLS layer"); 2250 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2251 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2252 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2253 return rte_flow_error_set(error, ENOTSUP, 2254 RTE_FLOW_ERROR_TYPE_ITEM, item, 2255 "multiple tunnel layers not" 2256 " supported"); 2257 if (!mask) 2258 mask = &rte_flow_item_mpls_mask; 2259 ret = mlx5_flow_item_acceptable 2260 (item, (const uint8_t *)mask, 2261 (const uint8_t *)&rte_flow_item_mpls_mask, 2262 sizeof(struct rte_flow_item_mpls), error); 2263 if (ret < 0) 2264 return ret; 2265 return 0; 2266 #endif 2267 return rte_flow_error_set(error, ENOTSUP, 2268 RTE_FLOW_ERROR_TYPE_ITEM, item, 2269 "MPLS is not supported by Verbs, please" 2270 " update."); 2271 } 2272 2273 /** 2274 * Validate NVGRE item. 2275 * 2276 * @param[in] item 2277 * Item specification. 2278 * @param[in] item_flags 2279 * Bit flags to mark detected items. 2280 * @param[in] target_protocol 2281 * The next protocol in the previous item. 2282 * @param[out] error 2283 * Pointer to error structure. 2284 * 2285 * @return 2286 * 0 on success, a negative errno value otherwise and rte_errno is set. 2287 */ 2288 int 2289 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2290 uint64_t item_flags, 2291 uint8_t target_protocol, 2292 struct rte_flow_error *error) 2293 { 2294 const struct rte_flow_item_nvgre *mask = item->mask; 2295 int ret; 2296 2297 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2298 return rte_flow_error_set(error, EINVAL, 2299 RTE_FLOW_ERROR_TYPE_ITEM, item, 2300 "protocol filtering not compatible" 2301 " with this GRE layer"); 2302 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2303 return rte_flow_error_set(error, ENOTSUP, 2304 RTE_FLOW_ERROR_TYPE_ITEM, item, 2305 "multiple tunnel layers not" 2306 " supported"); 2307 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2308 return rte_flow_error_set(error, ENOTSUP, 2309 RTE_FLOW_ERROR_TYPE_ITEM, item, 2310 "L3 Layer is missing"); 2311 if (!mask) 2312 mask = &rte_flow_item_nvgre_mask; 2313 ret = mlx5_flow_item_acceptable 2314 (item, (const uint8_t *)mask, 2315 (const uint8_t *)&rte_flow_item_nvgre_mask, 2316 sizeof(struct rte_flow_item_nvgre), error); 2317 if (ret < 0) 2318 return ret; 2319 return 0; 2320 } 2321 2322 /* Allocate unique ID for the split Q/RSS subflows. */ 2323 static uint32_t 2324 flow_qrss_get_id(struct rte_eth_dev *dev) 2325 { 2326 struct mlx5_priv *priv = dev->data->dev_private; 2327 uint32_t qrss_id, ret; 2328 2329 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2330 if (ret) 2331 return 0; 2332 MLX5_ASSERT(qrss_id); 2333 return qrss_id; 2334 } 2335 2336 /* Free unique ID for the split Q/RSS subflows. */ 2337 static void 2338 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2339 { 2340 struct mlx5_priv *priv = dev->data->dev_private; 2341 2342 if (qrss_id) 2343 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2344 } 2345 2346 /** 2347 * Release resource related QUEUE/RSS action split. 2348 * 2349 * @param dev 2350 * Pointer to Ethernet device. 2351 * @param flow 2352 * Flow to release id's from. 2353 */ 2354 static void 2355 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2356 struct rte_flow *flow) 2357 { 2358 struct mlx5_priv *priv = dev->data->dev_private; 2359 uint32_t handle_idx; 2360 struct mlx5_flow_handle *dev_handle; 2361 2362 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 2363 handle_idx, dev_handle, next) 2364 if (dev_handle->split_flow_id) 2365 flow_qrss_free_id(dev, dev_handle->split_flow_id); 2366 } 2367 2368 static int 2369 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2370 const struct rte_flow_attr *attr __rte_unused, 2371 const struct rte_flow_item items[] __rte_unused, 2372 const struct rte_flow_action actions[] __rte_unused, 2373 bool external __rte_unused, 2374 int hairpin __rte_unused, 2375 struct rte_flow_error *error) 2376 { 2377 return rte_flow_error_set(error, ENOTSUP, 2378 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2379 } 2380 2381 static struct mlx5_flow * 2382 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 2383 const struct rte_flow_attr *attr __rte_unused, 2384 const struct rte_flow_item items[] __rte_unused, 2385 const struct rte_flow_action actions[] __rte_unused, 2386 struct rte_flow_error *error) 2387 { 2388 rte_flow_error_set(error, ENOTSUP, 2389 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2390 return NULL; 2391 } 2392 2393 static int 2394 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2395 struct mlx5_flow *dev_flow __rte_unused, 2396 const struct rte_flow_attr *attr __rte_unused, 2397 const struct rte_flow_item items[] __rte_unused, 2398 const struct rte_flow_action actions[] __rte_unused, 2399 struct rte_flow_error *error) 2400 { 2401 return rte_flow_error_set(error, ENOTSUP, 2402 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2403 } 2404 2405 static int 2406 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2407 struct rte_flow *flow __rte_unused, 2408 struct rte_flow_error *error) 2409 { 2410 return rte_flow_error_set(error, ENOTSUP, 2411 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2412 } 2413 2414 static void 2415 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2416 struct rte_flow *flow __rte_unused) 2417 { 2418 } 2419 2420 static void 2421 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2422 struct rte_flow *flow __rte_unused) 2423 { 2424 } 2425 2426 static int 2427 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2428 struct rte_flow *flow __rte_unused, 2429 const struct rte_flow_action *actions __rte_unused, 2430 void *data __rte_unused, 2431 struct rte_flow_error *error) 2432 { 2433 return rte_flow_error_set(error, ENOTSUP, 2434 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2435 } 2436 2437 /* Void driver to protect from null pointer reference. */ 2438 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2439 .validate = flow_null_validate, 2440 .prepare = flow_null_prepare, 2441 .translate = flow_null_translate, 2442 .apply = flow_null_apply, 2443 .remove = flow_null_remove, 2444 .destroy = flow_null_destroy, 2445 .query = flow_null_query, 2446 }; 2447 2448 /** 2449 * Select flow driver type according to flow attributes and device 2450 * configuration. 2451 * 2452 * @param[in] dev 2453 * Pointer to the dev structure. 2454 * @param[in] attr 2455 * Pointer to the flow attributes. 2456 * 2457 * @return 2458 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2459 */ 2460 static enum mlx5_flow_drv_type 2461 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2462 { 2463 struct mlx5_priv *priv = dev->data->dev_private; 2464 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2465 2466 if (attr->transfer && priv->config.dv_esw_en) 2467 type = MLX5_FLOW_TYPE_DV; 2468 if (!attr->transfer) 2469 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2470 MLX5_FLOW_TYPE_VERBS; 2471 return type; 2472 } 2473 2474 #define flow_get_drv_ops(type) flow_drv_ops[type] 2475 2476 /** 2477 * Flow driver validation API. This abstracts calling driver specific functions. 2478 * The type of flow driver is determined according to flow attributes. 2479 * 2480 * @param[in] dev 2481 * Pointer to the dev structure. 2482 * @param[in] attr 2483 * Pointer to the flow attributes. 2484 * @param[in] items 2485 * Pointer to the list of items. 2486 * @param[in] actions 2487 * Pointer to the list of actions. 2488 * @param[in] external 2489 * This flow rule is created by request external to PMD. 2490 * @param[in] hairpin 2491 * Number of hairpin TX actions, 0 means classic flow. 2492 * @param[out] error 2493 * Pointer to the error structure. 2494 * 2495 * @return 2496 * 0 on success, a negative errno value otherwise and rte_errno is set. 2497 */ 2498 static inline int 2499 flow_drv_validate(struct rte_eth_dev *dev, 2500 const struct rte_flow_attr *attr, 2501 const struct rte_flow_item items[], 2502 const struct rte_flow_action actions[], 2503 bool external, int hairpin, struct rte_flow_error *error) 2504 { 2505 const struct mlx5_flow_driver_ops *fops; 2506 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2507 2508 fops = flow_get_drv_ops(type); 2509 return fops->validate(dev, attr, items, actions, external, 2510 hairpin, error); 2511 } 2512 2513 /** 2514 * Flow driver preparation API. This abstracts calling driver specific 2515 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2516 * calculates the size of memory required for device flow, allocates the memory, 2517 * initializes the device flow and returns the pointer. 2518 * 2519 * @note 2520 * This function initializes device flow structure such as dv or verbs in 2521 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2522 * rest. For example, adding returning device flow to flow->dev_flow list and 2523 * setting backward reference to the flow should be done out of this function. 2524 * layers field is not filled either. 2525 * 2526 * @param[in] dev 2527 * Pointer to the dev structure. 2528 * @param[in] attr 2529 * Pointer to the flow attributes. 2530 * @param[in] items 2531 * Pointer to the list of items. 2532 * @param[in] actions 2533 * Pointer to the list of actions. 2534 * @param[out] error 2535 * Pointer to the error structure. 2536 * 2537 * @return 2538 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2539 */ 2540 static inline struct mlx5_flow * 2541 flow_drv_prepare(struct rte_eth_dev *dev, 2542 const struct rte_flow *flow, 2543 const struct rte_flow_attr *attr, 2544 const struct rte_flow_item items[], 2545 const struct rte_flow_action actions[], 2546 struct rte_flow_error *error) 2547 { 2548 const struct mlx5_flow_driver_ops *fops; 2549 enum mlx5_flow_drv_type type = flow->drv_type; 2550 2551 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2552 fops = flow_get_drv_ops(type); 2553 return fops->prepare(dev, attr, items, actions, error); 2554 } 2555 2556 /** 2557 * Flow driver translation API. This abstracts calling driver specific 2558 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2559 * translates a generic flow into a driver flow. flow_drv_prepare() must 2560 * precede. 2561 * 2562 * @note 2563 * dev_flow->layers could be filled as a result of parsing during translation 2564 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2565 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2566 * flow->actions could be overwritten even though all the expanded dev_flows 2567 * have the same actions. 2568 * 2569 * @param[in] dev 2570 * Pointer to the rte dev structure. 2571 * @param[in, out] dev_flow 2572 * Pointer to the mlx5 flow. 2573 * @param[in] attr 2574 * Pointer to the flow attributes. 2575 * @param[in] items 2576 * Pointer to the list of items. 2577 * @param[in] actions 2578 * Pointer to the list of actions. 2579 * @param[out] error 2580 * Pointer to the error structure. 2581 * 2582 * @return 2583 * 0 on success, a negative errno value otherwise and rte_errno is set. 2584 */ 2585 static inline int 2586 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2587 const struct rte_flow_attr *attr, 2588 const struct rte_flow_item items[], 2589 const struct rte_flow_action actions[], 2590 struct rte_flow_error *error) 2591 { 2592 const struct mlx5_flow_driver_ops *fops; 2593 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2594 2595 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2596 fops = flow_get_drv_ops(type); 2597 return fops->translate(dev, dev_flow, attr, items, actions, error); 2598 } 2599 2600 /** 2601 * Flow driver apply API. This abstracts calling driver specific functions. 2602 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2603 * translated driver flows on to device. flow_drv_translate() must precede. 2604 * 2605 * @param[in] dev 2606 * Pointer to Ethernet device structure. 2607 * @param[in, out] flow 2608 * Pointer to flow structure. 2609 * @param[out] error 2610 * Pointer to error structure. 2611 * 2612 * @return 2613 * 0 on success, a negative errno value otherwise and rte_errno is set. 2614 */ 2615 static inline int 2616 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2617 struct rte_flow_error *error) 2618 { 2619 const struct mlx5_flow_driver_ops *fops; 2620 enum mlx5_flow_drv_type type = flow->drv_type; 2621 2622 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2623 fops = flow_get_drv_ops(type); 2624 return fops->apply(dev, flow, error); 2625 } 2626 2627 /** 2628 * Flow driver remove API. This abstracts calling driver specific functions. 2629 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2630 * on device. All the resources of the flow should be freed by calling 2631 * flow_drv_destroy(). 2632 * 2633 * @param[in] dev 2634 * Pointer to Ethernet device. 2635 * @param[in, out] flow 2636 * Pointer to flow structure. 2637 */ 2638 static inline void 2639 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2640 { 2641 const struct mlx5_flow_driver_ops *fops; 2642 enum mlx5_flow_drv_type type = flow->drv_type; 2643 2644 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2645 fops = flow_get_drv_ops(type); 2646 fops->remove(dev, flow); 2647 } 2648 2649 /** 2650 * Flow driver destroy API. This abstracts calling driver specific functions. 2651 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2652 * on device and releases resources of the flow. 2653 * 2654 * @param[in] dev 2655 * Pointer to Ethernet device. 2656 * @param[in, out] flow 2657 * Pointer to flow structure. 2658 */ 2659 static inline void 2660 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2661 { 2662 const struct mlx5_flow_driver_ops *fops; 2663 enum mlx5_flow_drv_type type = flow->drv_type; 2664 2665 flow_mreg_split_qrss_release(dev, flow); 2666 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2667 fops = flow_get_drv_ops(type); 2668 fops->destroy(dev, flow); 2669 } 2670 2671 /** 2672 * Get RSS action from the action list. 2673 * 2674 * @param[in] actions 2675 * Pointer to the list of actions. 2676 * 2677 * @return 2678 * Pointer to the RSS action if exist, else return NULL. 2679 */ 2680 static const struct rte_flow_action_rss* 2681 flow_get_rss_action(const struct rte_flow_action actions[]) 2682 { 2683 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2684 switch (actions->type) { 2685 case RTE_FLOW_ACTION_TYPE_RSS: 2686 return (const struct rte_flow_action_rss *) 2687 actions->conf; 2688 default: 2689 break; 2690 } 2691 } 2692 return NULL; 2693 } 2694 2695 static unsigned int 2696 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2697 { 2698 const struct rte_flow_item *item; 2699 unsigned int has_vlan = 0; 2700 2701 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2702 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2703 has_vlan = 1; 2704 break; 2705 } 2706 } 2707 if (has_vlan) 2708 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2709 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2710 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2711 MLX5_EXPANSION_ROOT_OUTER; 2712 } 2713 2714 /** 2715 * Get layer flags from the prefix flow. 2716 * 2717 * Some flows may be split to several subflows, the prefix subflow gets the 2718 * match items and the suffix sub flow gets the actions. 2719 * Some actions need the user defined match item flags to get the detail for 2720 * the action. 2721 * This function helps the suffix flow to get the item layer flags from prefix 2722 * subflow. 2723 * 2724 * @param[in] dev_flow 2725 * Pointer the created preifx subflow. 2726 * 2727 * @return 2728 * The layers get from prefix subflow. 2729 */ 2730 static inline uint64_t 2731 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 2732 { 2733 uint64_t layers = 0; 2734 2735 /* 2736 * Layers bits could be localization, but usually the compiler will 2737 * help to do the optimization work for source code. 2738 * If no decap actions, use the layers directly. 2739 */ 2740 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 2741 return dev_flow->handle->layers; 2742 /* Convert L3 layers with decap action. */ 2743 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 2744 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2745 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 2746 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2747 /* Convert L4 layers with decap action. */ 2748 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 2749 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 2750 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 2751 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 2752 return layers; 2753 } 2754 2755 /** 2756 * Get metadata split action information. 2757 * 2758 * @param[in] actions 2759 * Pointer to the list of actions. 2760 * @param[out] qrss 2761 * Pointer to the return pointer. 2762 * @param[out] qrss_type 2763 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2764 * if no QUEUE/RSS is found. 2765 * @param[out] encap_idx 2766 * Pointer to the index of the encap action if exists, otherwise the last 2767 * action index. 2768 * 2769 * @return 2770 * Total number of actions. 2771 */ 2772 static int 2773 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 2774 const struct rte_flow_action **qrss, 2775 int *encap_idx) 2776 { 2777 const struct rte_flow_action_raw_encap *raw_encap; 2778 int actions_n = 0; 2779 int raw_decap_idx = -1; 2780 2781 *encap_idx = -1; 2782 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2783 switch (actions->type) { 2784 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2785 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2786 *encap_idx = actions_n; 2787 break; 2788 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 2789 raw_decap_idx = actions_n; 2790 break; 2791 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2792 raw_encap = actions->conf; 2793 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2794 *encap_idx = raw_decap_idx != -1 ? 2795 raw_decap_idx : actions_n; 2796 break; 2797 case RTE_FLOW_ACTION_TYPE_QUEUE: 2798 case RTE_FLOW_ACTION_TYPE_RSS: 2799 *qrss = actions; 2800 break; 2801 default: 2802 break; 2803 } 2804 actions_n++; 2805 } 2806 if (*encap_idx == -1) 2807 *encap_idx = actions_n; 2808 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2809 return actions_n + 1; 2810 } 2811 2812 /** 2813 * Check meter action from the action list. 2814 * 2815 * @param[in] actions 2816 * Pointer to the list of actions. 2817 * @param[out] mtr 2818 * Pointer to the meter exist flag. 2819 * 2820 * @return 2821 * Total number of actions. 2822 */ 2823 static int 2824 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2825 { 2826 int actions_n = 0; 2827 2828 MLX5_ASSERT(mtr); 2829 *mtr = 0; 2830 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2831 switch (actions->type) { 2832 case RTE_FLOW_ACTION_TYPE_METER: 2833 *mtr = 1; 2834 break; 2835 default: 2836 break; 2837 } 2838 actions_n++; 2839 } 2840 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2841 return actions_n + 1; 2842 } 2843 2844 /** 2845 * Check if the flow should be splited due to hairpin. 2846 * The reason for the split is that in current HW we can't 2847 * support encap on Rx, so if a flow have encap we move it 2848 * to Tx. 2849 * 2850 * @param dev 2851 * Pointer to Ethernet device. 2852 * @param[in] attr 2853 * Flow rule attributes. 2854 * @param[in] actions 2855 * Associated actions (list terminated by the END action). 2856 * 2857 * @return 2858 * > 0 the number of actions and the flow should be split, 2859 * 0 when no split required. 2860 */ 2861 static int 2862 flow_check_hairpin_split(struct rte_eth_dev *dev, 2863 const struct rte_flow_attr *attr, 2864 const struct rte_flow_action actions[]) 2865 { 2866 int queue_action = 0; 2867 int action_n = 0; 2868 int encap = 0; 2869 const struct rte_flow_action_queue *queue; 2870 const struct rte_flow_action_rss *rss; 2871 const struct rte_flow_action_raw_encap *raw_encap; 2872 2873 if (!attr->ingress) 2874 return 0; 2875 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2876 switch (actions->type) { 2877 case RTE_FLOW_ACTION_TYPE_QUEUE: 2878 queue = actions->conf; 2879 if (queue == NULL) 2880 return 0; 2881 if (mlx5_rxq_get_type(dev, queue->index) != 2882 MLX5_RXQ_TYPE_HAIRPIN) 2883 return 0; 2884 queue_action = 1; 2885 action_n++; 2886 break; 2887 case RTE_FLOW_ACTION_TYPE_RSS: 2888 rss = actions->conf; 2889 if (rss == NULL || rss->queue_num == 0) 2890 return 0; 2891 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2892 MLX5_RXQ_TYPE_HAIRPIN) 2893 return 0; 2894 queue_action = 1; 2895 action_n++; 2896 break; 2897 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2898 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2899 encap = 1; 2900 action_n++; 2901 break; 2902 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2903 raw_encap = actions->conf; 2904 if (raw_encap->size > 2905 (sizeof(struct rte_flow_item_eth) + 2906 sizeof(struct rte_flow_item_ipv4))) 2907 encap = 1; 2908 action_n++; 2909 break; 2910 default: 2911 action_n++; 2912 break; 2913 } 2914 } 2915 if (encap == 1 && queue_action) 2916 return action_n; 2917 return 0; 2918 } 2919 2920 /* Declare flow create/destroy prototype in advance. */ 2921 static uint32_t 2922 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 2923 const struct rte_flow_attr *attr, 2924 const struct rte_flow_item items[], 2925 const struct rte_flow_action actions[], 2926 bool external, struct rte_flow_error *error); 2927 2928 static void 2929 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 2930 uint32_t flow_idx); 2931 2932 /** 2933 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2934 * 2935 * As mark_id is unique, if there's already a registered flow for the mark_id, 2936 * return by increasing the reference counter of the resource. Otherwise, create 2937 * the resource (mcp_res) and flow. 2938 * 2939 * Flow looks like, 2940 * - If ingress port is ANY and reg_c[1] is mark_id, 2941 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2942 * 2943 * For default flow (zero mark_id), flow is like, 2944 * - If ingress port is ANY, 2945 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2946 * 2947 * @param dev 2948 * Pointer to Ethernet device. 2949 * @param mark_id 2950 * ID of MARK action, zero means default flow for META. 2951 * @param[out] error 2952 * Perform verbose error reporting if not NULL. 2953 * 2954 * @return 2955 * Associated resource on success, NULL otherwise and rte_errno is set. 2956 */ 2957 static struct mlx5_flow_mreg_copy_resource * 2958 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2959 struct rte_flow_error *error) 2960 { 2961 struct mlx5_priv *priv = dev->data->dev_private; 2962 struct rte_flow_attr attr = { 2963 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2964 .ingress = 1, 2965 }; 2966 struct mlx5_rte_flow_item_tag tag_spec = { 2967 .data = mark_id, 2968 }; 2969 struct rte_flow_item items[] = { 2970 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2971 }; 2972 struct rte_flow_action_mark ftag = { 2973 .id = mark_id, 2974 }; 2975 struct mlx5_flow_action_copy_mreg cp_mreg = { 2976 .dst = REG_B, 2977 .src = 0, 2978 }; 2979 struct rte_flow_action_jump jump = { 2980 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2981 }; 2982 struct rte_flow_action actions[] = { 2983 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2984 }; 2985 struct mlx5_flow_mreg_copy_resource *mcp_res; 2986 uint32_t idx = 0; 2987 int ret; 2988 2989 /* Fill the register fileds in the flow. */ 2990 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2991 if (ret < 0) 2992 return NULL; 2993 tag_spec.id = ret; 2994 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 2995 if (ret < 0) 2996 return NULL; 2997 cp_mreg.src = ret; 2998 /* Check if already registered. */ 2999 MLX5_ASSERT(priv->mreg_cp_tbl); 3000 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 3001 if (mcp_res) { 3002 /* For non-default rule. */ 3003 if (mark_id != MLX5_DEFAULT_COPY_ID) 3004 mcp_res->refcnt++; 3005 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 3006 mcp_res->refcnt == 1); 3007 return mcp_res; 3008 } 3009 /* Provide the full width of FLAG specific value. */ 3010 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3011 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3012 /* Build a new flow. */ 3013 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3014 items[0] = (struct rte_flow_item){ 3015 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3016 .spec = &tag_spec, 3017 }; 3018 items[1] = (struct rte_flow_item){ 3019 .type = RTE_FLOW_ITEM_TYPE_END, 3020 }; 3021 actions[0] = (struct rte_flow_action){ 3022 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3023 .conf = &ftag, 3024 }; 3025 actions[1] = (struct rte_flow_action){ 3026 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3027 .conf = &cp_mreg, 3028 }; 3029 actions[2] = (struct rte_flow_action){ 3030 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3031 .conf = &jump, 3032 }; 3033 actions[3] = (struct rte_flow_action){ 3034 .type = RTE_FLOW_ACTION_TYPE_END, 3035 }; 3036 } else { 3037 /* Default rule, wildcard match. */ 3038 attr.priority = MLX5_FLOW_PRIO_RSVD; 3039 items[0] = (struct rte_flow_item){ 3040 .type = RTE_FLOW_ITEM_TYPE_END, 3041 }; 3042 actions[0] = (struct rte_flow_action){ 3043 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3044 .conf = &cp_mreg, 3045 }; 3046 actions[1] = (struct rte_flow_action){ 3047 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3048 .conf = &jump, 3049 }; 3050 actions[2] = (struct rte_flow_action){ 3051 .type = RTE_FLOW_ACTION_TYPE_END, 3052 }; 3053 } 3054 /* Build a new entry. */ 3055 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 3056 if (!mcp_res) { 3057 rte_errno = ENOMEM; 3058 return NULL; 3059 } 3060 mcp_res->idx = idx; 3061 /* 3062 * The copy Flows are not included in any list. There 3063 * ones are referenced from other Flows and can not 3064 * be applied, removed, deleted in ardbitrary order 3065 * by list traversing. 3066 */ 3067 mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items, 3068 actions, false, error); 3069 if (!mcp_res->rix_flow) 3070 goto error; 3071 mcp_res->refcnt++; 3072 mcp_res->hlist_ent.key = mark_id; 3073 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3074 &mcp_res->hlist_ent); 3075 MLX5_ASSERT(!ret); 3076 if (ret) 3077 goto error; 3078 return mcp_res; 3079 error: 3080 if (mcp_res->rix_flow) 3081 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3082 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3083 return NULL; 3084 } 3085 3086 /** 3087 * Release flow in RX_CP_TBL. 3088 * 3089 * @param dev 3090 * Pointer to Ethernet device. 3091 * @flow 3092 * Parent flow for wich copying is provided. 3093 */ 3094 static void 3095 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3096 struct rte_flow *flow) 3097 { 3098 struct mlx5_flow_mreg_copy_resource *mcp_res; 3099 struct mlx5_priv *priv = dev->data->dev_private; 3100 3101 if (!flow->rix_mreg_copy) 3102 return; 3103 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3104 flow->rix_mreg_copy); 3105 if (!mcp_res || !priv->mreg_cp_tbl) 3106 return; 3107 if (flow->copy_applied) { 3108 MLX5_ASSERT(mcp_res->appcnt); 3109 flow->copy_applied = 0; 3110 --mcp_res->appcnt; 3111 if (!mcp_res->appcnt) { 3112 struct rte_flow *mcp_flow = mlx5_ipool_get 3113 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3114 mcp_res->rix_flow); 3115 3116 if (mcp_flow) 3117 flow_drv_remove(dev, mcp_flow); 3118 } 3119 } 3120 /* 3121 * We do not check availability of metadata registers here, 3122 * because copy resources are not allocated in this case. 3123 */ 3124 if (--mcp_res->refcnt) 3125 return; 3126 MLX5_ASSERT(mcp_res->rix_flow); 3127 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3128 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3129 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3130 flow->rix_mreg_copy = 0; 3131 } 3132 3133 /** 3134 * Start flow in RX_CP_TBL. 3135 * 3136 * @param dev 3137 * Pointer to Ethernet device. 3138 * @flow 3139 * Parent flow for wich copying is provided. 3140 * 3141 * @return 3142 * 0 on success, a negative errno value otherwise and rte_errno is set. 3143 */ 3144 static int 3145 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3146 struct rte_flow *flow) 3147 { 3148 struct mlx5_flow_mreg_copy_resource *mcp_res; 3149 struct mlx5_priv *priv = dev->data->dev_private; 3150 int ret; 3151 3152 if (!flow->rix_mreg_copy || flow->copy_applied) 3153 return 0; 3154 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3155 flow->rix_mreg_copy); 3156 if (!mcp_res) 3157 return 0; 3158 if (!mcp_res->appcnt) { 3159 struct rte_flow *mcp_flow = mlx5_ipool_get 3160 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3161 mcp_res->rix_flow); 3162 3163 if (mcp_flow) { 3164 ret = flow_drv_apply(dev, mcp_flow, NULL); 3165 if (ret) 3166 return ret; 3167 } 3168 } 3169 ++mcp_res->appcnt; 3170 flow->copy_applied = 1; 3171 return 0; 3172 } 3173 3174 /** 3175 * Stop flow in RX_CP_TBL. 3176 * 3177 * @param dev 3178 * Pointer to Ethernet device. 3179 * @flow 3180 * Parent flow for wich copying is provided. 3181 */ 3182 static void 3183 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3184 struct rte_flow *flow) 3185 { 3186 struct mlx5_flow_mreg_copy_resource *mcp_res; 3187 struct mlx5_priv *priv = dev->data->dev_private; 3188 3189 if (!flow->rix_mreg_copy || !flow->copy_applied) 3190 return; 3191 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 3192 flow->rix_mreg_copy); 3193 if (!mcp_res) 3194 return; 3195 MLX5_ASSERT(mcp_res->appcnt); 3196 --mcp_res->appcnt; 3197 flow->copy_applied = 0; 3198 if (!mcp_res->appcnt) { 3199 struct rte_flow *mcp_flow = mlx5_ipool_get 3200 (priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 3201 mcp_res->rix_flow); 3202 3203 if (mcp_flow) 3204 flow_drv_remove(dev, mcp_flow); 3205 } 3206 } 3207 3208 /** 3209 * Remove the default copy action from RX_CP_TBL. 3210 * 3211 * @param dev 3212 * Pointer to Ethernet device. 3213 */ 3214 static void 3215 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3216 { 3217 struct mlx5_flow_mreg_copy_resource *mcp_res; 3218 struct mlx5_priv *priv = dev->data->dev_private; 3219 3220 /* Check if default flow is registered. */ 3221 if (!priv->mreg_cp_tbl) 3222 return; 3223 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3224 MLX5_DEFAULT_COPY_ID); 3225 if (!mcp_res) 3226 return; 3227 MLX5_ASSERT(mcp_res->rix_flow); 3228 flow_list_destroy(dev, NULL, mcp_res->rix_flow); 3229 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3230 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 3231 } 3232 3233 /** 3234 * Add the default copy action in in RX_CP_TBL. 3235 * 3236 * @param dev 3237 * Pointer to Ethernet device. 3238 * @param[out] error 3239 * Perform verbose error reporting if not NULL. 3240 * 3241 * @return 3242 * 0 for success, negative value otherwise and rte_errno is set. 3243 */ 3244 static int 3245 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3246 struct rte_flow_error *error) 3247 { 3248 struct mlx5_priv *priv = dev->data->dev_private; 3249 struct mlx5_flow_mreg_copy_resource *mcp_res; 3250 3251 /* Check whether extensive metadata feature is engaged. */ 3252 if (!priv->config.dv_flow_en || 3253 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3254 !mlx5_flow_ext_mreg_supported(dev) || 3255 !priv->sh->dv_regc0_mask) 3256 return 0; 3257 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3258 if (!mcp_res) 3259 return -rte_errno; 3260 return 0; 3261 } 3262 3263 /** 3264 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3265 * 3266 * All the flow having Q/RSS action should be split by 3267 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3268 * performs the following, 3269 * - CQE->flow_tag := reg_c[1] (MARK) 3270 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3271 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3272 * but there should be a flow per each MARK ID set by MARK action. 3273 * 3274 * For the aforementioned reason, if there's a MARK action in flow's action 3275 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3276 * the MARK ID to CQE's flow_tag like, 3277 * - If reg_c[1] is mark_id, 3278 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3279 * 3280 * For SET_META action which stores value in reg_c[0], as the destination is 3281 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3282 * MARK ID means the default flow. The default flow looks like, 3283 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3284 * 3285 * @param dev 3286 * Pointer to Ethernet device. 3287 * @param flow 3288 * Pointer to flow structure. 3289 * @param[in] actions 3290 * Pointer to the list of actions. 3291 * @param[out] error 3292 * Perform verbose error reporting if not NULL. 3293 * 3294 * @return 3295 * 0 on success, negative value otherwise and rte_errno is set. 3296 */ 3297 static int 3298 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3299 struct rte_flow *flow, 3300 const struct rte_flow_action *actions, 3301 struct rte_flow_error *error) 3302 { 3303 struct mlx5_priv *priv = dev->data->dev_private; 3304 struct mlx5_dev_config *config = &priv->config; 3305 struct mlx5_flow_mreg_copy_resource *mcp_res; 3306 const struct rte_flow_action_mark *mark; 3307 3308 /* Check whether extensive metadata feature is engaged. */ 3309 if (!config->dv_flow_en || 3310 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3311 !mlx5_flow_ext_mreg_supported(dev) || 3312 !priv->sh->dv_regc0_mask) 3313 return 0; 3314 /* Find MARK action. */ 3315 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3316 switch (actions->type) { 3317 case RTE_FLOW_ACTION_TYPE_FLAG: 3318 mcp_res = flow_mreg_add_copy_action 3319 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3320 if (!mcp_res) 3321 return -rte_errno; 3322 flow->rix_mreg_copy = mcp_res->idx; 3323 if (dev->data->dev_started) { 3324 mcp_res->appcnt++; 3325 flow->copy_applied = 1; 3326 } 3327 return 0; 3328 case RTE_FLOW_ACTION_TYPE_MARK: 3329 mark = (const struct rte_flow_action_mark *) 3330 actions->conf; 3331 mcp_res = 3332 flow_mreg_add_copy_action(dev, mark->id, error); 3333 if (!mcp_res) 3334 return -rte_errno; 3335 flow->rix_mreg_copy = mcp_res->idx; 3336 if (dev->data->dev_started) { 3337 mcp_res->appcnt++; 3338 flow->copy_applied = 1; 3339 } 3340 return 0; 3341 default: 3342 break; 3343 } 3344 } 3345 return 0; 3346 } 3347 3348 #define MLX5_MAX_SPLIT_ACTIONS 24 3349 #define MLX5_MAX_SPLIT_ITEMS 24 3350 3351 /** 3352 * Split the hairpin flow. 3353 * Since HW can't support encap on Rx we move the encap to Tx. 3354 * If the count action is after the encap then we also 3355 * move the count action. in this case the count will also measure 3356 * the outer bytes. 3357 * 3358 * @param dev 3359 * Pointer to Ethernet device. 3360 * @param[in] actions 3361 * Associated actions (list terminated by the END action). 3362 * @param[out] actions_rx 3363 * Rx flow actions. 3364 * @param[out] actions_tx 3365 * Tx flow actions.. 3366 * @param[out] pattern_tx 3367 * The pattern items for the Tx flow. 3368 * @param[out] flow_id 3369 * The flow ID connected to this flow. 3370 * 3371 * @return 3372 * 0 on success. 3373 */ 3374 static int 3375 flow_hairpin_split(struct rte_eth_dev *dev, 3376 const struct rte_flow_action actions[], 3377 struct rte_flow_action actions_rx[], 3378 struct rte_flow_action actions_tx[], 3379 struct rte_flow_item pattern_tx[], 3380 uint32_t *flow_id) 3381 { 3382 struct mlx5_priv *priv = dev->data->dev_private; 3383 const struct rte_flow_action_raw_encap *raw_encap; 3384 const struct rte_flow_action_raw_decap *raw_decap; 3385 struct mlx5_rte_flow_action_set_tag *set_tag; 3386 struct rte_flow_action *tag_action; 3387 struct mlx5_rte_flow_item_tag *tag_item; 3388 struct rte_flow_item *item; 3389 char *addr; 3390 int encap = 0; 3391 3392 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3393 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3394 switch (actions->type) { 3395 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3396 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3397 rte_memcpy(actions_tx, actions, 3398 sizeof(struct rte_flow_action)); 3399 actions_tx++; 3400 break; 3401 case RTE_FLOW_ACTION_TYPE_COUNT: 3402 if (encap) { 3403 rte_memcpy(actions_tx, actions, 3404 sizeof(struct rte_flow_action)); 3405 actions_tx++; 3406 } else { 3407 rte_memcpy(actions_rx, actions, 3408 sizeof(struct rte_flow_action)); 3409 actions_rx++; 3410 } 3411 break; 3412 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3413 raw_encap = actions->conf; 3414 if (raw_encap->size > 3415 (sizeof(struct rte_flow_item_eth) + 3416 sizeof(struct rte_flow_item_ipv4))) { 3417 memcpy(actions_tx, actions, 3418 sizeof(struct rte_flow_action)); 3419 actions_tx++; 3420 encap = 1; 3421 } else { 3422 rte_memcpy(actions_rx, actions, 3423 sizeof(struct rte_flow_action)); 3424 actions_rx++; 3425 } 3426 break; 3427 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3428 raw_decap = actions->conf; 3429 if (raw_decap->size < 3430 (sizeof(struct rte_flow_item_eth) + 3431 sizeof(struct rte_flow_item_ipv4))) { 3432 memcpy(actions_tx, actions, 3433 sizeof(struct rte_flow_action)); 3434 actions_tx++; 3435 } else { 3436 rte_memcpy(actions_rx, actions, 3437 sizeof(struct rte_flow_action)); 3438 actions_rx++; 3439 } 3440 break; 3441 default: 3442 rte_memcpy(actions_rx, actions, 3443 sizeof(struct rte_flow_action)); 3444 actions_rx++; 3445 break; 3446 } 3447 } 3448 /* Add set meta action and end action for the Rx flow. */ 3449 tag_action = actions_rx; 3450 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3451 actions_rx++; 3452 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3453 actions_rx++; 3454 set_tag = (void *)actions_rx; 3455 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3456 MLX5_ASSERT(set_tag->id > REG_NONE); 3457 set_tag->data = *flow_id; 3458 tag_action->conf = set_tag; 3459 /* Create Tx item list. */ 3460 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3461 addr = (void *)&pattern_tx[2]; 3462 item = pattern_tx; 3463 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3464 tag_item = (void *)addr; 3465 tag_item->data = *flow_id; 3466 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3467 MLX5_ASSERT(set_tag->id > REG_NONE); 3468 item->spec = tag_item; 3469 addr += sizeof(struct mlx5_rte_flow_item_tag); 3470 tag_item = (void *)addr; 3471 tag_item->data = UINT32_MAX; 3472 tag_item->id = UINT16_MAX; 3473 item->mask = tag_item; 3474 addr += sizeof(struct mlx5_rte_flow_item_tag); 3475 item->last = NULL; 3476 item++; 3477 item->type = RTE_FLOW_ITEM_TYPE_END; 3478 return 0; 3479 } 3480 3481 /** 3482 * The last stage of splitting chain, just creates the subflow 3483 * without any modification. 3484 * 3485 * @param[in] dev 3486 * Pointer to Ethernet device. 3487 * @param[in] flow 3488 * Parent flow structure pointer. 3489 * @param[in, out] sub_flow 3490 * Pointer to return the created subflow, may be NULL. 3491 * @param[in] prefix_layers 3492 * Prefix subflow layers, may be 0. 3493 * @param[in] attr 3494 * Flow rule attributes. 3495 * @param[in] items 3496 * Pattern specification (list terminated by the END pattern item). 3497 * @param[in] actions 3498 * Associated actions (list terminated by the END action). 3499 * @param[in] external 3500 * This flow rule is created by request external to PMD. 3501 * @param[out] error 3502 * Perform verbose error reporting if not NULL. 3503 * @return 3504 * 0 on success, negative value otherwise 3505 */ 3506 static int 3507 flow_create_split_inner(struct rte_eth_dev *dev, 3508 struct rte_flow *flow, 3509 struct mlx5_flow **sub_flow, 3510 uint64_t prefix_layers, 3511 const struct rte_flow_attr *attr, 3512 const struct rte_flow_item items[], 3513 const struct rte_flow_action actions[], 3514 bool external, struct rte_flow_error *error) 3515 { 3516 struct mlx5_flow *dev_flow; 3517 3518 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, error); 3519 if (!dev_flow) 3520 return -rte_errno; 3521 dev_flow->flow = flow; 3522 dev_flow->external = external; 3523 /* Subflow object was created, we must include one in the list. */ 3524 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 3525 dev_flow->handle, next); 3526 /* 3527 * If dev_flow is as one of the suffix flow, some actions in suffix 3528 * flow may need some user defined item layer flags. 3529 */ 3530 if (prefix_layers) 3531 dev_flow->handle->layers = prefix_layers; 3532 if (sub_flow) 3533 *sub_flow = dev_flow; 3534 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3535 } 3536 3537 /** 3538 * Split the meter flow. 3539 * 3540 * As meter flow will split to three sub flow, other than meter 3541 * action, the other actions make sense to only meter accepts 3542 * the packet. If it need to be dropped, no other additional 3543 * actions should be take. 3544 * 3545 * One kind of special action which decapsulates the L3 tunnel 3546 * header will be in the prefix sub flow, as not to take the 3547 * L3 tunnel header into account. 3548 * 3549 * @param dev 3550 * Pointer to Ethernet device. 3551 * @param[in] items 3552 * Pattern specification (list terminated by the END pattern item). 3553 * @param[out] sfx_items 3554 * Suffix flow match items (list terminated by the END pattern item). 3555 * @param[in] actions 3556 * Associated actions (list terminated by the END action). 3557 * @param[out] actions_sfx 3558 * Suffix flow actions. 3559 * @param[out] actions_pre 3560 * Prefix flow actions. 3561 * @param[out] pattern_sfx 3562 * The pattern items for the suffix flow. 3563 * @param[out] tag_sfx 3564 * Pointer to suffix flow tag. 3565 * 3566 * @return 3567 * 0 on success. 3568 */ 3569 static int 3570 flow_meter_split_prep(struct rte_eth_dev *dev, 3571 const struct rte_flow_item items[], 3572 struct rte_flow_item sfx_items[], 3573 const struct rte_flow_action actions[], 3574 struct rte_flow_action actions_sfx[], 3575 struct rte_flow_action actions_pre[]) 3576 { 3577 struct rte_flow_action *tag_action = NULL; 3578 struct rte_flow_item *tag_item; 3579 struct mlx5_rte_flow_action_set_tag *set_tag; 3580 struct rte_flow_error error; 3581 const struct rte_flow_action_raw_encap *raw_encap; 3582 const struct rte_flow_action_raw_decap *raw_decap; 3583 struct mlx5_rte_flow_item_tag *tag_spec; 3584 struct mlx5_rte_flow_item_tag *tag_mask; 3585 uint32_t tag_id; 3586 bool copy_vlan = false; 3587 3588 /* Prepare the actions for prefix and suffix flow. */ 3589 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3590 struct rte_flow_action **action_cur = NULL; 3591 3592 switch (actions->type) { 3593 case RTE_FLOW_ACTION_TYPE_METER: 3594 /* Add the extra tag action first. */ 3595 tag_action = actions_pre; 3596 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3597 actions_pre++; 3598 action_cur = &actions_pre; 3599 break; 3600 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3601 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3602 action_cur = &actions_pre; 3603 break; 3604 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3605 raw_encap = actions->conf; 3606 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 3607 action_cur = &actions_pre; 3608 break; 3609 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3610 raw_decap = actions->conf; 3611 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3612 action_cur = &actions_pre; 3613 break; 3614 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3615 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3616 copy_vlan = true; 3617 break; 3618 default: 3619 break; 3620 } 3621 if (!action_cur) 3622 action_cur = &actions_sfx; 3623 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 3624 (*action_cur)++; 3625 } 3626 /* Add end action to the actions. */ 3627 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3628 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3629 actions_pre++; 3630 /* Set the tag. */ 3631 set_tag = (void *)actions_pre; 3632 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3633 /* 3634 * Get the id from the qrss_pool to make qrss share the id with meter. 3635 */ 3636 tag_id = flow_qrss_get_id(dev); 3637 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 3638 assert(tag_action); 3639 tag_action->conf = set_tag; 3640 /* Prepare the suffix subflow items. */ 3641 tag_item = sfx_items++; 3642 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 3643 int item_type = items->type; 3644 3645 switch (item_type) { 3646 case RTE_FLOW_ITEM_TYPE_PORT_ID: 3647 memcpy(sfx_items, items, sizeof(*sfx_items)); 3648 sfx_items++; 3649 break; 3650 case RTE_FLOW_ITEM_TYPE_VLAN: 3651 if (copy_vlan) { 3652 memcpy(sfx_items, items, sizeof(*sfx_items)); 3653 /* 3654 * Convert to internal match item, it is used 3655 * for vlan push and set vid. 3656 */ 3657 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 3658 sfx_items++; 3659 } 3660 break; 3661 default: 3662 break; 3663 } 3664 } 3665 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 3666 sfx_items++; 3667 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 3668 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 3669 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3670 tag_mask = tag_spec + 1; 3671 tag_mask->data = 0xffffff00; 3672 tag_item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3673 tag_item->spec = tag_spec; 3674 tag_item->last = NULL; 3675 tag_item->mask = tag_mask; 3676 return tag_id; 3677 } 3678 3679 /** 3680 * Split action list having QUEUE/RSS for metadata register copy. 3681 * 3682 * Once Q/RSS action is detected in user's action list, the flow action 3683 * should be split in order to copy metadata registers, which will happen in 3684 * RX_CP_TBL like, 3685 * - CQE->flow_tag := reg_c[1] (MARK) 3686 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3687 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3688 * This is because the last action of each flow must be a terminal action 3689 * (QUEUE, RSS or DROP). 3690 * 3691 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3692 * stored and kept in the mlx5_flow structure per each sub_flow. 3693 * 3694 * The Q/RSS action is replaced with, 3695 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3696 * And the following JUMP action is added at the end, 3697 * - JUMP, to RX_CP_TBL. 3698 * 3699 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3700 * flow_create_split_metadata() routine. The flow will look like, 3701 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3702 * 3703 * @param dev 3704 * Pointer to Ethernet device. 3705 * @param[out] split_actions 3706 * Pointer to store split actions to jump to CP_TBL. 3707 * @param[in] actions 3708 * Pointer to the list of original flow actions. 3709 * @param[in] qrss 3710 * Pointer to the Q/RSS action. 3711 * @param[in] actions_n 3712 * Number of original actions. 3713 * @param[out] error 3714 * Perform verbose error reporting if not NULL. 3715 * 3716 * @return 3717 * non-zero unique flow_id on success, otherwise 0 and 3718 * error/rte_error are set. 3719 */ 3720 static uint32_t 3721 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3722 struct rte_flow_action *split_actions, 3723 const struct rte_flow_action *actions, 3724 const struct rte_flow_action *qrss, 3725 int actions_n, struct rte_flow_error *error) 3726 { 3727 struct mlx5_rte_flow_action_set_tag *set_tag; 3728 struct rte_flow_action_jump *jump; 3729 const int qrss_idx = qrss - actions; 3730 uint32_t flow_id = 0; 3731 int ret = 0; 3732 3733 /* 3734 * Given actions will be split 3735 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3736 * - Add jump to mreg CP_TBL. 3737 * As a result, there will be one more action. 3738 */ 3739 ++actions_n; 3740 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3741 set_tag = (void *)(split_actions + actions_n); 3742 /* 3743 * If tag action is not set to void(it means we are not the meter 3744 * suffix flow), add the tag action. Since meter suffix flow already 3745 * has the tag added. 3746 */ 3747 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3748 /* 3749 * Allocate the new subflow ID. This one is unique within 3750 * device and not shared with representors. Otherwise, 3751 * we would have to resolve multi-thread access synch 3752 * issue. Each flow on the shared device is appended 3753 * with source vport identifier, so the resulting 3754 * flows will be unique in the shared (by master and 3755 * representors) domain even if they have coinciding 3756 * IDs. 3757 */ 3758 flow_id = flow_qrss_get_id(dev); 3759 if (!flow_id) 3760 return rte_flow_error_set(error, ENOMEM, 3761 RTE_FLOW_ERROR_TYPE_ACTION, 3762 NULL, "can't allocate id " 3763 "for split Q/RSS subflow"); 3764 /* Internal SET_TAG action to set flow ID. */ 3765 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3766 .data = flow_id, 3767 }; 3768 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3769 if (ret < 0) 3770 return ret; 3771 set_tag->id = ret; 3772 /* Construct new actions array. */ 3773 /* Replace QUEUE/RSS action. */ 3774 split_actions[qrss_idx] = (struct rte_flow_action){ 3775 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3776 .conf = set_tag, 3777 }; 3778 } 3779 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3780 jump = (void *)(set_tag + 1); 3781 *jump = (struct rte_flow_action_jump){ 3782 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3783 }; 3784 split_actions[actions_n - 2] = (struct rte_flow_action){ 3785 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3786 .conf = jump, 3787 }; 3788 split_actions[actions_n - 1] = (struct rte_flow_action){ 3789 .type = RTE_FLOW_ACTION_TYPE_END, 3790 }; 3791 return flow_id; 3792 } 3793 3794 /** 3795 * Extend the given action list for Tx metadata copy. 3796 * 3797 * Copy the given action list to the ext_actions and add flow metadata register 3798 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3799 * 3800 * @param[out] ext_actions 3801 * Pointer to the extended action list. 3802 * @param[in] actions 3803 * Pointer to the list of actions. 3804 * @param[in] actions_n 3805 * Number of actions in the list. 3806 * @param[out] error 3807 * Perform verbose error reporting if not NULL. 3808 * @param[in] encap_idx 3809 * The encap action inndex. 3810 * 3811 * @return 3812 * 0 on success, negative value otherwise 3813 */ 3814 static int 3815 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3816 struct rte_flow_action *ext_actions, 3817 const struct rte_flow_action *actions, 3818 int actions_n, struct rte_flow_error *error, 3819 int encap_idx) 3820 { 3821 struct mlx5_flow_action_copy_mreg *cp_mreg = 3822 (struct mlx5_flow_action_copy_mreg *) 3823 (ext_actions + actions_n + 1); 3824 int ret; 3825 3826 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3827 if (ret < 0) 3828 return ret; 3829 cp_mreg->dst = ret; 3830 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3831 if (ret < 0) 3832 return ret; 3833 cp_mreg->src = ret; 3834 if (encap_idx != 0) 3835 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 3836 if (encap_idx == actions_n - 1) { 3837 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3838 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3839 .conf = cp_mreg, 3840 }; 3841 ext_actions[actions_n] = (struct rte_flow_action){ 3842 .type = RTE_FLOW_ACTION_TYPE_END, 3843 }; 3844 } else { 3845 ext_actions[encap_idx] = (struct rte_flow_action){ 3846 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3847 .conf = cp_mreg, 3848 }; 3849 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 3850 sizeof(*ext_actions) * (actions_n - encap_idx)); 3851 } 3852 return 0; 3853 } 3854 3855 /** 3856 * The splitting for metadata feature. 3857 * 3858 * - Q/RSS action on NIC Rx should be split in order to pass by 3859 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3860 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3861 * 3862 * - All the actions on NIC Tx should have a mreg copy action to 3863 * copy reg_a from WQE to reg_c[0]. 3864 * 3865 * @param dev 3866 * Pointer to Ethernet device. 3867 * @param[in] flow 3868 * Parent flow structure pointer. 3869 * @param[in] prefix_layers 3870 * Prefix flow layer flags. 3871 * @param[in] attr 3872 * Flow rule attributes. 3873 * @param[in] items 3874 * Pattern specification (list terminated by the END pattern item). 3875 * @param[in] actions 3876 * Associated actions (list terminated by the END action). 3877 * @param[in] external 3878 * This flow rule is created by request external to PMD. 3879 * @param[out] error 3880 * Perform verbose error reporting if not NULL. 3881 * @return 3882 * 0 on success, negative value otherwise 3883 */ 3884 static int 3885 flow_create_split_metadata(struct rte_eth_dev *dev, 3886 struct rte_flow *flow, 3887 uint64_t prefix_layers, 3888 const struct rte_flow_attr *attr, 3889 const struct rte_flow_item items[], 3890 const struct rte_flow_action actions[], 3891 bool external, struct rte_flow_error *error) 3892 { 3893 struct mlx5_priv *priv = dev->data->dev_private; 3894 struct mlx5_dev_config *config = &priv->config; 3895 const struct rte_flow_action *qrss = NULL; 3896 struct rte_flow_action *ext_actions = NULL; 3897 struct mlx5_flow *dev_flow = NULL; 3898 uint32_t qrss_id = 0; 3899 int mtr_sfx = 0; 3900 size_t act_size; 3901 int actions_n; 3902 int encap_idx; 3903 int ret; 3904 3905 /* Check whether extensive metadata feature is engaged. */ 3906 if (!config->dv_flow_en || 3907 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3908 !mlx5_flow_ext_mreg_supported(dev)) 3909 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 3910 attr, items, actions, external, 3911 error); 3912 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 3913 &encap_idx); 3914 if (qrss) { 3915 /* Exclude hairpin flows from splitting. */ 3916 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3917 const struct rte_flow_action_queue *queue; 3918 3919 queue = qrss->conf; 3920 if (mlx5_rxq_get_type(dev, queue->index) == 3921 MLX5_RXQ_TYPE_HAIRPIN) 3922 qrss = NULL; 3923 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3924 const struct rte_flow_action_rss *rss; 3925 3926 rss = qrss->conf; 3927 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3928 MLX5_RXQ_TYPE_HAIRPIN) 3929 qrss = NULL; 3930 } 3931 } 3932 if (qrss) { 3933 /* Check if it is in meter suffix table. */ 3934 mtr_sfx = attr->group == (attr->transfer ? 3935 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3936 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3937 /* 3938 * Q/RSS action on NIC Rx should be split in order to pass by 3939 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3940 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3941 */ 3942 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3943 sizeof(struct rte_flow_action_set_tag) + 3944 sizeof(struct rte_flow_action_jump); 3945 ext_actions = rte_zmalloc(__func__, act_size, 0); 3946 if (!ext_actions) 3947 return rte_flow_error_set(error, ENOMEM, 3948 RTE_FLOW_ERROR_TYPE_ACTION, 3949 NULL, "no memory to split " 3950 "metadata flow"); 3951 /* 3952 * If we are the suffix flow of meter, tag already exist. 3953 * Set the tag action to void. 3954 */ 3955 if (mtr_sfx) 3956 ext_actions[qrss - actions].type = 3957 RTE_FLOW_ACTION_TYPE_VOID; 3958 else 3959 ext_actions[qrss - actions].type = 3960 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3961 /* 3962 * Create the new actions list with removed Q/RSS action 3963 * and appended set tag and jump to register copy table 3964 * (RX_CP_TBL). We should preallocate unique tag ID here 3965 * in advance, because it is needed for set tag action. 3966 */ 3967 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3968 qrss, actions_n, error); 3969 if (!mtr_sfx && !qrss_id) { 3970 ret = -rte_errno; 3971 goto exit; 3972 } 3973 } else if (attr->egress && !attr->transfer) { 3974 /* 3975 * All the actions on NIC Tx should have a metadata register 3976 * copy action to copy reg_a from WQE to reg_c[meta] 3977 */ 3978 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3979 sizeof(struct mlx5_flow_action_copy_mreg); 3980 ext_actions = rte_zmalloc(__func__, act_size, 0); 3981 if (!ext_actions) 3982 return rte_flow_error_set(error, ENOMEM, 3983 RTE_FLOW_ERROR_TYPE_ACTION, 3984 NULL, "no memory to split " 3985 "metadata flow"); 3986 /* Create the action list appended with copy register. */ 3987 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 3988 actions_n, error, encap_idx); 3989 if (ret < 0) 3990 goto exit; 3991 } 3992 /* Add the unmodified original or prefix subflow. */ 3993 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr, 3994 items, ext_actions ? ext_actions : 3995 actions, external, error); 3996 if (ret < 0) 3997 goto exit; 3998 MLX5_ASSERT(dev_flow); 3999 if (qrss) { 4000 const struct rte_flow_attr q_attr = { 4001 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4002 .ingress = 1, 4003 }; 4004 /* Internal PMD action to set register. */ 4005 struct mlx5_rte_flow_item_tag q_tag_spec = { 4006 .data = qrss_id, 4007 .id = 0, 4008 }; 4009 struct rte_flow_item q_items[] = { 4010 { 4011 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4012 .spec = &q_tag_spec, 4013 .last = NULL, 4014 .mask = NULL, 4015 }, 4016 { 4017 .type = RTE_FLOW_ITEM_TYPE_END, 4018 }, 4019 }; 4020 struct rte_flow_action q_actions[] = { 4021 { 4022 .type = qrss->type, 4023 .conf = qrss->conf, 4024 }, 4025 { 4026 .type = RTE_FLOW_ACTION_TYPE_END, 4027 }, 4028 }; 4029 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 4030 4031 /* 4032 * Configure the tag item only if there is no meter subflow. 4033 * Since tag is already marked in the meter suffix subflow 4034 * we can just use the meter suffix items as is. 4035 */ 4036 if (qrss_id) { 4037 /* Not meter subflow. */ 4038 MLX5_ASSERT(!mtr_sfx); 4039 /* 4040 * Put unique id in prefix flow due to it is destroyed 4041 * after suffix flow and id will be freed after there 4042 * is no actual flows with this id and identifier 4043 * reallocation becomes possible (for example, for 4044 * other flows in other threads). 4045 */ 4046 dev_flow->handle->split_flow_id = qrss_id; 4047 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4048 error); 4049 if (ret < 0) 4050 goto exit; 4051 q_tag_spec.id = ret; 4052 } 4053 dev_flow = NULL; 4054 /* Add suffix subflow to execute Q/RSS. */ 4055 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 4056 &q_attr, mtr_sfx ? items : 4057 q_items, q_actions, 4058 external, error); 4059 if (ret < 0) 4060 goto exit; 4061 /* qrss ID should be freed if failed. */ 4062 qrss_id = 0; 4063 MLX5_ASSERT(dev_flow); 4064 } 4065 4066 exit: 4067 /* 4068 * We do not destroy the partially created sub_flows in case of error. 4069 * These ones are included into parent flow list and will be destroyed 4070 * by flow_drv_destroy. 4071 */ 4072 flow_qrss_free_id(dev, qrss_id); 4073 rte_free(ext_actions); 4074 return ret; 4075 } 4076 4077 /** 4078 * The splitting for meter feature. 4079 * 4080 * - The meter flow will be split to two flows as prefix and 4081 * suffix flow. The packets make sense only it pass the prefix 4082 * meter action. 4083 * 4084 * - Reg_C_5 is used for the packet to match betweend prefix and 4085 * suffix flow. 4086 * 4087 * @param dev 4088 * Pointer to Ethernet device. 4089 * @param[in] flow 4090 * Parent flow structure pointer. 4091 * @param[in] attr 4092 * Flow rule attributes. 4093 * @param[in] items 4094 * Pattern specification (list terminated by the END pattern item). 4095 * @param[in] actions 4096 * Associated actions (list terminated by the END action). 4097 * @param[in] external 4098 * This flow rule is created by request external to PMD. 4099 * @param[out] error 4100 * Perform verbose error reporting if not NULL. 4101 * @return 4102 * 0 on success, negative value otherwise 4103 */ 4104 static int 4105 flow_create_split_meter(struct rte_eth_dev *dev, 4106 struct rte_flow *flow, 4107 const struct rte_flow_attr *attr, 4108 const struct rte_flow_item items[], 4109 const struct rte_flow_action actions[], 4110 bool external, struct rte_flow_error *error) 4111 { 4112 struct mlx5_priv *priv = dev->data->dev_private; 4113 struct rte_flow_action *sfx_actions = NULL; 4114 struct rte_flow_action *pre_actions = NULL; 4115 struct rte_flow_item *sfx_items = NULL; 4116 struct mlx5_flow *dev_flow = NULL; 4117 struct rte_flow_attr sfx_attr = *attr; 4118 uint32_t mtr = 0; 4119 uint32_t mtr_tag_id = 0; 4120 size_t act_size; 4121 size_t item_size; 4122 int actions_n = 0; 4123 int ret; 4124 4125 if (priv->mtr_en) 4126 actions_n = flow_check_meter_action(actions, &mtr); 4127 if (mtr) { 4128 /* The five prefix actions: meter, decap, encap, tag, end. */ 4129 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4130 sizeof(struct mlx5_rte_flow_action_set_tag); 4131 /* tag, vlan, port id, end. */ 4132 #define METER_SUFFIX_ITEM 4 4133 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4134 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4135 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 4136 if (!sfx_actions) 4137 return rte_flow_error_set(error, ENOMEM, 4138 RTE_FLOW_ERROR_TYPE_ACTION, 4139 NULL, "no memory to split " 4140 "meter flow"); 4141 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4142 act_size); 4143 pre_actions = sfx_actions + actions_n; 4144 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4145 actions, sfx_actions, 4146 pre_actions); 4147 if (!mtr_tag_id) { 4148 ret = -rte_errno; 4149 goto exit; 4150 } 4151 /* Add the prefix subflow. */ 4152 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr, 4153 items, pre_actions, external, 4154 error); 4155 if (ret) { 4156 ret = -rte_errno; 4157 goto exit; 4158 } 4159 dev_flow->handle->split_flow_id = mtr_tag_id; 4160 /* Setting the sfx group atrr. */ 4161 sfx_attr.group = sfx_attr.transfer ? 4162 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4163 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4164 } 4165 /* Add the prefix subflow. */ 4166 ret = flow_create_split_metadata(dev, flow, dev_flow ? 4167 flow_get_prefix_layer_flags(dev_flow) : 4168 0, &sfx_attr, 4169 sfx_items ? sfx_items : items, 4170 sfx_actions ? sfx_actions : actions, 4171 external, error); 4172 exit: 4173 if (sfx_actions) 4174 rte_free(sfx_actions); 4175 return ret; 4176 } 4177 4178 /** 4179 * Split the flow to subflow set. The splitters might be linked 4180 * in the chain, like this: 4181 * flow_create_split_outer() calls: 4182 * flow_create_split_meter() calls: 4183 * flow_create_split_metadata(meter_subflow_0) calls: 4184 * flow_create_split_inner(metadata_subflow_0) 4185 * flow_create_split_inner(metadata_subflow_1) 4186 * flow_create_split_inner(metadata_subflow_2) 4187 * flow_create_split_metadata(meter_subflow_1) calls: 4188 * flow_create_split_inner(metadata_subflow_0) 4189 * flow_create_split_inner(metadata_subflow_1) 4190 * flow_create_split_inner(metadata_subflow_2) 4191 * 4192 * This provide flexible way to add new levels of flow splitting. 4193 * The all of successfully created subflows are included to the 4194 * parent flow dev_flow list. 4195 * 4196 * @param dev 4197 * Pointer to Ethernet device. 4198 * @param[in] flow 4199 * Parent flow structure pointer. 4200 * @param[in] attr 4201 * Flow rule attributes. 4202 * @param[in] items 4203 * Pattern specification (list terminated by the END pattern item). 4204 * @param[in] actions 4205 * Associated actions (list terminated by the END action). 4206 * @param[in] external 4207 * This flow rule is created by request external to PMD. 4208 * @param[out] error 4209 * Perform verbose error reporting if not NULL. 4210 * @return 4211 * 0 on success, negative value otherwise 4212 */ 4213 static int 4214 flow_create_split_outer(struct rte_eth_dev *dev, 4215 struct rte_flow *flow, 4216 const struct rte_flow_attr *attr, 4217 const struct rte_flow_item items[], 4218 const struct rte_flow_action actions[], 4219 bool external, struct rte_flow_error *error) 4220 { 4221 int ret; 4222 4223 ret = flow_create_split_meter(dev, flow, attr, items, 4224 actions, external, error); 4225 MLX5_ASSERT(ret <= 0); 4226 return ret; 4227 } 4228 4229 /** 4230 * Create a flow and add it to @p list. 4231 * 4232 * @param dev 4233 * Pointer to Ethernet device. 4234 * @param list 4235 * Pointer to a TAILQ flow list. If this parameter NULL, 4236 * no list insertion occurred, flow is just created, 4237 * this is caller's responsibility to track the 4238 * created flow. 4239 * @param[in] attr 4240 * Flow rule attributes. 4241 * @param[in] items 4242 * Pattern specification (list terminated by the END pattern item). 4243 * @param[in] actions 4244 * Associated actions (list terminated by the END action). 4245 * @param[in] external 4246 * This flow rule is created by request external to PMD. 4247 * @param[out] error 4248 * Perform verbose error reporting if not NULL. 4249 * 4250 * @return 4251 * A flow index on success, 0 otherwise and rte_errno is set. 4252 */ 4253 static uint32_t 4254 flow_list_create(struct rte_eth_dev *dev, uint32_t *list, 4255 const struct rte_flow_attr *attr, 4256 const struct rte_flow_item items[], 4257 const struct rte_flow_action actions[], 4258 bool external, struct rte_flow_error *error) 4259 { 4260 struct mlx5_priv *priv = dev->data->dev_private; 4261 struct rte_flow *flow = NULL; 4262 struct mlx5_flow *dev_flow; 4263 const struct rte_flow_action_rss *rss; 4264 union { 4265 struct rte_flow_expand_rss buf; 4266 uint8_t buffer[2048]; 4267 } expand_buffer; 4268 union { 4269 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4270 uint8_t buffer[2048]; 4271 } actions_rx; 4272 union { 4273 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4274 uint8_t buffer[2048]; 4275 } actions_hairpin_tx; 4276 union { 4277 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4278 uint8_t buffer[2048]; 4279 } items_tx; 4280 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4281 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 4282 priv->rss_desc)[!!priv->flow_idx]; 4283 const struct rte_flow_action *p_actions_rx = actions; 4284 uint32_t i; 4285 uint32_t idx = 0; 4286 int hairpin_flow; 4287 uint32_t hairpin_id = 0; 4288 struct rte_flow_attr attr_tx = { .priority = 0 }; 4289 int ret; 4290 4291 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4292 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 4293 external, hairpin_flow, error); 4294 if (ret < 0) 4295 return 0; 4296 if (hairpin_flow > 0) { 4297 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4298 rte_errno = EINVAL; 4299 return 0; 4300 } 4301 flow_hairpin_split(dev, actions, actions_rx.actions, 4302 actions_hairpin_tx.actions, items_tx.items, 4303 &hairpin_id); 4304 p_actions_rx = actions_rx.actions; 4305 } 4306 flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx); 4307 if (!flow) { 4308 rte_errno = ENOMEM; 4309 goto error_before_flow; 4310 } 4311 flow->drv_type = flow_get_drv_type(dev, attr); 4312 if (hairpin_id != 0) 4313 flow->hairpin_flow_id = hairpin_id; 4314 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4315 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4316 memset(rss_desc, 0, sizeof(*rss_desc)); 4317 rss = flow_get_rss_action(p_actions_rx); 4318 if (rss) { 4319 /* 4320 * The following information is required by 4321 * mlx5_flow_hashfields_adjust() in advance. 4322 */ 4323 rss_desc->level = rss->level; 4324 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4325 rss_desc->types = !rss->types ? ETH_RSS_IP : rss->types; 4326 } 4327 flow->dev_handles = 0; 4328 if (rss && rss->types) { 4329 unsigned int graph_root; 4330 4331 graph_root = find_graph_root(items, rss->level); 4332 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4333 items, rss->types, 4334 mlx5_support_expansion, 4335 graph_root); 4336 MLX5_ASSERT(ret > 0 && 4337 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4338 } else { 4339 buf->entries = 1; 4340 buf->entry[0].pattern = (void *)(uintptr_t)items; 4341 } 4342 /* 4343 * Record the start index when there is a nested call. All sub-flows 4344 * need to be translated before another calling. 4345 * No need to use ping-pong buffer to save memory here. 4346 */ 4347 if (priv->flow_idx) { 4348 MLX5_ASSERT(!priv->flow_nested_idx); 4349 priv->flow_nested_idx = priv->flow_idx; 4350 } 4351 for (i = 0; i < buf->entries; ++i) { 4352 /* 4353 * The splitter may create multiple dev_flows, 4354 * depending on configuration. In the simplest 4355 * case it just creates unmodified original flow. 4356 */ 4357 ret = flow_create_split_outer(dev, flow, attr, 4358 buf->entry[i].pattern, 4359 p_actions_rx, external, 4360 error); 4361 if (ret < 0) 4362 goto error; 4363 } 4364 /* Create the tx flow. */ 4365 if (hairpin_flow) { 4366 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4367 attr_tx.ingress = 0; 4368 attr_tx.egress = 1; 4369 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 4370 actions_hairpin_tx.actions, error); 4371 if (!dev_flow) 4372 goto error; 4373 dev_flow->flow = flow; 4374 dev_flow->external = 0; 4375 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4376 dev_flow->handle, next); 4377 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4378 items_tx.items, 4379 actions_hairpin_tx.actions, error); 4380 if (ret < 0) 4381 goto error; 4382 } 4383 /* 4384 * Update the metadata register copy table. If extensive 4385 * metadata feature is enabled and registers are supported 4386 * we might create the extra rte_flow for each unique 4387 * MARK/FLAG action ID. 4388 * 4389 * The table is updated for ingress Flows only, because 4390 * the egress Flows belong to the different device and 4391 * copy table should be updated in peer NIC Rx domain. 4392 */ 4393 if (attr->ingress && 4394 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4395 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4396 if (ret) 4397 goto error; 4398 } 4399 /* 4400 * If the flow is external (from application) OR device is started, then 4401 * the flow will be applied immediately. 4402 */ 4403 if (external || dev->data->dev_started) { 4404 ret = flow_drv_apply(dev, flow, error); 4405 if (ret < 0) 4406 goto error; 4407 } 4408 if (list) 4409 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx, 4410 flow, next); 4411 flow_rxq_flags_set(dev, flow); 4412 /* Nested flow creation index recovery. */ 4413 priv->flow_idx = priv->flow_nested_idx; 4414 if (priv->flow_nested_idx) 4415 priv->flow_nested_idx = 0; 4416 return idx; 4417 error: 4418 MLX5_ASSERT(flow); 4419 ret = rte_errno; /* Save rte_errno before cleanup. */ 4420 flow_mreg_del_copy_action(dev, flow); 4421 flow_drv_destroy(dev, flow); 4422 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx); 4423 rte_errno = ret; /* Restore rte_errno. */ 4424 error_before_flow: 4425 ret = rte_errno; 4426 if (hairpin_id) 4427 mlx5_flow_id_release(priv->sh->flow_id_pool, 4428 hairpin_id); 4429 rte_errno = ret; 4430 priv->flow_idx = priv->flow_nested_idx; 4431 if (priv->flow_nested_idx) 4432 priv->flow_nested_idx = 0; 4433 return 0; 4434 } 4435 4436 /** 4437 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4438 * incoming packets to table 1. 4439 * 4440 * Other flow rules, requested for group n, will be created in 4441 * e-switch table n+1. 4442 * Jump action to e-switch group n will be created to group n+1. 4443 * 4444 * Used when working in switchdev mode, to utilise advantages of table 1 4445 * and above. 4446 * 4447 * @param dev 4448 * Pointer to Ethernet device. 4449 * 4450 * @return 4451 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4452 */ 4453 struct rte_flow * 4454 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4455 { 4456 const struct rte_flow_attr attr = { 4457 .group = 0, 4458 .priority = 0, 4459 .ingress = 1, 4460 .egress = 0, 4461 .transfer = 1, 4462 }; 4463 const struct rte_flow_item pattern = { 4464 .type = RTE_FLOW_ITEM_TYPE_END, 4465 }; 4466 struct rte_flow_action_jump jump = { 4467 .group = 1, 4468 }; 4469 const struct rte_flow_action actions[] = { 4470 { 4471 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4472 .conf = &jump, 4473 }, 4474 { 4475 .type = RTE_FLOW_ACTION_TYPE_END, 4476 }, 4477 }; 4478 struct mlx5_priv *priv = dev->data->dev_private; 4479 struct rte_flow_error error; 4480 4481 return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows, 4482 &attr, &pattern, 4483 actions, false, &error); 4484 } 4485 4486 /** 4487 * Validate a flow supported by the NIC. 4488 * 4489 * @see rte_flow_validate() 4490 * @see rte_flow_ops 4491 */ 4492 int 4493 mlx5_flow_validate(struct rte_eth_dev *dev, 4494 const struct rte_flow_attr *attr, 4495 const struct rte_flow_item items[], 4496 const struct rte_flow_action actions[], 4497 struct rte_flow_error *error) 4498 { 4499 int hairpin_flow; 4500 4501 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4502 return flow_drv_validate(dev, attr, items, actions, 4503 true, hairpin_flow, error); 4504 } 4505 4506 /** 4507 * Create a flow. 4508 * 4509 * @see rte_flow_create() 4510 * @see rte_flow_ops 4511 */ 4512 struct rte_flow * 4513 mlx5_flow_create(struct rte_eth_dev *dev, 4514 const struct rte_flow_attr *attr, 4515 const struct rte_flow_item items[], 4516 const struct rte_flow_action actions[], 4517 struct rte_flow_error *error) 4518 { 4519 struct mlx5_priv *priv = dev->data->dev_private; 4520 4521 /* 4522 * If the device is not started yet, it is not allowed to created a 4523 * flow from application. PMD default flows and traffic control flows 4524 * are not affected. 4525 */ 4526 if (unlikely(!dev->data->dev_started)) { 4527 DRV_LOG(DEBUG, "port %u is not started when " 4528 "inserting a flow", dev->data->port_id); 4529 rte_flow_error_set(error, ENODEV, 4530 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4531 NULL, 4532 "port not started"); 4533 return NULL; 4534 } 4535 return (void *)(uintptr_t)flow_list_create(dev, &priv->flows, 4536 attr, items, actions, true, error); 4537 } 4538 4539 /** 4540 * Destroy a flow in a list. 4541 * 4542 * @param dev 4543 * Pointer to Ethernet device. 4544 * @param list 4545 * Pointer to the Indexed flow list. If this parameter NULL, 4546 * there is no flow removal from the list. Be noted that as 4547 * flow is add to the indexed list, memory of the indexed 4548 * list points to maybe changed as flow destroyed. 4549 * @param[in] flow_idx 4550 * Index of flow to destroy. 4551 */ 4552 static void 4553 flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list, 4554 uint32_t flow_idx) 4555 { 4556 struct mlx5_priv *priv = dev->data->dev_private; 4557 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 4558 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 4559 [MLX5_IPOOL_RTE_FLOW], flow_idx); 4560 4561 if (!flow) 4562 return; 4563 /* 4564 * Update RX queue flags only if port is started, otherwise it is 4565 * already clean. 4566 */ 4567 if (dev->data->dev_started) 4568 flow_rxq_flags_trim(dev, flow); 4569 if (flow->hairpin_flow_id) 4570 mlx5_flow_id_release(priv->sh->flow_id_pool, 4571 flow->hairpin_flow_id); 4572 flow_drv_destroy(dev, flow); 4573 if (list) 4574 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, 4575 flow_idx, flow, next); 4576 flow_mreg_del_copy_action(dev, flow); 4577 if (flow->fdir) { 4578 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 4579 if (priv_fdir_flow->rix_flow == flow_idx) 4580 break; 4581 } 4582 if (priv_fdir_flow) { 4583 LIST_REMOVE(priv_fdir_flow, next); 4584 rte_free(priv_fdir_flow->fdir); 4585 rte_free(priv_fdir_flow); 4586 } 4587 } 4588 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 4589 } 4590 4591 /** 4592 * Destroy all flows. 4593 * 4594 * @param dev 4595 * Pointer to Ethernet device. 4596 * @param list 4597 * Pointer to the Indexed flow list. 4598 * @param active 4599 * If flushing is called avtively. 4600 */ 4601 void 4602 mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active) 4603 { 4604 uint32_t num_flushed = 0; 4605 4606 while (*list) { 4607 flow_list_destroy(dev, list, *list); 4608 num_flushed++; 4609 } 4610 if (active) { 4611 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 4612 dev->data->port_id, num_flushed); 4613 } 4614 } 4615 4616 /** 4617 * Remove all flows. 4618 * 4619 * @param dev 4620 * Pointer to Ethernet device. 4621 * @param list 4622 * Pointer to the Indexed flow list. 4623 */ 4624 void 4625 mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list) 4626 { 4627 struct mlx5_priv *priv = dev->data->dev_private; 4628 struct rte_flow *flow = NULL; 4629 uint32_t idx; 4630 4631 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4632 flow, next) { 4633 flow_drv_remove(dev, flow); 4634 flow_mreg_stop_copy_action(dev, flow); 4635 } 4636 flow_mreg_del_default_copy_action(dev); 4637 flow_rxq_flags_clear(dev); 4638 } 4639 4640 /** 4641 * Add all flows. 4642 * 4643 * @param dev 4644 * Pointer to Ethernet device. 4645 * @param list 4646 * Pointer to the Indexed flow list. 4647 * 4648 * @return 4649 * 0 on success, a negative errno value otherwise and rte_errno is set. 4650 */ 4651 int 4652 mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list) 4653 { 4654 struct mlx5_priv *priv = dev->data->dev_private; 4655 struct rte_flow *flow = NULL; 4656 struct rte_flow_error error; 4657 uint32_t idx; 4658 int ret = 0; 4659 4660 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4661 ret = flow_mreg_add_default_copy_action(dev, &error); 4662 if (ret < 0) 4663 return -rte_errno; 4664 /* Apply Flows created by application. */ 4665 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], *list, idx, 4666 flow, next) { 4667 ret = flow_mreg_start_copy_action(dev, flow); 4668 if (ret < 0) 4669 goto error; 4670 ret = flow_drv_apply(dev, flow, &error); 4671 if (ret < 0) 4672 goto error; 4673 flow_rxq_flags_set(dev, flow); 4674 } 4675 return 0; 4676 error: 4677 ret = rte_errno; /* Save rte_errno before cleanup. */ 4678 mlx5_flow_stop(dev, list); 4679 rte_errno = ret; /* Restore rte_errno. */ 4680 return -rte_errno; 4681 } 4682 4683 /** 4684 * Stop all default actions for flows. 4685 * 4686 * @param dev 4687 * Pointer to Ethernet device. 4688 */ 4689 void 4690 mlx5_flow_stop_default(struct rte_eth_dev *dev) 4691 { 4692 flow_mreg_del_default_copy_action(dev); 4693 flow_rxq_flags_clear(dev); 4694 } 4695 4696 /** 4697 * Start all default actions for flows. 4698 * 4699 * @param dev 4700 * Pointer to Ethernet device. 4701 * @return 4702 * 0 on success, a negative errno value otherwise and rte_errno is set. 4703 */ 4704 int 4705 mlx5_flow_start_default(struct rte_eth_dev *dev) 4706 { 4707 struct rte_flow_error error; 4708 4709 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4710 return flow_mreg_add_default_copy_action(dev, &error); 4711 } 4712 4713 /** 4714 * Allocate intermediate resources for flow creation. 4715 * 4716 * @param dev 4717 * Pointer to Ethernet device. 4718 */ 4719 void 4720 mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) 4721 { 4722 struct mlx5_priv *priv = dev->data->dev_private; 4723 4724 if (!priv->inter_flows) { 4725 priv->inter_flows = rte_calloc(__func__, 1, 4726 MLX5_NUM_MAX_DEV_FLOWS * 4727 sizeof(struct mlx5_flow) + 4728 (sizeof(struct mlx5_flow_rss_desc) + 4729 sizeof(uint16_t) * UINT16_MAX) * 2, 0); 4730 if (!priv->inter_flows) { 4731 DRV_LOG(ERR, "can't allocate intermediate memory."); 4732 return; 4733 } 4734 } 4735 priv->rss_desc = &((struct mlx5_flow *)priv->inter_flows) 4736 [MLX5_NUM_MAX_DEV_FLOWS]; 4737 /* Reset the index. */ 4738 priv->flow_idx = 0; 4739 priv->flow_nested_idx = 0; 4740 } 4741 4742 /** 4743 * Free intermediate resources for flows. 4744 * 4745 * @param dev 4746 * Pointer to Ethernet device. 4747 */ 4748 void 4749 mlx5_flow_free_intermediate(struct rte_eth_dev *dev) 4750 { 4751 struct mlx5_priv *priv = dev->data->dev_private; 4752 4753 rte_free(priv->inter_flows); 4754 priv->inter_flows = NULL; 4755 } 4756 4757 /** 4758 * Verify the flow list is empty 4759 * 4760 * @param dev 4761 * Pointer to Ethernet device. 4762 * 4763 * @return the number of flows not released. 4764 */ 4765 int 4766 mlx5_flow_verify(struct rte_eth_dev *dev) 4767 { 4768 struct mlx5_priv *priv = dev->data->dev_private; 4769 struct rte_flow *flow; 4770 uint32_t idx; 4771 int ret = 0; 4772 4773 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx, 4774 flow, next) { 4775 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4776 dev->data->port_id, (void *)flow); 4777 ++ret; 4778 } 4779 return ret; 4780 } 4781 4782 /** 4783 * Enable default hairpin egress flow. 4784 * 4785 * @param dev 4786 * Pointer to Ethernet device. 4787 * @param queue 4788 * The queue index. 4789 * 4790 * @return 4791 * 0 on success, a negative errno value otherwise and rte_errno is set. 4792 */ 4793 int 4794 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4795 uint32_t queue) 4796 { 4797 struct mlx5_priv *priv = dev->data->dev_private; 4798 const struct rte_flow_attr attr = { 4799 .egress = 1, 4800 .priority = 0, 4801 }; 4802 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4803 .queue = queue, 4804 }; 4805 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4806 .queue = UINT32_MAX, 4807 }; 4808 struct rte_flow_item items[] = { 4809 { 4810 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4811 .spec = &queue_spec, 4812 .last = NULL, 4813 .mask = &queue_mask, 4814 }, 4815 { 4816 .type = RTE_FLOW_ITEM_TYPE_END, 4817 }, 4818 }; 4819 struct rte_flow_action_jump jump = { 4820 .group = MLX5_HAIRPIN_TX_TABLE, 4821 }; 4822 struct rte_flow_action actions[2]; 4823 uint32_t flow_idx; 4824 struct rte_flow_error error; 4825 4826 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4827 actions[0].conf = &jump; 4828 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4829 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4830 &attr, items, actions, false, &error); 4831 if (!flow_idx) { 4832 DRV_LOG(DEBUG, 4833 "Failed to create ctrl flow: rte_errno(%d)," 4834 " type(%d), message(%s)", 4835 rte_errno, error.type, 4836 error.message ? error.message : " (no stated reason)"); 4837 return -rte_errno; 4838 } 4839 return 0; 4840 } 4841 4842 /** 4843 * Enable a control flow configured from the control plane. 4844 * 4845 * @param dev 4846 * Pointer to Ethernet device. 4847 * @param eth_spec 4848 * An Ethernet flow spec to apply. 4849 * @param eth_mask 4850 * An Ethernet flow mask to apply. 4851 * @param vlan_spec 4852 * A VLAN flow spec to apply. 4853 * @param vlan_mask 4854 * A VLAN flow mask to apply. 4855 * 4856 * @return 4857 * 0 on success, a negative errno value otherwise and rte_errno is set. 4858 */ 4859 int 4860 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4861 struct rte_flow_item_eth *eth_spec, 4862 struct rte_flow_item_eth *eth_mask, 4863 struct rte_flow_item_vlan *vlan_spec, 4864 struct rte_flow_item_vlan *vlan_mask) 4865 { 4866 struct mlx5_priv *priv = dev->data->dev_private; 4867 const struct rte_flow_attr attr = { 4868 .ingress = 1, 4869 .priority = MLX5_FLOW_PRIO_RSVD, 4870 }; 4871 struct rte_flow_item items[] = { 4872 { 4873 .type = RTE_FLOW_ITEM_TYPE_ETH, 4874 .spec = eth_spec, 4875 .last = NULL, 4876 .mask = eth_mask, 4877 }, 4878 { 4879 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4880 RTE_FLOW_ITEM_TYPE_END, 4881 .spec = vlan_spec, 4882 .last = NULL, 4883 .mask = vlan_mask, 4884 }, 4885 { 4886 .type = RTE_FLOW_ITEM_TYPE_END, 4887 }, 4888 }; 4889 uint16_t queue[priv->reta_idx_n]; 4890 struct rte_flow_action_rss action_rss = { 4891 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4892 .level = 0, 4893 .types = priv->rss_conf.rss_hf, 4894 .key_len = priv->rss_conf.rss_key_len, 4895 .queue_num = priv->reta_idx_n, 4896 .key = priv->rss_conf.rss_key, 4897 .queue = queue, 4898 }; 4899 struct rte_flow_action actions[] = { 4900 { 4901 .type = RTE_FLOW_ACTION_TYPE_RSS, 4902 .conf = &action_rss, 4903 }, 4904 { 4905 .type = RTE_FLOW_ACTION_TYPE_END, 4906 }, 4907 }; 4908 uint32_t flow_idx; 4909 struct rte_flow_error error; 4910 unsigned int i; 4911 4912 if (!priv->reta_idx_n || !priv->rxqs_n) { 4913 return 0; 4914 } 4915 for (i = 0; i != priv->reta_idx_n; ++i) 4916 queue[i] = (*priv->reta_idx)[i]; 4917 flow_idx = flow_list_create(dev, &priv->ctrl_flows, 4918 &attr, items, actions, false, &error); 4919 if (!flow_idx) 4920 return -rte_errno; 4921 return 0; 4922 } 4923 4924 /** 4925 * Enable a flow control configured from the control plane. 4926 * 4927 * @param dev 4928 * Pointer to Ethernet device. 4929 * @param eth_spec 4930 * An Ethernet flow spec to apply. 4931 * @param eth_mask 4932 * An Ethernet flow mask to apply. 4933 * 4934 * @return 4935 * 0 on success, a negative errno value otherwise and rte_errno is set. 4936 */ 4937 int 4938 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4939 struct rte_flow_item_eth *eth_spec, 4940 struct rte_flow_item_eth *eth_mask) 4941 { 4942 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4943 } 4944 4945 /** 4946 * Destroy a flow. 4947 * 4948 * @see rte_flow_destroy() 4949 * @see rte_flow_ops 4950 */ 4951 int 4952 mlx5_flow_destroy(struct rte_eth_dev *dev, 4953 struct rte_flow *flow, 4954 struct rte_flow_error *error __rte_unused) 4955 { 4956 struct mlx5_priv *priv = dev->data->dev_private; 4957 4958 flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow); 4959 return 0; 4960 } 4961 4962 /** 4963 * Destroy all flows. 4964 * 4965 * @see rte_flow_flush() 4966 * @see rte_flow_ops 4967 */ 4968 int 4969 mlx5_flow_flush(struct rte_eth_dev *dev, 4970 struct rte_flow_error *error __rte_unused) 4971 { 4972 struct mlx5_priv *priv = dev->data->dev_private; 4973 4974 mlx5_flow_list_flush(dev, &priv->flows, false); 4975 return 0; 4976 } 4977 4978 /** 4979 * Isolated mode. 4980 * 4981 * @see rte_flow_isolate() 4982 * @see rte_flow_ops 4983 */ 4984 int 4985 mlx5_flow_isolate(struct rte_eth_dev *dev, 4986 int enable, 4987 struct rte_flow_error *error) 4988 { 4989 struct mlx5_priv *priv = dev->data->dev_private; 4990 4991 if (dev->data->dev_started) { 4992 rte_flow_error_set(error, EBUSY, 4993 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4994 NULL, 4995 "port must be stopped first"); 4996 return -rte_errno; 4997 } 4998 priv->isolated = !!enable; 4999 if (enable) 5000 dev->dev_ops = &mlx5_dev_ops_isolate; 5001 else 5002 dev->dev_ops = &mlx5_dev_ops; 5003 return 0; 5004 } 5005 5006 /** 5007 * Query a flow. 5008 * 5009 * @see rte_flow_query() 5010 * @see rte_flow_ops 5011 */ 5012 static int 5013 flow_drv_query(struct rte_eth_dev *dev, 5014 uint32_t flow_idx, 5015 const struct rte_flow_action *actions, 5016 void *data, 5017 struct rte_flow_error *error) 5018 { 5019 struct mlx5_priv *priv = dev->data->dev_private; 5020 const struct mlx5_flow_driver_ops *fops; 5021 struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool 5022 [MLX5_IPOOL_RTE_FLOW], 5023 flow_idx); 5024 enum mlx5_flow_drv_type ftype; 5025 5026 if (!flow) { 5027 return rte_flow_error_set(error, ENOENT, 5028 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5029 NULL, 5030 "invalid flow handle"); 5031 } 5032 ftype = flow->drv_type; 5033 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 5034 fops = flow_get_drv_ops(ftype); 5035 5036 return fops->query(dev, flow, actions, data, error); 5037 } 5038 5039 /** 5040 * Query a flow. 5041 * 5042 * @see rte_flow_query() 5043 * @see rte_flow_ops 5044 */ 5045 int 5046 mlx5_flow_query(struct rte_eth_dev *dev, 5047 struct rte_flow *flow, 5048 const struct rte_flow_action *actions, 5049 void *data, 5050 struct rte_flow_error *error) 5051 { 5052 int ret; 5053 5054 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 5055 error); 5056 if (ret < 0) 5057 return ret; 5058 return 0; 5059 } 5060 5061 /** 5062 * Convert a flow director filter to a generic flow. 5063 * 5064 * @param dev 5065 * Pointer to Ethernet device. 5066 * @param fdir_filter 5067 * Flow director filter to add. 5068 * @param attributes 5069 * Generic flow parameters structure. 5070 * 5071 * @return 5072 * 0 on success, a negative errno value otherwise and rte_errno is set. 5073 */ 5074 static int 5075 flow_fdir_filter_convert(struct rte_eth_dev *dev, 5076 const struct rte_eth_fdir_filter *fdir_filter, 5077 struct mlx5_fdir *attributes) 5078 { 5079 struct mlx5_priv *priv = dev->data->dev_private; 5080 const struct rte_eth_fdir_input *input = &fdir_filter->input; 5081 const struct rte_eth_fdir_masks *mask = 5082 &dev->data->dev_conf.fdir_conf.mask; 5083 5084 /* Validate queue number. */ 5085 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 5086 DRV_LOG(ERR, "port %u invalid queue number %d", 5087 dev->data->port_id, fdir_filter->action.rx_queue); 5088 rte_errno = EINVAL; 5089 return -rte_errno; 5090 } 5091 attributes->attr.ingress = 1; 5092 attributes->items[0] = (struct rte_flow_item) { 5093 .type = RTE_FLOW_ITEM_TYPE_ETH, 5094 .spec = &attributes->l2, 5095 .mask = &attributes->l2_mask, 5096 }; 5097 switch (fdir_filter->action.behavior) { 5098 case RTE_ETH_FDIR_ACCEPT: 5099 attributes->actions[0] = (struct rte_flow_action){ 5100 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 5101 .conf = &attributes->queue, 5102 }; 5103 break; 5104 case RTE_ETH_FDIR_REJECT: 5105 attributes->actions[0] = (struct rte_flow_action){ 5106 .type = RTE_FLOW_ACTION_TYPE_DROP, 5107 }; 5108 break; 5109 default: 5110 DRV_LOG(ERR, "port %u invalid behavior %d", 5111 dev->data->port_id, 5112 fdir_filter->action.behavior); 5113 rte_errno = ENOTSUP; 5114 return -rte_errno; 5115 } 5116 attributes->queue.index = fdir_filter->action.rx_queue; 5117 /* Handle L3. */ 5118 switch (fdir_filter->input.flow_type) { 5119 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5120 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5121 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5122 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 5123 .src_addr = input->flow.ip4_flow.src_ip, 5124 .dst_addr = input->flow.ip4_flow.dst_ip, 5125 .time_to_live = input->flow.ip4_flow.ttl, 5126 .type_of_service = input->flow.ip4_flow.tos, 5127 }; 5128 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 5129 .src_addr = mask->ipv4_mask.src_ip, 5130 .dst_addr = mask->ipv4_mask.dst_ip, 5131 .time_to_live = mask->ipv4_mask.ttl, 5132 .type_of_service = mask->ipv4_mask.tos, 5133 .next_proto_id = mask->ipv4_mask.proto, 5134 }; 5135 attributes->items[1] = (struct rte_flow_item){ 5136 .type = RTE_FLOW_ITEM_TYPE_IPV4, 5137 .spec = &attributes->l3, 5138 .mask = &attributes->l3_mask, 5139 }; 5140 break; 5141 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5142 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5143 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5144 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 5145 .hop_limits = input->flow.ipv6_flow.hop_limits, 5146 .proto = input->flow.ipv6_flow.proto, 5147 }; 5148 5149 memcpy(attributes->l3.ipv6.hdr.src_addr, 5150 input->flow.ipv6_flow.src_ip, 5151 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5152 memcpy(attributes->l3.ipv6.hdr.dst_addr, 5153 input->flow.ipv6_flow.dst_ip, 5154 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 5155 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 5156 mask->ipv6_mask.src_ip, 5157 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5158 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 5159 mask->ipv6_mask.dst_ip, 5160 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 5161 attributes->items[1] = (struct rte_flow_item){ 5162 .type = RTE_FLOW_ITEM_TYPE_IPV6, 5163 .spec = &attributes->l3, 5164 .mask = &attributes->l3_mask, 5165 }; 5166 break; 5167 default: 5168 DRV_LOG(ERR, "port %u invalid flow type%d", 5169 dev->data->port_id, fdir_filter->input.flow_type); 5170 rte_errno = ENOTSUP; 5171 return -rte_errno; 5172 } 5173 /* Handle L4. */ 5174 switch (fdir_filter->input.flow_type) { 5175 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 5176 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5177 .src_port = input->flow.udp4_flow.src_port, 5178 .dst_port = input->flow.udp4_flow.dst_port, 5179 }; 5180 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5181 .src_port = mask->src_port_mask, 5182 .dst_port = mask->dst_port_mask, 5183 }; 5184 attributes->items[2] = (struct rte_flow_item){ 5185 .type = RTE_FLOW_ITEM_TYPE_UDP, 5186 .spec = &attributes->l4, 5187 .mask = &attributes->l4_mask, 5188 }; 5189 break; 5190 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 5191 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5192 .src_port = input->flow.tcp4_flow.src_port, 5193 .dst_port = input->flow.tcp4_flow.dst_port, 5194 }; 5195 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5196 .src_port = mask->src_port_mask, 5197 .dst_port = mask->dst_port_mask, 5198 }; 5199 attributes->items[2] = (struct rte_flow_item){ 5200 .type = RTE_FLOW_ITEM_TYPE_TCP, 5201 .spec = &attributes->l4, 5202 .mask = &attributes->l4_mask, 5203 }; 5204 break; 5205 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 5206 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 5207 .src_port = input->flow.udp6_flow.src_port, 5208 .dst_port = input->flow.udp6_flow.dst_port, 5209 }; 5210 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 5211 .src_port = mask->src_port_mask, 5212 .dst_port = mask->dst_port_mask, 5213 }; 5214 attributes->items[2] = (struct rte_flow_item){ 5215 .type = RTE_FLOW_ITEM_TYPE_UDP, 5216 .spec = &attributes->l4, 5217 .mask = &attributes->l4_mask, 5218 }; 5219 break; 5220 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5221 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5222 .src_port = input->flow.tcp6_flow.src_port, 5223 .dst_port = input->flow.tcp6_flow.dst_port, 5224 }; 5225 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5226 .src_port = mask->src_port_mask, 5227 .dst_port = mask->dst_port_mask, 5228 }; 5229 attributes->items[2] = (struct rte_flow_item){ 5230 .type = RTE_FLOW_ITEM_TYPE_TCP, 5231 .spec = &attributes->l4, 5232 .mask = &attributes->l4_mask, 5233 }; 5234 break; 5235 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5236 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5237 break; 5238 default: 5239 DRV_LOG(ERR, "port %u invalid flow type%d", 5240 dev->data->port_id, fdir_filter->input.flow_type); 5241 rte_errno = ENOTSUP; 5242 return -rte_errno; 5243 } 5244 return 0; 5245 } 5246 5247 #define FLOW_FDIR_CMP(f1, f2, fld) \ 5248 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 5249 5250 /** 5251 * Compare two FDIR flows. If items and actions are identical, the two flows are 5252 * regarded as same. 5253 * 5254 * @param dev 5255 * Pointer to Ethernet device. 5256 * @param f1 5257 * FDIR flow to compare. 5258 * @param f2 5259 * FDIR flow to compare. 5260 * 5261 * @return 5262 * Zero on match, 1 otherwise. 5263 */ 5264 static int 5265 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 5266 { 5267 if (FLOW_FDIR_CMP(f1, f2, attr) || 5268 FLOW_FDIR_CMP(f1, f2, l2) || 5269 FLOW_FDIR_CMP(f1, f2, l2_mask) || 5270 FLOW_FDIR_CMP(f1, f2, l3) || 5271 FLOW_FDIR_CMP(f1, f2, l3_mask) || 5272 FLOW_FDIR_CMP(f1, f2, l4) || 5273 FLOW_FDIR_CMP(f1, f2, l4_mask) || 5274 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 5275 return 1; 5276 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 5277 FLOW_FDIR_CMP(f1, f2, queue)) 5278 return 1; 5279 return 0; 5280 } 5281 5282 /** 5283 * Search device flow list to find out a matched FDIR flow. 5284 * 5285 * @param dev 5286 * Pointer to Ethernet device. 5287 * @param fdir_flow 5288 * FDIR flow to lookup. 5289 * 5290 * @return 5291 * Index of flow if found, 0 otherwise. 5292 */ 5293 static uint32_t 5294 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5295 { 5296 struct mlx5_priv *priv = dev->data->dev_private; 5297 uint32_t flow_idx = 0; 5298 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5299 5300 MLX5_ASSERT(fdir_flow); 5301 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5302 if (!flow_fdir_cmp(priv_fdir_flow->fdir, fdir_flow)) { 5303 DRV_LOG(DEBUG, "port %u found FDIR flow %u", 5304 dev->data->port_id, flow_idx); 5305 flow_idx = priv_fdir_flow->rix_flow; 5306 break; 5307 } 5308 } 5309 return flow_idx; 5310 } 5311 5312 /** 5313 * Add new flow director filter and store it in list. 5314 * 5315 * @param dev 5316 * Pointer to Ethernet device. 5317 * @param fdir_filter 5318 * Flow director filter to add. 5319 * 5320 * @return 5321 * 0 on success, a negative errno value otherwise and rte_errno is set. 5322 */ 5323 static int 5324 flow_fdir_filter_add(struct rte_eth_dev *dev, 5325 const struct rte_eth_fdir_filter *fdir_filter) 5326 { 5327 struct mlx5_priv *priv = dev->data->dev_private; 5328 struct mlx5_fdir *fdir_flow; 5329 struct rte_flow *flow; 5330 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5331 uint32_t flow_idx; 5332 int ret; 5333 5334 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5335 if (!fdir_flow) { 5336 rte_errno = ENOMEM; 5337 return -rte_errno; 5338 } 5339 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5340 if (ret) 5341 goto error; 5342 flow_idx = flow_fdir_filter_lookup(dev, fdir_flow); 5343 if (flow_idx) { 5344 rte_errno = EEXIST; 5345 goto error; 5346 } 5347 priv_fdir_flow = rte_zmalloc(__func__, sizeof(struct mlx5_fdir_flow), 5348 0); 5349 if (!priv_fdir_flow) { 5350 rte_errno = ENOMEM; 5351 goto error; 5352 } 5353 flow_idx = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5354 fdir_flow->items, fdir_flow->actions, true, 5355 NULL); 5356 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx); 5357 if (!flow) 5358 goto error; 5359 flow->fdir = 1; 5360 priv_fdir_flow->fdir = fdir_flow; 5361 priv_fdir_flow->rix_flow = flow_idx; 5362 LIST_INSERT_HEAD(&priv->fdir_flows, priv_fdir_flow, next); 5363 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5364 dev->data->port_id, (void *)flow); 5365 return 0; 5366 error: 5367 rte_free(priv_fdir_flow); 5368 rte_free(fdir_flow); 5369 return -rte_errno; 5370 } 5371 5372 /** 5373 * Delete specific filter. 5374 * 5375 * @param dev 5376 * Pointer to Ethernet device. 5377 * @param fdir_filter 5378 * Filter to be deleted. 5379 * 5380 * @return 5381 * 0 on success, a negative errno value otherwise and rte_errno is set. 5382 */ 5383 static int 5384 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5385 const struct rte_eth_fdir_filter *fdir_filter) 5386 { 5387 struct mlx5_priv *priv = dev->data->dev_private; 5388 uint32_t flow_idx; 5389 struct mlx5_fdir fdir_flow = { 5390 .attr.group = 0, 5391 }; 5392 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5393 int ret; 5394 5395 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5396 if (ret) 5397 return -rte_errno; 5398 LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) { 5399 /* Find the fdir in priv list */ 5400 if (!flow_fdir_cmp(priv_fdir_flow->fdir, &fdir_flow)) 5401 break; 5402 } 5403 if (!priv_fdir_flow) 5404 return 0; 5405 LIST_REMOVE(priv_fdir_flow, next); 5406 flow_idx = priv_fdir_flow->rix_flow; 5407 flow_list_destroy(dev, &priv->flows, flow_idx); 5408 rte_free(priv_fdir_flow->fdir); 5409 rte_free(priv_fdir_flow); 5410 DRV_LOG(DEBUG, "port %u deleted FDIR flow %u", 5411 dev->data->port_id, flow_idx); 5412 return 0; 5413 } 5414 5415 /** 5416 * Update queue for specific filter. 5417 * 5418 * @param dev 5419 * Pointer to Ethernet device. 5420 * @param fdir_filter 5421 * Filter to be updated. 5422 * 5423 * @return 5424 * 0 on success, a negative errno value otherwise and rte_errno is set. 5425 */ 5426 static int 5427 flow_fdir_filter_update(struct rte_eth_dev *dev, 5428 const struct rte_eth_fdir_filter *fdir_filter) 5429 { 5430 int ret; 5431 5432 ret = flow_fdir_filter_delete(dev, fdir_filter); 5433 if (ret) 5434 return ret; 5435 return flow_fdir_filter_add(dev, fdir_filter); 5436 } 5437 5438 /** 5439 * Flush all filters. 5440 * 5441 * @param dev 5442 * Pointer to Ethernet device. 5443 */ 5444 static void 5445 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5446 { 5447 struct mlx5_priv *priv = dev->data->dev_private; 5448 struct mlx5_fdir_flow *priv_fdir_flow = NULL; 5449 5450 while (!LIST_EMPTY(&priv->fdir_flows)) { 5451 priv_fdir_flow = LIST_FIRST(&priv->fdir_flows); 5452 LIST_REMOVE(priv_fdir_flow, next); 5453 flow_list_destroy(dev, &priv->flows, priv_fdir_flow->rix_flow); 5454 rte_free(priv_fdir_flow->fdir); 5455 rte_free(priv_fdir_flow); 5456 } 5457 } 5458 5459 /** 5460 * Get flow director information. 5461 * 5462 * @param dev 5463 * Pointer to Ethernet device. 5464 * @param[out] fdir_info 5465 * Resulting flow director information. 5466 */ 5467 static void 5468 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5469 { 5470 struct rte_eth_fdir_masks *mask = 5471 &dev->data->dev_conf.fdir_conf.mask; 5472 5473 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5474 fdir_info->guarant_spc = 0; 5475 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5476 fdir_info->max_flexpayload = 0; 5477 fdir_info->flow_types_mask[0] = 0; 5478 fdir_info->flex_payload_unit = 0; 5479 fdir_info->max_flex_payload_segment_num = 0; 5480 fdir_info->flex_payload_limit = 0; 5481 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5482 } 5483 5484 /** 5485 * Deal with flow director operations. 5486 * 5487 * @param dev 5488 * Pointer to Ethernet device. 5489 * @param filter_op 5490 * Operation to perform. 5491 * @param arg 5492 * Pointer to operation-specific structure. 5493 * 5494 * @return 5495 * 0 on success, a negative errno value otherwise and rte_errno is set. 5496 */ 5497 static int 5498 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5499 void *arg) 5500 { 5501 enum rte_fdir_mode fdir_mode = 5502 dev->data->dev_conf.fdir_conf.mode; 5503 5504 if (filter_op == RTE_ETH_FILTER_NOP) 5505 return 0; 5506 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5507 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5508 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5509 dev->data->port_id, fdir_mode); 5510 rte_errno = EINVAL; 5511 return -rte_errno; 5512 } 5513 switch (filter_op) { 5514 case RTE_ETH_FILTER_ADD: 5515 return flow_fdir_filter_add(dev, arg); 5516 case RTE_ETH_FILTER_UPDATE: 5517 return flow_fdir_filter_update(dev, arg); 5518 case RTE_ETH_FILTER_DELETE: 5519 return flow_fdir_filter_delete(dev, arg); 5520 case RTE_ETH_FILTER_FLUSH: 5521 flow_fdir_filter_flush(dev); 5522 break; 5523 case RTE_ETH_FILTER_INFO: 5524 flow_fdir_info_get(dev, arg); 5525 break; 5526 default: 5527 DRV_LOG(DEBUG, "port %u unknown operation %u", 5528 dev->data->port_id, filter_op); 5529 rte_errno = EINVAL; 5530 return -rte_errno; 5531 } 5532 return 0; 5533 } 5534 5535 /** 5536 * Manage filter operations. 5537 * 5538 * @param dev 5539 * Pointer to Ethernet device structure. 5540 * @param filter_type 5541 * Filter type. 5542 * @param filter_op 5543 * Operation to perform. 5544 * @param arg 5545 * Pointer to operation-specific structure. 5546 * 5547 * @return 5548 * 0 on success, a negative errno value otherwise and rte_errno is set. 5549 */ 5550 int 5551 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5552 enum rte_filter_type filter_type, 5553 enum rte_filter_op filter_op, 5554 void *arg) 5555 { 5556 switch (filter_type) { 5557 case RTE_ETH_FILTER_GENERIC: 5558 if (filter_op != RTE_ETH_FILTER_GET) { 5559 rte_errno = EINVAL; 5560 return -rte_errno; 5561 } 5562 *(const void **)arg = &mlx5_flow_ops; 5563 return 0; 5564 case RTE_ETH_FILTER_FDIR: 5565 return flow_fdir_ctrl_func(dev, filter_op, arg); 5566 default: 5567 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5568 dev->data->port_id, filter_type); 5569 rte_errno = ENOTSUP; 5570 return -rte_errno; 5571 } 5572 return 0; 5573 } 5574 5575 /** 5576 * Create the needed meter and suffix tables. 5577 * 5578 * @param[in] dev 5579 * Pointer to Ethernet device. 5580 * @param[in] fm 5581 * Pointer to the flow meter. 5582 * 5583 * @return 5584 * Pointer to table set on success, NULL otherwise. 5585 */ 5586 struct mlx5_meter_domains_infos * 5587 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5588 const struct mlx5_flow_meter *fm) 5589 { 5590 const struct mlx5_flow_driver_ops *fops; 5591 5592 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5593 return fops->create_mtr_tbls(dev, fm); 5594 } 5595 5596 /** 5597 * Destroy the meter table set. 5598 * 5599 * @param[in] dev 5600 * Pointer to Ethernet device. 5601 * @param[in] tbl 5602 * Pointer to the meter table set. 5603 * 5604 * @return 5605 * 0 on success. 5606 */ 5607 int 5608 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5609 struct mlx5_meter_domains_infos *tbls) 5610 { 5611 const struct mlx5_flow_driver_ops *fops; 5612 5613 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5614 return fops->destroy_mtr_tbls(dev, tbls); 5615 } 5616 5617 /** 5618 * Create policer rules. 5619 * 5620 * @param[in] dev 5621 * Pointer to Ethernet device. 5622 * @param[in] fm 5623 * Pointer to flow meter structure. 5624 * @param[in] attr 5625 * Pointer to flow attributes. 5626 * 5627 * @return 5628 * 0 on success, -1 otherwise. 5629 */ 5630 int 5631 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5632 struct mlx5_flow_meter *fm, 5633 const struct rte_flow_attr *attr) 5634 { 5635 const struct mlx5_flow_driver_ops *fops; 5636 5637 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5638 return fops->create_policer_rules(dev, fm, attr); 5639 } 5640 5641 /** 5642 * Destroy policer rules. 5643 * 5644 * @param[in] fm 5645 * Pointer to flow meter structure. 5646 * @param[in] attr 5647 * Pointer to flow attributes. 5648 * 5649 * @return 5650 * 0 on success, -1 otherwise. 5651 */ 5652 int 5653 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5654 struct mlx5_flow_meter *fm, 5655 const struct rte_flow_attr *attr) 5656 { 5657 const struct mlx5_flow_driver_ops *fops; 5658 5659 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5660 return fops->destroy_policer_rules(dev, fm, attr); 5661 } 5662 5663 /** 5664 * Allocate a counter. 5665 * 5666 * @param[in] dev 5667 * Pointer to Ethernet device structure. 5668 * 5669 * @return 5670 * Index to allocated counter on success, 0 otherwise. 5671 */ 5672 uint32_t 5673 mlx5_counter_alloc(struct rte_eth_dev *dev) 5674 { 5675 const struct mlx5_flow_driver_ops *fops; 5676 struct rte_flow_attr attr = { .transfer = 0 }; 5677 5678 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5679 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5680 return fops->counter_alloc(dev); 5681 } 5682 DRV_LOG(ERR, 5683 "port %u counter allocate is not supported.", 5684 dev->data->port_id); 5685 return 0; 5686 } 5687 5688 /** 5689 * Free a counter. 5690 * 5691 * @param[in] dev 5692 * Pointer to Ethernet device structure. 5693 * @param[in] cnt 5694 * Index to counter to be free. 5695 */ 5696 void 5697 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 5698 { 5699 const struct mlx5_flow_driver_ops *fops; 5700 struct rte_flow_attr attr = { .transfer = 0 }; 5701 5702 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5703 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5704 fops->counter_free(dev, cnt); 5705 return; 5706 } 5707 DRV_LOG(ERR, 5708 "port %u counter free is not supported.", 5709 dev->data->port_id); 5710 } 5711 5712 /** 5713 * Query counter statistics. 5714 * 5715 * @param[in] dev 5716 * Pointer to Ethernet device structure. 5717 * @param[in] cnt 5718 * Index to counter to query. 5719 * @param[in] clear 5720 * Set to clear counter statistics. 5721 * @param[out] pkts 5722 * The counter hits packets number to save. 5723 * @param[out] bytes 5724 * The counter hits bytes number to save. 5725 * 5726 * @return 5727 * 0 on success, a negative errno value otherwise. 5728 */ 5729 int 5730 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 5731 bool clear, uint64_t *pkts, uint64_t *bytes) 5732 { 5733 const struct mlx5_flow_driver_ops *fops; 5734 struct rte_flow_attr attr = { .transfer = 0 }; 5735 5736 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5737 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5738 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5739 } 5740 DRV_LOG(ERR, 5741 "port %u counter query is not supported.", 5742 dev->data->port_id); 5743 return -ENOTSUP; 5744 } 5745 5746 #define MLX5_POOL_QUERY_FREQ_US 1000000 5747 5748 /** 5749 * Set the periodic procedure for triggering asynchronous batch queries for all 5750 * the counter pools. 5751 * 5752 * @param[in] sh 5753 * Pointer to mlx5_ibv_shared object. 5754 */ 5755 void 5756 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5757 { 5758 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0); 5759 uint32_t pools_n = rte_atomic16_read(&cont->n_valid); 5760 uint32_t us; 5761 5762 cont = MLX5_CNT_CONTAINER(sh, 1, 0); 5763 pools_n += rte_atomic16_read(&cont->n_valid); 5764 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5765 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5766 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5767 sh->cmng.query_thread_on = 0; 5768 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5769 } else { 5770 sh->cmng.query_thread_on = 1; 5771 } 5772 } 5773 5774 /** 5775 * The periodic procedure for triggering asynchronous batch queries for all the 5776 * counter pools. This function is probably called by the host thread. 5777 * 5778 * @param[in] arg 5779 * The parameter for the alarm process. 5780 */ 5781 void 5782 mlx5_flow_query_alarm(void *arg) 5783 { 5784 struct mlx5_ibv_shared *sh = arg; 5785 struct mlx5_devx_obj *dcs; 5786 uint16_t offset; 5787 int ret; 5788 uint8_t batch = sh->cmng.batch; 5789 uint16_t pool_index = sh->cmng.pool_index; 5790 struct mlx5_pools_container *cont; 5791 struct mlx5_pools_container *mcont; 5792 struct mlx5_flow_counter_pool *pool; 5793 5794 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5795 goto set_alarm; 5796 next_container: 5797 cont = MLX5_CNT_CONTAINER(sh, batch, 1); 5798 mcont = MLX5_CNT_CONTAINER(sh, batch, 0); 5799 /* Check if resize was done and need to flip a container. */ 5800 if (cont != mcont) { 5801 if (cont->pools) { 5802 /* Clean the old container. */ 5803 rte_free(cont->pools); 5804 memset(cont, 0, sizeof(*cont)); 5805 } 5806 rte_cio_wmb(); 5807 /* Flip the host container. */ 5808 sh->cmng.mhi[batch] ^= (uint8_t)2; 5809 cont = mcont; 5810 } 5811 if (!cont->pools) { 5812 /* 2 empty containers case is unexpected. */ 5813 if (unlikely(batch != sh->cmng.batch)) 5814 goto set_alarm; 5815 batch ^= 0x1; 5816 pool_index = 0; 5817 goto next_container; 5818 } 5819 pool = cont->pools[pool_index]; 5820 if (pool->raw_hw) 5821 /* There is a pool query in progress. */ 5822 goto set_alarm; 5823 pool->raw_hw = 5824 LIST_FIRST(&sh->cmng.free_stat_raws); 5825 if (!pool->raw_hw) 5826 /* No free counter statistics raw memory. */ 5827 goto set_alarm; 5828 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5829 (&pool->a64_dcs); 5830 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5831 /* 5832 * Identify the counters released between query trigger and query 5833 * handle more effiecntly. The counter released in this gap period 5834 * should wait for a new round of query as the new arrived packets 5835 * will not be taken into account. 5836 */ 5837 rte_atomic64_add(&pool->start_query_gen, 1); 5838 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5839 offset, NULL, NULL, 5840 pool->raw_hw->mem_mng->dm->id, 5841 (void *)(uintptr_t) 5842 (pool->raw_hw->data + offset), 5843 sh->devx_comp, 5844 (uint64_t)(uintptr_t)pool); 5845 if (ret) { 5846 rte_atomic64_sub(&pool->start_query_gen, 1); 5847 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5848 " %d", pool->min_dcs->id); 5849 pool->raw_hw = NULL; 5850 goto set_alarm; 5851 } 5852 pool->raw_hw->min_dcs_id = dcs->id; 5853 LIST_REMOVE(pool->raw_hw, next); 5854 sh->cmng.pending_queries++; 5855 pool_index++; 5856 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5857 batch ^= 0x1; 5858 pool_index = 0; 5859 } 5860 set_alarm: 5861 sh->cmng.batch = batch; 5862 sh->cmng.pool_index = pool_index; 5863 mlx5_set_query_alarm(sh); 5864 } 5865 5866 /** 5867 * Handler for the HW respond about ready values from an asynchronous batch 5868 * query. This function is probably called by the host thread. 5869 * 5870 * @param[in] sh 5871 * The pointer to the shared IB device context. 5872 * @param[in] async_id 5873 * The Devx async ID. 5874 * @param[in] status 5875 * The status of the completion. 5876 */ 5877 void 5878 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 5879 uint64_t async_id, int status) 5880 { 5881 struct mlx5_flow_counter_pool *pool = 5882 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 5883 struct mlx5_counter_stats_raw *raw_to_free; 5884 5885 if (unlikely(status)) { 5886 rte_atomic64_sub(&pool->start_query_gen, 1); 5887 raw_to_free = pool->raw_hw; 5888 } else { 5889 raw_to_free = pool->raw; 5890 rte_spinlock_lock(&pool->sl); 5891 pool->raw = pool->raw_hw; 5892 rte_spinlock_unlock(&pool->sl); 5893 MLX5_ASSERT(rte_atomic64_read(&pool->end_query_gen) + 1 == 5894 rte_atomic64_read(&pool->start_query_gen)); 5895 rte_atomic64_set(&pool->end_query_gen, 5896 rte_atomic64_read(&pool->start_query_gen)); 5897 /* Be sure the new raw counters data is updated in memory. */ 5898 rte_cio_wmb(); 5899 } 5900 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 5901 pool->raw_hw = NULL; 5902 sh->cmng.pending_queries--; 5903 } 5904 5905 /** 5906 * Translate the rte_flow group index to HW table value. 5907 * 5908 * @param[in] attributes 5909 * Pointer to flow attributes 5910 * @param[in] external 5911 * Value is part of flow rule created by request external to PMD. 5912 * @param[in] group 5913 * rte_flow group index value. 5914 * @param[out] fdb_def_rule 5915 * Whether fdb jump to table 1 is configured. 5916 * @param[out] table 5917 * HW table value. 5918 * @param[out] error 5919 * Pointer to error structure. 5920 * 5921 * @return 5922 * 0 on success, a negative errno value otherwise and rte_errno is set. 5923 */ 5924 int 5925 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 5926 uint32_t group, bool fdb_def_rule, uint32_t *table, 5927 struct rte_flow_error *error) 5928 { 5929 if (attributes->transfer && external && fdb_def_rule) { 5930 if (group == UINT32_MAX) 5931 return rte_flow_error_set 5932 (error, EINVAL, 5933 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 5934 NULL, 5935 "group index not supported"); 5936 *table = group + 1; 5937 } else { 5938 *table = group; 5939 } 5940 return 0; 5941 } 5942 5943 /** 5944 * Discover availability of metadata reg_c's. 5945 * 5946 * Iteratively use test flows to check availability. 5947 * 5948 * @param[in] dev 5949 * Pointer to the Ethernet device structure. 5950 * 5951 * @return 5952 * 0 on success, a negative errno value otherwise and rte_errno is set. 5953 */ 5954 int 5955 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 5956 { 5957 struct mlx5_priv *priv = dev->data->dev_private; 5958 struct mlx5_dev_config *config = &priv->config; 5959 enum modify_reg idx; 5960 int n = 0; 5961 5962 /* reg_c[0] and reg_c[1] are reserved. */ 5963 config->flow_mreg_c[n++] = REG_C_0; 5964 config->flow_mreg_c[n++] = REG_C_1; 5965 /* Discover availability of other reg_c's. */ 5966 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 5967 struct rte_flow_attr attr = { 5968 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5969 .priority = MLX5_FLOW_PRIO_RSVD, 5970 .ingress = 1, 5971 }; 5972 struct rte_flow_item items[] = { 5973 [0] = { 5974 .type = RTE_FLOW_ITEM_TYPE_END, 5975 }, 5976 }; 5977 struct rte_flow_action actions[] = { 5978 [0] = { 5979 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5980 .conf = &(struct mlx5_flow_action_copy_mreg){ 5981 .src = REG_C_1, 5982 .dst = idx, 5983 }, 5984 }, 5985 [1] = { 5986 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5987 .conf = &(struct rte_flow_action_jump){ 5988 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5989 }, 5990 }, 5991 [2] = { 5992 .type = RTE_FLOW_ACTION_TYPE_END, 5993 }, 5994 }; 5995 uint32_t flow_idx; 5996 struct rte_flow *flow; 5997 struct rte_flow_error error; 5998 5999 if (!config->dv_flow_en) 6000 break; 6001 /* Create internal flow, validation skips copy action. */ 6002 flow_idx = flow_list_create(dev, NULL, &attr, items, 6003 actions, false, &error); 6004 flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], 6005 flow_idx); 6006 if (!flow) 6007 continue; 6008 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 6009 config->flow_mreg_c[n++] = idx; 6010 flow_list_destroy(dev, NULL, flow_idx); 6011 } 6012 for (; n < MLX5_MREG_C_NUM; ++n) 6013 config->flow_mreg_c[n] = REG_NONE; 6014 return 0; 6015 } 6016 6017 /** 6018 * Dump flow raw hw data to file 6019 * 6020 * @param[in] dev 6021 * The pointer to Ethernet device. 6022 * @param[in] file 6023 * A pointer to a file for output. 6024 * @param[out] error 6025 * Perform verbose error reporting if not NULL. PMDs initialize this 6026 * structure in case of error only. 6027 * @return 6028 * 0 on success, a nagative value otherwise. 6029 */ 6030 int 6031 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 6032 FILE *file, 6033 struct rte_flow_error *error __rte_unused) 6034 { 6035 struct mlx5_priv *priv = dev->data->dev_private; 6036 struct mlx5_ibv_shared *sh = priv->sh; 6037 6038 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 6039 sh->tx_domain, file); 6040 } 6041