1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <netinet/in.h> 7 #include <sys/queue.h> 8 #include <stdalign.h> 9 #include <stdint.h> 10 #include <string.h> 11 12 /* Verbs header. */ 13 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 14 #ifdef PEDANTIC 15 #pragma GCC diagnostic ignored "-Wpedantic" 16 #endif 17 #include <infiniband/verbs.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_common.h> 23 #include <rte_ether.h> 24 #include <rte_ethdev_driver.h> 25 #include <rte_flow.h> 26 #include <rte_flow_driver.h> 27 #include <rte_malloc.h> 28 #include <rte_ip.h> 29 30 #include <mlx5_glue.h> 31 #include <mlx5_devx_cmds.h> 32 #include <mlx5_prm.h> 33 34 #include "mlx5_defs.h" 35 #include "mlx5.h" 36 #include "mlx5_flow.h" 37 #include "mlx5_rxtx.h" 38 39 /* Dev ops structure defined in mlx5.c */ 40 extern const struct eth_dev_ops mlx5_dev_ops; 41 extern const struct eth_dev_ops mlx5_dev_ops_isolate; 42 43 /** Device flow drivers. */ 44 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 45 extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops; 46 #endif 47 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 48 49 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 50 51 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 52 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 53 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 54 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 55 #endif 56 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 57 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 58 }; 59 60 enum mlx5_expansion { 61 MLX5_EXPANSION_ROOT, 62 MLX5_EXPANSION_ROOT_OUTER, 63 MLX5_EXPANSION_ROOT_ETH_VLAN, 64 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN, 65 MLX5_EXPANSION_OUTER_ETH, 66 MLX5_EXPANSION_OUTER_ETH_VLAN, 67 MLX5_EXPANSION_OUTER_VLAN, 68 MLX5_EXPANSION_OUTER_IPV4, 69 MLX5_EXPANSION_OUTER_IPV4_UDP, 70 MLX5_EXPANSION_OUTER_IPV4_TCP, 71 MLX5_EXPANSION_OUTER_IPV6, 72 MLX5_EXPANSION_OUTER_IPV6_UDP, 73 MLX5_EXPANSION_OUTER_IPV6_TCP, 74 MLX5_EXPANSION_VXLAN, 75 MLX5_EXPANSION_VXLAN_GPE, 76 MLX5_EXPANSION_GRE, 77 MLX5_EXPANSION_MPLS, 78 MLX5_EXPANSION_ETH, 79 MLX5_EXPANSION_ETH_VLAN, 80 MLX5_EXPANSION_VLAN, 81 MLX5_EXPANSION_IPV4, 82 MLX5_EXPANSION_IPV4_UDP, 83 MLX5_EXPANSION_IPV4_TCP, 84 MLX5_EXPANSION_IPV6, 85 MLX5_EXPANSION_IPV6_UDP, 86 MLX5_EXPANSION_IPV6_TCP, 87 }; 88 89 /** Supported expansion of items. */ 90 static const struct rte_flow_expand_node mlx5_support_expansion[] = { 91 [MLX5_EXPANSION_ROOT] = { 92 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 93 MLX5_EXPANSION_IPV4, 94 MLX5_EXPANSION_IPV6), 95 .type = RTE_FLOW_ITEM_TYPE_END, 96 }, 97 [MLX5_EXPANSION_ROOT_OUTER] = { 98 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 99 MLX5_EXPANSION_OUTER_IPV4, 100 MLX5_EXPANSION_OUTER_IPV6), 101 .type = RTE_FLOW_ITEM_TYPE_END, 102 }, 103 [MLX5_EXPANSION_ROOT_ETH_VLAN] = { 104 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN), 105 .type = RTE_FLOW_ITEM_TYPE_END, 106 }, 107 [MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = { 108 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN), 109 .type = RTE_FLOW_ITEM_TYPE_END, 110 }, 111 [MLX5_EXPANSION_OUTER_ETH] = { 112 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 113 MLX5_EXPANSION_OUTER_IPV6, 114 MLX5_EXPANSION_MPLS), 115 .type = RTE_FLOW_ITEM_TYPE_ETH, 116 .rss_types = 0, 117 }, 118 [MLX5_EXPANSION_OUTER_ETH_VLAN] = { 119 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 120 .type = RTE_FLOW_ITEM_TYPE_ETH, 121 .rss_types = 0, 122 }, 123 [MLX5_EXPANSION_OUTER_VLAN] = { 124 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 125 MLX5_EXPANSION_OUTER_IPV6), 126 .type = RTE_FLOW_ITEM_TYPE_VLAN, 127 }, 128 [MLX5_EXPANSION_OUTER_IPV4] = { 129 .next = RTE_FLOW_EXPAND_RSS_NEXT 130 (MLX5_EXPANSION_OUTER_IPV4_UDP, 131 MLX5_EXPANSION_OUTER_IPV4_TCP, 132 MLX5_EXPANSION_GRE, 133 MLX5_EXPANSION_IPV4, 134 MLX5_EXPANSION_IPV6), 135 .type = RTE_FLOW_ITEM_TYPE_IPV4, 136 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 137 ETH_RSS_NONFRAG_IPV4_OTHER, 138 }, 139 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 140 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 141 MLX5_EXPANSION_VXLAN_GPE), 142 .type = RTE_FLOW_ITEM_TYPE_UDP, 143 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 144 }, 145 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 146 .type = RTE_FLOW_ITEM_TYPE_TCP, 147 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 148 }, 149 [MLX5_EXPANSION_OUTER_IPV6] = { 150 .next = RTE_FLOW_EXPAND_RSS_NEXT 151 (MLX5_EXPANSION_OUTER_IPV6_UDP, 152 MLX5_EXPANSION_OUTER_IPV6_TCP, 153 MLX5_EXPANSION_IPV4, 154 MLX5_EXPANSION_IPV6), 155 .type = RTE_FLOW_ITEM_TYPE_IPV6, 156 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 157 ETH_RSS_NONFRAG_IPV6_OTHER, 158 }, 159 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 160 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 161 MLX5_EXPANSION_VXLAN_GPE), 162 .type = RTE_FLOW_ITEM_TYPE_UDP, 163 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 164 }, 165 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 166 .type = RTE_FLOW_ITEM_TYPE_TCP, 167 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 168 }, 169 [MLX5_EXPANSION_VXLAN] = { 170 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 171 MLX5_EXPANSION_IPV4, 172 MLX5_EXPANSION_IPV6), 173 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 174 }, 175 [MLX5_EXPANSION_VXLAN_GPE] = { 176 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 177 MLX5_EXPANSION_IPV4, 178 MLX5_EXPANSION_IPV6), 179 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 180 }, 181 [MLX5_EXPANSION_GRE] = { 182 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4), 183 .type = RTE_FLOW_ITEM_TYPE_GRE, 184 }, 185 [MLX5_EXPANSION_MPLS] = { 186 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 187 MLX5_EXPANSION_IPV6), 188 .type = RTE_FLOW_ITEM_TYPE_MPLS, 189 }, 190 [MLX5_EXPANSION_ETH] = { 191 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 192 MLX5_EXPANSION_IPV6), 193 .type = RTE_FLOW_ITEM_TYPE_ETH, 194 }, 195 [MLX5_EXPANSION_ETH_VLAN] = { 196 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 197 .type = RTE_FLOW_ITEM_TYPE_ETH, 198 }, 199 [MLX5_EXPANSION_VLAN] = { 200 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 201 MLX5_EXPANSION_IPV6), 202 .type = RTE_FLOW_ITEM_TYPE_VLAN, 203 }, 204 [MLX5_EXPANSION_IPV4] = { 205 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 206 MLX5_EXPANSION_IPV4_TCP), 207 .type = RTE_FLOW_ITEM_TYPE_IPV4, 208 .rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 209 ETH_RSS_NONFRAG_IPV4_OTHER, 210 }, 211 [MLX5_EXPANSION_IPV4_UDP] = { 212 .type = RTE_FLOW_ITEM_TYPE_UDP, 213 .rss_types = ETH_RSS_NONFRAG_IPV4_UDP, 214 }, 215 [MLX5_EXPANSION_IPV4_TCP] = { 216 .type = RTE_FLOW_ITEM_TYPE_TCP, 217 .rss_types = ETH_RSS_NONFRAG_IPV4_TCP, 218 }, 219 [MLX5_EXPANSION_IPV6] = { 220 .next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 221 MLX5_EXPANSION_IPV6_TCP), 222 .type = RTE_FLOW_ITEM_TYPE_IPV6, 223 .rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | 224 ETH_RSS_NONFRAG_IPV6_OTHER, 225 }, 226 [MLX5_EXPANSION_IPV6_UDP] = { 227 .type = RTE_FLOW_ITEM_TYPE_UDP, 228 .rss_types = ETH_RSS_NONFRAG_IPV6_UDP, 229 }, 230 [MLX5_EXPANSION_IPV6_TCP] = { 231 .type = RTE_FLOW_ITEM_TYPE_TCP, 232 .rss_types = ETH_RSS_NONFRAG_IPV6_TCP, 233 }, 234 }; 235 236 static const struct rte_flow_ops mlx5_flow_ops = { 237 .validate = mlx5_flow_validate, 238 .create = mlx5_flow_create, 239 .destroy = mlx5_flow_destroy, 240 .flush = mlx5_flow_flush, 241 .isolate = mlx5_flow_isolate, 242 .query = mlx5_flow_query, 243 .dev_dump = mlx5_flow_dev_dump, 244 }; 245 246 /* Convert FDIR request to Generic flow. */ 247 struct mlx5_fdir { 248 struct rte_flow_attr attr; 249 struct rte_flow_item items[4]; 250 struct rte_flow_item_eth l2; 251 struct rte_flow_item_eth l2_mask; 252 union { 253 struct rte_flow_item_ipv4 ipv4; 254 struct rte_flow_item_ipv6 ipv6; 255 } l3; 256 union { 257 struct rte_flow_item_ipv4 ipv4; 258 struct rte_flow_item_ipv6 ipv6; 259 } l3_mask; 260 union { 261 struct rte_flow_item_udp udp; 262 struct rte_flow_item_tcp tcp; 263 } l4; 264 union { 265 struct rte_flow_item_udp udp; 266 struct rte_flow_item_tcp tcp; 267 } l4_mask; 268 struct rte_flow_action actions[2]; 269 struct rte_flow_action_queue queue; 270 }; 271 272 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 273 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 274 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 275 }; 276 277 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 278 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 279 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 280 { 9, 10, 11 }, { 12, 13, 14 }, 281 }; 282 283 /* Tunnel information. */ 284 struct mlx5_flow_tunnel_info { 285 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 286 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 287 }; 288 289 static struct mlx5_flow_tunnel_info tunnels_info[] = { 290 { 291 .tunnel = MLX5_FLOW_LAYER_VXLAN, 292 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 293 }, 294 { 295 .tunnel = MLX5_FLOW_LAYER_GENEVE, 296 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 297 }, 298 { 299 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 300 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 301 }, 302 { 303 .tunnel = MLX5_FLOW_LAYER_GRE, 304 .ptype = RTE_PTYPE_TUNNEL_GRE, 305 }, 306 { 307 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 308 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 309 }, 310 { 311 .tunnel = MLX5_FLOW_LAYER_MPLS, 312 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 313 }, 314 { 315 .tunnel = MLX5_FLOW_LAYER_NVGRE, 316 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 317 }, 318 { 319 .tunnel = MLX5_FLOW_LAYER_IPIP, 320 .ptype = RTE_PTYPE_TUNNEL_IP, 321 }, 322 { 323 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 324 .ptype = RTE_PTYPE_TUNNEL_IP, 325 }, 326 { 327 .tunnel = MLX5_FLOW_LAYER_GTP, 328 .ptype = RTE_PTYPE_TUNNEL_GTPU, 329 }, 330 }; 331 332 /** 333 * Translate tag ID to register. 334 * 335 * @param[in] dev 336 * Pointer to the Ethernet device structure. 337 * @param[in] feature 338 * The feature that request the register. 339 * @param[in] id 340 * The request register ID. 341 * @param[out] error 342 * Error description in case of any. 343 * 344 * @return 345 * The request register on success, a negative errno 346 * value otherwise and rte_errno is set. 347 */ 348 int 349 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 350 enum mlx5_feature_name feature, 351 uint32_t id, 352 struct rte_flow_error *error) 353 { 354 struct mlx5_priv *priv = dev->data->dev_private; 355 struct mlx5_dev_config *config = &priv->config; 356 enum modify_reg start_reg; 357 bool skip_mtr_reg = false; 358 359 switch (feature) { 360 case MLX5_HAIRPIN_RX: 361 return REG_B; 362 case MLX5_HAIRPIN_TX: 363 return REG_A; 364 case MLX5_METADATA_RX: 365 switch (config->dv_xmeta_en) { 366 case MLX5_XMETA_MODE_LEGACY: 367 return REG_B; 368 case MLX5_XMETA_MODE_META16: 369 return REG_C_0; 370 case MLX5_XMETA_MODE_META32: 371 return REG_C_1; 372 } 373 break; 374 case MLX5_METADATA_TX: 375 return REG_A; 376 case MLX5_METADATA_FDB: 377 switch (config->dv_xmeta_en) { 378 case MLX5_XMETA_MODE_LEGACY: 379 return REG_NONE; 380 case MLX5_XMETA_MODE_META16: 381 return REG_C_0; 382 case MLX5_XMETA_MODE_META32: 383 return REG_C_1; 384 } 385 break; 386 case MLX5_FLOW_MARK: 387 switch (config->dv_xmeta_en) { 388 case MLX5_XMETA_MODE_LEGACY: 389 return REG_NONE; 390 case MLX5_XMETA_MODE_META16: 391 return REG_C_1; 392 case MLX5_XMETA_MODE_META32: 393 return REG_C_0; 394 } 395 break; 396 case MLX5_MTR_SFX: 397 /* 398 * If meter color and flow match share one register, flow match 399 * should use the meter color register for match. 400 */ 401 if (priv->mtr_reg_share) 402 return priv->mtr_color_reg; 403 else 404 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 405 REG_C_3; 406 case MLX5_MTR_COLOR: 407 MLX5_ASSERT(priv->mtr_color_reg != REG_NONE); 408 return priv->mtr_color_reg; 409 case MLX5_COPY_MARK: 410 /* 411 * Metadata COPY_MARK register using is in meter suffix sub 412 * flow while with meter. It's safe to share the same register. 413 */ 414 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 415 case MLX5_APP_TAG: 416 /* 417 * If meter is enable, it will engage the register for color 418 * match and flow match. If meter color match is not using the 419 * REG_C_2, need to skip the REG_C_x be used by meter color 420 * match. 421 * If meter is disable, free to use all available registers. 422 */ 423 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 424 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 425 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 426 if (id > (REG_C_7 - start_reg)) 427 return rte_flow_error_set(error, EINVAL, 428 RTE_FLOW_ERROR_TYPE_ITEM, 429 NULL, "invalid tag id"); 430 if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NONE) 431 return rte_flow_error_set(error, ENOTSUP, 432 RTE_FLOW_ERROR_TYPE_ITEM, 433 NULL, "unsupported tag id"); 434 /* 435 * This case means meter is using the REG_C_x great than 2. 436 * Take care not to conflict with meter color REG_C_x. 437 * If the available index REG_C_y >= REG_C_x, skip the 438 * color register. 439 */ 440 if (skip_mtr_reg && config->flow_mreg_c 441 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 442 if (config->flow_mreg_c 443 [id + 1 + start_reg - REG_C_0] != REG_NONE) 444 return config->flow_mreg_c 445 [id + 1 + start_reg - REG_C_0]; 446 return rte_flow_error_set(error, ENOTSUP, 447 RTE_FLOW_ERROR_TYPE_ITEM, 448 NULL, "unsupported tag id"); 449 } 450 return config->flow_mreg_c[id + start_reg - REG_C_0]; 451 } 452 MLX5_ASSERT(false); 453 return rte_flow_error_set(error, EINVAL, 454 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 455 NULL, "invalid feature name"); 456 } 457 458 /** 459 * Check extensive flow metadata register support. 460 * 461 * @param dev 462 * Pointer to rte_eth_dev structure. 463 * 464 * @return 465 * True if device supports extensive flow metadata register, otherwise false. 466 */ 467 bool 468 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 469 { 470 struct mlx5_priv *priv = dev->data->dev_private; 471 struct mlx5_dev_config *config = &priv->config; 472 473 /* 474 * Having available reg_c can be regarded inclusively as supporting 475 * extensive flow metadata register, which could mean, 476 * - metadata register copy action by modify header. 477 * - 16 modify header actions is supported. 478 * - reg_c's are preserved across different domain (FDB and NIC) on 479 * packet loopback by flow lookup miss. 480 */ 481 return config->flow_mreg_c[2] != REG_NONE; 482 } 483 484 /** 485 * Discover the maximum number of priority available. 486 * 487 * @param[in] dev 488 * Pointer to the Ethernet device structure. 489 * 490 * @return 491 * number of supported flow priority on success, a negative errno 492 * value otherwise and rte_errno is set. 493 */ 494 int 495 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 496 { 497 struct mlx5_priv *priv = dev->data->dev_private; 498 struct { 499 struct ibv_flow_attr attr; 500 struct ibv_flow_spec_eth eth; 501 struct ibv_flow_spec_action_drop drop; 502 } flow_attr = { 503 .attr = { 504 .num_of_specs = 2, 505 .port = (uint8_t)priv->ibv_port, 506 }, 507 .eth = { 508 .type = IBV_FLOW_SPEC_ETH, 509 .size = sizeof(struct ibv_flow_spec_eth), 510 }, 511 .drop = { 512 .size = sizeof(struct ibv_flow_spec_action_drop), 513 .type = IBV_FLOW_SPEC_ACTION_DROP, 514 }, 515 }; 516 struct ibv_flow *flow; 517 struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev); 518 uint16_t vprio[] = { 8, 16 }; 519 int i; 520 int priority = 0; 521 522 if (!drop) { 523 rte_errno = ENOTSUP; 524 return -rte_errno; 525 } 526 for (i = 0; i != RTE_DIM(vprio); i++) { 527 flow_attr.attr.priority = vprio[i] - 1; 528 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); 529 if (!flow) 530 break; 531 claim_zero(mlx5_glue->destroy_flow(flow)); 532 priority = vprio[i]; 533 } 534 mlx5_hrxq_drop_release(dev); 535 switch (priority) { 536 case 8: 537 priority = RTE_DIM(priority_map_3); 538 break; 539 case 16: 540 priority = RTE_DIM(priority_map_5); 541 break; 542 default: 543 rte_errno = ENOTSUP; 544 DRV_LOG(ERR, 545 "port %u verbs maximum priority: %d expected 8/16", 546 dev->data->port_id, priority); 547 return -rte_errno; 548 } 549 DRV_LOG(INFO, "port %u flow maximum priority: %d", 550 dev->data->port_id, priority); 551 return priority; 552 } 553 554 /** 555 * Adjust flow priority based on the highest layer and the request priority. 556 * 557 * @param[in] dev 558 * Pointer to the Ethernet device structure. 559 * @param[in] priority 560 * The rule base priority. 561 * @param[in] subpriority 562 * The priority based on the items. 563 * 564 * @return 565 * The new priority. 566 */ 567 uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 568 uint32_t subpriority) 569 { 570 uint32_t res = 0; 571 struct mlx5_priv *priv = dev->data->dev_private; 572 573 switch (priv->config.flow_prio) { 574 case RTE_DIM(priority_map_3): 575 res = priority_map_3[priority][subpriority]; 576 break; 577 case RTE_DIM(priority_map_5): 578 res = priority_map_5[priority][subpriority]; 579 break; 580 } 581 return res; 582 } 583 584 /** 585 * Verify the @p item specifications (spec, last, mask) are compatible with the 586 * NIC capabilities. 587 * 588 * @param[in] item 589 * Item specification. 590 * @param[in] mask 591 * @p item->mask or flow default bit-masks. 592 * @param[in] nic_mask 593 * Bit-masks covering supported fields by the NIC to compare with user mask. 594 * @param[in] size 595 * Bit-masks size in bytes. 596 * @param[out] error 597 * Pointer to error structure. 598 * 599 * @return 600 * 0 on success, a negative errno value otherwise and rte_errno is set. 601 */ 602 int 603 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 604 const uint8_t *mask, 605 const uint8_t *nic_mask, 606 unsigned int size, 607 struct rte_flow_error *error) 608 { 609 unsigned int i; 610 611 MLX5_ASSERT(nic_mask); 612 for (i = 0; i < size; ++i) 613 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 614 return rte_flow_error_set(error, ENOTSUP, 615 RTE_FLOW_ERROR_TYPE_ITEM, 616 item, 617 "mask enables non supported" 618 " bits"); 619 if (!item->spec && (item->mask || item->last)) 620 return rte_flow_error_set(error, EINVAL, 621 RTE_FLOW_ERROR_TYPE_ITEM, item, 622 "mask/last without a spec is not" 623 " supported"); 624 if (item->spec && item->last) { 625 uint8_t spec[size]; 626 uint8_t last[size]; 627 unsigned int i; 628 int ret; 629 630 for (i = 0; i < size; ++i) { 631 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 632 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 633 } 634 ret = memcmp(spec, last, size); 635 if (ret != 0) 636 return rte_flow_error_set(error, EINVAL, 637 RTE_FLOW_ERROR_TYPE_ITEM, 638 item, 639 "range is not valid"); 640 } 641 return 0; 642 } 643 644 /** 645 * Adjust the hash fields according to the @p flow information. 646 * 647 * @param[in] dev_flow. 648 * Pointer to the mlx5_flow. 649 * @param[in] tunnel 650 * 1 when the hash field is for a tunnel item. 651 * @param[in] layer_types 652 * ETH_RSS_* types. 653 * @param[in] hash_fields 654 * Item hash fields. 655 * 656 * @return 657 * The hash fields that should be used. 658 */ 659 uint64_t 660 mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, 661 int tunnel __rte_unused, uint64_t layer_types, 662 uint64_t hash_fields) 663 { 664 struct rte_flow *flow = dev_flow->flow; 665 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 666 int rss_request_inner = flow->rss.level >= 2; 667 668 /* Check RSS hash level for tunnel. */ 669 if (tunnel && rss_request_inner) 670 hash_fields |= IBV_RX_HASH_INNER; 671 else if (tunnel || rss_request_inner) 672 return 0; 673 #endif 674 /* Check if requested layer matches RSS hash fields. */ 675 if (!(flow->rss.types & layer_types)) 676 return 0; 677 return hash_fields; 678 } 679 680 /** 681 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 682 * if several tunnel rules are used on this queue, the tunnel ptype will be 683 * cleared. 684 * 685 * @param rxq_ctrl 686 * Rx queue to update. 687 */ 688 static void 689 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 690 { 691 unsigned int i; 692 uint32_t tunnel_ptype = 0; 693 694 /* Look up for the ptype to use. */ 695 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 696 if (!rxq_ctrl->flow_tunnels_n[i]) 697 continue; 698 if (!tunnel_ptype) { 699 tunnel_ptype = tunnels_info[i].ptype; 700 } else { 701 tunnel_ptype = 0; 702 break; 703 } 704 } 705 rxq_ctrl->rxq.tunnel = tunnel_ptype; 706 } 707 708 /** 709 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive 710 * flow. 711 * 712 * @param[in] dev 713 * Pointer to the Ethernet device structure. 714 * @param[in] dev_flow 715 * Pointer to device flow structure. 716 */ 717 static void 718 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 719 { 720 struct mlx5_priv *priv = dev->data->dev_private; 721 struct rte_flow *flow = dev_flow->flow; 722 const int mark = !!(dev_flow->actions & 723 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 724 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 725 unsigned int i; 726 727 for (i = 0; i != flow->rss.queue_num; ++i) { 728 int idx = (*flow->rss.queue)[i]; 729 struct mlx5_rxq_ctrl *rxq_ctrl = 730 container_of((*priv->rxqs)[idx], 731 struct mlx5_rxq_ctrl, rxq); 732 733 /* 734 * To support metadata register copy on Tx loopback, 735 * this must be always enabled (metadata may arive 736 * from other port - not from local flows only. 737 */ 738 if (priv->config.dv_flow_en && 739 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 740 mlx5_flow_ext_mreg_supported(dev)) { 741 rxq_ctrl->rxq.mark = 1; 742 rxq_ctrl->flow_mark_n = 1; 743 } else if (mark) { 744 rxq_ctrl->rxq.mark = 1; 745 rxq_ctrl->flow_mark_n++; 746 } 747 if (tunnel) { 748 unsigned int j; 749 750 /* Increase the counter matching the flow. */ 751 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 752 if ((tunnels_info[j].tunnel & 753 dev_flow->layers) == 754 tunnels_info[j].tunnel) { 755 rxq_ctrl->flow_tunnels_n[j]++; 756 break; 757 } 758 } 759 flow_rxq_tunnel_ptype_update(rxq_ctrl); 760 } 761 } 762 } 763 764 /** 765 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 766 * 767 * @param[in] dev 768 * Pointer to the Ethernet device structure. 769 * @param[in] flow 770 * Pointer to flow structure. 771 */ 772 static void 773 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 774 { 775 struct mlx5_flow *dev_flow; 776 777 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 778 flow_drv_rxq_flags_set(dev, dev_flow); 779 } 780 781 /** 782 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 783 * device flow if no other flow uses it with the same kind of request. 784 * 785 * @param dev 786 * Pointer to Ethernet device. 787 * @param[in] dev_flow 788 * Pointer to the device flow. 789 */ 790 static void 791 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) 792 { 793 struct mlx5_priv *priv = dev->data->dev_private; 794 struct rte_flow *flow = dev_flow->flow; 795 const int mark = !!(dev_flow->actions & 796 (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); 797 const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); 798 unsigned int i; 799 800 MLX5_ASSERT(dev->data->dev_started); 801 for (i = 0; i != flow->rss.queue_num; ++i) { 802 int idx = (*flow->rss.queue)[i]; 803 struct mlx5_rxq_ctrl *rxq_ctrl = 804 container_of((*priv->rxqs)[idx], 805 struct mlx5_rxq_ctrl, rxq); 806 807 if (priv->config.dv_flow_en && 808 priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 809 mlx5_flow_ext_mreg_supported(dev)) { 810 rxq_ctrl->rxq.mark = 1; 811 rxq_ctrl->flow_mark_n = 1; 812 } else if (mark) { 813 rxq_ctrl->flow_mark_n--; 814 rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n; 815 } 816 if (tunnel) { 817 unsigned int j; 818 819 /* Decrease the counter matching the flow. */ 820 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 821 if ((tunnels_info[j].tunnel & 822 dev_flow->layers) == 823 tunnels_info[j].tunnel) { 824 rxq_ctrl->flow_tunnels_n[j]--; 825 break; 826 } 827 } 828 flow_rxq_tunnel_ptype_update(rxq_ctrl); 829 } 830 } 831 } 832 833 /** 834 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 835 * @p flow if no other flow uses it with the same kind of request. 836 * 837 * @param dev 838 * Pointer to Ethernet device. 839 * @param[in] flow 840 * Pointer to the flow. 841 */ 842 static void 843 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 844 { 845 struct mlx5_flow *dev_flow; 846 847 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 848 flow_drv_rxq_flags_trim(dev, dev_flow); 849 } 850 851 /** 852 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 853 * 854 * @param dev 855 * Pointer to Ethernet device. 856 */ 857 static void 858 flow_rxq_flags_clear(struct rte_eth_dev *dev) 859 { 860 struct mlx5_priv *priv = dev->data->dev_private; 861 unsigned int i; 862 863 for (i = 0; i != priv->rxqs_n; ++i) { 864 struct mlx5_rxq_ctrl *rxq_ctrl; 865 unsigned int j; 866 867 if (!(*priv->rxqs)[i]) 868 continue; 869 rxq_ctrl = container_of((*priv->rxqs)[i], 870 struct mlx5_rxq_ctrl, rxq); 871 rxq_ctrl->flow_mark_n = 0; 872 rxq_ctrl->rxq.mark = 0; 873 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 874 rxq_ctrl->flow_tunnels_n[j] = 0; 875 rxq_ctrl->rxq.tunnel = 0; 876 } 877 } 878 879 /* 880 * return a pointer to the desired action in the list of actions. 881 * 882 * @param[in] actions 883 * The list of actions to search the action in. 884 * @param[in] action 885 * The action to find. 886 * 887 * @return 888 * Pointer to the action in the list, if found. NULL otherwise. 889 */ 890 const struct rte_flow_action * 891 mlx5_flow_find_action(const struct rte_flow_action *actions, 892 enum rte_flow_action_type action) 893 { 894 if (actions == NULL) 895 return NULL; 896 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 897 if (actions->type == action) 898 return actions; 899 return NULL; 900 } 901 902 /* 903 * Validate the flag action. 904 * 905 * @param[in] action_flags 906 * Bit-fields that holds the actions detected until now. 907 * @param[in] attr 908 * Attributes of flow that includes this action. 909 * @param[out] error 910 * Pointer to error structure. 911 * 912 * @return 913 * 0 on success, a negative errno value otherwise and rte_errno is set. 914 */ 915 int 916 mlx5_flow_validate_action_flag(uint64_t action_flags, 917 const struct rte_flow_attr *attr, 918 struct rte_flow_error *error) 919 { 920 if (action_flags & MLX5_FLOW_ACTION_MARK) 921 return rte_flow_error_set(error, EINVAL, 922 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 923 "can't mark and flag in same flow"); 924 if (action_flags & MLX5_FLOW_ACTION_FLAG) 925 return rte_flow_error_set(error, EINVAL, 926 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 927 "can't have 2 flag" 928 " actions in same flow"); 929 if (attr->egress) 930 return rte_flow_error_set(error, ENOTSUP, 931 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 932 "flag action not supported for " 933 "egress"); 934 return 0; 935 } 936 937 /* 938 * Validate the mark action. 939 * 940 * @param[in] action 941 * Pointer to the queue action. 942 * @param[in] action_flags 943 * Bit-fields that holds the actions detected until now. 944 * @param[in] attr 945 * Attributes of flow that includes this action. 946 * @param[out] error 947 * Pointer to error structure. 948 * 949 * @return 950 * 0 on success, a negative errno value otherwise and rte_errno is set. 951 */ 952 int 953 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 954 uint64_t action_flags, 955 const struct rte_flow_attr *attr, 956 struct rte_flow_error *error) 957 { 958 const struct rte_flow_action_mark *mark = action->conf; 959 960 if (!mark) 961 return rte_flow_error_set(error, EINVAL, 962 RTE_FLOW_ERROR_TYPE_ACTION, 963 action, 964 "configuration cannot be null"); 965 if (mark->id >= MLX5_FLOW_MARK_MAX) 966 return rte_flow_error_set(error, EINVAL, 967 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 968 &mark->id, 969 "mark id must in 0 <= id < " 970 RTE_STR(MLX5_FLOW_MARK_MAX)); 971 if (action_flags & MLX5_FLOW_ACTION_FLAG) 972 return rte_flow_error_set(error, EINVAL, 973 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 974 "can't flag and mark in same flow"); 975 if (action_flags & MLX5_FLOW_ACTION_MARK) 976 return rte_flow_error_set(error, EINVAL, 977 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 978 "can't have 2 mark actions in same" 979 " flow"); 980 if (attr->egress) 981 return rte_flow_error_set(error, ENOTSUP, 982 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 983 "mark action not supported for " 984 "egress"); 985 return 0; 986 } 987 988 /* 989 * Validate the drop action. 990 * 991 * @param[in] action_flags 992 * Bit-fields that holds the actions detected until now. 993 * @param[in] attr 994 * Attributes of flow that includes this action. 995 * @param[out] error 996 * Pointer to error structure. 997 * 998 * @return 999 * 0 on success, a negative errno value otherwise and rte_errno is set. 1000 */ 1001 int 1002 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1003 const struct rte_flow_attr *attr, 1004 struct rte_flow_error *error) 1005 { 1006 if (attr->egress) 1007 return rte_flow_error_set(error, ENOTSUP, 1008 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1009 "drop action not supported for " 1010 "egress"); 1011 return 0; 1012 } 1013 1014 /* 1015 * Validate the queue action. 1016 * 1017 * @param[in] action 1018 * Pointer to the queue action. 1019 * @param[in] action_flags 1020 * Bit-fields that holds the actions detected until now. 1021 * @param[in] dev 1022 * Pointer to the Ethernet device structure. 1023 * @param[in] attr 1024 * Attributes of flow that includes this action. 1025 * @param[out] error 1026 * Pointer to error structure. 1027 * 1028 * @return 1029 * 0 on success, a negative errno value otherwise and rte_errno is set. 1030 */ 1031 int 1032 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1033 uint64_t action_flags, 1034 struct rte_eth_dev *dev, 1035 const struct rte_flow_attr *attr, 1036 struct rte_flow_error *error) 1037 { 1038 struct mlx5_priv *priv = dev->data->dev_private; 1039 const struct rte_flow_action_queue *queue = action->conf; 1040 1041 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1042 return rte_flow_error_set(error, EINVAL, 1043 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1044 "can't have 2 fate actions in" 1045 " same flow"); 1046 if (!priv->rxqs_n) 1047 return rte_flow_error_set(error, EINVAL, 1048 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1049 NULL, "No Rx queues configured"); 1050 if (queue->index >= priv->rxqs_n) 1051 return rte_flow_error_set(error, EINVAL, 1052 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1053 &queue->index, 1054 "queue index out of range"); 1055 if (!(*priv->rxqs)[queue->index]) 1056 return rte_flow_error_set(error, EINVAL, 1057 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1058 &queue->index, 1059 "queue is not configured"); 1060 if (attr->egress) 1061 return rte_flow_error_set(error, ENOTSUP, 1062 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1063 "queue action not supported for " 1064 "egress"); 1065 return 0; 1066 } 1067 1068 /* 1069 * Validate the rss action. 1070 * 1071 * @param[in] action 1072 * Pointer to the queue action. 1073 * @param[in] action_flags 1074 * Bit-fields that holds the actions detected until now. 1075 * @param[in] dev 1076 * Pointer to the Ethernet device structure. 1077 * @param[in] attr 1078 * Attributes of flow that includes this action. 1079 * @param[in] item_flags 1080 * Items that were detected. 1081 * @param[out] error 1082 * Pointer to error structure. 1083 * 1084 * @return 1085 * 0 on success, a negative errno value otherwise and rte_errno is set. 1086 */ 1087 int 1088 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1089 uint64_t action_flags, 1090 struct rte_eth_dev *dev, 1091 const struct rte_flow_attr *attr, 1092 uint64_t item_flags, 1093 struct rte_flow_error *error) 1094 { 1095 struct mlx5_priv *priv = dev->data->dev_private; 1096 const struct rte_flow_action_rss *rss = action->conf; 1097 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1098 unsigned int i; 1099 1100 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1101 return rte_flow_error_set(error, EINVAL, 1102 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1103 "can't have 2 fate actions" 1104 " in same flow"); 1105 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1106 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1107 return rte_flow_error_set(error, ENOTSUP, 1108 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1109 &rss->func, 1110 "RSS hash function not supported"); 1111 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1112 if (rss->level > 2) 1113 #else 1114 if (rss->level > 1) 1115 #endif 1116 return rte_flow_error_set(error, ENOTSUP, 1117 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1118 &rss->level, 1119 "tunnel RSS is not supported"); 1120 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1121 if (rss->key_len == 0 && rss->key != NULL) 1122 return rte_flow_error_set(error, ENOTSUP, 1123 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1124 &rss->key_len, 1125 "RSS hash key length 0"); 1126 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1127 return rte_flow_error_set(error, ENOTSUP, 1128 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1129 &rss->key_len, 1130 "RSS hash key too small"); 1131 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1132 return rte_flow_error_set(error, ENOTSUP, 1133 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1134 &rss->key_len, 1135 "RSS hash key too large"); 1136 if (rss->queue_num > priv->config.ind_table_max_size) 1137 return rte_flow_error_set(error, ENOTSUP, 1138 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1139 &rss->queue_num, 1140 "number of queues too large"); 1141 if (rss->types & MLX5_RSS_HF_MASK) 1142 return rte_flow_error_set(error, ENOTSUP, 1143 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1144 &rss->types, 1145 "some RSS protocols are not" 1146 " supported"); 1147 if ((rss->types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) && 1148 !(rss->types & ETH_RSS_IP)) 1149 return rte_flow_error_set(error, EINVAL, 1150 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1151 "L3 partial RSS requested but L3 RSS" 1152 " type not specified"); 1153 if ((rss->types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) && 1154 !(rss->types & (ETH_RSS_UDP | ETH_RSS_TCP))) 1155 return rte_flow_error_set(error, EINVAL, 1156 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1157 "L4 partial RSS requested but L4 RSS" 1158 " type not specified"); 1159 if (!priv->rxqs_n) 1160 return rte_flow_error_set(error, EINVAL, 1161 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1162 NULL, "No Rx queues configured"); 1163 if (!rss->queue_num) 1164 return rte_flow_error_set(error, EINVAL, 1165 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1166 NULL, "No queues configured"); 1167 for (i = 0; i != rss->queue_num; ++i) { 1168 if (rss->queue[i] >= priv->rxqs_n) 1169 return rte_flow_error_set 1170 (error, EINVAL, 1171 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1172 &rss->queue[i], "queue index out of range"); 1173 if (!(*priv->rxqs)[rss->queue[i]]) 1174 return rte_flow_error_set 1175 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1176 &rss->queue[i], "queue is not configured"); 1177 } 1178 if (attr->egress) 1179 return rte_flow_error_set(error, ENOTSUP, 1180 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1181 "rss action not supported for " 1182 "egress"); 1183 if (rss->level > 1 && !tunnel) 1184 return rte_flow_error_set(error, EINVAL, 1185 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1186 "inner RSS is not supported for " 1187 "non-tunnel flows"); 1188 return 0; 1189 } 1190 1191 /* 1192 * Validate the count action. 1193 * 1194 * @param[in] dev 1195 * Pointer to the Ethernet device structure. 1196 * @param[in] attr 1197 * Attributes of flow that includes this action. 1198 * @param[out] error 1199 * Pointer to error structure. 1200 * 1201 * @return 1202 * 0 on success, a negative errno value otherwise and rte_errno is set. 1203 */ 1204 int 1205 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1206 const struct rte_flow_attr *attr, 1207 struct rte_flow_error *error) 1208 { 1209 if (attr->egress) 1210 return rte_flow_error_set(error, ENOTSUP, 1211 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1212 "count action not supported for " 1213 "egress"); 1214 return 0; 1215 } 1216 1217 /** 1218 * Verify the @p attributes will be correctly understood by the NIC and store 1219 * them in the @p flow if everything is correct. 1220 * 1221 * @param[in] dev 1222 * Pointer to the Ethernet device structure. 1223 * @param[in] attributes 1224 * Pointer to flow attributes 1225 * @param[out] error 1226 * Pointer to error structure. 1227 * 1228 * @return 1229 * 0 on success, a negative errno value otherwise and rte_errno is set. 1230 */ 1231 int 1232 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 1233 const struct rte_flow_attr *attributes, 1234 struct rte_flow_error *error) 1235 { 1236 struct mlx5_priv *priv = dev->data->dev_private; 1237 uint32_t priority_max = priv->config.flow_prio - 1; 1238 1239 if (attributes->group) 1240 return rte_flow_error_set(error, ENOTSUP, 1241 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 1242 NULL, "groups is not supported"); 1243 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 1244 attributes->priority >= priority_max) 1245 return rte_flow_error_set(error, ENOTSUP, 1246 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 1247 NULL, "priority out of range"); 1248 if (attributes->egress) 1249 return rte_flow_error_set(error, ENOTSUP, 1250 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1251 "egress is not supported"); 1252 if (attributes->transfer && !priv->config.dv_esw_en) 1253 return rte_flow_error_set(error, ENOTSUP, 1254 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1255 NULL, "transfer is not supported"); 1256 if (!attributes->ingress) 1257 return rte_flow_error_set(error, EINVAL, 1258 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1259 NULL, 1260 "ingress attribute is mandatory"); 1261 return 0; 1262 } 1263 1264 /** 1265 * Validate ICMP6 item. 1266 * 1267 * @param[in] item 1268 * Item specification. 1269 * @param[in] item_flags 1270 * Bit-fields that holds the items detected until now. 1271 * @param[out] error 1272 * Pointer to error structure. 1273 * 1274 * @return 1275 * 0 on success, a negative errno value otherwise and rte_errno is set. 1276 */ 1277 int 1278 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 1279 uint64_t item_flags, 1280 uint8_t target_protocol, 1281 struct rte_flow_error *error) 1282 { 1283 const struct rte_flow_item_icmp6 *mask = item->mask; 1284 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1285 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 1286 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 1287 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1288 MLX5_FLOW_LAYER_OUTER_L4; 1289 int ret; 1290 1291 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 1292 return rte_flow_error_set(error, EINVAL, 1293 RTE_FLOW_ERROR_TYPE_ITEM, item, 1294 "protocol filtering not compatible" 1295 " with ICMP6 layer"); 1296 if (!(item_flags & l3m)) 1297 return rte_flow_error_set(error, EINVAL, 1298 RTE_FLOW_ERROR_TYPE_ITEM, item, 1299 "IPv6 is mandatory to filter on" 1300 " ICMP6"); 1301 if (item_flags & l4m) 1302 return rte_flow_error_set(error, EINVAL, 1303 RTE_FLOW_ERROR_TYPE_ITEM, item, 1304 "multiple L4 layers not supported"); 1305 if (!mask) 1306 mask = &rte_flow_item_icmp6_mask; 1307 ret = mlx5_flow_item_acceptable 1308 (item, (const uint8_t *)mask, 1309 (const uint8_t *)&rte_flow_item_icmp6_mask, 1310 sizeof(struct rte_flow_item_icmp6), error); 1311 if (ret < 0) 1312 return ret; 1313 return 0; 1314 } 1315 1316 /** 1317 * Validate ICMP item. 1318 * 1319 * @param[in] item 1320 * Item specification. 1321 * @param[in] item_flags 1322 * Bit-fields that holds the items detected until now. 1323 * @param[out] error 1324 * Pointer to error structure. 1325 * 1326 * @return 1327 * 0 on success, a negative errno value otherwise and rte_errno is set. 1328 */ 1329 int 1330 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 1331 uint64_t item_flags, 1332 uint8_t target_protocol, 1333 struct rte_flow_error *error) 1334 { 1335 const struct rte_flow_item_icmp *mask = item->mask; 1336 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1337 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 1338 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 1339 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1340 MLX5_FLOW_LAYER_OUTER_L4; 1341 int ret; 1342 1343 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 1344 return rte_flow_error_set(error, EINVAL, 1345 RTE_FLOW_ERROR_TYPE_ITEM, item, 1346 "protocol filtering not compatible" 1347 " with ICMP layer"); 1348 if (!(item_flags & l3m)) 1349 return rte_flow_error_set(error, EINVAL, 1350 RTE_FLOW_ERROR_TYPE_ITEM, item, 1351 "IPv4 is mandatory to filter" 1352 " on ICMP"); 1353 if (item_flags & l4m) 1354 return rte_flow_error_set(error, EINVAL, 1355 RTE_FLOW_ERROR_TYPE_ITEM, item, 1356 "multiple L4 layers not supported"); 1357 if (!mask) 1358 mask = &rte_flow_item_icmp_mask; 1359 ret = mlx5_flow_item_acceptable 1360 (item, (const uint8_t *)mask, 1361 (const uint8_t *)&rte_flow_item_icmp_mask, 1362 sizeof(struct rte_flow_item_icmp), error); 1363 if (ret < 0) 1364 return ret; 1365 return 0; 1366 } 1367 1368 /** 1369 * Validate Ethernet item. 1370 * 1371 * @param[in] item 1372 * Item specification. 1373 * @param[in] item_flags 1374 * Bit-fields that holds the items detected until now. 1375 * @param[out] error 1376 * Pointer to error structure. 1377 * 1378 * @return 1379 * 0 on success, a negative errno value otherwise and rte_errno is set. 1380 */ 1381 int 1382 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 1383 uint64_t item_flags, 1384 struct rte_flow_error *error) 1385 { 1386 const struct rte_flow_item_eth *mask = item->mask; 1387 const struct rte_flow_item_eth nic_mask = { 1388 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1389 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1390 .type = RTE_BE16(0xffff), 1391 }; 1392 int ret; 1393 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1394 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 1395 MLX5_FLOW_LAYER_OUTER_L2; 1396 1397 if (item_flags & ethm) 1398 return rte_flow_error_set(error, ENOTSUP, 1399 RTE_FLOW_ERROR_TYPE_ITEM, item, 1400 "multiple L2 layers not supported"); 1401 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 1402 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 1403 return rte_flow_error_set(error, EINVAL, 1404 RTE_FLOW_ERROR_TYPE_ITEM, item, 1405 "L2 layer should not follow " 1406 "L3 layers"); 1407 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 1408 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 1409 return rte_flow_error_set(error, EINVAL, 1410 RTE_FLOW_ERROR_TYPE_ITEM, item, 1411 "L2 layer should not follow VLAN"); 1412 if (!mask) 1413 mask = &rte_flow_item_eth_mask; 1414 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1415 (const uint8_t *)&nic_mask, 1416 sizeof(struct rte_flow_item_eth), 1417 error); 1418 return ret; 1419 } 1420 1421 /** 1422 * Validate VLAN item. 1423 * 1424 * @param[in] item 1425 * Item specification. 1426 * @param[in] item_flags 1427 * Bit-fields that holds the items detected until now. 1428 * @param[in] dev 1429 * Ethernet device flow is being created on. 1430 * @param[out] error 1431 * Pointer to error structure. 1432 * 1433 * @return 1434 * 0 on success, a negative errno value otherwise and rte_errno is set. 1435 */ 1436 int 1437 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 1438 uint64_t item_flags, 1439 struct rte_eth_dev *dev, 1440 struct rte_flow_error *error) 1441 { 1442 const struct rte_flow_item_vlan *spec = item->spec; 1443 const struct rte_flow_item_vlan *mask = item->mask; 1444 const struct rte_flow_item_vlan nic_mask = { 1445 .tci = RTE_BE16(UINT16_MAX), 1446 .inner_type = RTE_BE16(UINT16_MAX), 1447 }; 1448 uint16_t vlan_tag = 0; 1449 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1450 int ret; 1451 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1452 MLX5_FLOW_LAYER_INNER_L4) : 1453 (MLX5_FLOW_LAYER_OUTER_L3 | 1454 MLX5_FLOW_LAYER_OUTER_L4); 1455 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1456 MLX5_FLOW_LAYER_OUTER_VLAN; 1457 1458 if (item_flags & vlanm) 1459 return rte_flow_error_set(error, EINVAL, 1460 RTE_FLOW_ERROR_TYPE_ITEM, item, 1461 "multiple VLAN layers not supported"); 1462 else if ((item_flags & l34m) != 0) 1463 return rte_flow_error_set(error, EINVAL, 1464 RTE_FLOW_ERROR_TYPE_ITEM, item, 1465 "VLAN cannot follow L3/L4 layer"); 1466 if (!mask) 1467 mask = &rte_flow_item_vlan_mask; 1468 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1469 (const uint8_t *)&nic_mask, 1470 sizeof(struct rte_flow_item_vlan), 1471 error); 1472 if (ret) 1473 return ret; 1474 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1475 struct mlx5_priv *priv = dev->data->dev_private; 1476 1477 if (priv->vmwa_context) { 1478 /* 1479 * Non-NULL context means we have a virtual machine 1480 * and SR-IOV enabled, we have to create VLAN interface 1481 * to make hypervisor to setup E-Switch vport 1482 * context correctly. We avoid creating the multiple 1483 * VLAN interfaces, so we cannot support VLAN tag mask. 1484 */ 1485 return rte_flow_error_set(error, EINVAL, 1486 RTE_FLOW_ERROR_TYPE_ITEM, 1487 item, 1488 "VLAN tag mask is not" 1489 " supported in virtual" 1490 " environment"); 1491 } 1492 } 1493 if (spec) { 1494 vlan_tag = spec->tci; 1495 vlan_tag &= mask->tci; 1496 } 1497 /* 1498 * From verbs perspective an empty VLAN is equivalent 1499 * to a packet without VLAN layer. 1500 */ 1501 if (!vlan_tag) 1502 return rte_flow_error_set(error, EINVAL, 1503 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1504 item->spec, 1505 "VLAN cannot be empty"); 1506 return 0; 1507 } 1508 1509 /** 1510 * Validate IPV4 item. 1511 * 1512 * @param[in] item 1513 * Item specification. 1514 * @param[in] item_flags 1515 * Bit-fields that holds the items detected until now. 1516 * @param[in] acc_mask 1517 * Acceptable mask, if NULL default internal default mask 1518 * will be used to check whether item fields are supported. 1519 * @param[out] error 1520 * Pointer to error structure. 1521 * 1522 * @return 1523 * 0 on success, a negative errno value otherwise and rte_errno is set. 1524 */ 1525 int 1526 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 1527 uint64_t item_flags, 1528 uint64_t last_item, 1529 uint16_t ether_type, 1530 const struct rte_flow_item_ipv4 *acc_mask, 1531 struct rte_flow_error *error) 1532 { 1533 const struct rte_flow_item_ipv4 *mask = item->mask; 1534 const struct rte_flow_item_ipv4 *spec = item->spec; 1535 const struct rte_flow_item_ipv4 nic_mask = { 1536 .hdr = { 1537 .src_addr = RTE_BE32(0xffffffff), 1538 .dst_addr = RTE_BE32(0xffffffff), 1539 .type_of_service = 0xff, 1540 .next_proto_id = 0xff, 1541 }, 1542 }; 1543 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1544 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1545 MLX5_FLOW_LAYER_OUTER_L3; 1546 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1547 MLX5_FLOW_LAYER_OUTER_L4; 1548 int ret; 1549 uint8_t next_proto = 0xFF; 1550 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1551 MLX5_FLOW_LAYER_OUTER_VLAN | 1552 MLX5_FLOW_LAYER_INNER_VLAN); 1553 1554 if ((last_item & l2_vlan) && ether_type && 1555 ether_type != RTE_ETHER_TYPE_IPV4) 1556 return rte_flow_error_set(error, EINVAL, 1557 RTE_FLOW_ERROR_TYPE_ITEM, item, 1558 "IPv4 cannot follow L2/VLAN layer " 1559 "which ether type is not IPv4"); 1560 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 1561 if (mask && spec) 1562 next_proto = mask->hdr.next_proto_id & 1563 spec->hdr.next_proto_id; 1564 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1565 return rte_flow_error_set(error, EINVAL, 1566 RTE_FLOW_ERROR_TYPE_ITEM, 1567 item, 1568 "multiple tunnel " 1569 "not supported"); 1570 } 1571 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 1572 return rte_flow_error_set(error, EINVAL, 1573 RTE_FLOW_ERROR_TYPE_ITEM, item, 1574 "wrong tunnel type - IPv6 specified " 1575 "but IPv4 item provided"); 1576 if (item_flags & l3m) 1577 return rte_flow_error_set(error, ENOTSUP, 1578 RTE_FLOW_ERROR_TYPE_ITEM, item, 1579 "multiple L3 layers not supported"); 1580 else if (item_flags & l4m) 1581 return rte_flow_error_set(error, EINVAL, 1582 RTE_FLOW_ERROR_TYPE_ITEM, item, 1583 "L3 cannot follow an L4 layer."); 1584 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1585 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1586 return rte_flow_error_set(error, EINVAL, 1587 RTE_FLOW_ERROR_TYPE_ITEM, item, 1588 "L3 cannot follow an NVGRE layer."); 1589 if (!mask) 1590 mask = &rte_flow_item_ipv4_mask; 1591 else if (mask->hdr.next_proto_id != 0 && 1592 mask->hdr.next_proto_id != 0xff) 1593 return rte_flow_error_set(error, EINVAL, 1594 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 1595 "partial mask is not supported" 1596 " for protocol"); 1597 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1598 acc_mask ? (const uint8_t *)acc_mask 1599 : (const uint8_t *)&nic_mask, 1600 sizeof(struct rte_flow_item_ipv4), 1601 error); 1602 if (ret < 0) 1603 return ret; 1604 return 0; 1605 } 1606 1607 /** 1608 * Validate IPV6 item. 1609 * 1610 * @param[in] item 1611 * Item specification. 1612 * @param[in] item_flags 1613 * Bit-fields that holds the items detected until now. 1614 * @param[in] acc_mask 1615 * Acceptable mask, if NULL default internal default mask 1616 * will be used to check whether item fields are supported. 1617 * @param[out] error 1618 * Pointer to error structure. 1619 * 1620 * @return 1621 * 0 on success, a negative errno value otherwise and rte_errno is set. 1622 */ 1623 int 1624 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 1625 uint64_t item_flags, 1626 uint64_t last_item, 1627 uint16_t ether_type, 1628 const struct rte_flow_item_ipv6 *acc_mask, 1629 struct rte_flow_error *error) 1630 { 1631 const struct rte_flow_item_ipv6 *mask = item->mask; 1632 const struct rte_flow_item_ipv6 *spec = item->spec; 1633 const struct rte_flow_item_ipv6 nic_mask = { 1634 .hdr = { 1635 .src_addr = 1636 "\xff\xff\xff\xff\xff\xff\xff\xff" 1637 "\xff\xff\xff\xff\xff\xff\xff\xff", 1638 .dst_addr = 1639 "\xff\xff\xff\xff\xff\xff\xff\xff" 1640 "\xff\xff\xff\xff\xff\xff\xff\xff", 1641 .vtc_flow = RTE_BE32(0xffffffff), 1642 .proto = 0xff, 1643 }, 1644 }; 1645 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1646 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1647 MLX5_FLOW_LAYER_OUTER_L3; 1648 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1649 MLX5_FLOW_LAYER_OUTER_L4; 1650 int ret; 1651 uint8_t next_proto = 0xFF; 1652 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 1653 MLX5_FLOW_LAYER_OUTER_VLAN | 1654 MLX5_FLOW_LAYER_INNER_VLAN); 1655 1656 if ((last_item & l2_vlan) && ether_type && 1657 ether_type != RTE_ETHER_TYPE_IPV6) 1658 return rte_flow_error_set(error, EINVAL, 1659 RTE_FLOW_ERROR_TYPE_ITEM, item, 1660 "IPv6 cannot follow L2/VLAN layer " 1661 "which ether type is not IPv6"); 1662 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) { 1663 if (mask && spec) 1664 next_proto = mask->hdr.proto & spec->hdr.proto; 1665 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 1666 return rte_flow_error_set(error, EINVAL, 1667 RTE_FLOW_ERROR_TYPE_ITEM, 1668 item, 1669 "multiple tunnel " 1670 "not supported"); 1671 } 1672 if (item_flags & MLX5_FLOW_LAYER_IPIP) 1673 return rte_flow_error_set(error, EINVAL, 1674 RTE_FLOW_ERROR_TYPE_ITEM, item, 1675 "wrong tunnel type - IPv4 specified " 1676 "but IPv6 item provided"); 1677 if (item_flags & l3m) 1678 return rte_flow_error_set(error, ENOTSUP, 1679 RTE_FLOW_ERROR_TYPE_ITEM, item, 1680 "multiple L3 layers not supported"); 1681 else if (item_flags & l4m) 1682 return rte_flow_error_set(error, EINVAL, 1683 RTE_FLOW_ERROR_TYPE_ITEM, item, 1684 "L3 cannot follow an L4 layer."); 1685 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 1686 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 1687 return rte_flow_error_set(error, EINVAL, 1688 RTE_FLOW_ERROR_TYPE_ITEM, item, 1689 "L3 cannot follow an NVGRE layer."); 1690 if (!mask) 1691 mask = &rte_flow_item_ipv6_mask; 1692 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1693 acc_mask ? (const uint8_t *)acc_mask 1694 : (const uint8_t *)&nic_mask, 1695 sizeof(struct rte_flow_item_ipv6), 1696 error); 1697 if (ret < 0) 1698 return ret; 1699 return 0; 1700 } 1701 1702 /** 1703 * Validate UDP item. 1704 * 1705 * @param[in] item 1706 * Item specification. 1707 * @param[in] item_flags 1708 * Bit-fields that holds the items detected until now. 1709 * @param[in] target_protocol 1710 * The next protocol in the previous item. 1711 * @param[in] flow_mask 1712 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 1713 * @param[out] error 1714 * Pointer to error structure. 1715 * 1716 * @return 1717 * 0 on success, a negative errno value otherwise and rte_errno is set. 1718 */ 1719 int 1720 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 1721 uint64_t item_flags, 1722 uint8_t target_protocol, 1723 struct rte_flow_error *error) 1724 { 1725 const struct rte_flow_item_udp *mask = item->mask; 1726 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1727 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1728 MLX5_FLOW_LAYER_OUTER_L3; 1729 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1730 MLX5_FLOW_LAYER_OUTER_L4; 1731 int ret; 1732 1733 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 1734 return rte_flow_error_set(error, EINVAL, 1735 RTE_FLOW_ERROR_TYPE_ITEM, item, 1736 "protocol filtering not compatible" 1737 " with UDP layer"); 1738 if (!(item_flags & l3m)) 1739 return rte_flow_error_set(error, EINVAL, 1740 RTE_FLOW_ERROR_TYPE_ITEM, item, 1741 "L3 is mandatory to filter on L4"); 1742 if (item_flags & l4m) 1743 return rte_flow_error_set(error, EINVAL, 1744 RTE_FLOW_ERROR_TYPE_ITEM, item, 1745 "multiple L4 layers not supported"); 1746 if (!mask) 1747 mask = &rte_flow_item_udp_mask; 1748 ret = mlx5_flow_item_acceptable 1749 (item, (const uint8_t *)mask, 1750 (const uint8_t *)&rte_flow_item_udp_mask, 1751 sizeof(struct rte_flow_item_udp), error); 1752 if (ret < 0) 1753 return ret; 1754 return 0; 1755 } 1756 1757 /** 1758 * Validate TCP item. 1759 * 1760 * @param[in] item 1761 * Item specification. 1762 * @param[in] item_flags 1763 * Bit-fields that holds the items detected until now. 1764 * @param[in] target_protocol 1765 * The next protocol in the previous item. 1766 * @param[out] error 1767 * Pointer to error structure. 1768 * 1769 * @return 1770 * 0 on success, a negative errno value otherwise and rte_errno is set. 1771 */ 1772 int 1773 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 1774 uint64_t item_flags, 1775 uint8_t target_protocol, 1776 const struct rte_flow_item_tcp *flow_mask, 1777 struct rte_flow_error *error) 1778 { 1779 const struct rte_flow_item_tcp *mask = item->mask; 1780 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1781 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 1782 MLX5_FLOW_LAYER_OUTER_L3; 1783 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 1784 MLX5_FLOW_LAYER_OUTER_L4; 1785 int ret; 1786 1787 MLX5_ASSERT(flow_mask); 1788 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 1789 return rte_flow_error_set(error, EINVAL, 1790 RTE_FLOW_ERROR_TYPE_ITEM, item, 1791 "protocol filtering not compatible" 1792 " with TCP layer"); 1793 if (!(item_flags & l3m)) 1794 return rte_flow_error_set(error, EINVAL, 1795 RTE_FLOW_ERROR_TYPE_ITEM, item, 1796 "L3 is mandatory to filter on L4"); 1797 if (item_flags & l4m) 1798 return rte_flow_error_set(error, EINVAL, 1799 RTE_FLOW_ERROR_TYPE_ITEM, item, 1800 "multiple L4 layers not supported"); 1801 if (!mask) 1802 mask = &rte_flow_item_tcp_mask; 1803 ret = mlx5_flow_item_acceptable 1804 (item, (const uint8_t *)mask, 1805 (const uint8_t *)flow_mask, 1806 sizeof(struct rte_flow_item_tcp), error); 1807 if (ret < 0) 1808 return ret; 1809 return 0; 1810 } 1811 1812 /** 1813 * Validate VXLAN item. 1814 * 1815 * @param[in] item 1816 * Item specification. 1817 * @param[in] item_flags 1818 * Bit-fields that holds the items detected until now. 1819 * @param[in] target_protocol 1820 * The next protocol in the previous item. 1821 * @param[out] error 1822 * Pointer to error structure. 1823 * 1824 * @return 1825 * 0 on success, a negative errno value otherwise and rte_errno is set. 1826 */ 1827 int 1828 mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item, 1829 uint64_t item_flags, 1830 struct rte_flow_error *error) 1831 { 1832 const struct rte_flow_item_vxlan *spec = item->spec; 1833 const struct rte_flow_item_vxlan *mask = item->mask; 1834 int ret; 1835 union vni { 1836 uint32_t vlan_id; 1837 uint8_t vni[4]; 1838 } id = { .vlan_id = 0, }; 1839 uint32_t vlan_id = 0; 1840 1841 1842 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1843 return rte_flow_error_set(error, ENOTSUP, 1844 RTE_FLOW_ERROR_TYPE_ITEM, item, 1845 "multiple tunnel layers not" 1846 " supported"); 1847 /* 1848 * Verify only UDPv4 is present as defined in 1849 * https://tools.ietf.org/html/rfc7348 1850 */ 1851 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1852 return rte_flow_error_set(error, EINVAL, 1853 RTE_FLOW_ERROR_TYPE_ITEM, item, 1854 "no outer UDP layer found"); 1855 if (!mask) 1856 mask = &rte_flow_item_vxlan_mask; 1857 ret = mlx5_flow_item_acceptable 1858 (item, (const uint8_t *)mask, 1859 (const uint8_t *)&rte_flow_item_vxlan_mask, 1860 sizeof(struct rte_flow_item_vxlan), 1861 error); 1862 if (ret < 0) 1863 return ret; 1864 if (spec) { 1865 memcpy(&id.vni[1], spec->vni, 3); 1866 vlan_id = id.vlan_id; 1867 memcpy(&id.vni[1], mask->vni, 3); 1868 vlan_id &= id.vlan_id; 1869 } 1870 /* 1871 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if 1872 * only this layer is defined in the Verbs specification it is 1873 * interpreted as wildcard and all packets will match this 1874 * rule, if it follows a full stack layer (ex: eth / ipv4 / 1875 * udp), all packets matching the layers before will also 1876 * match this rule. To avoid such situation, VNI 0 is 1877 * currently refused. 1878 */ 1879 if (!vlan_id) 1880 return rte_flow_error_set(error, ENOTSUP, 1881 RTE_FLOW_ERROR_TYPE_ITEM, item, 1882 "VXLAN vni cannot be 0"); 1883 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1884 return rte_flow_error_set(error, ENOTSUP, 1885 RTE_FLOW_ERROR_TYPE_ITEM, item, 1886 "VXLAN tunnel must be fully defined"); 1887 return 0; 1888 } 1889 1890 /** 1891 * Validate VXLAN_GPE item. 1892 * 1893 * @param[in] item 1894 * Item specification. 1895 * @param[in] item_flags 1896 * Bit-fields that holds the items detected until now. 1897 * @param[in] priv 1898 * Pointer to the private data structure. 1899 * @param[in] target_protocol 1900 * The next protocol in the previous item. 1901 * @param[out] error 1902 * Pointer to error structure. 1903 * 1904 * @return 1905 * 0 on success, a negative errno value otherwise and rte_errno is set. 1906 */ 1907 int 1908 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 1909 uint64_t item_flags, 1910 struct rte_eth_dev *dev, 1911 struct rte_flow_error *error) 1912 { 1913 struct mlx5_priv *priv = dev->data->dev_private; 1914 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 1915 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 1916 int ret; 1917 union vni { 1918 uint32_t vlan_id; 1919 uint8_t vni[4]; 1920 } id = { .vlan_id = 0, }; 1921 uint32_t vlan_id = 0; 1922 1923 if (!priv->config.l3_vxlan_en) 1924 return rte_flow_error_set(error, ENOTSUP, 1925 RTE_FLOW_ERROR_TYPE_ITEM, item, 1926 "L3 VXLAN is not enabled by device" 1927 " parameter and/or not configured in" 1928 " firmware"); 1929 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1930 return rte_flow_error_set(error, ENOTSUP, 1931 RTE_FLOW_ERROR_TYPE_ITEM, item, 1932 "multiple tunnel layers not" 1933 " supported"); 1934 /* 1935 * Verify only UDPv4 is present as defined in 1936 * https://tools.ietf.org/html/rfc7348 1937 */ 1938 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1939 return rte_flow_error_set(error, EINVAL, 1940 RTE_FLOW_ERROR_TYPE_ITEM, item, 1941 "no outer UDP layer found"); 1942 if (!mask) 1943 mask = &rte_flow_item_vxlan_gpe_mask; 1944 ret = mlx5_flow_item_acceptable 1945 (item, (const uint8_t *)mask, 1946 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 1947 sizeof(struct rte_flow_item_vxlan_gpe), 1948 error); 1949 if (ret < 0) 1950 return ret; 1951 if (spec) { 1952 if (spec->protocol) 1953 return rte_flow_error_set(error, ENOTSUP, 1954 RTE_FLOW_ERROR_TYPE_ITEM, 1955 item, 1956 "VxLAN-GPE protocol" 1957 " not supported"); 1958 memcpy(&id.vni[1], spec->vni, 3); 1959 vlan_id = id.vlan_id; 1960 memcpy(&id.vni[1], mask->vni, 3); 1961 vlan_id &= id.vlan_id; 1962 } 1963 /* 1964 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this 1965 * layer is defined in the Verbs specification it is interpreted as 1966 * wildcard and all packets will match this rule, if it follows a full 1967 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers 1968 * before will also match this rule. To avoid such situation, VNI 0 1969 * is currently refused. 1970 */ 1971 if (!vlan_id) 1972 return rte_flow_error_set(error, ENOTSUP, 1973 RTE_FLOW_ERROR_TYPE_ITEM, item, 1974 "VXLAN-GPE vni cannot be 0"); 1975 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 1976 return rte_flow_error_set(error, ENOTSUP, 1977 RTE_FLOW_ERROR_TYPE_ITEM, item, 1978 "VXLAN-GPE tunnel must be fully" 1979 " defined"); 1980 return 0; 1981 } 1982 /** 1983 * Validate GRE Key item. 1984 * 1985 * @param[in] item 1986 * Item specification. 1987 * @param[in] item_flags 1988 * Bit flags to mark detected items. 1989 * @param[in] gre_item 1990 * Pointer to gre_item 1991 * @param[out] error 1992 * Pointer to error structure. 1993 * 1994 * @return 1995 * 0 on success, a negative errno value otherwise and rte_errno is set. 1996 */ 1997 int 1998 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 1999 uint64_t item_flags, 2000 const struct rte_flow_item *gre_item, 2001 struct rte_flow_error *error) 2002 { 2003 const rte_be32_t *mask = item->mask; 2004 int ret = 0; 2005 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2006 const struct rte_flow_item_gre *gre_spec; 2007 const struct rte_flow_item_gre *gre_mask; 2008 2009 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2010 return rte_flow_error_set(error, ENOTSUP, 2011 RTE_FLOW_ERROR_TYPE_ITEM, item, 2012 "Multiple GRE key not support"); 2013 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2014 return rte_flow_error_set(error, ENOTSUP, 2015 RTE_FLOW_ERROR_TYPE_ITEM, item, 2016 "No preceding GRE header"); 2017 if (item_flags & MLX5_FLOW_LAYER_INNER) 2018 return rte_flow_error_set(error, ENOTSUP, 2019 RTE_FLOW_ERROR_TYPE_ITEM, item, 2020 "GRE key following a wrong item"); 2021 gre_mask = gre_item->mask; 2022 if (!gre_mask) 2023 gre_mask = &rte_flow_item_gre_mask; 2024 gre_spec = gre_item->spec; 2025 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2026 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2027 return rte_flow_error_set(error, EINVAL, 2028 RTE_FLOW_ERROR_TYPE_ITEM, item, 2029 "Key bit must be on"); 2030 2031 if (!mask) 2032 mask = &gre_key_default_mask; 2033 ret = mlx5_flow_item_acceptable 2034 (item, (const uint8_t *)mask, 2035 (const uint8_t *)&gre_key_default_mask, 2036 sizeof(rte_be32_t), error); 2037 return ret; 2038 } 2039 2040 /** 2041 * Validate GRE item. 2042 * 2043 * @param[in] item 2044 * Item specification. 2045 * @param[in] item_flags 2046 * Bit flags to mark detected items. 2047 * @param[in] target_protocol 2048 * The next protocol in the previous item. 2049 * @param[out] error 2050 * Pointer to error structure. 2051 * 2052 * @return 2053 * 0 on success, a negative errno value otherwise and rte_errno is set. 2054 */ 2055 int 2056 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2057 uint64_t item_flags, 2058 uint8_t target_protocol, 2059 struct rte_flow_error *error) 2060 { 2061 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2062 const struct rte_flow_item_gre *mask = item->mask; 2063 int ret; 2064 const struct rte_flow_item_gre nic_mask = { 2065 .c_rsvd0_ver = RTE_BE16(0xB000), 2066 .protocol = RTE_BE16(UINT16_MAX), 2067 }; 2068 2069 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2070 return rte_flow_error_set(error, EINVAL, 2071 RTE_FLOW_ERROR_TYPE_ITEM, item, 2072 "protocol filtering not compatible" 2073 " with this GRE layer"); 2074 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2075 return rte_flow_error_set(error, ENOTSUP, 2076 RTE_FLOW_ERROR_TYPE_ITEM, item, 2077 "multiple tunnel layers not" 2078 " supported"); 2079 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2080 return rte_flow_error_set(error, ENOTSUP, 2081 RTE_FLOW_ERROR_TYPE_ITEM, item, 2082 "L3 Layer is missing"); 2083 if (!mask) 2084 mask = &rte_flow_item_gre_mask; 2085 ret = mlx5_flow_item_acceptable 2086 (item, (const uint8_t *)mask, 2087 (const uint8_t *)&nic_mask, 2088 sizeof(struct rte_flow_item_gre), error); 2089 if (ret < 0) 2090 return ret; 2091 #ifndef HAVE_MLX5DV_DR 2092 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2093 if (spec && (spec->protocol & mask->protocol)) 2094 return rte_flow_error_set(error, ENOTSUP, 2095 RTE_FLOW_ERROR_TYPE_ITEM, item, 2096 "without MPLS support the" 2097 " specification cannot be used for" 2098 " filtering"); 2099 #endif 2100 #endif 2101 return 0; 2102 } 2103 2104 /** 2105 * Validate Geneve item. 2106 * 2107 * @param[in] item 2108 * Item specification. 2109 * @param[in] itemFlags 2110 * Bit-fields that holds the items detected until now. 2111 * @param[in] enPriv 2112 * Pointer to the private data structure. 2113 * @param[out] error 2114 * Pointer to error structure. 2115 * 2116 * @return 2117 * 0 on success, a negative errno value otherwise and rte_errno is set. 2118 */ 2119 2120 int 2121 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2122 uint64_t item_flags, 2123 struct rte_eth_dev *dev, 2124 struct rte_flow_error *error) 2125 { 2126 struct mlx5_priv *priv = dev->data->dev_private; 2127 const struct rte_flow_item_geneve *spec = item->spec; 2128 const struct rte_flow_item_geneve *mask = item->mask; 2129 int ret; 2130 uint16_t gbhdr; 2131 uint8_t opt_len = priv->config.hca_attr.geneve_max_opt_len ? 2132 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2133 const struct rte_flow_item_geneve nic_mask = { 2134 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2135 .vni = "\xff\xff\xff", 2136 .protocol = RTE_BE16(UINT16_MAX), 2137 }; 2138 2139 if (!priv->config.hca_attr.tunnel_stateless_geneve_rx) 2140 return rte_flow_error_set(error, ENOTSUP, 2141 RTE_FLOW_ERROR_TYPE_ITEM, item, 2142 "L3 Geneve is not enabled by device" 2143 " parameter and/or not configured in" 2144 " firmware"); 2145 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2146 return rte_flow_error_set(error, ENOTSUP, 2147 RTE_FLOW_ERROR_TYPE_ITEM, item, 2148 "multiple tunnel layers not" 2149 " supported"); 2150 /* 2151 * Verify only UDPv4 is present as defined in 2152 * https://tools.ietf.org/html/rfc7348 2153 */ 2154 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2155 return rte_flow_error_set(error, EINVAL, 2156 RTE_FLOW_ERROR_TYPE_ITEM, item, 2157 "no outer UDP layer found"); 2158 if (!mask) 2159 mask = &rte_flow_item_geneve_mask; 2160 ret = mlx5_flow_item_acceptable 2161 (item, (const uint8_t *)mask, 2162 (const uint8_t *)&nic_mask, 2163 sizeof(struct rte_flow_item_geneve), error); 2164 if (ret) 2165 return ret; 2166 if (spec) { 2167 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2168 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2169 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2170 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2171 return rte_flow_error_set(error, ENOTSUP, 2172 RTE_FLOW_ERROR_TYPE_ITEM, 2173 item, 2174 "Geneve protocol unsupported" 2175 " fields are being used"); 2176 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2177 return rte_flow_error_set 2178 (error, ENOTSUP, 2179 RTE_FLOW_ERROR_TYPE_ITEM, 2180 item, 2181 "Unsupported Geneve options length"); 2182 } 2183 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2184 return rte_flow_error_set 2185 (error, ENOTSUP, 2186 RTE_FLOW_ERROR_TYPE_ITEM, item, 2187 "Geneve tunnel must be fully defined"); 2188 return 0; 2189 } 2190 2191 /** 2192 * Validate MPLS item. 2193 * 2194 * @param[in] dev 2195 * Pointer to the rte_eth_dev structure. 2196 * @param[in] item 2197 * Item specification. 2198 * @param[in] item_flags 2199 * Bit-fields that holds the items detected until now. 2200 * @param[in] prev_layer 2201 * The protocol layer indicated in previous item. 2202 * @param[out] error 2203 * Pointer to error structure. 2204 * 2205 * @return 2206 * 0 on success, a negative errno value otherwise and rte_errno is set. 2207 */ 2208 int 2209 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 2210 const struct rte_flow_item *item __rte_unused, 2211 uint64_t item_flags __rte_unused, 2212 uint64_t prev_layer __rte_unused, 2213 struct rte_flow_error *error) 2214 { 2215 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 2216 const struct rte_flow_item_mpls *mask = item->mask; 2217 struct mlx5_priv *priv = dev->data->dev_private; 2218 int ret; 2219 2220 if (!priv->config.mpls_en) 2221 return rte_flow_error_set(error, ENOTSUP, 2222 RTE_FLOW_ERROR_TYPE_ITEM, item, 2223 "MPLS not supported or" 2224 " disabled in firmware" 2225 " configuration."); 2226 /* MPLS over IP, UDP, GRE is allowed */ 2227 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L3 | 2228 MLX5_FLOW_LAYER_OUTER_L4_UDP | 2229 MLX5_FLOW_LAYER_GRE))) 2230 return rte_flow_error_set(error, EINVAL, 2231 RTE_FLOW_ERROR_TYPE_ITEM, item, 2232 "protocol filtering not compatible" 2233 " with MPLS layer"); 2234 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 2235 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 2236 !(item_flags & MLX5_FLOW_LAYER_GRE)) 2237 return rte_flow_error_set(error, ENOTSUP, 2238 RTE_FLOW_ERROR_TYPE_ITEM, item, 2239 "multiple tunnel layers not" 2240 " supported"); 2241 if (!mask) 2242 mask = &rte_flow_item_mpls_mask; 2243 ret = mlx5_flow_item_acceptable 2244 (item, (const uint8_t *)mask, 2245 (const uint8_t *)&rte_flow_item_mpls_mask, 2246 sizeof(struct rte_flow_item_mpls), error); 2247 if (ret < 0) 2248 return ret; 2249 return 0; 2250 #endif 2251 return rte_flow_error_set(error, ENOTSUP, 2252 RTE_FLOW_ERROR_TYPE_ITEM, item, 2253 "MPLS is not supported by Verbs, please" 2254 " update."); 2255 } 2256 2257 /** 2258 * Validate NVGRE item. 2259 * 2260 * @param[in] item 2261 * Item specification. 2262 * @param[in] item_flags 2263 * Bit flags to mark detected items. 2264 * @param[in] target_protocol 2265 * The next protocol in the previous item. 2266 * @param[out] error 2267 * Pointer to error structure. 2268 * 2269 * @return 2270 * 0 on success, a negative errno value otherwise and rte_errno is set. 2271 */ 2272 int 2273 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 2274 uint64_t item_flags, 2275 uint8_t target_protocol, 2276 struct rte_flow_error *error) 2277 { 2278 const struct rte_flow_item_nvgre *mask = item->mask; 2279 int ret; 2280 2281 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2282 return rte_flow_error_set(error, EINVAL, 2283 RTE_FLOW_ERROR_TYPE_ITEM, item, 2284 "protocol filtering not compatible" 2285 " with this GRE layer"); 2286 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2287 return rte_flow_error_set(error, ENOTSUP, 2288 RTE_FLOW_ERROR_TYPE_ITEM, item, 2289 "multiple tunnel layers not" 2290 " supported"); 2291 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2292 return rte_flow_error_set(error, ENOTSUP, 2293 RTE_FLOW_ERROR_TYPE_ITEM, item, 2294 "L3 Layer is missing"); 2295 if (!mask) 2296 mask = &rte_flow_item_nvgre_mask; 2297 ret = mlx5_flow_item_acceptable 2298 (item, (const uint8_t *)mask, 2299 (const uint8_t *)&rte_flow_item_nvgre_mask, 2300 sizeof(struct rte_flow_item_nvgre), error); 2301 if (ret < 0) 2302 return ret; 2303 return 0; 2304 } 2305 2306 /* Allocate unique ID for the split Q/RSS subflows. */ 2307 static uint32_t 2308 flow_qrss_get_id(struct rte_eth_dev *dev) 2309 { 2310 struct mlx5_priv *priv = dev->data->dev_private; 2311 uint32_t qrss_id, ret; 2312 2313 ret = mlx5_flow_id_get(priv->qrss_id_pool, &qrss_id); 2314 if (ret) 2315 return 0; 2316 MLX5_ASSERT(qrss_id); 2317 return qrss_id; 2318 } 2319 2320 /* Free unique ID for the split Q/RSS subflows. */ 2321 static void 2322 flow_qrss_free_id(struct rte_eth_dev *dev, uint32_t qrss_id) 2323 { 2324 struct mlx5_priv *priv = dev->data->dev_private; 2325 2326 if (qrss_id) 2327 mlx5_flow_id_release(priv->qrss_id_pool, qrss_id); 2328 } 2329 2330 /** 2331 * Release resource related QUEUE/RSS action split. 2332 * 2333 * @param dev 2334 * Pointer to Ethernet device. 2335 * @param flow 2336 * Flow to release id's from. 2337 */ 2338 static void 2339 flow_mreg_split_qrss_release(struct rte_eth_dev *dev, 2340 struct rte_flow *flow) 2341 { 2342 struct mlx5_flow *dev_flow; 2343 2344 LIST_FOREACH(dev_flow, &flow->dev_flows, next) 2345 if (dev_flow->qrss_id) 2346 flow_qrss_free_id(dev, dev_flow->qrss_id); 2347 } 2348 2349 static int 2350 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 2351 const struct rte_flow_attr *attr __rte_unused, 2352 const struct rte_flow_item items[] __rte_unused, 2353 const struct rte_flow_action actions[] __rte_unused, 2354 bool external __rte_unused, 2355 struct rte_flow_error *error) 2356 { 2357 return rte_flow_error_set(error, ENOTSUP, 2358 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2359 } 2360 2361 static struct mlx5_flow * 2362 flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, 2363 const struct rte_flow_item items[] __rte_unused, 2364 const struct rte_flow_action actions[] __rte_unused, 2365 struct rte_flow_error *error) 2366 { 2367 rte_flow_error_set(error, ENOTSUP, 2368 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2369 return NULL; 2370 } 2371 2372 static int 2373 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 2374 struct mlx5_flow *dev_flow __rte_unused, 2375 const struct rte_flow_attr *attr __rte_unused, 2376 const struct rte_flow_item items[] __rte_unused, 2377 const struct rte_flow_action actions[] __rte_unused, 2378 struct rte_flow_error *error) 2379 { 2380 return rte_flow_error_set(error, ENOTSUP, 2381 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2382 } 2383 2384 static int 2385 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 2386 struct rte_flow *flow __rte_unused, 2387 struct rte_flow_error *error) 2388 { 2389 return rte_flow_error_set(error, ENOTSUP, 2390 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2391 } 2392 2393 static void 2394 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 2395 struct rte_flow *flow __rte_unused) 2396 { 2397 } 2398 2399 static void 2400 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 2401 struct rte_flow *flow __rte_unused) 2402 { 2403 } 2404 2405 static int 2406 flow_null_query(struct rte_eth_dev *dev __rte_unused, 2407 struct rte_flow *flow __rte_unused, 2408 const struct rte_flow_action *actions __rte_unused, 2409 void *data __rte_unused, 2410 struct rte_flow_error *error) 2411 { 2412 return rte_flow_error_set(error, ENOTSUP, 2413 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 2414 } 2415 2416 /* Void driver to protect from null pointer reference. */ 2417 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 2418 .validate = flow_null_validate, 2419 .prepare = flow_null_prepare, 2420 .translate = flow_null_translate, 2421 .apply = flow_null_apply, 2422 .remove = flow_null_remove, 2423 .destroy = flow_null_destroy, 2424 .query = flow_null_query, 2425 }; 2426 2427 /** 2428 * Select flow driver type according to flow attributes and device 2429 * configuration. 2430 * 2431 * @param[in] dev 2432 * Pointer to the dev structure. 2433 * @param[in] attr 2434 * Pointer to the flow attributes. 2435 * 2436 * @return 2437 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 2438 */ 2439 static enum mlx5_flow_drv_type 2440 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 2441 { 2442 struct mlx5_priv *priv = dev->data->dev_private; 2443 enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; 2444 2445 if (attr->transfer && priv->config.dv_esw_en) 2446 type = MLX5_FLOW_TYPE_DV; 2447 if (!attr->transfer) 2448 type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 2449 MLX5_FLOW_TYPE_VERBS; 2450 return type; 2451 } 2452 2453 #define flow_get_drv_ops(type) flow_drv_ops[type] 2454 2455 /** 2456 * Flow driver validation API. This abstracts calling driver specific functions. 2457 * The type of flow driver is determined according to flow attributes. 2458 * 2459 * @param[in] dev 2460 * Pointer to the dev structure. 2461 * @param[in] attr 2462 * Pointer to the flow attributes. 2463 * @param[in] items 2464 * Pointer to the list of items. 2465 * @param[in] actions 2466 * Pointer to the list of actions. 2467 * @param[in] external 2468 * This flow rule is created by request external to PMD. 2469 * @param[out] error 2470 * Pointer to the error structure. 2471 * 2472 * @return 2473 * 0 on success, a negative errno value otherwise and rte_errno is set. 2474 */ 2475 static inline int 2476 flow_drv_validate(struct rte_eth_dev *dev, 2477 const struct rte_flow_attr *attr, 2478 const struct rte_flow_item items[], 2479 const struct rte_flow_action actions[], 2480 bool external, struct rte_flow_error *error) 2481 { 2482 const struct mlx5_flow_driver_ops *fops; 2483 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 2484 2485 fops = flow_get_drv_ops(type); 2486 return fops->validate(dev, attr, items, actions, external, error); 2487 } 2488 2489 /** 2490 * Flow driver preparation API. This abstracts calling driver specific 2491 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2492 * calculates the size of memory required for device flow, allocates the memory, 2493 * initializes the device flow and returns the pointer. 2494 * 2495 * @note 2496 * This function initializes device flow structure such as dv or verbs in 2497 * struct mlx5_flow. However, it is caller's responsibility to initialize the 2498 * rest. For example, adding returning device flow to flow->dev_flow list and 2499 * setting backward reference to the flow should be done out of this function. 2500 * layers field is not filled either. 2501 * 2502 * @param[in] attr 2503 * Pointer to the flow attributes. 2504 * @param[in] items 2505 * Pointer to the list of items. 2506 * @param[in] actions 2507 * Pointer to the list of actions. 2508 * @param[out] error 2509 * Pointer to the error structure. 2510 * 2511 * @return 2512 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 2513 */ 2514 static inline struct mlx5_flow * 2515 flow_drv_prepare(const struct rte_flow *flow, 2516 const struct rte_flow_attr *attr, 2517 const struct rte_flow_item items[], 2518 const struct rte_flow_action actions[], 2519 struct rte_flow_error *error) 2520 { 2521 const struct mlx5_flow_driver_ops *fops; 2522 enum mlx5_flow_drv_type type = flow->drv_type; 2523 2524 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2525 fops = flow_get_drv_ops(type); 2526 return fops->prepare(attr, items, actions, error); 2527 } 2528 2529 /** 2530 * Flow driver translation API. This abstracts calling driver specific 2531 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 2532 * translates a generic flow into a driver flow. flow_drv_prepare() must 2533 * precede. 2534 * 2535 * @note 2536 * dev_flow->layers could be filled as a result of parsing during translation 2537 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 2538 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 2539 * flow->actions could be overwritten even though all the expanded dev_flows 2540 * have the same actions. 2541 * 2542 * @param[in] dev 2543 * Pointer to the rte dev structure. 2544 * @param[in, out] dev_flow 2545 * Pointer to the mlx5 flow. 2546 * @param[in] attr 2547 * Pointer to the flow attributes. 2548 * @param[in] items 2549 * Pointer to the list of items. 2550 * @param[in] actions 2551 * Pointer to the list of actions. 2552 * @param[out] error 2553 * Pointer to the error structure. 2554 * 2555 * @return 2556 * 0 on success, a negative errno value otherwise and rte_errno is set. 2557 */ 2558 static inline int 2559 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 2560 const struct rte_flow_attr *attr, 2561 const struct rte_flow_item items[], 2562 const struct rte_flow_action actions[], 2563 struct rte_flow_error *error) 2564 { 2565 const struct mlx5_flow_driver_ops *fops; 2566 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 2567 2568 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2569 fops = flow_get_drv_ops(type); 2570 return fops->translate(dev, dev_flow, attr, items, actions, error); 2571 } 2572 2573 /** 2574 * Flow driver apply API. This abstracts calling driver specific functions. 2575 * Parent flow (rte_flow) should have driver type (drv_type). It applies 2576 * translated driver flows on to device. flow_drv_translate() must precede. 2577 * 2578 * @param[in] dev 2579 * Pointer to Ethernet device structure. 2580 * @param[in, out] flow 2581 * Pointer to flow structure. 2582 * @param[out] error 2583 * Pointer to error structure. 2584 * 2585 * @return 2586 * 0 on success, a negative errno value otherwise and rte_errno is set. 2587 */ 2588 static inline int 2589 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2590 struct rte_flow_error *error) 2591 { 2592 const struct mlx5_flow_driver_ops *fops; 2593 enum mlx5_flow_drv_type type = flow->drv_type; 2594 2595 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2596 fops = flow_get_drv_ops(type); 2597 return fops->apply(dev, flow, error); 2598 } 2599 2600 /** 2601 * Flow driver remove API. This abstracts calling driver specific functions. 2602 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2603 * on device. All the resources of the flow should be freed by calling 2604 * flow_drv_destroy(). 2605 * 2606 * @param[in] dev 2607 * Pointer to Ethernet device. 2608 * @param[in, out] flow 2609 * Pointer to flow structure. 2610 */ 2611 static inline void 2612 flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2613 { 2614 const struct mlx5_flow_driver_ops *fops; 2615 enum mlx5_flow_drv_type type = flow->drv_type; 2616 2617 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2618 fops = flow_get_drv_ops(type); 2619 fops->remove(dev, flow); 2620 } 2621 2622 /** 2623 * Flow driver destroy API. This abstracts calling driver specific functions. 2624 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 2625 * on device and releases resources of the flow. 2626 * 2627 * @param[in] dev 2628 * Pointer to Ethernet device. 2629 * @param[in, out] flow 2630 * Pointer to flow structure. 2631 */ 2632 static inline void 2633 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2634 { 2635 const struct mlx5_flow_driver_ops *fops; 2636 enum mlx5_flow_drv_type type = flow->drv_type; 2637 2638 flow_mreg_split_qrss_release(dev, flow); 2639 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 2640 fops = flow_get_drv_ops(type); 2641 fops->destroy(dev, flow); 2642 } 2643 2644 /** 2645 * Validate a flow supported by the NIC. 2646 * 2647 * @see rte_flow_validate() 2648 * @see rte_flow_ops 2649 */ 2650 int 2651 mlx5_flow_validate(struct rte_eth_dev *dev, 2652 const struct rte_flow_attr *attr, 2653 const struct rte_flow_item items[], 2654 const struct rte_flow_action actions[], 2655 struct rte_flow_error *error) 2656 { 2657 int ret; 2658 2659 ret = flow_drv_validate(dev, attr, items, actions, true, error); 2660 if (ret < 0) 2661 return ret; 2662 return 0; 2663 } 2664 2665 /** 2666 * Get RSS action from the action list. 2667 * 2668 * @param[in] actions 2669 * Pointer to the list of actions. 2670 * 2671 * @return 2672 * Pointer to the RSS action if exist, else return NULL. 2673 */ 2674 static const struct rte_flow_action_rss* 2675 flow_get_rss_action(const struct rte_flow_action actions[]) 2676 { 2677 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2678 switch (actions->type) { 2679 case RTE_FLOW_ACTION_TYPE_RSS: 2680 return (const struct rte_flow_action_rss *) 2681 actions->conf; 2682 default: 2683 break; 2684 } 2685 } 2686 return NULL; 2687 } 2688 2689 static unsigned int 2690 find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) 2691 { 2692 const struct rte_flow_item *item; 2693 unsigned int has_vlan = 0; 2694 2695 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 2696 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { 2697 has_vlan = 1; 2698 break; 2699 } 2700 } 2701 if (has_vlan) 2702 return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN : 2703 MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN; 2704 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 2705 MLX5_EXPANSION_ROOT_OUTER; 2706 } 2707 2708 /** 2709 * Get layer flags from the prefix flow. 2710 * 2711 * Some flows may be split to several subflows, the prefix subflow gets the 2712 * match items and the suffix sub flow gets the actions. 2713 * Some actions need the user defined match item flags to get the detail for 2714 * the action. 2715 * This function helps the suffix flow to get the item layer flags from prefix 2716 * subflow. 2717 * 2718 * @param[in] dev_flow 2719 * Pointer the created preifx subflow. 2720 * 2721 * @return 2722 * The layers get from prefix subflow. 2723 */ 2724 static inline uint64_t 2725 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 2726 { 2727 uint64_t layers = 0; 2728 2729 /* If no decap actions, use the layers directly. */ 2730 if (!(dev_flow->actions & MLX5_FLOW_ACTION_DECAP)) 2731 return dev_flow->layers; 2732 /* Convert L3 layers with decap action. */ 2733 if (dev_flow->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 2734 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2735 else if (dev_flow->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 2736 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2737 /* Convert L4 layers with decap action. */ 2738 if (dev_flow->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 2739 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 2740 else if (dev_flow->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 2741 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 2742 return layers; 2743 } 2744 2745 /** 2746 * Get metadata split action information. 2747 * 2748 * @param[in] actions 2749 * Pointer to the list of actions. 2750 * @param[out] qrss 2751 * Pointer to the return pointer. 2752 * @param[out] qrss_type 2753 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 2754 * if no QUEUE/RSS is found. 2755 * @param[out] encap_idx 2756 * Pointer to the index of the encap action if exists, otherwise the last 2757 * action index. 2758 * 2759 * @return 2760 * Total number of actions. 2761 */ 2762 static int 2763 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 2764 const struct rte_flow_action **qrss, 2765 int *encap_idx) 2766 { 2767 const struct rte_flow_action_raw_encap *raw_encap; 2768 int actions_n = 0; 2769 int raw_decap_idx = -1; 2770 2771 *encap_idx = -1; 2772 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2773 switch (actions->type) { 2774 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2775 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2776 *encap_idx = actions_n; 2777 break; 2778 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 2779 raw_decap_idx = actions_n; 2780 break; 2781 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2782 raw_encap = actions->conf; 2783 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2784 *encap_idx = raw_decap_idx != -1 ? 2785 raw_decap_idx : actions_n; 2786 break; 2787 case RTE_FLOW_ACTION_TYPE_QUEUE: 2788 case RTE_FLOW_ACTION_TYPE_RSS: 2789 *qrss = actions; 2790 break; 2791 default: 2792 break; 2793 } 2794 actions_n++; 2795 } 2796 if (*encap_idx == -1) 2797 *encap_idx = actions_n; 2798 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2799 return actions_n + 1; 2800 } 2801 2802 /** 2803 * Check meter action from the action list. 2804 * 2805 * @param[in] actions 2806 * Pointer to the list of actions. 2807 * @param[out] mtr 2808 * Pointer to the meter exist flag. 2809 * 2810 * @return 2811 * Total number of actions. 2812 */ 2813 static int 2814 flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) 2815 { 2816 int actions_n = 0; 2817 2818 MLX5_ASSERT(mtr); 2819 *mtr = 0; 2820 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2821 switch (actions->type) { 2822 case RTE_FLOW_ACTION_TYPE_METER: 2823 *mtr = 1; 2824 break; 2825 default: 2826 break; 2827 } 2828 actions_n++; 2829 } 2830 /* Count RTE_FLOW_ACTION_TYPE_END. */ 2831 return actions_n + 1; 2832 } 2833 2834 /** 2835 * Check if the flow should be splited due to hairpin. 2836 * The reason for the split is that in current HW we can't 2837 * support encap on Rx, so if a flow have encap we move it 2838 * to Tx. 2839 * 2840 * @param dev 2841 * Pointer to Ethernet device. 2842 * @param[in] attr 2843 * Flow rule attributes. 2844 * @param[in] actions 2845 * Associated actions (list terminated by the END action). 2846 * 2847 * @return 2848 * > 0 the number of actions and the flow should be split, 2849 * 0 when no split required. 2850 */ 2851 static int 2852 flow_check_hairpin_split(struct rte_eth_dev *dev, 2853 const struct rte_flow_attr *attr, 2854 const struct rte_flow_action actions[]) 2855 { 2856 int queue_action = 0; 2857 int action_n = 0; 2858 int encap = 0; 2859 const struct rte_flow_action_queue *queue; 2860 const struct rte_flow_action_rss *rss; 2861 const struct rte_flow_action_raw_encap *raw_encap; 2862 2863 if (!attr->ingress) 2864 return 0; 2865 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 2866 switch (actions->type) { 2867 case RTE_FLOW_ACTION_TYPE_QUEUE: 2868 queue = actions->conf; 2869 if (queue == NULL) 2870 return 0; 2871 if (mlx5_rxq_get_type(dev, queue->index) != 2872 MLX5_RXQ_TYPE_HAIRPIN) 2873 return 0; 2874 queue_action = 1; 2875 action_n++; 2876 break; 2877 case RTE_FLOW_ACTION_TYPE_RSS: 2878 rss = actions->conf; 2879 if (rss == NULL || rss->queue_num == 0) 2880 return 0; 2881 if (mlx5_rxq_get_type(dev, rss->queue[0]) != 2882 MLX5_RXQ_TYPE_HAIRPIN) 2883 return 0; 2884 queue_action = 1; 2885 action_n++; 2886 break; 2887 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 2888 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 2889 encap = 1; 2890 action_n++; 2891 break; 2892 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 2893 raw_encap = actions->conf; 2894 if (raw_encap->size > 2895 (sizeof(struct rte_flow_item_eth) + 2896 sizeof(struct rte_flow_item_ipv4))) 2897 encap = 1; 2898 action_n++; 2899 break; 2900 default: 2901 action_n++; 2902 break; 2903 } 2904 } 2905 if (encap == 1 && queue_action) 2906 return action_n; 2907 return 0; 2908 } 2909 2910 /* Declare flow create/destroy prototype in advance. */ 2911 static struct rte_flow * 2912 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 2913 const struct rte_flow_attr *attr, 2914 const struct rte_flow_item items[], 2915 const struct rte_flow_action actions[], 2916 bool external, struct rte_flow_error *error); 2917 2918 static void 2919 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 2920 struct rte_flow *flow); 2921 2922 /** 2923 * Add a flow of copying flow metadata registers in RX_CP_TBL. 2924 * 2925 * As mark_id is unique, if there's already a registered flow for the mark_id, 2926 * return by increasing the reference counter of the resource. Otherwise, create 2927 * the resource (mcp_res) and flow. 2928 * 2929 * Flow looks like, 2930 * - If ingress port is ANY and reg_c[1] is mark_id, 2931 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 2932 * 2933 * For default flow (zero mark_id), flow is like, 2934 * - If ingress port is ANY, 2935 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 2936 * 2937 * @param dev 2938 * Pointer to Ethernet device. 2939 * @param mark_id 2940 * ID of MARK action, zero means default flow for META. 2941 * @param[out] error 2942 * Perform verbose error reporting if not NULL. 2943 * 2944 * @return 2945 * Associated resource on success, NULL otherwise and rte_errno is set. 2946 */ 2947 static struct mlx5_flow_mreg_copy_resource * 2948 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 2949 struct rte_flow_error *error) 2950 { 2951 struct mlx5_priv *priv = dev->data->dev_private; 2952 struct rte_flow_attr attr = { 2953 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 2954 .ingress = 1, 2955 }; 2956 struct mlx5_rte_flow_item_tag tag_spec = { 2957 .data = mark_id, 2958 }; 2959 struct rte_flow_item items[] = { 2960 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 2961 }; 2962 struct rte_flow_action_mark ftag = { 2963 .id = mark_id, 2964 }; 2965 struct mlx5_flow_action_copy_mreg cp_mreg = { 2966 .dst = REG_B, 2967 .src = 0, 2968 }; 2969 struct rte_flow_action_jump jump = { 2970 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 2971 }; 2972 struct rte_flow_action actions[] = { 2973 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 2974 }; 2975 struct mlx5_flow_mreg_copy_resource *mcp_res; 2976 int ret; 2977 2978 /* Fill the register fileds in the flow. */ 2979 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2980 if (ret < 0) 2981 return NULL; 2982 tag_spec.id = ret; 2983 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 2984 if (ret < 0) 2985 return NULL; 2986 cp_mreg.src = ret; 2987 /* Check if already registered. */ 2988 MLX5_ASSERT(priv->mreg_cp_tbl); 2989 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); 2990 if (mcp_res) { 2991 /* For non-default rule. */ 2992 if (mark_id != MLX5_DEFAULT_COPY_ID) 2993 mcp_res->refcnt++; 2994 MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID || 2995 mcp_res->refcnt == 1); 2996 return mcp_res; 2997 } 2998 /* Provide the full width of FLAG specific value. */ 2999 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 3000 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 3001 /* Build a new flow. */ 3002 if (mark_id != MLX5_DEFAULT_COPY_ID) { 3003 items[0] = (struct rte_flow_item){ 3004 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3005 .spec = &tag_spec, 3006 }; 3007 items[1] = (struct rte_flow_item){ 3008 .type = RTE_FLOW_ITEM_TYPE_END, 3009 }; 3010 actions[0] = (struct rte_flow_action){ 3011 .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK, 3012 .conf = &ftag, 3013 }; 3014 actions[1] = (struct rte_flow_action){ 3015 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3016 .conf = &cp_mreg, 3017 }; 3018 actions[2] = (struct rte_flow_action){ 3019 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3020 .conf = &jump, 3021 }; 3022 actions[3] = (struct rte_flow_action){ 3023 .type = RTE_FLOW_ACTION_TYPE_END, 3024 }; 3025 } else { 3026 /* Default rule, wildcard match. */ 3027 attr.priority = MLX5_FLOW_PRIO_RSVD; 3028 items[0] = (struct rte_flow_item){ 3029 .type = RTE_FLOW_ITEM_TYPE_END, 3030 }; 3031 actions[0] = (struct rte_flow_action){ 3032 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3033 .conf = &cp_mreg, 3034 }; 3035 actions[1] = (struct rte_flow_action){ 3036 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3037 .conf = &jump, 3038 }; 3039 actions[2] = (struct rte_flow_action){ 3040 .type = RTE_FLOW_ACTION_TYPE_END, 3041 }; 3042 } 3043 /* Build a new entry. */ 3044 mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0); 3045 if (!mcp_res) { 3046 rte_errno = ENOMEM; 3047 return NULL; 3048 } 3049 /* 3050 * The copy Flows are not included in any list. There 3051 * ones are referenced from other Flows and can not 3052 * be applied, removed, deleted in ardbitrary order 3053 * by list traversing. 3054 */ 3055 mcp_res->flow = flow_list_create(dev, NULL, &attr, items, 3056 actions, false, error); 3057 if (!mcp_res->flow) 3058 goto error; 3059 mcp_res->refcnt++; 3060 mcp_res->hlist_ent.key = mark_id; 3061 ret = mlx5_hlist_insert(priv->mreg_cp_tbl, 3062 &mcp_res->hlist_ent); 3063 MLX5_ASSERT(!ret); 3064 if (ret) 3065 goto error; 3066 return mcp_res; 3067 error: 3068 if (mcp_res->flow) 3069 flow_list_destroy(dev, NULL, mcp_res->flow); 3070 rte_free(mcp_res); 3071 return NULL; 3072 } 3073 3074 /** 3075 * Release flow in RX_CP_TBL. 3076 * 3077 * @param dev 3078 * Pointer to Ethernet device. 3079 * @flow 3080 * Parent flow for wich copying is provided. 3081 */ 3082 static void 3083 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 3084 struct rte_flow *flow) 3085 { 3086 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3087 struct mlx5_priv *priv = dev->data->dev_private; 3088 3089 if (!mcp_res || !priv->mreg_cp_tbl) 3090 return; 3091 if (flow->copy_applied) { 3092 MLX5_ASSERT(mcp_res->appcnt); 3093 flow->copy_applied = 0; 3094 --mcp_res->appcnt; 3095 if (!mcp_res->appcnt) 3096 flow_drv_remove(dev, mcp_res->flow); 3097 } 3098 /* 3099 * We do not check availability of metadata registers here, 3100 * because copy resources are not allocated in this case. 3101 */ 3102 if (--mcp_res->refcnt) 3103 return; 3104 MLX5_ASSERT(mcp_res->flow); 3105 flow_list_destroy(dev, NULL, mcp_res->flow); 3106 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3107 rte_free(mcp_res); 3108 flow->mreg_copy = NULL; 3109 } 3110 3111 /** 3112 * Start flow in RX_CP_TBL. 3113 * 3114 * @param dev 3115 * Pointer to Ethernet device. 3116 * @flow 3117 * Parent flow for wich copying is provided. 3118 * 3119 * @return 3120 * 0 on success, a negative errno value otherwise and rte_errno is set. 3121 */ 3122 static int 3123 flow_mreg_start_copy_action(struct rte_eth_dev *dev, 3124 struct rte_flow *flow) 3125 { 3126 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3127 int ret; 3128 3129 if (!mcp_res || flow->copy_applied) 3130 return 0; 3131 if (!mcp_res->appcnt) { 3132 ret = flow_drv_apply(dev, mcp_res->flow, NULL); 3133 if (ret) 3134 return ret; 3135 } 3136 ++mcp_res->appcnt; 3137 flow->copy_applied = 1; 3138 return 0; 3139 } 3140 3141 /** 3142 * Stop flow in RX_CP_TBL. 3143 * 3144 * @param dev 3145 * Pointer to Ethernet device. 3146 * @flow 3147 * Parent flow for wich copying is provided. 3148 */ 3149 static void 3150 flow_mreg_stop_copy_action(struct rte_eth_dev *dev, 3151 struct rte_flow *flow) 3152 { 3153 struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy; 3154 3155 if (!mcp_res || !flow->copy_applied) 3156 return; 3157 MLX5_ASSERT(mcp_res->appcnt); 3158 --mcp_res->appcnt; 3159 flow->copy_applied = 0; 3160 if (!mcp_res->appcnt) 3161 flow_drv_remove(dev, mcp_res->flow); 3162 } 3163 3164 /** 3165 * Remove the default copy action from RX_CP_TBL. 3166 * 3167 * @param dev 3168 * Pointer to Ethernet device. 3169 */ 3170 static void 3171 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 3172 { 3173 struct mlx5_flow_mreg_copy_resource *mcp_res; 3174 struct mlx5_priv *priv = dev->data->dev_private; 3175 3176 /* Check if default flow is registered. */ 3177 if (!priv->mreg_cp_tbl) 3178 return; 3179 mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, 3180 MLX5_DEFAULT_COPY_ID); 3181 if (!mcp_res) 3182 return; 3183 MLX5_ASSERT(mcp_res->flow); 3184 flow_list_destroy(dev, NULL, mcp_res->flow); 3185 mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 3186 rte_free(mcp_res); 3187 } 3188 3189 /** 3190 * Add the default copy action in in RX_CP_TBL. 3191 * 3192 * @param dev 3193 * Pointer to Ethernet device. 3194 * @param[out] error 3195 * Perform verbose error reporting if not NULL. 3196 * 3197 * @return 3198 * 0 for success, negative value otherwise and rte_errno is set. 3199 */ 3200 static int 3201 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 3202 struct rte_flow_error *error) 3203 { 3204 struct mlx5_priv *priv = dev->data->dev_private; 3205 struct mlx5_flow_mreg_copy_resource *mcp_res; 3206 3207 /* Check whether extensive metadata feature is engaged. */ 3208 if (!priv->config.dv_flow_en || 3209 priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3210 !mlx5_flow_ext_mreg_supported(dev) || 3211 !priv->sh->dv_regc0_mask) 3212 return 0; 3213 mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error); 3214 if (!mcp_res) 3215 return -rte_errno; 3216 return 0; 3217 } 3218 3219 /** 3220 * Add a flow of copying flow metadata registers in RX_CP_TBL. 3221 * 3222 * All the flow having Q/RSS action should be split by 3223 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 3224 * performs the following, 3225 * - CQE->flow_tag := reg_c[1] (MARK) 3226 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3227 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 3228 * but there should be a flow per each MARK ID set by MARK action. 3229 * 3230 * For the aforementioned reason, if there's a MARK action in flow's action 3231 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 3232 * the MARK ID to CQE's flow_tag like, 3233 * - If reg_c[1] is mark_id, 3234 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3235 * 3236 * For SET_META action which stores value in reg_c[0], as the destination is 3237 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 3238 * MARK ID means the default flow. The default flow looks like, 3239 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 3240 * 3241 * @param dev 3242 * Pointer to Ethernet device. 3243 * @param flow 3244 * Pointer to flow structure. 3245 * @param[in] actions 3246 * Pointer to the list of actions. 3247 * @param[out] error 3248 * Perform verbose error reporting if not NULL. 3249 * 3250 * @return 3251 * 0 on success, negative value otherwise and rte_errno is set. 3252 */ 3253 static int 3254 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 3255 struct rte_flow *flow, 3256 const struct rte_flow_action *actions, 3257 struct rte_flow_error *error) 3258 { 3259 struct mlx5_priv *priv = dev->data->dev_private; 3260 struct mlx5_dev_config *config = &priv->config; 3261 struct mlx5_flow_mreg_copy_resource *mcp_res; 3262 const struct rte_flow_action_mark *mark; 3263 3264 /* Check whether extensive metadata feature is engaged. */ 3265 if (!config->dv_flow_en || 3266 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3267 !mlx5_flow_ext_mreg_supported(dev) || 3268 !priv->sh->dv_regc0_mask) 3269 return 0; 3270 /* Find MARK action. */ 3271 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3272 switch (actions->type) { 3273 case RTE_FLOW_ACTION_TYPE_FLAG: 3274 mcp_res = flow_mreg_add_copy_action 3275 (dev, MLX5_FLOW_MARK_DEFAULT, error); 3276 if (!mcp_res) 3277 return -rte_errno; 3278 flow->mreg_copy = mcp_res; 3279 if (dev->data->dev_started) { 3280 mcp_res->appcnt++; 3281 flow->copy_applied = 1; 3282 } 3283 return 0; 3284 case RTE_FLOW_ACTION_TYPE_MARK: 3285 mark = (const struct rte_flow_action_mark *) 3286 actions->conf; 3287 mcp_res = 3288 flow_mreg_add_copy_action(dev, mark->id, error); 3289 if (!mcp_res) 3290 return -rte_errno; 3291 flow->mreg_copy = mcp_res; 3292 if (dev->data->dev_started) { 3293 mcp_res->appcnt++; 3294 flow->copy_applied = 1; 3295 } 3296 return 0; 3297 default: 3298 break; 3299 } 3300 } 3301 return 0; 3302 } 3303 3304 #define MLX5_MAX_SPLIT_ACTIONS 24 3305 #define MLX5_MAX_SPLIT_ITEMS 24 3306 3307 /** 3308 * Split the hairpin flow. 3309 * Since HW can't support encap on Rx we move the encap to Tx. 3310 * If the count action is after the encap then we also 3311 * move the count action. in this case the count will also measure 3312 * the outer bytes. 3313 * 3314 * @param dev 3315 * Pointer to Ethernet device. 3316 * @param[in] actions 3317 * Associated actions (list terminated by the END action). 3318 * @param[out] actions_rx 3319 * Rx flow actions. 3320 * @param[out] actions_tx 3321 * Tx flow actions.. 3322 * @param[out] pattern_tx 3323 * The pattern items for the Tx flow. 3324 * @param[out] flow_id 3325 * The flow ID connected to this flow. 3326 * 3327 * @return 3328 * 0 on success. 3329 */ 3330 static int 3331 flow_hairpin_split(struct rte_eth_dev *dev, 3332 const struct rte_flow_action actions[], 3333 struct rte_flow_action actions_rx[], 3334 struct rte_flow_action actions_tx[], 3335 struct rte_flow_item pattern_tx[], 3336 uint32_t *flow_id) 3337 { 3338 struct mlx5_priv *priv = dev->data->dev_private; 3339 const struct rte_flow_action_raw_encap *raw_encap; 3340 const struct rte_flow_action_raw_decap *raw_decap; 3341 struct mlx5_rte_flow_action_set_tag *set_tag; 3342 struct rte_flow_action *tag_action; 3343 struct mlx5_rte_flow_item_tag *tag_item; 3344 struct rte_flow_item *item; 3345 char *addr; 3346 int encap = 0; 3347 3348 mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id); 3349 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3350 switch (actions->type) { 3351 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3352 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3353 rte_memcpy(actions_tx, actions, 3354 sizeof(struct rte_flow_action)); 3355 actions_tx++; 3356 break; 3357 case RTE_FLOW_ACTION_TYPE_COUNT: 3358 if (encap) { 3359 rte_memcpy(actions_tx, actions, 3360 sizeof(struct rte_flow_action)); 3361 actions_tx++; 3362 } else { 3363 rte_memcpy(actions_rx, actions, 3364 sizeof(struct rte_flow_action)); 3365 actions_rx++; 3366 } 3367 break; 3368 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3369 raw_encap = actions->conf; 3370 if (raw_encap->size > 3371 (sizeof(struct rte_flow_item_eth) + 3372 sizeof(struct rte_flow_item_ipv4))) { 3373 memcpy(actions_tx, actions, 3374 sizeof(struct rte_flow_action)); 3375 actions_tx++; 3376 encap = 1; 3377 } else { 3378 rte_memcpy(actions_rx, actions, 3379 sizeof(struct rte_flow_action)); 3380 actions_rx++; 3381 } 3382 break; 3383 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3384 raw_decap = actions->conf; 3385 if (raw_decap->size < 3386 (sizeof(struct rte_flow_item_eth) + 3387 sizeof(struct rte_flow_item_ipv4))) { 3388 memcpy(actions_tx, actions, 3389 sizeof(struct rte_flow_action)); 3390 actions_tx++; 3391 } else { 3392 rte_memcpy(actions_rx, actions, 3393 sizeof(struct rte_flow_action)); 3394 actions_rx++; 3395 } 3396 break; 3397 default: 3398 rte_memcpy(actions_rx, actions, 3399 sizeof(struct rte_flow_action)); 3400 actions_rx++; 3401 break; 3402 } 3403 } 3404 /* Add set meta action and end action for the Rx flow. */ 3405 tag_action = actions_rx; 3406 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3407 actions_rx++; 3408 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 3409 actions_rx++; 3410 set_tag = (void *)actions_rx; 3411 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); 3412 MLX5_ASSERT(set_tag->id > REG_NONE); 3413 set_tag->data = *flow_id; 3414 tag_action->conf = set_tag; 3415 /* Create Tx item list. */ 3416 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 3417 addr = (void *)&pattern_tx[2]; 3418 item = pattern_tx; 3419 item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3420 tag_item = (void *)addr; 3421 tag_item->data = *flow_id; 3422 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 3423 MLX5_ASSERT(set_tag->id > REG_NONE); 3424 item->spec = tag_item; 3425 addr += sizeof(struct mlx5_rte_flow_item_tag); 3426 tag_item = (void *)addr; 3427 tag_item->data = UINT32_MAX; 3428 tag_item->id = UINT16_MAX; 3429 item->mask = tag_item; 3430 addr += sizeof(struct mlx5_rte_flow_item_tag); 3431 item->last = NULL; 3432 item++; 3433 item->type = RTE_FLOW_ITEM_TYPE_END; 3434 return 0; 3435 } 3436 3437 /** 3438 * The last stage of splitting chain, just creates the subflow 3439 * without any modification. 3440 * 3441 * @param dev 3442 * Pointer to Ethernet device. 3443 * @param[in] flow 3444 * Parent flow structure pointer. 3445 * @param[in, out] sub_flow 3446 * Pointer to return the created subflow, may be NULL. 3447 * @param[in] prefix_layers 3448 * Prefix subflow layers, may be 0. 3449 * @param[in] attr 3450 * Flow rule attributes. 3451 * @param[in] items 3452 * Pattern specification (list terminated by the END pattern item). 3453 * @param[in] actions 3454 * Associated actions (list terminated by the END action). 3455 * @param[in] external 3456 * This flow rule is created by request external to PMD. 3457 * @param[out] error 3458 * Perform verbose error reporting if not NULL. 3459 * @return 3460 * 0 on success, negative value otherwise 3461 */ 3462 static int 3463 flow_create_split_inner(struct rte_eth_dev *dev, 3464 struct rte_flow *flow, 3465 struct mlx5_flow **sub_flow, 3466 uint64_t prefix_layers, 3467 const struct rte_flow_attr *attr, 3468 const struct rte_flow_item items[], 3469 const struct rte_flow_action actions[], 3470 bool external, struct rte_flow_error *error) 3471 { 3472 struct mlx5_flow *dev_flow; 3473 3474 dev_flow = flow_drv_prepare(flow, attr, items, actions, error); 3475 if (!dev_flow) 3476 return -rte_errno; 3477 dev_flow->flow = flow; 3478 dev_flow->external = external; 3479 /* Subflow object was created, we must include one in the list. */ 3480 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 3481 /* 3482 * If dev_flow is as one of the suffix flow, some actions in suffix 3483 * flow may need some user defined item layer flags. 3484 */ 3485 if (prefix_layers) 3486 dev_flow->layers = prefix_layers; 3487 if (sub_flow) 3488 *sub_flow = dev_flow; 3489 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 3490 } 3491 3492 /** 3493 * Split the meter flow. 3494 * 3495 * As meter flow will split to three sub flow, other than meter 3496 * action, the other actions make sense to only meter accepts 3497 * the packet. If it need to be dropped, no other additional 3498 * actions should be take. 3499 * 3500 * One kind of special action which decapsulates the L3 tunnel 3501 * header will be in the prefix sub flow, as not to take the 3502 * L3 tunnel header into account. 3503 * 3504 * @param dev 3505 * Pointer to Ethernet device. 3506 * @param[in] items 3507 * Pattern specification (list terminated by the END pattern item). 3508 * @param[out] sfx_items 3509 * Suffix flow match items (list terminated by the END pattern item). 3510 * @param[in] actions 3511 * Associated actions (list terminated by the END action). 3512 * @param[out] actions_sfx 3513 * Suffix flow actions. 3514 * @param[out] actions_pre 3515 * Prefix flow actions. 3516 * @param[out] pattern_sfx 3517 * The pattern items for the suffix flow. 3518 * @param[out] tag_sfx 3519 * Pointer to suffix flow tag. 3520 * 3521 * @return 3522 * 0 on success. 3523 */ 3524 static int 3525 flow_meter_split_prep(struct rte_eth_dev *dev, 3526 const struct rte_flow_item items[], 3527 struct rte_flow_item sfx_items[], 3528 const struct rte_flow_action actions[], 3529 struct rte_flow_action actions_sfx[], 3530 struct rte_flow_action actions_pre[]) 3531 { 3532 struct rte_flow_action *tag_action = NULL; 3533 struct rte_flow_item *tag_item; 3534 struct mlx5_rte_flow_action_set_tag *set_tag; 3535 struct rte_flow_error error; 3536 const struct rte_flow_action_raw_encap *raw_encap; 3537 const struct rte_flow_action_raw_decap *raw_decap; 3538 struct mlx5_rte_flow_item_tag *tag_spec; 3539 struct mlx5_rte_flow_item_tag *tag_mask; 3540 uint32_t tag_id; 3541 bool copy_vlan = false; 3542 3543 /* Prepare the actions for prefix and suffix flow. */ 3544 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3545 struct rte_flow_action **action_cur = NULL; 3546 3547 switch (actions->type) { 3548 case RTE_FLOW_ACTION_TYPE_METER: 3549 /* Add the extra tag action first. */ 3550 tag_action = actions_pre; 3551 tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3552 actions_pre++; 3553 action_cur = &actions_pre; 3554 break; 3555 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3556 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3557 action_cur = &actions_pre; 3558 break; 3559 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3560 raw_encap = actions->conf; 3561 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 3562 action_cur = &actions_pre; 3563 break; 3564 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3565 raw_decap = actions->conf; 3566 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 3567 action_cur = &actions_pre; 3568 break; 3569 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 3570 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 3571 copy_vlan = true; 3572 break; 3573 default: 3574 break; 3575 } 3576 if (!action_cur) 3577 action_cur = &actions_sfx; 3578 memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); 3579 (*action_cur)++; 3580 } 3581 /* Add end action to the actions. */ 3582 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 3583 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 3584 actions_pre++; 3585 /* Set the tag. */ 3586 set_tag = (void *)actions_pre; 3587 set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3588 /* 3589 * Get the id from the qrss_pool to make qrss share the id with meter. 3590 */ 3591 tag_id = flow_qrss_get_id(dev); 3592 set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; 3593 assert(tag_action); 3594 tag_action->conf = set_tag; 3595 /* Prepare the suffix subflow items. */ 3596 tag_item = sfx_items++; 3597 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 3598 int item_type = items->type; 3599 3600 switch (item_type) { 3601 case RTE_FLOW_ITEM_TYPE_PORT_ID: 3602 memcpy(sfx_items, items, sizeof(*sfx_items)); 3603 sfx_items++; 3604 break; 3605 case RTE_FLOW_ITEM_TYPE_VLAN: 3606 if (copy_vlan) { 3607 memcpy(sfx_items, items, sizeof(*sfx_items)); 3608 /* 3609 * Convert to internal match item, it is used 3610 * for vlan push and set vid. 3611 */ 3612 sfx_items->type = MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 3613 sfx_items++; 3614 } 3615 break; 3616 default: 3617 break; 3618 } 3619 } 3620 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 3621 sfx_items++; 3622 tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 3623 tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; 3624 tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); 3625 tag_mask = tag_spec + 1; 3626 tag_mask->data = 0xffffff00; 3627 tag_item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG; 3628 tag_item->spec = tag_spec; 3629 tag_item->last = NULL; 3630 tag_item->mask = tag_mask; 3631 return tag_id; 3632 } 3633 3634 /** 3635 * Split action list having QUEUE/RSS for metadata register copy. 3636 * 3637 * Once Q/RSS action is detected in user's action list, the flow action 3638 * should be split in order to copy metadata registers, which will happen in 3639 * RX_CP_TBL like, 3640 * - CQE->flow_tag := reg_c[1] (MARK) 3641 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 3642 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 3643 * This is because the last action of each flow must be a terminal action 3644 * (QUEUE, RSS or DROP). 3645 * 3646 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 3647 * stored and kept in the mlx5_flow structure per each sub_flow. 3648 * 3649 * The Q/RSS action is replaced with, 3650 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 3651 * And the following JUMP action is added at the end, 3652 * - JUMP, to RX_CP_TBL. 3653 * 3654 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 3655 * flow_create_split_metadata() routine. The flow will look like, 3656 * - If flow ID matches (reg_c[2]), perform Q/RSS. 3657 * 3658 * @param dev 3659 * Pointer to Ethernet device. 3660 * @param[out] split_actions 3661 * Pointer to store split actions to jump to CP_TBL. 3662 * @param[in] actions 3663 * Pointer to the list of original flow actions. 3664 * @param[in] qrss 3665 * Pointer to the Q/RSS action. 3666 * @param[in] actions_n 3667 * Number of original actions. 3668 * @param[out] error 3669 * Perform verbose error reporting if not NULL. 3670 * 3671 * @return 3672 * non-zero unique flow_id on success, otherwise 0 and 3673 * error/rte_error are set. 3674 */ 3675 static uint32_t 3676 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 3677 struct rte_flow_action *split_actions, 3678 const struct rte_flow_action *actions, 3679 const struct rte_flow_action *qrss, 3680 int actions_n, struct rte_flow_error *error) 3681 { 3682 struct mlx5_rte_flow_action_set_tag *set_tag; 3683 struct rte_flow_action_jump *jump; 3684 const int qrss_idx = qrss - actions; 3685 uint32_t flow_id = 0; 3686 int ret = 0; 3687 3688 /* 3689 * Given actions will be split 3690 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 3691 * - Add jump to mreg CP_TBL. 3692 * As a result, there will be one more action. 3693 */ 3694 ++actions_n; 3695 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 3696 set_tag = (void *)(split_actions + actions_n); 3697 /* 3698 * If tag action is not set to void(it means we are not the meter 3699 * suffix flow), add the tag action. Since meter suffix flow already 3700 * has the tag added. 3701 */ 3702 if (split_actions[qrss_idx].type != RTE_FLOW_ACTION_TYPE_VOID) { 3703 /* 3704 * Allocate the new subflow ID. This one is unique within 3705 * device and not shared with representors. Otherwise, 3706 * we would have to resolve multi-thread access synch 3707 * issue. Each flow on the shared device is appended 3708 * with source vport identifier, so the resulting 3709 * flows will be unique in the shared (by master and 3710 * representors) domain even if they have coinciding 3711 * IDs. 3712 */ 3713 flow_id = flow_qrss_get_id(dev); 3714 if (!flow_id) 3715 return rte_flow_error_set(error, ENOMEM, 3716 RTE_FLOW_ERROR_TYPE_ACTION, 3717 NULL, "can't allocate id " 3718 "for split Q/RSS subflow"); 3719 /* Internal SET_TAG action to set flow ID. */ 3720 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 3721 .data = flow_id, 3722 }; 3723 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 3724 if (ret < 0) 3725 return ret; 3726 set_tag->id = ret; 3727 /* Construct new actions array. */ 3728 /* Replace QUEUE/RSS action. */ 3729 split_actions[qrss_idx] = (struct rte_flow_action){ 3730 .type = MLX5_RTE_FLOW_ACTION_TYPE_TAG, 3731 .conf = set_tag, 3732 }; 3733 } 3734 /* JUMP action to jump to mreg copy table (CP_TBL). */ 3735 jump = (void *)(set_tag + 1); 3736 *jump = (struct rte_flow_action_jump){ 3737 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 3738 }; 3739 split_actions[actions_n - 2] = (struct rte_flow_action){ 3740 .type = RTE_FLOW_ACTION_TYPE_JUMP, 3741 .conf = jump, 3742 }; 3743 split_actions[actions_n - 1] = (struct rte_flow_action){ 3744 .type = RTE_FLOW_ACTION_TYPE_END, 3745 }; 3746 return flow_id; 3747 } 3748 3749 /** 3750 * Extend the given action list for Tx metadata copy. 3751 * 3752 * Copy the given action list to the ext_actions and add flow metadata register 3753 * copy action in order to copy reg_a set by WQE to reg_c[0]. 3754 * 3755 * @param[out] ext_actions 3756 * Pointer to the extended action list. 3757 * @param[in] actions 3758 * Pointer to the list of actions. 3759 * @param[in] actions_n 3760 * Number of actions in the list. 3761 * @param[out] error 3762 * Perform verbose error reporting if not NULL. 3763 * @param[in] encap_idx 3764 * The encap action inndex. 3765 * 3766 * @return 3767 * 0 on success, negative value otherwise 3768 */ 3769 static int 3770 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 3771 struct rte_flow_action *ext_actions, 3772 const struct rte_flow_action *actions, 3773 int actions_n, struct rte_flow_error *error, 3774 int encap_idx) 3775 { 3776 struct mlx5_flow_action_copy_mreg *cp_mreg = 3777 (struct mlx5_flow_action_copy_mreg *) 3778 (ext_actions + actions_n + 1); 3779 int ret; 3780 3781 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 3782 if (ret < 0) 3783 return ret; 3784 cp_mreg->dst = ret; 3785 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 3786 if (ret < 0) 3787 return ret; 3788 cp_mreg->src = ret; 3789 if (encap_idx != 0) 3790 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 3791 if (encap_idx == actions_n - 1) { 3792 ext_actions[actions_n - 1] = (struct rte_flow_action){ 3793 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3794 .conf = cp_mreg, 3795 }; 3796 ext_actions[actions_n] = (struct rte_flow_action){ 3797 .type = RTE_FLOW_ACTION_TYPE_END, 3798 }; 3799 } else { 3800 ext_actions[encap_idx] = (struct rte_flow_action){ 3801 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 3802 .conf = cp_mreg, 3803 }; 3804 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 3805 sizeof(*ext_actions) * (actions_n - encap_idx)); 3806 } 3807 return 0; 3808 } 3809 3810 /** 3811 * The splitting for metadata feature. 3812 * 3813 * - Q/RSS action on NIC Rx should be split in order to pass by 3814 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3815 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3816 * 3817 * - All the actions on NIC Tx should have a mreg copy action to 3818 * copy reg_a from WQE to reg_c[0]. 3819 * 3820 * @param dev 3821 * Pointer to Ethernet device. 3822 * @param[in] flow 3823 * Parent flow structure pointer. 3824 * @param[in] prefix_layers 3825 * Prefix flow layer flags. 3826 * @param[in] attr 3827 * Flow rule attributes. 3828 * @param[in] items 3829 * Pattern specification (list terminated by the END pattern item). 3830 * @param[in] actions 3831 * Associated actions (list terminated by the END action). 3832 * @param[in] external 3833 * This flow rule is created by request external to PMD. 3834 * @param[out] error 3835 * Perform verbose error reporting if not NULL. 3836 * @return 3837 * 0 on success, negative value otherwise 3838 */ 3839 static int 3840 flow_create_split_metadata(struct rte_eth_dev *dev, 3841 struct rte_flow *flow, 3842 uint64_t prefix_layers, 3843 const struct rte_flow_attr *attr, 3844 const struct rte_flow_item items[], 3845 const struct rte_flow_action actions[], 3846 bool external, struct rte_flow_error *error) 3847 { 3848 struct mlx5_priv *priv = dev->data->dev_private; 3849 struct mlx5_dev_config *config = &priv->config; 3850 const struct rte_flow_action *qrss = NULL; 3851 struct rte_flow_action *ext_actions = NULL; 3852 struct mlx5_flow *dev_flow = NULL; 3853 uint32_t qrss_id = 0; 3854 int mtr_sfx = 0; 3855 size_t act_size; 3856 int actions_n; 3857 int encap_idx; 3858 int ret; 3859 3860 /* Check whether extensive metadata feature is engaged. */ 3861 if (!config->dv_flow_en || 3862 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 3863 !mlx5_flow_ext_mreg_supported(dev)) 3864 return flow_create_split_inner(dev, flow, NULL, prefix_layers, 3865 attr, items, actions, external, 3866 error); 3867 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 3868 &encap_idx); 3869 if (qrss) { 3870 /* Exclude hairpin flows from splitting. */ 3871 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 3872 const struct rte_flow_action_queue *queue; 3873 3874 queue = qrss->conf; 3875 if (mlx5_rxq_get_type(dev, queue->index) == 3876 MLX5_RXQ_TYPE_HAIRPIN) 3877 qrss = NULL; 3878 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 3879 const struct rte_flow_action_rss *rss; 3880 3881 rss = qrss->conf; 3882 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 3883 MLX5_RXQ_TYPE_HAIRPIN) 3884 qrss = NULL; 3885 } 3886 } 3887 if (qrss) { 3888 /* Check if it is in meter suffix table. */ 3889 mtr_sfx = attr->group == (attr->transfer ? 3890 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 3891 MLX5_FLOW_TABLE_LEVEL_SUFFIX); 3892 /* 3893 * Q/RSS action on NIC Rx should be split in order to pass by 3894 * the mreg copy table (RX_CP_TBL) and then it jumps to the 3895 * action table (RX_ACT_TBL) which has the split Q/RSS action. 3896 */ 3897 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3898 sizeof(struct rte_flow_action_set_tag) + 3899 sizeof(struct rte_flow_action_jump); 3900 ext_actions = rte_zmalloc(__func__, act_size, 0); 3901 if (!ext_actions) 3902 return rte_flow_error_set(error, ENOMEM, 3903 RTE_FLOW_ERROR_TYPE_ACTION, 3904 NULL, "no memory to split " 3905 "metadata flow"); 3906 /* 3907 * If we are the suffix flow of meter, tag already exist. 3908 * Set the tag action to void. 3909 */ 3910 if (mtr_sfx) 3911 ext_actions[qrss - actions].type = 3912 RTE_FLOW_ACTION_TYPE_VOID; 3913 else 3914 ext_actions[qrss - actions].type = 3915 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 3916 /* 3917 * Create the new actions list with removed Q/RSS action 3918 * and appended set tag and jump to register copy table 3919 * (RX_CP_TBL). We should preallocate unique tag ID here 3920 * in advance, because it is needed for set tag action. 3921 */ 3922 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 3923 qrss, actions_n, error); 3924 if (!mtr_sfx && !qrss_id) { 3925 ret = -rte_errno; 3926 goto exit; 3927 } 3928 } else if (attr->egress && !attr->transfer) { 3929 /* 3930 * All the actions on NIC Tx should have a metadata register 3931 * copy action to copy reg_a from WQE to reg_c[meta] 3932 */ 3933 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 3934 sizeof(struct mlx5_flow_action_copy_mreg); 3935 ext_actions = rte_zmalloc(__func__, act_size, 0); 3936 if (!ext_actions) 3937 return rte_flow_error_set(error, ENOMEM, 3938 RTE_FLOW_ERROR_TYPE_ACTION, 3939 NULL, "no memory to split " 3940 "metadata flow"); 3941 /* Create the action list appended with copy register. */ 3942 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 3943 actions_n, error, encap_idx); 3944 if (ret < 0) 3945 goto exit; 3946 } 3947 /* Add the unmodified original or prefix subflow. */ 3948 ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr, 3949 items, ext_actions ? ext_actions : 3950 actions, external, error); 3951 if (ret < 0) 3952 goto exit; 3953 MLX5_ASSERT(dev_flow); 3954 if (qrss) { 3955 const struct rte_flow_attr q_attr = { 3956 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 3957 .ingress = 1, 3958 }; 3959 /* Internal PMD action to set register. */ 3960 struct mlx5_rte_flow_item_tag q_tag_spec = { 3961 .data = qrss_id, 3962 .id = 0, 3963 }; 3964 struct rte_flow_item q_items[] = { 3965 { 3966 .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG, 3967 .spec = &q_tag_spec, 3968 .last = NULL, 3969 .mask = NULL, 3970 }, 3971 { 3972 .type = RTE_FLOW_ITEM_TYPE_END, 3973 }, 3974 }; 3975 struct rte_flow_action q_actions[] = { 3976 { 3977 .type = qrss->type, 3978 .conf = qrss->conf, 3979 }, 3980 { 3981 .type = RTE_FLOW_ACTION_TYPE_END, 3982 }, 3983 }; 3984 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 3985 3986 /* 3987 * Configure the tag item only if there is no meter subflow. 3988 * Since tag is already marked in the meter suffix subflow 3989 * we can just use the meter suffix items as is. 3990 */ 3991 if (qrss_id) { 3992 /* Not meter subflow. */ 3993 MLX5_ASSERT(!mtr_sfx); 3994 /* 3995 * Put unique id in prefix flow due to it is destroyed 3996 * after suffix flow and id will be freed after there 3997 * is no actual flows with this id and identifier 3998 * reallocation becomes possible (for example, for 3999 * other flows in other threads). 4000 */ 4001 dev_flow->qrss_id = qrss_id; 4002 qrss_id = 0; 4003 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 4004 error); 4005 if (ret < 0) 4006 goto exit; 4007 q_tag_spec.id = ret; 4008 } 4009 dev_flow = NULL; 4010 /* Add suffix subflow to execute Q/RSS. */ 4011 ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 4012 &q_attr, mtr_sfx ? items : 4013 q_items, q_actions, 4014 external, error); 4015 if (ret < 0) 4016 goto exit; 4017 MLX5_ASSERT(dev_flow); 4018 } 4019 4020 exit: 4021 /* 4022 * We do not destroy the partially created sub_flows in case of error. 4023 * These ones are included into parent flow list and will be destroyed 4024 * by flow_drv_destroy. 4025 */ 4026 flow_qrss_free_id(dev, qrss_id); 4027 rte_free(ext_actions); 4028 return ret; 4029 } 4030 4031 /** 4032 * The splitting for meter feature. 4033 * 4034 * - The meter flow will be split to two flows as prefix and 4035 * suffix flow. The packets make sense only it pass the prefix 4036 * meter action. 4037 * 4038 * - Reg_C_5 is used for the packet to match betweend prefix and 4039 * suffix flow. 4040 * 4041 * @param dev 4042 * Pointer to Ethernet device. 4043 * @param[in] flow 4044 * Parent flow structure pointer. 4045 * @param[in] attr 4046 * Flow rule attributes. 4047 * @param[in] items 4048 * Pattern specification (list terminated by the END pattern item). 4049 * @param[in] actions 4050 * Associated actions (list terminated by the END action). 4051 * @param[in] external 4052 * This flow rule is created by request external to PMD. 4053 * @param[out] error 4054 * Perform verbose error reporting if not NULL. 4055 * @return 4056 * 0 on success, negative value otherwise 4057 */ 4058 static int 4059 flow_create_split_meter(struct rte_eth_dev *dev, 4060 struct rte_flow *flow, 4061 const struct rte_flow_attr *attr, 4062 const struct rte_flow_item items[], 4063 const struct rte_flow_action actions[], 4064 bool external, struct rte_flow_error *error) 4065 { 4066 struct mlx5_priv *priv = dev->data->dev_private; 4067 struct rte_flow_action *sfx_actions = NULL; 4068 struct rte_flow_action *pre_actions = NULL; 4069 struct rte_flow_item *sfx_items = NULL; 4070 struct mlx5_flow *dev_flow = NULL; 4071 struct rte_flow_attr sfx_attr = *attr; 4072 uint32_t mtr = 0; 4073 uint32_t mtr_tag_id = 0; 4074 size_t act_size; 4075 size_t item_size; 4076 int actions_n = 0; 4077 int ret; 4078 4079 if (priv->mtr_en) 4080 actions_n = flow_check_meter_action(actions, &mtr); 4081 if (mtr) { 4082 /* The five prefix actions: meter, decap, encap, tag, end. */ 4083 act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + 4084 sizeof(struct mlx5_rte_flow_action_set_tag); 4085 /* tag, vlan, port id, end. */ 4086 #define METER_SUFFIX_ITEM 4 4087 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 4088 sizeof(struct mlx5_rte_flow_item_tag) * 2; 4089 sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0); 4090 if (!sfx_actions) 4091 return rte_flow_error_set(error, ENOMEM, 4092 RTE_FLOW_ERROR_TYPE_ACTION, 4093 NULL, "no memory to split " 4094 "meter flow"); 4095 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 4096 act_size); 4097 pre_actions = sfx_actions + actions_n; 4098 mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, 4099 actions, sfx_actions, 4100 pre_actions); 4101 if (!mtr_tag_id) { 4102 ret = -rte_errno; 4103 goto exit; 4104 } 4105 /* Add the prefix subflow. */ 4106 ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr, 4107 items, pre_actions, external, 4108 error); 4109 if (ret) { 4110 ret = -rte_errno; 4111 goto exit; 4112 } 4113 dev_flow->mtr_flow_id = mtr_tag_id; 4114 /* Setting the sfx group atrr. */ 4115 sfx_attr.group = sfx_attr.transfer ? 4116 (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : 4117 MLX5_FLOW_TABLE_LEVEL_SUFFIX; 4118 } 4119 /* Add the prefix subflow. */ 4120 ret = flow_create_split_metadata(dev, flow, dev_flow ? 4121 flow_get_prefix_layer_flags(dev_flow) : 4122 0, &sfx_attr, 4123 sfx_items ? sfx_items : items, 4124 sfx_actions ? sfx_actions : actions, 4125 external, error); 4126 exit: 4127 if (sfx_actions) 4128 rte_free(sfx_actions); 4129 return ret; 4130 } 4131 4132 /** 4133 * Split the flow to subflow set. The splitters might be linked 4134 * in the chain, like this: 4135 * flow_create_split_outer() calls: 4136 * flow_create_split_meter() calls: 4137 * flow_create_split_metadata(meter_subflow_0) calls: 4138 * flow_create_split_inner(metadata_subflow_0) 4139 * flow_create_split_inner(metadata_subflow_1) 4140 * flow_create_split_inner(metadata_subflow_2) 4141 * flow_create_split_metadata(meter_subflow_1) calls: 4142 * flow_create_split_inner(metadata_subflow_0) 4143 * flow_create_split_inner(metadata_subflow_1) 4144 * flow_create_split_inner(metadata_subflow_2) 4145 * 4146 * This provide flexible way to add new levels of flow splitting. 4147 * The all of successfully created subflows are included to the 4148 * parent flow dev_flow list. 4149 * 4150 * @param dev 4151 * Pointer to Ethernet device. 4152 * @param[in] flow 4153 * Parent flow structure pointer. 4154 * @param[in] attr 4155 * Flow rule attributes. 4156 * @param[in] items 4157 * Pattern specification (list terminated by the END pattern item). 4158 * @param[in] actions 4159 * Associated actions (list terminated by the END action). 4160 * @param[in] external 4161 * This flow rule is created by request external to PMD. 4162 * @param[out] error 4163 * Perform verbose error reporting if not NULL. 4164 * @return 4165 * 0 on success, negative value otherwise 4166 */ 4167 static int 4168 flow_create_split_outer(struct rte_eth_dev *dev, 4169 struct rte_flow *flow, 4170 const struct rte_flow_attr *attr, 4171 const struct rte_flow_item items[], 4172 const struct rte_flow_action actions[], 4173 bool external, struct rte_flow_error *error) 4174 { 4175 int ret; 4176 4177 ret = flow_create_split_meter(dev, flow, attr, items, 4178 actions, external, error); 4179 MLX5_ASSERT(ret <= 0); 4180 return ret; 4181 } 4182 4183 /** 4184 * Create a flow and add it to @p list. 4185 * 4186 * @param dev 4187 * Pointer to Ethernet device. 4188 * @param list 4189 * Pointer to a TAILQ flow list. If this parameter NULL, 4190 * no list insertion occurred, flow is just created, 4191 * this is caller's responsibility to track the 4192 * created flow. 4193 * @param[in] attr 4194 * Flow rule attributes. 4195 * @param[in] items 4196 * Pattern specification (list terminated by the END pattern item). 4197 * @param[in] actions 4198 * Associated actions (list terminated by the END action). 4199 * @param[in] external 4200 * This flow rule is created by request external to PMD. 4201 * @param[out] error 4202 * Perform verbose error reporting if not NULL. 4203 * 4204 * @return 4205 * A flow on success, NULL otherwise and rte_errno is set. 4206 */ 4207 static struct rte_flow * 4208 flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, 4209 const struct rte_flow_attr *attr, 4210 const struct rte_flow_item items[], 4211 const struct rte_flow_action actions[], 4212 bool external, struct rte_flow_error *error) 4213 { 4214 struct mlx5_priv *priv = dev->data->dev_private; 4215 struct rte_flow *flow = NULL; 4216 struct mlx5_flow *dev_flow; 4217 const struct rte_flow_action_rss *rss; 4218 union { 4219 struct rte_flow_expand_rss buf; 4220 uint8_t buffer[2048]; 4221 } expand_buffer; 4222 union { 4223 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4224 uint8_t buffer[2048]; 4225 } actions_rx; 4226 union { 4227 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 4228 uint8_t buffer[2048]; 4229 } actions_hairpin_tx; 4230 union { 4231 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 4232 uint8_t buffer[2048]; 4233 } items_tx; 4234 struct rte_flow_expand_rss *buf = &expand_buffer.buf; 4235 const struct rte_flow_action *p_actions_rx = actions; 4236 uint32_t i; 4237 uint32_t flow_size; 4238 int hairpin_flow = 0; 4239 uint32_t hairpin_id = 0; 4240 struct rte_flow_attr attr_tx = { .priority = 0 }; 4241 int ret = flow_drv_validate(dev, attr, items, p_actions_rx, external, 4242 error); 4243 4244 if (ret < 0) 4245 return NULL; 4246 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 4247 if (hairpin_flow > 0) { 4248 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 4249 rte_errno = EINVAL; 4250 return NULL; 4251 } 4252 flow_hairpin_split(dev, actions, actions_rx.actions, 4253 actions_hairpin_tx.actions, items_tx.items, 4254 &hairpin_id); 4255 p_actions_rx = actions_rx.actions; 4256 } 4257 flow_size = sizeof(struct rte_flow); 4258 rss = flow_get_rss_action(p_actions_rx); 4259 if (rss) 4260 flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t), 4261 sizeof(void *)); 4262 else 4263 flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *)); 4264 flow = rte_calloc(__func__, 1, flow_size, 0); 4265 if (!flow) { 4266 rte_errno = ENOMEM; 4267 goto error_before_flow; 4268 } 4269 flow->drv_type = flow_get_drv_type(dev, attr); 4270 if (hairpin_id != 0) 4271 flow->hairpin_flow_id = hairpin_id; 4272 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 4273 flow->drv_type < MLX5_FLOW_TYPE_MAX); 4274 flow->rss.queue = (void *)(flow + 1); 4275 if (rss) { 4276 /* 4277 * The following information is required by 4278 * mlx5_flow_hashfields_adjust() in advance. 4279 */ 4280 flow->rss.level = rss->level; 4281 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ 4282 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; 4283 } 4284 LIST_INIT(&flow->dev_flows); 4285 if (rss && rss->types) { 4286 unsigned int graph_root; 4287 4288 graph_root = find_graph_root(items, rss->level); 4289 ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 4290 items, rss->types, 4291 mlx5_support_expansion, 4292 graph_root); 4293 MLX5_ASSERT(ret > 0 && 4294 (unsigned int)ret < sizeof(expand_buffer.buffer)); 4295 } else { 4296 buf->entries = 1; 4297 buf->entry[0].pattern = (void *)(uintptr_t)items; 4298 } 4299 for (i = 0; i < buf->entries; ++i) { 4300 /* 4301 * The splitter may create multiple dev_flows, 4302 * depending on configuration. In the simplest 4303 * case it just creates unmodified original flow. 4304 */ 4305 ret = flow_create_split_outer(dev, flow, attr, 4306 buf->entry[i].pattern, 4307 p_actions_rx, external, 4308 error); 4309 if (ret < 0) 4310 goto error; 4311 } 4312 /* Create the tx flow. */ 4313 if (hairpin_flow) { 4314 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 4315 attr_tx.ingress = 0; 4316 attr_tx.egress = 1; 4317 dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items, 4318 actions_hairpin_tx.actions, error); 4319 if (!dev_flow) 4320 goto error; 4321 dev_flow->flow = flow; 4322 dev_flow->external = 0; 4323 LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); 4324 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 4325 items_tx.items, 4326 actions_hairpin_tx.actions, error); 4327 if (ret < 0) 4328 goto error; 4329 } 4330 /* 4331 * Update the metadata register copy table. If extensive 4332 * metadata feature is enabled and registers are supported 4333 * we might create the extra rte_flow for each unique 4334 * MARK/FLAG action ID. 4335 * 4336 * The table is updated for ingress Flows only, because 4337 * the egress Flows belong to the different device and 4338 * copy table should be updated in peer NIC Rx domain. 4339 */ 4340 if (attr->ingress && 4341 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 4342 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 4343 if (ret) 4344 goto error; 4345 } 4346 if (dev->data->dev_started) { 4347 ret = flow_drv_apply(dev, flow, error); 4348 if (ret < 0) 4349 goto error; 4350 } 4351 if (list) 4352 TAILQ_INSERT_TAIL(list, flow, next); 4353 flow_rxq_flags_set(dev, flow); 4354 return flow; 4355 error_before_flow: 4356 if (hairpin_id) 4357 mlx5_flow_id_release(priv->sh->flow_id_pool, 4358 hairpin_id); 4359 return NULL; 4360 error: 4361 MLX5_ASSERT(flow); 4362 flow_mreg_del_copy_action(dev, flow); 4363 ret = rte_errno; /* Save rte_errno before cleanup. */ 4364 if (flow->hairpin_flow_id) 4365 mlx5_flow_id_release(priv->sh->flow_id_pool, 4366 flow->hairpin_flow_id); 4367 MLX5_ASSERT(flow); 4368 flow_drv_destroy(dev, flow); 4369 rte_free(flow); 4370 rte_errno = ret; /* Restore rte_errno. */ 4371 return NULL; 4372 } 4373 4374 /** 4375 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 4376 * incoming packets to table 1. 4377 * 4378 * Other flow rules, requested for group n, will be created in 4379 * e-switch table n+1. 4380 * Jump action to e-switch group n will be created to group n+1. 4381 * 4382 * Used when working in switchdev mode, to utilise advantages of table 1 4383 * and above. 4384 * 4385 * @param dev 4386 * Pointer to Ethernet device. 4387 * 4388 * @return 4389 * Pointer to flow on success, NULL otherwise and rte_errno is set. 4390 */ 4391 struct rte_flow * 4392 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 4393 { 4394 const struct rte_flow_attr attr = { 4395 .group = 0, 4396 .priority = 0, 4397 .ingress = 1, 4398 .egress = 0, 4399 .transfer = 1, 4400 }; 4401 const struct rte_flow_item pattern = { 4402 .type = RTE_FLOW_ITEM_TYPE_END, 4403 }; 4404 struct rte_flow_action_jump jump = { 4405 .group = 1, 4406 }; 4407 const struct rte_flow_action actions[] = { 4408 { 4409 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4410 .conf = &jump, 4411 }, 4412 { 4413 .type = RTE_FLOW_ACTION_TYPE_END, 4414 }, 4415 }; 4416 struct mlx5_priv *priv = dev->data->dev_private; 4417 struct rte_flow_error error; 4418 4419 return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern, 4420 actions, false, &error); 4421 } 4422 4423 /** 4424 * Create a flow. 4425 * 4426 * @see rte_flow_create() 4427 * @see rte_flow_ops 4428 */ 4429 struct rte_flow * 4430 mlx5_flow_create(struct rte_eth_dev *dev, 4431 const struct rte_flow_attr *attr, 4432 const struct rte_flow_item items[], 4433 const struct rte_flow_action actions[], 4434 struct rte_flow_error *error) 4435 { 4436 struct mlx5_priv *priv = dev->data->dev_private; 4437 4438 return flow_list_create(dev, &priv->flows, 4439 attr, items, actions, true, error); 4440 } 4441 4442 /** 4443 * Destroy a flow in a list. 4444 * 4445 * @param dev 4446 * Pointer to Ethernet device. 4447 * @param list 4448 * Pointer to a TAILQ flow list. If this parameter NULL, 4449 * there is no flow removal from the list. 4450 * @param[in] flow 4451 * Flow to destroy. 4452 */ 4453 static void 4454 flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, 4455 struct rte_flow *flow) 4456 { 4457 struct mlx5_priv *priv = dev->data->dev_private; 4458 4459 /* 4460 * Update RX queue flags only if port is started, otherwise it is 4461 * already clean. 4462 */ 4463 if (dev->data->dev_started) 4464 flow_rxq_flags_trim(dev, flow); 4465 if (flow->hairpin_flow_id) 4466 mlx5_flow_id_release(priv->sh->flow_id_pool, 4467 flow->hairpin_flow_id); 4468 flow_drv_destroy(dev, flow); 4469 if (list) 4470 TAILQ_REMOVE(list, flow, next); 4471 flow_mreg_del_copy_action(dev, flow); 4472 rte_free(flow->fdir); 4473 rte_free(flow); 4474 } 4475 4476 /** 4477 * Destroy all flows. 4478 * 4479 * @param dev 4480 * Pointer to Ethernet device. 4481 * @param list 4482 * Pointer to a TAILQ flow list. 4483 */ 4484 void 4485 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) 4486 { 4487 while (!TAILQ_EMPTY(list)) { 4488 struct rte_flow *flow; 4489 4490 flow = TAILQ_FIRST(list); 4491 flow_list_destroy(dev, list, flow); 4492 } 4493 } 4494 4495 /** 4496 * Remove all flows. 4497 * 4498 * @param dev 4499 * Pointer to Ethernet device. 4500 * @param list 4501 * Pointer to a TAILQ flow list. 4502 */ 4503 void 4504 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) 4505 { 4506 struct rte_flow *flow; 4507 4508 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { 4509 flow_drv_remove(dev, flow); 4510 flow_mreg_stop_copy_action(dev, flow); 4511 } 4512 flow_mreg_del_default_copy_action(dev); 4513 flow_rxq_flags_clear(dev); 4514 } 4515 4516 /** 4517 * Add all flows. 4518 * 4519 * @param dev 4520 * Pointer to Ethernet device. 4521 * @param list 4522 * Pointer to a TAILQ flow list. 4523 * 4524 * @return 4525 * 0 on success, a negative errno value otherwise and rte_errno is set. 4526 */ 4527 int 4528 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) 4529 { 4530 struct rte_flow *flow; 4531 struct rte_flow_error error; 4532 int ret = 0; 4533 4534 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 4535 ret = flow_mreg_add_default_copy_action(dev, &error); 4536 if (ret < 0) 4537 return -rte_errno; 4538 /* Apply Flows created by application. */ 4539 TAILQ_FOREACH(flow, list, next) { 4540 ret = flow_mreg_start_copy_action(dev, flow); 4541 if (ret < 0) 4542 goto error; 4543 ret = flow_drv_apply(dev, flow, &error); 4544 if (ret < 0) 4545 goto error; 4546 flow_rxq_flags_set(dev, flow); 4547 } 4548 return 0; 4549 error: 4550 ret = rte_errno; /* Save rte_errno before cleanup. */ 4551 mlx5_flow_stop(dev, list); 4552 rte_errno = ret; /* Restore rte_errno. */ 4553 return -rte_errno; 4554 } 4555 4556 /** 4557 * Verify the flow list is empty 4558 * 4559 * @param dev 4560 * Pointer to Ethernet device. 4561 * 4562 * @return the number of flows not released. 4563 */ 4564 int 4565 mlx5_flow_verify(struct rte_eth_dev *dev) 4566 { 4567 struct mlx5_priv *priv = dev->data->dev_private; 4568 struct rte_flow *flow; 4569 int ret = 0; 4570 4571 TAILQ_FOREACH(flow, &priv->flows, next) { 4572 DRV_LOG(DEBUG, "port %u flow %p still referenced", 4573 dev->data->port_id, (void *)flow); 4574 ++ret; 4575 } 4576 return ret; 4577 } 4578 4579 /** 4580 * Enable default hairpin egress flow. 4581 * 4582 * @param dev 4583 * Pointer to Ethernet device. 4584 * @param queue 4585 * The queue index. 4586 * 4587 * @return 4588 * 0 on success, a negative errno value otherwise and rte_errno is set. 4589 */ 4590 int 4591 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 4592 uint32_t queue) 4593 { 4594 struct mlx5_priv *priv = dev->data->dev_private; 4595 const struct rte_flow_attr attr = { 4596 .egress = 1, 4597 .priority = 0, 4598 }; 4599 struct mlx5_rte_flow_item_tx_queue queue_spec = { 4600 .queue = queue, 4601 }; 4602 struct mlx5_rte_flow_item_tx_queue queue_mask = { 4603 .queue = UINT32_MAX, 4604 }; 4605 struct rte_flow_item items[] = { 4606 { 4607 .type = MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 4608 .spec = &queue_spec, 4609 .last = NULL, 4610 .mask = &queue_mask, 4611 }, 4612 { 4613 .type = RTE_FLOW_ITEM_TYPE_END, 4614 }, 4615 }; 4616 struct rte_flow_action_jump jump = { 4617 .group = MLX5_HAIRPIN_TX_TABLE, 4618 }; 4619 struct rte_flow_action actions[2]; 4620 struct rte_flow *flow; 4621 struct rte_flow_error error; 4622 4623 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 4624 actions[0].conf = &jump; 4625 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 4626 flow = flow_list_create(dev, &priv->ctrl_flows, 4627 &attr, items, actions, false, &error); 4628 if (!flow) { 4629 DRV_LOG(DEBUG, 4630 "Failed to create ctrl flow: rte_errno(%d)," 4631 " type(%d), message(%s)", 4632 rte_errno, error.type, 4633 error.message ? error.message : " (no stated reason)"); 4634 return -rte_errno; 4635 } 4636 return 0; 4637 } 4638 4639 /** 4640 * Enable a control flow configured from the control plane. 4641 * 4642 * @param dev 4643 * Pointer to Ethernet device. 4644 * @param eth_spec 4645 * An Ethernet flow spec to apply. 4646 * @param eth_mask 4647 * An Ethernet flow mask to apply. 4648 * @param vlan_spec 4649 * A VLAN flow spec to apply. 4650 * @param vlan_mask 4651 * A VLAN flow mask to apply. 4652 * 4653 * @return 4654 * 0 on success, a negative errno value otherwise and rte_errno is set. 4655 */ 4656 int 4657 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 4658 struct rte_flow_item_eth *eth_spec, 4659 struct rte_flow_item_eth *eth_mask, 4660 struct rte_flow_item_vlan *vlan_spec, 4661 struct rte_flow_item_vlan *vlan_mask) 4662 { 4663 struct mlx5_priv *priv = dev->data->dev_private; 4664 const struct rte_flow_attr attr = { 4665 .ingress = 1, 4666 .priority = MLX5_FLOW_PRIO_RSVD, 4667 }; 4668 struct rte_flow_item items[] = { 4669 { 4670 .type = RTE_FLOW_ITEM_TYPE_ETH, 4671 .spec = eth_spec, 4672 .last = NULL, 4673 .mask = eth_mask, 4674 }, 4675 { 4676 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 4677 RTE_FLOW_ITEM_TYPE_END, 4678 .spec = vlan_spec, 4679 .last = NULL, 4680 .mask = vlan_mask, 4681 }, 4682 { 4683 .type = RTE_FLOW_ITEM_TYPE_END, 4684 }, 4685 }; 4686 uint16_t queue[priv->reta_idx_n]; 4687 struct rte_flow_action_rss action_rss = { 4688 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 4689 .level = 0, 4690 .types = priv->rss_conf.rss_hf, 4691 .key_len = priv->rss_conf.rss_key_len, 4692 .queue_num = priv->reta_idx_n, 4693 .key = priv->rss_conf.rss_key, 4694 .queue = queue, 4695 }; 4696 struct rte_flow_action actions[] = { 4697 { 4698 .type = RTE_FLOW_ACTION_TYPE_RSS, 4699 .conf = &action_rss, 4700 }, 4701 { 4702 .type = RTE_FLOW_ACTION_TYPE_END, 4703 }, 4704 }; 4705 struct rte_flow *flow; 4706 struct rte_flow_error error; 4707 unsigned int i; 4708 4709 if (!priv->reta_idx_n || !priv->rxqs_n) { 4710 return 0; 4711 } 4712 for (i = 0; i != priv->reta_idx_n; ++i) 4713 queue[i] = (*priv->reta_idx)[i]; 4714 flow = flow_list_create(dev, &priv->ctrl_flows, 4715 &attr, items, actions, false, &error); 4716 if (!flow) 4717 return -rte_errno; 4718 return 0; 4719 } 4720 4721 /** 4722 * Enable a flow control configured from the control plane. 4723 * 4724 * @param dev 4725 * Pointer to Ethernet device. 4726 * @param eth_spec 4727 * An Ethernet flow spec to apply. 4728 * @param eth_mask 4729 * An Ethernet flow mask to apply. 4730 * 4731 * @return 4732 * 0 on success, a negative errno value otherwise and rte_errno is set. 4733 */ 4734 int 4735 mlx5_ctrl_flow(struct rte_eth_dev *dev, 4736 struct rte_flow_item_eth *eth_spec, 4737 struct rte_flow_item_eth *eth_mask) 4738 { 4739 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 4740 } 4741 4742 /** 4743 * Destroy a flow. 4744 * 4745 * @see rte_flow_destroy() 4746 * @see rte_flow_ops 4747 */ 4748 int 4749 mlx5_flow_destroy(struct rte_eth_dev *dev, 4750 struct rte_flow *flow, 4751 struct rte_flow_error *error __rte_unused) 4752 { 4753 struct mlx5_priv *priv = dev->data->dev_private; 4754 4755 flow_list_destroy(dev, &priv->flows, flow); 4756 return 0; 4757 } 4758 4759 /** 4760 * Destroy all flows. 4761 * 4762 * @see rte_flow_flush() 4763 * @see rte_flow_ops 4764 */ 4765 int 4766 mlx5_flow_flush(struct rte_eth_dev *dev, 4767 struct rte_flow_error *error __rte_unused) 4768 { 4769 struct mlx5_priv *priv = dev->data->dev_private; 4770 4771 mlx5_flow_list_flush(dev, &priv->flows); 4772 return 0; 4773 } 4774 4775 /** 4776 * Isolated mode. 4777 * 4778 * @see rte_flow_isolate() 4779 * @see rte_flow_ops 4780 */ 4781 int 4782 mlx5_flow_isolate(struct rte_eth_dev *dev, 4783 int enable, 4784 struct rte_flow_error *error) 4785 { 4786 struct mlx5_priv *priv = dev->data->dev_private; 4787 4788 if (dev->data->dev_started) { 4789 rte_flow_error_set(error, EBUSY, 4790 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4791 NULL, 4792 "port must be stopped first"); 4793 return -rte_errno; 4794 } 4795 priv->isolated = !!enable; 4796 if (enable) 4797 dev->dev_ops = &mlx5_dev_ops_isolate; 4798 else 4799 dev->dev_ops = &mlx5_dev_ops; 4800 return 0; 4801 } 4802 4803 /** 4804 * Query a flow. 4805 * 4806 * @see rte_flow_query() 4807 * @see rte_flow_ops 4808 */ 4809 static int 4810 flow_drv_query(struct rte_eth_dev *dev, 4811 struct rte_flow *flow, 4812 const struct rte_flow_action *actions, 4813 void *data, 4814 struct rte_flow_error *error) 4815 { 4816 const struct mlx5_flow_driver_ops *fops; 4817 enum mlx5_flow_drv_type ftype = flow->drv_type; 4818 4819 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 4820 fops = flow_get_drv_ops(ftype); 4821 4822 return fops->query(dev, flow, actions, data, error); 4823 } 4824 4825 /** 4826 * Query a flow. 4827 * 4828 * @see rte_flow_query() 4829 * @see rte_flow_ops 4830 */ 4831 int 4832 mlx5_flow_query(struct rte_eth_dev *dev, 4833 struct rte_flow *flow, 4834 const struct rte_flow_action *actions, 4835 void *data, 4836 struct rte_flow_error *error) 4837 { 4838 int ret; 4839 4840 ret = flow_drv_query(dev, flow, actions, data, error); 4841 if (ret < 0) 4842 return ret; 4843 return 0; 4844 } 4845 4846 /** 4847 * Convert a flow director filter to a generic flow. 4848 * 4849 * @param dev 4850 * Pointer to Ethernet device. 4851 * @param fdir_filter 4852 * Flow director filter to add. 4853 * @param attributes 4854 * Generic flow parameters structure. 4855 * 4856 * @return 4857 * 0 on success, a negative errno value otherwise and rte_errno is set. 4858 */ 4859 static int 4860 flow_fdir_filter_convert(struct rte_eth_dev *dev, 4861 const struct rte_eth_fdir_filter *fdir_filter, 4862 struct mlx5_fdir *attributes) 4863 { 4864 struct mlx5_priv *priv = dev->data->dev_private; 4865 const struct rte_eth_fdir_input *input = &fdir_filter->input; 4866 const struct rte_eth_fdir_masks *mask = 4867 &dev->data->dev_conf.fdir_conf.mask; 4868 4869 /* Validate queue number. */ 4870 if (fdir_filter->action.rx_queue >= priv->rxqs_n) { 4871 DRV_LOG(ERR, "port %u invalid queue number %d", 4872 dev->data->port_id, fdir_filter->action.rx_queue); 4873 rte_errno = EINVAL; 4874 return -rte_errno; 4875 } 4876 attributes->attr.ingress = 1; 4877 attributes->items[0] = (struct rte_flow_item) { 4878 .type = RTE_FLOW_ITEM_TYPE_ETH, 4879 .spec = &attributes->l2, 4880 .mask = &attributes->l2_mask, 4881 }; 4882 switch (fdir_filter->action.behavior) { 4883 case RTE_ETH_FDIR_ACCEPT: 4884 attributes->actions[0] = (struct rte_flow_action){ 4885 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 4886 .conf = &attributes->queue, 4887 }; 4888 break; 4889 case RTE_ETH_FDIR_REJECT: 4890 attributes->actions[0] = (struct rte_flow_action){ 4891 .type = RTE_FLOW_ACTION_TYPE_DROP, 4892 }; 4893 break; 4894 default: 4895 DRV_LOG(ERR, "port %u invalid behavior %d", 4896 dev->data->port_id, 4897 fdir_filter->action.behavior); 4898 rte_errno = ENOTSUP; 4899 return -rte_errno; 4900 } 4901 attributes->queue.index = fdir_filter->action.rx_queue; 4902 /* Handle L3. */ 4903 switch (fdir_filter->input.flow_type) { 4904 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4905 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4906 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 4907 attributes->l3.ipv4.hdr = (struct rte_ipv4_hdr){ 4908 .src_addr = input->flow.ip4_flow.src_ip, 4909 .dst_addr = input->flow.ip4_flow.dst_ip, 4910 .time_to_live = input->flow.ip4_flow.ttl, 4911 .type_of_service = input->flow.ip4_flow.tos, 4912 }; 4913 attributes->l3_mask.ipv4.hdr = (struct rte_ipv4_hdr){ 4914 .src_addr = mask->ipv4_mask.src_ip, 4915 .dst_addr = mask->ipv4_mask.dst_ip, 4916 .time_to_live = mask->ipv4_mask.ttl, 4917 .type_of_service = mask->ipv4_mask.tos, 4918 .next_proto_id = mask->ipv4_mask.proto, 4919 }; 4920 attributes->items[1] = (struct rte_flow_item){ 4921 .type = RTE_FLOW_ITEM_TYPE_IPV4, 4922 .spec = &attributes->l3, 4923 .mask = &attributes->l3_mask, 4924 }; 4925 break; 4926 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4927 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 4928 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 4929 attributes->l3.ipv6.hdr = (struct rte_ipv6_hdr){ 4930 .hop_limits = input->flow.ipv6_flow.hop_limits, 4931 .proto = input->flow.ipv6_flow.proto, 4932 }; 4933 4934 memcpy(attributes->l3.ipv6.hdr.src_addr, 4935 input->flow.ipv6_flow.src_ip, 4936 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4937 memcpy(attributes->l3.ipv6.hdr.dst_addr, 4938 input->flow.ipv6_flow.dst_ip, 4939 RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); 4940 memcpy(attributes->l3_mask.ipv6.hdr.src_addr, 4941 mask->ipv6_mask.src_ip, 4942 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4943 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, 4944 mask->ipv6_mask.dst_ip, 4945 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); 4946 attributes->items[1] = (struct rte_flow_item){ 4947 .type = RTE_FLOW_ITEM_TYPE_IPV6, 4948 .spec = &attributes->l3, 4949 .mask = &attributes->l3_mask, 4950 }; 4951 break; 4952 default: 4953 DRV_LOG(ERR, "port %u invalid flow type%d", 4954 dev->data->port_id, fdir_filter->input.flow_type); 4955 rte_errno = ENOTSUP; 4956 return -rte_errno; 4957 } 4958 /* Handle L4. */ 4959 switch (fdir_filter->input.flow_type) { 4960 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: 4961 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4962 .src_port = input->flow.udp4_flow.src_port, 4963 .dst_port = input->flow.udp4_flow.dst_port, 4964 }; 4965 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4966 .src_port = mask->src_port_mask, 4967 .dst_port = mask->dst_port_mask, 4968 }; 4969 attributes->items[2] = (struct rte_flow_item){ 4970 .type = RTE_FLOW_ITEM_TYPE_UDP, 4971 .spec = &attributes->l4, 4972 .mask = &attributes->l4_mask, 4973 }; 4974 break; 4975 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: 4976 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 4977 .src_port = input->flow.tcp4_flow.src_port, 4978 .dst_port = input->flow.tcp4_flow.dst_port, 4979 }; 4980 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 4981 .src_port = mask->src_port_mask, 4982 .dst_port = mask->dst_port_mask, 4983 }; 4984 attributes->items[2] = (struct rte_flow_item){ 4985 .type = RTE_FLOW_ITEM_TYPE_TCP, 4986 .spec = &attributes->l4, 4987 .mask = &attributes->l4_mask, 4988 }; 4989 break; 4990 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: 4991 attributes->l4.udp.hdr = (struct rte_udp_hdr){ 4992 .src_port = input->flow.udp6_flow.src_port, 4993 .dst_port = input->flow.udp6_flow.dst_port, 4994 }; 4995 attributes->l4_mask.udp.hdr = (struct rte_udp_hdr){ 4996 .src_port = mask->src_port_mask, 4997 .dst_port = mask->dst_port_mask, 4998 }; 4999 attributes->items[2] = (struct rte_flow_item){ 5000 .type = RTE_FLOW_ITEM_TYPE_UDP, 5001 .spec = &attributes->l4, 5002 .mask = &attributes->l4_mask, 5003 }; 5004 break; 5005 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: 5006 attributes->l4.tcp.hdr = (struct rte_tcp_hdr){ 5007 .src_port = input->flow.tcp6_flow.src_port, 5008 .dst_port = input->flow.tcp6_flow.dst_port, 5009 }; 5010 attributes->l4_mask.tcp.hdr = (struct rte_tcp_hdr){ 5011 .src_port = mask->src_port_mask, 5012 .dst_port = mask->dst_port_mask, 5013 }; 5014 attributes->items[2] = (struct rte_flow_item){ 5015 .type = RTE_FLOW_ITEM_TYPE_TCP, 5016 .spec = &attributes->l4, 5017 .mask = &attributes->l4_mask, 5018 }; 5019 break; 5020 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: 5021 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: 5022 break; 5023 default: 5024 DRV_LOG(ERR, "port %u invalid flow type%d", 5025 dev->data->port_id, fdir_filter->input.flow_type); 5026 rte_errno = ENOTSUP; 5027 return -rte_errno; 5028 } 5029 return 0; 5030 } 5031 5032 #define FLOW_FDIR_CMP(f1, f2, fld) \ 5033 memcmp(&(f1)->fld, &(f2)->fld, sizeof(f1->fld)) 5034 5035 /** 5036 * Compare two FDIR flows. If items and actions are identical, the two flows are 5037 * regarded as same. 5038 * 5039 * @param dev 5040 * Pointer to Ethernet device. 5041 * @param f1 5042 * FDIR flow to compare. 5043 * @param f2 5044 * FDIR flow to compare. 5045 * 5046 * @return 5047 * Zero on match, 1 otherwise. 5048 */ 5049 static int 5050 flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) 5051 { 5052 if (FLOW_FDIR_CMP(f1, f2, attr) || 5053 FLOW_FDIR_CMP(f1, f2, l2) || 5054 FLOW_FDIR_CMP(f1, f2, l2_mask) || 5055 FLOW_FDIR_CMP(f1, f2, l3) || 5056 FLOW_FDIR_CMP(f1, f2, l3_mask) || 5057 FLOW_FDIR_CMP(f1, f2, l4) || 5058 FLOW_FDIR_CMP(f1, f2, l4_mask) || 5059 FLOW_FDIR_CMP(f1, f2, actions[0].type)) 5060 return 1; 5061 if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE && 5062 FLOW_FDIR_CMP(f1, f2, queue)) 5063 return 1; 5064 return 0; 5065 } 5066 5067 /** 5068 * Search device flow list to find out a matched FDIR flow. 5069 * 5070 * @param dev 5071 * Pointer to Ethernet device. 5072 * @param fdir_flow 5073 * FDIR flow to lookup. 5074 * 5075 * @return 5076 * Pointer of flow if found, NULL otherwise. 5077 */ 5078 static struct rte_flow * 5079 flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) 5080 { 5081 struct mlx5_priv *priv = dev->data->dev_private; 5082 struct rte_flow *flow = NULL; 5083 5084 MLX5_ASSERT(fdir_flow); 5085 TAILQ_FOREACH(flow, &priv->flows, next) { 5086 if (flow->fdir && !flow_fdir_cmp(flow->fdir, fdir_flow)) { 5087 DRV_LOG(DEBUG, "port %u found FDIR flow %p", 5088 dev->data->port_id, (void *)flow); 5089 break; 5090 } 5091 } 5092 return flow; 5093 } 5094 5095 /** 5096 * Add new flow director filter and store it in list. 5097 * 5098 * @param dev 5099 * Pointer to Ethernet device. 5100 * @param fdir_filter 5101 * Flow director filter to add. 5102 * 5103 * @return 5104 * 0 on success, a negative errno value otherwise and rte_errno is set. 5105 */ 5106 static int 5107 flow_fdir_filter_add(struct rte_eth_dev *dev, 5108 const struct rte_eth_fdir_filter *fdir_filter) 5109 { 5110 struct mlx5_priv *priv = dev->data->dev_private; 5111 struct mlx5_fdir *fdir_flow; 5112 struct rte_flow *flow; 5113 int ret; 5114 5115 fdir_flow = rte_zmalloc(__func__, sizeof(*fdir_flow), 0); 5116 if (!fdir_flow) { 5117 rte_errno = ENOMEM; 5118 return -rte_errno; 5119 } 5120 ret = flow_fdir_filter_convert(dev, fdir_filter, fdir_flow); 5121 if (ret) 5122 goto error; 5123 flow = flow_fdir_filter_lookup(dev, fdir_flow); 5124 if (flow) { 5125 rte_errno = EEXIST; 5126 goto error; 5127 } 5128 flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, 5129 fdir_flow->items, fdir_flow->actions, true, 5130 NULL); 5131 if (!flow) 5132 goto error; 5133 MLX5_ASSERT(!flow->fdir); 5134 flow->fdir = fdir_flow; 5135 DRV_LOG(DEBUG, "port %u created FDIR flow %p", 5136 dev->data->port_id, (void *)flow); 5137 return 0; 5138 error: 5139 rte_free(fdir_flow); 5140 return -rte_errno; 5141 } 5142 5143 /** 5144 * Delete specific filter. 5145 * 5146 * @param dev 5147 * Pointer to Ethernet device. 5148 * @param fdir_filter 5149 * Filter to be deleted. 5150 * 5151 * @return 5152 * 0 on success, a negative errno value otherwise and rte_errno is set. 5153 */ 5154 static int 5155 flow_fdir_filter_delete(struct rte_eth_dev *dev, 5156 const struct rte_eth_fdir_filter *fdir_filter) 5157 { 5158 struct mlx5_priv *priv = dev->data->dev_private; 5159 struct rte_flow *flow; 5160 struct mlx5_fdir fdir_flow = { 5161 .attr.group = 0, 5162 }; 5163 int ret; 5164 5165 ret = flow_fdir_filter_convert(dev, fdir_filter, &fdir_flow); 5166 if (ret) 5167 return -rte_errno; 5168 flow = flow_fdir_filter_lookup(dev, &fdir_flow); 5169 if (!flow) { 5170 rte_errno = ENOENT; 5171 return -rte_errno; 5172 } 5173 flow_list_destroy(dev, &priv->flows, flow); 5174 DRV_LOG(DEBUG, "port %u deleted FDIR flow %p", 5175 dev->data->port_id, (void *)flow); 5176 return 0; 5177 } 5178 5179 /** 5180 * Update queue for specific filter. 5181 * 5182 * @param dev 5183 * Pointer to Ethernet device. 5184 * @param fdir_filter 5185 * Filter to be updated. 5186 * 5187 * @return 5188 * 0 on success, a negative errno value otherwise and rte_errno is set. 5189 */ 5190 static int 5191 flow_fdir_filter_update(struct rte_eth_dev *dev, 5192 const struct rte_eth_fdir_filter *fdir_filter) 5193 { 5194 int ret; 5195 5196 ret = flow_fdir_filter_delete(dev, fdir_filter); 5197 if (ret) 5198 return ret; 5199 return flow_fdir_filter_add(dev, fdir_filter); 5200 } 5201 5202 /** 5203 * Flush all filters. 5204 * 5205 * @param dev 5206 * Pointer to Ethernet device. 5207 */ 5208 static void 5209 flow_fdir_filter_flush(struct rte_eth_dev *dev) 5210 { 5211 struct mlx5_priv *priv = dev->data->dev_private; 5212 5213 mlx5_flow_list_flush(dev, &priv->flows); 5214 } 5215 5216 /** 5217 * Get flow director information. 5218 * 5219 * @param dev 5220 * Pointer to Ethernet device. 5221 * @param[out] fdir_info 5222 * Resulting flow director information. 5223 */ 5224 static void 5225 flow_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 5226 { 5227 struct rte_eth_fdir_masks *mask = 5228 &dev->data->dev_conf.fdir_conf.mask; 5229 5230 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 5231 fdir_info->guarant_spc = 0; 5232 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); 5233 fdir_info->max_flexpayload = 0; 5234 fdir_info->flow_types_mask[0] = 0; 5235 fdir_info->flex_payload_unit = 0; 5236 fdir_info->max_flex_payload_segment_num = 0; 5237 fdir_info->flex_payload_limit = 0; 5238 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); 5239 } 5240 5241 /** 5242 * Deal with flow director operations. 5243 * 5244 * @param dev 5245 * Pointer to Ethernet device. 5246 * @param filter_op 5247 * Operation to perform. 5248 * @param arg 5249 * Pointer to operation-specific structure. 5250 * 5251 * @return 5252 * 0 on success, a negative errno value otherwise and rte_errno is set. 5253 */ 5254 static int 5255 flow_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, 5256 void *arg) 5257 { 5258 enum rte_fdir_mode fdir_mode = 5259 dev->data->dev_conf.fdir_conf.mode; 5260 5261 if (filter_op == RTE_ETH_FILTER_NOP) 5262 return 0; 5263 if (fdir_mode != RTE_FDIR_MODE_PERFECT && 5264 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 5265 DRV_LOG(ERR, "port %u flow director mode %d not supported", 5266 dev->data->port_id, fdir_mode); 5267 rte_errno = EINVAL; 5268 return -rte_errno; 5269 } 5270 switch (filter_op) { 5271 case RTE_ETH_FILTER_ADD: 5272 return flow_fdir_filter_add(dev, arg); 5273 case RTE_ETH_FILTER_UPDATE: 5274 return flow_fdir_filter_update(dev, arg); 5275 case RTE_ETH_FILTER_DELETE: 5276 return flow_fdir_filter_delete(dev, arg); 5277 case RTE_ETH_FILTER_FLUSH: 5278 flow_fdir_filter_flush(dev); 5279 break; 5280 case RTE_ETH_FILTER_INFO: 5281 flow_fdir_info_get(dev, arg); 5282 break; 5283 default: 5284 DRV_LOG(DEBUG, "port %u unknown operation %u", 5285 dev->data->port_id, filter_op); 5286 rte_errno = EINVAL; 5287 return -rte_errno; 5288 } 5289 return 0; 5290 } 5291 5292 /** 5293 * Manage filter operations. 5294 * 5295 * @param dev 5296 * Pointer to Ethernet device structure. 5297 * @param filter_type 5298 * Filter type. 5299 * @param filter_op 5300 * Operation to perform. 5301 * @param arg 5302 * Pointer to operation-specific structure. 5303 * 5304 * @return 5305 * 0 on success, a negative errno value otherwise and rte_errno is set. 5306 */ 5307 int 5308 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, 5309 enum rte_filter_type filter_type, 5310 enum rte_filter_op filter_op, 5311 void *arg) 5312 { 5313 switch (filter_type) { 5314 case RTE_ETH_FILTER_GENERIC: 5315 if (filter_op != RTE_ETH_FILTER_GET) { 5316 rte_errno = EINVAL; 5317 return -rte_errno; 5318 } 5319 *(const void **)arg = &mlx5_flow_ops; 5320 return 0; 5321 case RTE_ETH_FILTER_FDIR: 5322 return flow_fdir_ctrl_func(dev, filter_op, arg); 5323 default: 5324 DRV_LOG(ERR, "port %u filter type (%d) not supported", 5325 dev->data->port_id, filter_type); 5326 rte_errno = ENOTSUP; 5327 return -rte_errno; 5328 } 5329 return 0; 5330 } 5331 5332 /** 5333 * Create the needed meter and suffix tables. 5334 * 5335 * @param[in] dev 5336 * Pointer to Ethernet device. 5337 * @param[in] fm 5338 * Pointer to the flow meter. 5339 * 5340 * @return 5341 * Pointer to table set on success, NULL otherwise. 5342 */ 5343 struct mlx5_meter_domains_infos * 5344 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 5345 const struct mlx5_flow_meter *fm) 5346 { 5347 const struct mlx5_flow_driver_ops *fops; 5348 5349 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5350 return fops->create_mtr_tbls(dev, fm); 5351 } 5352 5353 /** 5354 * Destroy the meter table set. 5355 * 5356 * @param[in] dev 5357 * Pointer to Ethernet device. 5358 * @param[in] tbl 5359 * Pointer to the meter table set. 5360 * 5361 * @return 5362 * 0 on success. 5363 */ 5364 int 5365 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 5366 struct mlx5_meter_domains_infos *tbls) 5367 { 5368 const struct mlx5_flow_driver_ops *fops; 5369 5370 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5371 return fops->destroy_mtr_tbls(dev, tbls); 5372 } 5373 5374 /** 5375 * Create policer rules. 5376 * 5377 * @param[in] dev 5378 * Pointer to Ethernet device. 5379 * @param[in] fm 5380 * Pointer to flow meter structure. 5381 * @param[in] attr 5382 * Pointer to flow attributes. 5383 * 5384 * @return 5385 * 0 on success, -1 otherwise. 5386 */ 5387 int 5388 mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, 5389 struct mlx5_flow_meter *fm, 5390 const struct rte_flow_attr *attr) 5391 { 5392 const struct mlx5_flow_driver_ops *fops; 5393 5394 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5395 return fops->create_policer_rules(dev, fm, attr); 5396 } 5397 5398 /** 5399 * Destroy policer rules. 5400 * 5401 * @param[in] fm 5402 * Pointer to flow meter structure. 5403 * @param[in] attr 5404 * Pointer to flow attributes. 5405 * 5406 * @return 5407 * 0 on success, -1 otherwise. 5408 */ 5409 int 5410 mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, 5411 struct mlx5_flow_meter *fm, 5412 const struct rte_flow_attr *attr) 5413 { 5414 const struct mlx5_flow_driver_ops *fops; 5415 5416 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5417 return fops->destroy_policer_rules(dev, fm, attr); 5418 } 5419 5420 /** 5421 * Allocate a counter. 5422 * 5423 * @param[in] dev 5424 * Pointer to Ethernet device structure. 5425 * 5426 * @return 5427 * Pointer to allocated counter on success, NULL otherwise. 5428 */ 5429 struct mlx5_flow_counter * 5430 mlx5_counter_alloc(struct rte_eth_dev *dev) 5431 { 5432 const struct mlx5_flow_driver_ops *fops; 5433 struct rte_flow_attr attr = { .transfer = 0 }; 5434 5435 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5436 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5437 return fops->counter_alloc(dev); 5438 } 5439 DRV_LOG(ERR, 5440 "port %u counter allocate is not supported.", 5441 dev->data->port_id); 5442 return NULL; 5443 } 5444 5445 /** 5446 * Free a counter. 5447 * 5448 * @param[in] dev 5449 * Pointer to Ethernet device structure. 5450 * @param[in] cnt 5451 * Pointer to counter to be free. 5452 */ 5453 void 5454 mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) 5455 { 5456 const struct mlx5_flow_driver_ops *fops; 5457 struct rte_flow_attr attr = { .transfer = 0 }; 5458 5459 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5460 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5461 fops->counter_free(dev, cnt); 5462 return; 5463 } 5464 DRV_LOG(ERR, 5465 "port %u counter free is not supported.", 5466 dev->data->port_id); 5467 } 5468 5469 /** 5470 * Query counter statistics. 5471 * 5472 * @param[in] dev 5473 * Pointer to Ethernet device structure. 5474 * @param[in] cnt 5475 * Pointer to counter to query. 5476 * @param[in] clear 5477 * Set to clear counter statistics. 5478 * @param[out] pkts 5479 * The counter hits packets number to save. 5480 * @param[out] bytes 5481 * The counter hits bytes number to save. 5482 * 5483 * @return 5484 * 0 on success, a negative errno value otherwise. 5485 */ 5486 int 5487 mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt, 5488 bool clear, uint64_t *pkts, uint64_t *bytes) 5489 { 5490 const struct mlx5_flow_driver_ops *fops; 5491 struct rte_flow_attr attr = { .transfer = 0 }; 5492 5493 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 5494 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 5495 return fops->counter_query(dev, cnt, clear, pkts, bytes); 5496 } 5497 DRV_LOG(ERR, 5498 "port %u counter query is not supported.", 5499 dev->data->port_id); 5500 return -ENOTSUP; 5501 } 5502 5503 #define MLX5_POOL_QUERY_FREQ_US 1000000 5504 5505 /** 5506 * Set the periodic procedure for triggering asynchronous batch queries for all 5507 * the counter pools. 5508 * 5509 * @param[in] sh 5510 * Pointer to mlx5_ibv_shared object. 5511 */ 5512 void 5513 mlx5_set_query_alarm(struct mlx5_ibv_shared *sh) 5514 { 5515 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(sh, 0, 0); 5516 uint32_t pools_n = rte_atomic16_read(&cont->n_valid); 5517 uint32_t us; 5518 5519 cont = MLX5_CNT_CONTAINER(sh, 1, 0); 5520 pools_n += rte_atomic16_read(&cont->n_valid); 5521 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 5522 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 5523 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 5524 sh->cmng.query_thread_on = 0; 5525 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 5526 } else { 5527 sh->cmng.query_thread_on = 1; 5528 } 5529 } 5530 5531 /** 5532 * The periodic procedure for triggering asynchronous batch queries for all the 5533 * counter pools. This function is probably called by the host thread. 5534 * 5535 * @param[in] arg 5536 * The parameter for the alarm process. 5537 */ 5538 void 5539 mlx5_flow_query_alarm(void *arg) 5540 { 5541 struct mlx5_ibv_shared *sh = arg; 5542 struct mlx5_devx_obj *dcs; 5543 uint16_t offset; 5544 int ret; 5545 uint8_t batch = sh->cmng.batch; 5546 uint16_t pool_index = sh->cmng.pool_index; 5547 struct mlx5_pools_container *cont; 5548 struct mlx5_pools_container *mcont; 5549 struct mlx5_flow_counter_pool *pool; 5550 5551 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 5552 goto set_alarm; 5553 next_container: 5554 cont = MLX5_CNT_CONTAINER(sh, batch, 1); 5555 mcont = MLX5_CNT_CONTAINER(sh, batch, 0); 5556 /* Check if resize was done and need to flip a container. */ 5557 if (cont != mcont) { 5558 if (cont->pools) { 5559 /* Clean the old container. */ 5560 rte_free(cont->pools); 5561 memset(cont, 0, sizeof(*cont)); 5562 } 5563 rte_cio_wmb(); 5564 /* Flip the host container. */ 5565 sh->cmng.mhi[batch] ^= (uint8_t)2; 5566 cont = mcont; 5567 } 5568 if (!cont->pools) { 5569 /* 2 empty containers case is unexpected. */ 5570 if (unlikely(batch != sh->cmng.batch)) 5571 goto set_alarm; 5572 batch ^= 0x1; 5573 pool_index = 0; 5574 goto next_container; 5575 } 5576 pool = cont->pools[pool_index]; 5577 if (pool->raw_hw) 5578 /* There is a pool query in progress. */ 5579 goto set_alarm; 5580 pool->raw_hw = 5581 LIST_FIRST(&sh->cmng.free_stat_raws); 5582 if (!pool->raw_hw) 5583 /* No free counter statistics raw memory. */ 5584 goto set_alarm; 5585 dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read 5586 (&pool->a64_dcs); 5587 offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; 5588 ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - 5589 offset, NULL, NULL, 5590 pool->raw_hw->mem_mng->dm->id, 5591 (void *)(uintptr_t) 5592 (pool->raw_hw->data + offset), 5593 sh->devx_comp, 5594 (uint64_t)(uintptr_t)pool); 5595 if (ret) { 5596 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 5597 " %d", pool->min_dcs->id); 5598 pool->raw_hw = NULL; 5599 goto set_alarm; 5600 } 5601 pool->raw_hw->min_dcs_id = dcs->id; 5602 LIST_REMOVE(pool->raw_hw, next); 5603 sh->cmng.pending_queries++; 5604 pool_index++; 5605 if (pool_index >= rte_atomic16_read(&cont->n_valid)) { 5606 batch ^= 0x1; 5607 pool_index = 0; 5608 } 5609 set_alarm: 5610 sh->cmng.batch = batch; 5611 sh->cmng.pool_index = pool_index; 5612 mlx5_set_query_alarm(sh); 5613 } 5614 5615 /** 5616 * Handler for the HW respond about ready values from an asynchronous batch 5617 * query. This function is probably called by the host thread. 5618 * 5619 * @param[in] sh 5620 * The pointer to the shared IB device context. 5621 * @param[in] async_id 5622 * The Devx async ID. 5623 * @param[in] status 5624 * The status of the completion. 5625 */ 5626 void 5627 mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, 5628 uint64_t async_id, int status) 5629 { 5630 struct mlx5_flow_counter_pool *pool = 5631 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 5632 struct mlx5_counter_stats_raw *raw_to_free; 5633 5634 if (unlikely(status)) { 5635 raw_to_free = pool->raw_hw; 5636 } else { 5637 raw_to_free = pool->raw; 5638 rte_spinlock_lock(&pool->sl); 5639 pool->raw = pool->raw_hw; 5640 rte_spinlock_unlock(&pool->sl); 5641 rte_atomic64_add(&pool->query_gen, 1); 5642 /* Be sure the new raw counters data is updated in memory. */ 5643 rte_cio_wmb(); 5644 } 5645 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 5646 pool->raw_hw = NULL; 5647 sh->cmng.pending_queries--; 5648 } 5649 5650 /** 5651 * Translate the rte_flow group index to HW table value. 5652 * 5653 * @param[in] attributes 5654 * Pointer to flow attributes 5655 * @param[in] external 5656 * Value is part of flow rule created by request external to PMD. 5657 * @param[in] group 5658 * rte_flow group index value. 5659 * @param[out] fdb_def_rule 5660 * Whether fdb jump to table 1 is configured. 5661 * @param[out] table 5662 * HW table value. 5663 * @param[out] error 5664 * Pointer to error structure. 5665 * 5666 * @return 5667 * 0 on success, a negative errno value otherwise and rte_errno is set. 5668 */ 5669 int 5670 mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, 5671 uint32_t group, bool fdb_def_rule, uint32_t *table, 5672 struct rte_flow_error *error) 5673 { 5674 if (attributes->transfer && external && fdb_def_rule) { 5675 if (group == UINT32_MAX) 5676 return rte_flow_error_set 5677 (error, EINVAL, 5678 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 5679 NULL, 5680 "group index not supported"); 5681 *table = group + 1; 5682 } else { 5683 *table = group; 5684 } 5685 return 0; 5686 } 5687 5688 /** 5689 * Discover availability of metadata reg_c's. 5690 * 5691 * Iteratively use test flows to check availability. 5692 * 5693 * @param[in] dev 5694 * Pointer to the Ethernet device structure. 5695 * 5696 * @return 5697 * 0 on success, a negative errno value otherwise and rte_errno is set. 5698 */ 5699 int 5700 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 5701 { 5702 struct mlx5_priv *priv = dev->data->dev_private; 5703 struct mlx5_dev_config *config = &priv->config; 5704 enum modify_reg idx; 5705 int n = 0; 5706 5707 /* reg_c[0] and reg_c[1] are reserved. */ 5708 config->flow_mreg_c[n++] = REG_C_0; 5709 config->flow_mreg_c[n++] = REG_C_1; 5710 /* Discover availability of other reg_c's. */ 5711 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 5712 struct rte_flow_attr attr = { 5713 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5714 .priority = MLX5_FLOW_PRIO_RSVD, 5715 .ingress = 1, 5716 }; 5717 struct rte_flow_item items[] = { 5718 [0] = { 5719 .type = RTE_FLOW_ITEM_TYPE_END, 5720 }, 5721 }; 5722 struct rte_flow_action actions[] = { 5723 [0] = { 5724 .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5725 .conf = &(struct mlx5_flow_action_copy_mreg){ 5726 .src = REG_C_1, 5727 .dst = idx, 5728 }, 5729 }, 5730 [1] = { 5731 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5732 .conf = &(struct rte_flow_action_jump){ 5733 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5734 }, 5735 }, 5736 [2] = { 5737 .type = RTE_FLOW_ACTION_TYPE_END, 5738 }, 5739 }; 5740 struct rte_flow *flow; 5741 struct rte_flow_error error; 5742 5743 if (!config->dv_flow_en) 5744 break; 5745 /* Create internal flow, validation skips copy action. */ 5746 flow = flow_list_create(dev, NULL, &attr, items, 5747 actions, false, &error); 5748 if (!flow) 5749 continue; 5750 if (dev->data->dev_started || !flow_drv_apply(dev, flow, NULL)) 5751 config->flow_mreg_c[n++] = idx; 5752 flow_list_destroy(dev, NULL, flow); 5753 } 5754 for (; n < MLX5_MREG_C_NUM; ++n) 5755 config->flow_mreg_c[n] = REG_NONE; 5756 return 0; 5757 } 5758 5759 /** 5760 * Dump flow raw hw data to file 5761 * 5762 * @param[in] dev 5763 * The pointer to Ethernet device. 5764 * @param[in] file 5765 * A pointer to a file for output. 5766 * @param[out] error 5767 * Perform verbose error reporting if not NULL. PMDs initialize this 5768 * structure in case of error only. 5769 * @return 5770 * 0 on success, a nagative value otherwise. 5771 */ 5772 int 5773 mlx5_flow_dev_dump(struct rte_eth_dev *dev, 5774 FILE *file, 5775 struct rte_flow_error *error __rte_unused) 5776 { 5777 struct mlx5_priv *priv = dev->data->dev_private; 5778 struct mlx5_ibv_shared *sh = priv->sh; 5779 5780 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, sh->rx_domain, 5781 sh->tx_domain, file); 5782 } 5783